Changeset 37833

Timestamp:

Jan 12, 2015, 12:53:36 PM (11 years ago)

Author:

eugene

Message:

merging changes from ipp-pv3-20140717 (via branches/eam_branches/ipp-pv3-20140717-merge)

Location:

trunk

Files:

• ippScripts/Build.PL (modified) (1 diff, 1 prop)
• ippScripts/MANIFEST (modified) (1 diff, 1 prop)
• ippScripts/scripts (modified) (1 prop)
• ippScripts/scripts/dist_defineruns.pl (modified) (1 diff)
• ippScripts/scripts/lap_science.pl (modified) (5 diffs, 1 prop)
• ippScripts/scripts/permcheck.pl (modified) (1 diff)
• ippScripts/scripts/queuestaticsky.pl (modified) (1 prop)
• ippScripts/scripts/sc_mk_stack_mdc.pl (modified) (1 diff)
• ippScripts/scripts/sc_mk_staticsky_mdc.pl (copied) (copied from branches/eam_branches/ipp-pv3-20140717-merge/ippScripts/scripts/sc_mk_staticsky_mdc.pl )
• ippScripts/scripts/sc_prepare_camera.pl (modified) (16 diffs)
• ippScripts/scripts/sc_prepare_chip.pl (modified) (16 diffs)
• ippScripts/scripts/sc_prepare_run.pl (copied) (copied from branches/eam_branches/ipp-pv3-20140717-merge/ippScripts/scripts/sc_prepare_run.pl )
• ippScripts/scripts/sc_prepare_stack.pl (modified) (15 diffs)
• ippScripts/scripts/sc_prepare_staticsky.pl (copied) (copied from branches/eam_branches/ipp-pv3-20140717-merge/ippScripts/scripts/sc_prepare_staticsky.pl )
• ippScripts/scripts/sc_prepare_warp.pl (modified) (15 diffs)
• ippScripts/scripts/sc_remote_exec.pl (modified) (12 diffs, 1 prop)
• ippScripts/scripts/sc_remote_poll.pl (copied) (copied from branches/eam_branches/ipp-pv3-20140717-merge/ippScripts/scripts/sc_remote_poll.pl )
• ippScripts/scripts/sc_retry_return.pl (copied) (copied from branches/eam_branches/ipp-pv3-20140717-merge/ippScripts/scripts/sc_retry_return.pl )
• ippScripts/scripts/sc_transfer_tool.pl (modified) (6 diffs)
• ippScripts/scripts/stack_skycell.pl (modified) (1 diff)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/laptool_exposures.sql (modified) (1 prop)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff)
• ippTools/share/remotetool_definebyquery_cam.sql (modified) (1 diff)
• ippTools/share/remotetool_definebyquery_chip.sql (modified) (2 diffs)
• ippTools/share/remotetool_definebyquery_stack.sql (modified) (1 diff)
• ippTools/share/remotetool_definebyquery_staticsky.sql (copied) (copied from branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/remotetool_definebyquery_staticsky.sql )
• ippTools/share/remotetool_definebyquery_warp.sql (modified) (1 diff)
• ippTools/share/remotetool_doneprep.sql (copied) (copied from branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/remotetool_doneprep.sql )
• ippTools/share/remotetool_listcomponent.sql (copied) (copied from branches/eam_branches/ipp-pv3-20140717-merge/ippTools/share/remotetool_listcomponent.sql )
• ippTools/src/remotetool.c (modified) (27 diffs)
• ippTools/src/remotetool.h (modified) (1 diff)
• ippTools/src/remotetoolConfig.c (modified) (6 diffs)
• ippTools/src/stacktool.c (modified) (1 prop)
• ippTools/src/stacktoolConfig.c (modified) (1 prop)

trunk/ippScripts/Build.PL

@@ -138 +138 @@
         scripts/stack_bkg_mk_mdc.pl
         scripts/sc_mk_stack_mdc.pl
+        scripts/sc_mk_staticsky_mdc.pl
         scripts/sc_prepare_camera.pl
         scripts/sc_prepare_chip.pl
         scripts/sc_prepare_stack.pl
+        scripts/sc_prepare_staticsky.pl
         scripts/sc_prepare_warp.pl
+        scripts/sc_prepare_run.pl
         scripts/sc_remote_exec.pl
+        scripts/sc_remote_poll.pl
+        scripts/sc_retry_return.pl
         scripts/sc_transfer_tool.pl
     )],

trunk/ippScripts/MANIFEST

@@ -51 +51 @@
 scripts/regenerate_background.pl
 scripts/sc_mk_stack_mdc.pl
+scripts/sc_mk_staticsky_mdc.pl
 scripts/sc_prepare_camera.pl
 scripts/sc_prepare_chip.pl
 scripts/sc_prepare_stack.pl
+scripts/sc_prepare_staticsky.pl
 scripts/sc_prepare_warp.pl
 scripts/sc_remote_exec.pl

trunk/ippScripts/scripts/dist_defineruns.pl

@@ -90 +90 @@
     push @stages, $stage;
 } else {
-    @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle skycal ff);
+    @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle ff);
 }
 

trunk/ippScripts/scripts/lap_science.pl

@@ -34 +34 @@
 my $qstack_threshold     = 0.05; # Only make a quickstack if more than 5% of the exposures require it.
 my $minimum_stack_inputs = 2;    # We can avoid magicking stack inputs if we have more than this number.
-my $maximum_stack_inputs = 25;   # This is the place-holder guess on how big a stack we should process locally
+my $maximum_stack_inputs = 25;   # This is the place-holder guess on how big a stack we should process locally.  25 is the best guess.
 my $do_updates_pv2       = 0;    # Do updates using the calls needed for PV2.  
 
@@ -1058 +1058 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id);
+        &my_die("Unable to perform stacktool -definebyquery: $error_code :: $command", $lap_id);
     }
     my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or
@@ -1066 +1066 @@
     ## Queue the "deep" stacks that will run locally.
     my $deep_minimum = $maximum_stack_inputs + 1;
+    if ($deep_minimum < 2) { $deep_minimum = 2; }
     $command = "$stacktool ";
     $command .= " -pretend " if defined $debug;
@@ -1079 +1080 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id);
-    }
-    my $stacks_made_deep .= $mdcParser->parse_list(join "", @$stdout_buf) or
+        &my_die("Unable to perform stacktool -definebyquery: $error_code :: $command", $lap_id);
+    }
+    my $stacks_made_deep = $mdcParser->parse_list(join "", @$stdout_buf) or
         &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, "");    
 
     # Append deep stacks to stacks so we can get a complete count later.
-    push @{ $stacks_made}, @{ $stacks_made_deep };
+    push @{ $stacks_made }, @{ $stacks_made_deep };
     
     ## Determine the sass_id so we can record it in the database.
@@ -1449 +1450 @@
 
     my @clean_modes = (
-#       'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@',
+        'chiptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -chip_id @CHIP_ID@',
 #       'warptool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -warp_id @WARP_ID@',
         'difftool -dbname @DBNAME@ -updaterun -set_state goto_cleaned -set_label goto_cleaned -label @LABEL@ -diff_id @DIFF_ID@',

trunk/ippScripts/scripts/permcheck.pl

@@ -62 +62 @@
 if ($stage eq 'warp') { # These things are destined to have their only copy on the stsciXX nodes.
     %backup_hosts = ('stsci00' => 1, 'stsci01' => 1, 'stsci02' => 1,
-                     'stsci03' => 1, 'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1,
+                     #'stsci03' => 1, 
+                     'stsci04' => 1, 'stsci05' => 1, 'stsci06' => 1,
                      'stsci07' => 1, 'stsci08' => 1, 'stsci09' => 1,
                      'stsci10' => 1, 'stsci11' => 1, 'stsci12' => 1,

trunk/ippScripts/scripts/sc_mk_stack_mdc.pl

@@ -1 +1 @@
 #! /usr/bin/env perl                                                                                                                                                       
-
 use Carp;
 use warnings;
 use strict;
 
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+my $remote_root = '/scratch3/watersc1/';
+
+my ($compmap_file);
+GetOptions(
+    'compmap=s'   => \$compmap_file,
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Required options: --compmap", -exitval => 3) unless
+    defined($compmap_file);
+
+my %component_map = ();
+open(CM,$compmap_file) || die "Cannot find requested compmap file";
+while(<CM>) {
+    chomp;
+    my ($symbolic_name,$disk_name) = split /\s+/;
+    $component_map{$symbolic_name} = $disk_name;
+}
+close(CM);
+
 my $counter = 0;
 
 foreach my $path_base (@ARGV) {
-    print "INPUT${counter}  METADATA\n";
-    print "   IMAGE           STR     ${path_base}.fits\n";
-    print "   MASK            STR     ${path_base}.mask.fits\n";
-    print "   VARIANCE        STR     ${path_base}.wt.fits\n";
-    print "   PSF             STR     ${path_base}.psf\n";
-    print "   SOURCES         STR     ${path_base}.cmf\n";
-    print "   BKGMODEL        STR     ${path_base}.mdl.fits\n";
-    print "END\n\n";
+    my $mdc_content = "";
+    my $file = "";
+    my $error = 0;
+
+#   print "PB: $path_base\n";
+
+    #header
+    $mdc_content .= "INPUT${counter}  METADATA\n";
+
+    # check each file against its possible locations, incrementing the error counter if not found
+    $file = do_checks("${path_base}.fits",$error);
+    $mdc_content .=  "   IMAGE           STR     $file\n";
+
+    $file = do_checks("${path_base}.mask.fits",$error);
+    $mdc_content .=  "   MASK            STR     $file\n";
+
+    $file = do_checks("${path_base}.wt.fits",$error);
+    $mdc_content .=  "   VARIANCE        STR     $file\n";
+    
+    $file = do_checks("${path_base}.psf",$error);
+    $mdc_content .=  "   PSF             STR     $file\n";
+    
+    $file = do_checks("${path_base}.cmf",$error);
+    $mdc_content .=  "   SOURCES         STR     $file\n";
+
+    # This is static, and isn't used by ppStack, but I believe needs to exist in the mdc. 2014-08-28 czw
+    $mdc_content .=  "   BKGMODEL        STR     ${path_base}.mdl.fits\n";
+    $mdc_content .=  "END\n\n";
+#    print "MDC: $mdc_content\n";
+    # If we had no errors, print it and move on.
+    if ($error == 0) {
+        print $mdc_content;
+    }
+
     $counter++;
 }
+
+sub do_checks {
+    my ($cfile,$error) = @_;
+#    print  ">$cfile<\n";
+    if (! ((-e $cfile)&&(-s $cfile))) { # This cfile does not exist or is zero size.
+        my $disk = $component_map{$cfile};
+        $cfile = "${disk}";
+        if (! ((-e $cfile)&&(-s $cfile))) { # The other expected cfile did not exist.
+#           $cfile = '';
+            $error++;
+            die "Missing cfile $cfile!";
+        }
+    }
+    return($cfile);
+}
+    

trunk/ippScripts/scripts/sc_prepare_camera.pl

@@ -1 +1 @@
 #! /usr/bin/env perl
+
+# generate the input & output files lists and commands for a single camRun
 
 use Carp;
@@ -16 +18 @@
 use Pod::Usage qw( pod2usage );
 
-
 # Hard coded values
-# my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
-my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-$remote_root = '/scratch3/watersc1/';
-my $threads       = 2;                      # How many threads are we going to use?
-my $job_cost      = 1700 / 60 / 60;         # Estimate of how long a job runs, in hours.
-my $proc_per_node = 24;                     # processors per node
-my $min_nodes     = 1;                      # smallest allocation to ask for
-my $max_nodes     = 1000;                   # largest allocation to ask for
-my $min_time      = 1;                      # shortest allocation to ask for
-my $max_time      = 8;                      # longest allocation to ask for
-my $remote_hostname= "LANL/Mustang";         # Name of the remote node.
-my @return_component_list = ("DBINFO.EXP", "PSASTRO.CONFIG", "PSASTRO.OUTPUT", "LOG.EXP", "PSASTRO.STATS");
+# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
+my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
+my $threads_req     = 0;                      # How many threads are we going to use?
+
+my $fail_state = "prep_fail";
+
 # Look for programs we need
 my $missing_tools;
-
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
 my $camtool    = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
@@ -41 +36 @@
 }
 
-my ($remote_id,$cam_id,,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$cam_id,$camera,$dbname,$verbose,$path_base,$no_update);
 GetOptions(
     'remote_id=s'    => \$remote_id,
+    'cam_id=s'       => \$cam_id,
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
@@ -52 +48 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --cam_id --camera --dbname --path_base", -exitval => 3) unless
     defined($remote_id) and
+    defined($cam_id) and
     defined($camera) and
     defined($path_base) and
     defined($dbname);
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
+
+my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $cam_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-#
-# Step 1: Get a list of the components that make up this remoteRun
-
-# SHould this call listrun to ensure we're in state new?
-my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id";
-$rt_cmd   .= " -dbname $dbname " if defined($dbname);
-
-my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-    run(command => $rt_cmd, verbose => $verbose);
-unless ($success) {
-    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to run remotetool to determine stage parameters.",
-            $remote_id,$error_code);
-}
-my $compData = $mdcParser->parse(join "", @$stdout_buf) or
-    &my_die("Unable to determine component information.",
-            $remote_id,$PS_EXIT_PROG_ERROR);
-my $compData2= parse_md_list($compData);
-
-#
-# Step 1b: Open output files
-my $uri_command = $path_base . ".cmd";
+my @return_component_list = ("DBINFO.EXP", "PSASTRO.CONFIG", "PSASTRO.OUTPUT", "LOG.EXP", "PSASTRO.STATS");
+
+# STEP 0: Open output files
 my $uri_transfer= $path_base . ".transfer";
 my $uri_check   = $path_base . ".check";
@@ -89 +68 @@
 my $uri_return  = $path_base . ".return";
 
-my $disk_command = $ipprc->file_resolve($uri_command,1);
 my $disk_transfer= $ipprc->file_resolve($uri_transfer,1);
 my $disk_check   = $ipprc->file_resolve($uri_check,1);
@@ -96 +74 @@
 my $disk_return  = $ipprc->file_resolve($uri_return,1);
 
-my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
-
-open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$remote_id,$PS_EXIT_SYS_ERROR);
-open(CHECK,    ">$disk_check")  || &my_die("Couldn't open file? $disk_check",$remote_id,$PS_EXIT_SYS_ERROR);
-open(CONFIG,   ">$disk_config")  || &my_die("Couldn't open file? $disk_config",$remote_id,$PS_EXIT_SYS_ERROR);
-open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id,$PS_EXIT_SYS_ERROR);
-open(RETURN,   ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR);
+my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it.
+
+open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CHECK,    ">$disk_check")     || &my_die("Couldn't open file? $disk_check",   $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CONFIG,   ">$disk_config")    || &my_die("Couldn't open file? $disk_config",  $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(GENERATE, ">$disk_generate")  || &my_die("Couldn't open file? $disk_generate",$remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(RETURN,   ">$disk_return")    || &my_die("Couldn't open file? $disk_return",  $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state);
 my %file_filter = ();
 
-#
-# Step 2: Iterate over all componenets in this remote run.
 my $job_index = 0;
-my @pre_commands = ();
-my @main_commands = ();
-my @post_commands = ();
-
-foreach my $compEntry (@$compData2) {
-    my $cam_id = $compEntry->{stage_id};
-
-# Get exposure level information from the camRun we're working from.
-    
+
+# STEP 1: Get exposure level information for this camRun
+my ($workdir,$exp_tag,$reduction);
+{
     my $command = "$camtool -pendingexp -cam_id $cam_id ";
     $command   .= " -dbname $dbname " if defined($dbname);
-    
+
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 0);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        
-        &my_die("Unable to run camtool to determine stage parameters.",
-                $cam_id,$error_code);
-    }
-
-    my $camData = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to determine cam component information.",
-                $cam_id,$PS_EXIT_PROG_ERROR);
-    my $camData2= parse_md_list($camData);
-    my $camEntry = ${ $camData2 }[0];
-    
-    my $workdir = $camEntry->{workdir};
-    my $exp_tag = $camEntry->{exp_tag};
+
+        &my_die("Unable to run camtool to determine stage parameters.", $remote_id, $cam_id, $error_code, $fail_state);
+    }
+
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state);
+    my $metadata = parse_md_list($MDlist);
+    my $camEntry = $metadata->[0];
+
+    $workdir   = $camEntry->{workdir};
+    $exp_tag   = $camEntry->{exp_tag};
+    $reduction = $camEntry->{reduction};
+    $reduction = 'DEFAULT' unless defined $reduction;
     unless (defined($workdir)) {
-        while( my ($k, $v) = each %$camEntry ) {
-            print "key: $k, value: $v.\n";
-        }
-
-        print "%{ $camEntry }\n";
-        die;
-    }
-    my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.cm.${cam_id}";
-    my $remote_outroot = uri_local_to_remote($ipp_outroot);
-
-
-#
+        while( my ($k, $v) = each %$camEntry ) {
+            print "key: $k, value: $v.\n";
+        }
+        print "%{ $camEntry }\n";
+        die;
+    }
+}
+
+my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.cm.${cam_id}";
+my $remote_outroot = uri_local_to_remote($ipp_outroot);
+
 # Step 3: Iterate over the sub-components
-
-    my $reg_command = "$camtool -pendingimfile -cam_id $cam_id ";
-    $reg_command   .= " -dbname $dbname " if defined($dbname);
-
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $reg_command, verbose => 0);
+{
+    my $command = "$camtool -pendingimfile -cam_id $cam_id ";
+    $command   .= " -dbname $dbname " if defined($dbname);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 0);
     unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to run camtool -pendingimfile ",
-                $cam_id,$error_code);
-    }
-    
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to run camtool -pendingimfile ", $remote_id, $cam_id, $error_code, $fail_state);
+    }
 
     # We don't actually care about the input cam data other than to know which mask files to instantiate.
-    my $inpData = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to determine cam component information.",
-                $cam_id, $PS_EXIT_PROG_ERROR);
-    my $inpData2=parse_md_list($inpData);
-    my $chipProto= ${ $inpData2 }[0];
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to determine cam component information.", $remote_id, $cam_id, $PS_EXIT_PROG_ERROR, $fail_state);
+
+    my $metadata = parse_md_list($MDlist);
+    my $chipProto = $metadata->[0];
     my $chip_path = $chipProto->{path_base};
-    my $remote_chip_path = uri_local_to_remote( $chip_path );
+    my $remote_chip_path = &uri_local_to_remote( $chip_path );
     my $pre_cmd_cmfs = "ls -1 ${remote_chip_path}*.cmf > ${remote_outroot}.cmflist";
     my $pre_cmd_masks= "ls -1 ${remote_chip_path}*.mk.fits > ${remote_outroot}.masklist";
 
-    my $reduction = 'DEFAULT' unless defined $camEntry->{reduction};
-    my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use                                                                          
+    my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use
+    if ($recipe_psastro eq "") {
+        &my_die("Unable to determine PSASTRO recipe", $cam_id, $PS_EXIT_PROG_ERROR);
+    }
 
     my $psastro_command = " psastro -list ${remote_outroot}.cmflist ";
     $psastro_command   .= " -masklist ${remote_outroot}.masklist ${remote_outroot} ";
-#    $psastro_command   .= " -refmasklist ${remote_outroot}.masklist ${remote_outroot} "; # This is used to do edge calculations.  It should probably be smart enough to know that it can just use the masks I just specified, but it's not.
-    # Uncomment when remote psastro supports this
+#   $psastro_command   .= " -refmasklist ${remote_outroot}.masklist ${remote_outroot} ";
+    $psastro_command   .= " -refmasklist /turquoise/usr/projects/ps1/watersc1/references/gpc1.refmask.list ";
     $psastro_command   .= " -kh-correct /turquoise/usr/projects/ps1/watersc1/references/khcorrect.20140606.v0.fits ";
-    $psastro_command   .= " -refmasklist /turquoise/usr/projects/ps1/watersc1/references/gpc1.refmask.list ";
-#    $psastro_command   .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20080909.asm ";
+#   $psastro_command   .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20080909.asm ";
     $psastro_command   .= " -astrommodel /turquoise/usr/projects/ps1/watersc1/references/gpc1.20140505.asm ";
     $psastro_command   .= " -recipe PSASTRO $recipe_psastro ";
@@ -190 +159 @@
     $psastro_command   .= " -dumpconfig ${remote_outroot}.psastro.mdc -stats ${remote_outroot}.stats ";
     $psastro_command   .= " -recipe PPSTATS CAMSTATS ";
-    
+
     my $camtool_post_cmd = "camtool -cam_id $cam_id -addprocessedexp -uri UNKNOWN ";
-    $camtool_post_cmd  .=  " -dbname $dbname " if defined $dbname;                                                                                                            $camtool_post_cmd  .=  " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 ";
+    $camtool_post_cmd  .=  " -dbname $dbname " if defined $dbname;
+    $camtool_post_cmd  .=  " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 ";
     my $post_cmd_echo = " echo -n \"$camtool_post_cmd\" > ${remote_outroot}.dbinfo ";
     my $post_cmd_SfM  = " ppStatsFromMetadata ${remote_outroot}.stats - CAMERA_EXP_FPA >> ${remote_outroot}.dbinfo ";
@@ -201 +171 @@
     # Determine which output files need to be returned
     foreach my $component(@return_component_list) {
-        my $filename = $ipprc->filename($component,$ipp_outroot);
-        my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename);
-        my $remote_outroot_dir = dirname($ipp_disk);
-        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
-    }
-    foreach my $chipInfo (@{ $inpData2 }) {
-        my $filename = $ipprc->filename("PSASTRO.OUTPUT.MASK",$ipp_outroot,$chipInfo->{class_id});
-        my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename);
-        my $remote_outroot_dir = dirname($ipp_disk);
-        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
+        my $filename = $ipprc->filename($component,$ipp_outroot);
+        my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename);
+        my $remote_outroot_dir = dirname($ipp_disk);
+        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
+    }
+    foreach my $chipInfo (@{ $metadata }) {
+        my $filename = $ipprc->filename("PSASTRO.OUTPUT.MASK",$ipp_outroot,$chipInfo->{class_id});
+        my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename);
+        my $remote_outroot_dir = dirname($ipp_disk);
+        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
     }
     print CONFIG "\n";
@@ -220 +190 @@
 close(GENERATE);
 
-#
-# Construct the moab command last, so we can use the job_index counter to estimate resources.  Somehow.
-my $proc_need = $job_index * $threads;       # how many total processors do we need?
-my $node_need = $proc_need / $proc_per_node; # this equals how many nodes?
-my $time_need = $job_index * $job_cost;      # How many seconds will this take?
-
-my $fill_factor = 0.8;  # This is the factor of how much of the time allocation we'd like to fill
-my ($time_req,$node_req);
-if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) {
-    $time_req = $min_time;
-    $node_req = $min_nodes;
-}
-elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) {
-    $time_req = $max_time;
-    $node_req = $max_nodes;
-    print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job.  This exceeds the max limits ($max_nodes, $max_time).  Using those max values instead.  Good luck.\n";
-}
-else {
-    $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1;
-    $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1;
-}
-$time_req += 1; # Safety addition.
-
-open(COMMAND,  ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR);
-print COMMAND "#!/bin/tcsh\n";
-print COMMAND "##### Moab controll lines\n";
-print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES
-print COMMAND "#MSUB -j oe\n";
-print COMMAND "#MSUB -V\n";
-print COMMAND "#MSUB -o ${remote_root}cam.${remote_id}.out\n";
-print COMMAND "date\n";
-print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n";
-print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n";
-print COMMAND "date\n";
-close(COMMAND);
-
 unless($no_update) {
-    my $command = "remotetool -updaterun -remote_id $remote_id ";
-    $command .= " -set_state pending ";
-    $command .= " -dbname $dbname " if defined $dbname;
-
+    my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $cam_id ";
+    $command   .= " -set_jobs $job_index";
+    $command   .= " -set_path_base $path_base";
+    $command   .= " -set_state prep_done";
+    $command   .= " -dbname $dbname " if defined $dbname;
     system($command);
 }
 
+exit (0);
 
 ## Common SC routines
@@ -271 +207 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri );
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -287 +223 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $cam_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -302 +238 @@
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
-    
+
     unless (exists($file_filter{$neb_uri})) {
         $file_filter{$neb_uri} = 1;
@@ -332 +268 @@
     return($remote_uri);
 }
- 
+
 sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri);
     # This needs to replace the remote root directory with the nebulous tag.
@@ -338 +274 @@
     $remote_uri =~ s%${remote_root}%%;
     my $local_uri  = "neb:///" . $remote_uri;
-    
+
     return($local_uri);
 }
@@ -345 +281 @@
 sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status);
     my $msg = shift;
-    my $id  = shift;
+    my $remote_id  = shift;
+    my $stage_id  = shift;
     my $exit_code = shift;
     my $exit_state = shift;
@@ -352 +289 @@
 
     carp($msg);
-    
-    if (defined $id and not $no_update) {
-        my $command = "remotetool -updaterun -remote_id $id";
-        $command .= " -fault $exit_code " if defined $exit_code;
+
+    if (defined $remote_id and defined $stage_id and not $no_update) {
+        my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id";
         $command .= " -set_state $exit_state " if defined $exit_state;
         $command .= " -dbname $dbname " if defined $dbname;

trunk/ippScripts/scripts/sc_prepare_chip.pl

@@ -1 +1 @@
 #! /usr/bin/env perl
+
+# generate the input & output files lists and commands for a single chipRun
 
 use Carp;
@@ -16 +18 @@
 use Pod::Usage qw( pod2usage );
 
-
 # Hard coded values
-# my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
-my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-$remote_root = '/scratch3/watersc1/';
-my $remote_raw    = "${remote_root}/raw/";  # Directory to find raw data in.
-my $threads       = 2;                      # How many threads are we going to use?
-my $job_cost      = 150 / 60 / 60;          # Estimate of how long a job runs, in hours.
-my $proc_per_node = 24;                     # processors per node
-my $min_nodes     = 1;                      # smallest allocation to ask for
-my $max_nodes     = 1000;                   # largest allocation to ask for
-my $min_time      = 1;                      # shortest allocation to ask for
-my $max_time      = 8;                      # longest allocation to ask for
-
-# We need to ensure we only ever try to transfer a file once.
-my %file_filter = ();
+# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
+my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
+my $remote_raw      = "${remote_root}/tmp/";  # Directory to find raw data in.
+my $threads_req     = 4;
+
+my $fail_state = "prep_fail";
+my $hard_fail_state = "fail";
 
 # Look for programs we need
 my $missing_tools;
-my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
 my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1);
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
+my $ipp_burntool_fix = can_run('ipp_apply_burntool_fix.pl') or (warn "Can't find ipp_apply_burntool_fix.pl" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -46 +41 @@
 
 # Options
-my ($exp_id,$exp_name,$remote_id,$chip_id,$detrends,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose);
 GetOptions(
     'remote_id=s'    => \$remote_id,
+    'chip_id=s'      => \$chip_id,
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
@@ -54 +50 @@
     'no_update'      => \$no_update,
     'verbose'        => \$verbose,
+    'dbverbose'      => \$dbverbose,
     ) or pod2usage( 2 );
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --chip_id --camera --dbname --path_base", -exitval => 3) unless
     defined($remote_id) and
+    defined($chip_id) and
     defined($camera) and
     defined($path_base) and
     defined($dbname);
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id);
+
+my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-#
-# Step 1: Get a list of the components that make up this remoteRun
-
-# SHould this call listrun to ensure we're in state new?
-my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id";
-$rt_cmd   .= " -dbname $dbname " if defined($dbname);
-
-my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-    run(command => $rt_cmd, verbose => $verbose);
-unless ($success) {
-    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    
-    &my_die("Unable to run chiptool to determine stage parameters.",
-            $remote_id,$error_code);
-}
-my $compData = $mdcParser->parse(join "", @$stdout_buf) or
-    &my_die("Unable to determine component information.",
-            $remote_id,$PS_EXIT_PROG_ERROR);
-my $compData2= parse_md_list($compData);
-
-#
-# Step 1a: Load up the catalog of detrends 
-# Detrend concept holders
-my %detrends = ();
-my %det_types = ('MASK' => '-mask',
-                 'FLAT' => '-flat',
-                 'DARK' => '-dark',
-                 'VIDEODARK' => '-dark',
-                 'LINEARITY' => '-linearity',
-                 'FRINGE'    => '-fringe',
-                 'NOISEMAP'  => '-noisemap');
-
-foreach my $det_type (keys (%det_types)) {
-    my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type ";
-    $dt_cmd   .= " -dbname $dbname " if defined($dbname);
-
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $dt_cmd, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("UNable to run detselect to determine detrend catalog",
-                $remote_id,$error_code);
-    }
-    my $detData = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to determine detrend information.",
-                $remote_id,$PS_EXIT_PROG_ERROR);
-    my $detData2= parse_md_list($detData);
-
-    # Data structure for this:
-    # $detrend{NAME} = \@list_of_detselect_shows
-    # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element)
-    # $detrend{NAME}[0]{class_id} = uri
-
-    # At this point, I have the list of detselect shows, assign it
-    $detrends{$det_type} = $detData2;
-
-    for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) {
-        my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id};
-        my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration};
-
-        my $detselect_command2 = "detselect -select ";
-        $detselect_command2   .= " -det_id $det_id ";
-        $detselect_command2   .= " -iteration $det_iter ";
-        $detselect_command2   .= " -dbname $dbname " if defined($dbname);
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $detselect_command2, verbose => 0);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            
-            &my_die("No valid detrend available for this image: $det_type $det_id $det_iter",
-                    $chip_id,$error_code);
-        }
-        
-        my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or 
-            &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter",
-                    $chip_id,$error_code);
-        my $detImfile2= parse_md_list($detImfile);
-        foreach $detImfile (@$detImfile2) {
-            my $class_id = $detImfile->{class_id};
-            my $duri = $detImfile->{uri};
-            
-            ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri;
-        }
-    }
-}
-
-#print Dumper(%detrends);
+# Detrend concept holders (name, ppImage option)
+my %det_types = (
+    'MASK'      => '-mask',
+    'FLAT'      => '-flat',
+    'DARK'      => '-dark',
+    'VIDEODARK' => '-dark',
+    'LINEARITY' => '-linearity',
+    'FRINGE'    => '-fringe',
+    'NOISEMAP'  => '-noisemap'
+    );
 
 my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN");
 
-#
-# Step 1b: Open output files
-my $uri_command = $path_base . ".cmd";
+# STEP 0: Open output files
 my $uri_transfer= $path_base . ".transfer";
 my $uri_check   = $path_base . ".check";
@@ -165 +85 @@
 my $uri_return  = $path_base . ".return";
 
-my $disk_command = $ipprc->file_resolve($uri_command,1);
 my $disk_transfer= $ipprc->file_resolve($uri_transfer,1);
 my $disk_check   = $ipprc->file_resolve($uri_check,1);
@@ -174 +93 @@
 my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
 
-open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR);
-open(CHECK,    ">$disk_check")  || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR);
-open(CONFIG,   ">$disk_config")  || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR);
-open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR);
-open(RETURN,   ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR);
-
-
-#
-# Step 2: Iterate over all componenets in this remote run.
+open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CHECK,    ">$disk_check")     || &my_die("Couldn't open file? $disk_check",    $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CONFIG,   ">$disk_config")    || &my_die("Couldn't open file? $disk_config",   $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(GENERATE, ">$disk_generate")  || &my_die("Couldn't open file? $disk_generate", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(RETURN,   ">$disk_return")    || &my_die("Couldn't open file? $disk_return",   $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+
 my $job_index = 0;
-my @pre_commands = ();
-my @main_commands = ();
-my @post_commands = ();
-
-foreach my $compEntry (@$compData2) {
-
-    my $chip_id = $compEntry->{stage_id};
-    my $exp_id ;
-# Get exposure level information from the chipRun we're working from.
+
+# STEP 1 : Get exposure level information for this chipRun
+my ($exp_id, $filter, $altfilt, $dateobs);
+{
     my $command = "$chiptool -listrun -chip_id $chip_id ";
     $command   .= " -dbname $dbname " if defined($dbname);
-    
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 0);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $dbverbose);
     unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        
-        &my_die("Unable to run chiptool to determine stage parameters.",
-                $chip_id,$error_code);
-    }
-    
-# Parse chipRun level data to determine detrend information 
-    my %detrends_to_use = ();
-    {
-        my $chipData = $mdcParser->parse(join "", @$stdout_buf) or
-            &my_die("Unable to determine chip component information.",
-                    $chip_id,$PS_EXIT_PROG_ERROR);
-        my $chipData2= parse_md_list($chipData);
-        my $chipEntry = ${ $chipData2 }[0];
-        
-        $exp_id = $chipEntry->{exp_id};    
-        my $filter = $chipEntry->{filter};
-        my $altfilt= $filter;
-        $altfilt =~ s/.00000//;
-        my $dateobs= $chipEntry->{dateobs};
-        my ($date, $time) = split /T/, $dateobs;
-        my ($year,$month,$day) = split /-/, $date;
-        my ($hour,$minute,$second) = split /:/, $time; #/;
-        my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second);
-
-        foreach my $det_type (%detrends) {
-            if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y
-
-            foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options
-                if (defined($det_obj->{filter})) { 
-                    if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) {
-                        next; # Skip detrends with a filter that we can't use.
-                    }
-                }
-                if (defined($det_obj->{time_begin})) {
-                    if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) {
-                        next; # Skip detrends that come into effect after this exposure (d1 < d2)
-                    }
-                }
-                if (defined($det_obj->{time_end})) {
-                    if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) {
-                        next; # Skip detrends that stop working before this exposure (d1 > d2)
-                    }
-                }
-                $detrends_to_use{$det_type} = $det_obj;
-                last;
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+        &my_die("Unable to run chiptool to determine stage parameters.", $remote_id, $chip_id, $error_code, $fail_state);
+    }
+
+    # Parse chipRun level data to determine exposure information
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to determine chip component information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
+    my $metadata = parse_md_list($MDlist);
+    my $chipEntry = $metadata->[0];
+
+    $exp_id  = $chipEntry->{exp_id};
+    $filter  = $chipEntry->{filter};
+    $altfilt = $filter;
+    $altfilt =~ s/.00000//;
+    $dateobs = $chipEntry->{dateobs};
+}
+
+# STEP 2 : select the det_ids for each desired det_type (appropriate to this exposure)
+my %det_ids = ();
+my %det_iters = ();
+
+foreach my $det_type (keys (%det_types)) {
+    if (($filter !~ /y/) && ($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y
+
+    my $command = "detselect -search -det_type $det_type -time $dateobs";
+    $command   .= " -dbname $dbname " if defined($dbname);
+
+    if (($det_type eq 'FLAT') || ($det_type eq 'FRINGE')) {
+        $command .= " -filter $altfilt";
+    }
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $dbverbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+        &my_die("No valid detrend available for this image: $det_type", $remote_id, $chip_id, $error_code, $fail_state);
+    }
+
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Could not parse detrend information for this image: $det_type", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
+    my $metadata = parse_md_list($MDlist);
+    my $detEntry = $metadata->[0];
+
+    if ($verbose) {
+        print STDERR "det_id: $detEntry->{det_id}\n";
+        print STDERR "det_iter: $detEntry->{iteration}\n";
+    }
+
+    # save det_id and iteration
+    $det_ids{$det_type} = $detEntry->{det_id};
+    $det_iters{$det_type} = $detEntry->{iteration};
+}
+
+# Step 3: Iterate over the sub-components
+{
+    # select chipProcessedImfile entries
+    my $command = "$chiptool -pendingimfile -chip_id $chip_id";
+    $command   .= " -dbname $dbname "   if defined($dbname);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $dbverbose);
+
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+        &my_die("Unable to run chiptool -pendingimfile", $remote_id, $chip_id, $error_code, $fail_state);
+    }
+
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse chiptool -pendingimfile information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
+    my $metadata = parse_md_list($MDlist);
+
+    # Iterate over the chipProcessedImfile level data.
+    foreach my $chipEntry (@$metadata) {
+        # Get information we need to pass to ppImage
+        my $uri            = $chipEntry->{uri};
+        my $class_id       = $chipEntry->{class_id};
+        my $video_cells    = $chipEntry->{video_cells};
+        my $reduction      = $chipEntry->{reduction};
+        my $exp_tag        = $chipEntry->{exp_tag};
+        my $workdir        = $chipEntry->{workdir};
+        my $chip_imfile_id = $chipEntry->{chip_imfile_id};
+
+        # Process the image and burntool table
+        # IMAGE CHECK: Look up the image location, prove it exists, and if not, abort the prep for this object.
+        my ($ipp_uri, $remote_uri) = uri_convert($uri);
+        unless (-e $ipp_uri) {
+            # Try to fix it.
+            system("neb-repair $uri");
+            ($ipp_uri, $remote_uri) = uri_convert($uri);
+            unless (-e $ipp_uri) {
+                # This image file is missing, so it will require manual intervention.
+                &my_die("Required image file $ipp_uri ($chip_id, $class_id) not found.  Stopping this prep.",
+                        $remote_id,$chip_id,$PS_EXIT_CONFIG_ERROR,$hard_fail_state);
             }
         }
-    }
-
-#
-# Step 3: Iterate over the sub-components
-# Iterate over the chipProcessedImfile level data.
-    my $reg_command = "$chiptool -pendingimfile ";
-    $reg_command   .= " -chip_id $chip_id " if defined($chip_id);
-    $reg_command   .= " -dbname $dbname "   if defined($dbname);
-    
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $reg_command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        
-        &my_die("Unable to run chiptool -pendingimfile",
-                $chip_id,$error_code);
-    }
-    
-    my $chipData = $mdcParser->parse(join "", @$stdout_buf) or 
-        next;
-#       &my_die("Unable to parse chiptool -pendingimfile information.",
-#               $chip_id,$error_code);
-    my $chipData2= parse_md_list($chipData);
-
-    
-    foreach my $chipEntry (@$chipData2) {
-        # Get information we need to pass to ppImage
-        my $uri            = $chipEntry->{uri};
-        my $class_id       = $chipEntry->{class_id};
-        my $video_cells    = $chipEntry->{video_cells};
-        my $reduction      = $chipEntry->{reduction};
-        my $exp_tag        = $chipEntry->{exp_tag};
-        my $workdir        = $chipEntry->{workdir};
-        my $chip_imfile_id = $chipEntry->{chip_imfile_id};
-        
-        # Process the image and burntool table
-        my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri);
-        my $btt = $uri;
-        $btt =~ s/fits$/burn.tbl/;
-        my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt);
-        
-        # Initialize the ppI command
-        my $ppImage_command = "ppImage -file $remote_uri";
-        $ppImage_command   .= " -burntool $remote_btt ";
-        
-        # Add detrend information to the command line
-        foreach my $det (keys %detrends_to_use) {
-            if ((($video_cells)&&($det eq 'DARK'))||
-                ((!$video_cells)&&($det eq 'VIDEODARK'))) {
-                next;
+
+        # The image exists, so continue.  This resets the variables, but also outputs the file locations.
+        ($ipp_uri, $remote_uri) = uri_to_outputs_raw($uri);
+        my $btt = $uri;
+        $btt =~ s/fits$/burn.tbl/;
+
+        # Check burntool table for existance, and if it doesn't, regenerate it.
+        my ($ipp_btt, $remote_btt) = uri_convert($btt);
+        unless (-e $ipp_btt) {
+            my $exp_name = $chipEntry->{exp_name};
+            my $burntool_command = "$ipp_burntool_fix --exp_name $exp_name --class_id $class_id";
+            $burntool_command   .= " -dbname $dbname" if defined($dbname);
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $burntool_command, verbose => $dbverbose);
+            # That command repairs the tables, but the instance we learned about may not be correct.  
+            # Look up the new one.
+            ($ipp_btt, $remote_btt) = uri_convert($btt);
+            
+            unless (-e $ipp_btt) {
+                &my_die("Attempted regeneration of burntool table ($chip_id, $class_id) has failed.  Stopping this prep.", 
+                        $remote_id,$chip_id, $PS_EXIT_CONFIG_ERROR, $hard_fail_state);
             }
-            my $det_code = $det_types{$det};
-            my $duri = $detrends_to_use{$det}->{$class_id};
-            my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
-            $ppImage_command .= " $det_code $remote_det_uri ";
         }
-            
-        
-        # Add output root
-        my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}";
-        my $remote_outroot = uri_local_to_remote($ipp_outroot);
-        $ppImage_command .= " $remote_outroot ";
-        print STDERR "$remote_outroot $ipp_outroot $class_id\n";
-        # Complete reduction information.
-        $reduction = 'DEFAULT' unless defined $reduction;
-        my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage
-        my $recipe_psphot  = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot
-        
-        $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage ";
-        $ppImage_command .= " -recipe PSPHOT $recipe_psphot ";
-        $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats ";
-        $ppImage_command .= " -threads 4 ";
-        $ppImage_command .= " -image_id $chip_imfile_id ";
-        $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace ";
-        $ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
-
-#       push @main_commands, $ppImage_command;
-
-        # Calculate pre and post commands
-        my $remote_outroot_dir = dirname($remote_outroot);
-        my $pre_command =  "mkdir -p $remote_outroot_dir";
-        
-        my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
-        $post_commandA   .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot ";
-        $post_commandA   .= " -magicked 0 -hostname REMOTE -dtime_script 0 ";
-        $post_commandA   .= " -dbname $dbname " if defined $dbname;
-
-        my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
-        my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
-        
-#       push @post_commands, "$post_commandB $post_commandC";
-
-        $job_index++;
-
-        print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
-
-
-        # Determine which output files need to be returned.
-        foreach my $component (@return_component_list) {
-#           uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) );
-            my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
-            my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
-            my $remote_outroot_dir = dirname($ipp_disk);
-            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
-#           print "  $filename $ipp_disk $remote_disk\n";
-#           die;
-        }
-
-        print CONFIG "\n";
-    }
-}
-
-# Actually put the commands into the output file.
-# my %unique_cmds = ();
-# foreach my $cmd (@pre_commands) {
-#     unless(exists($unique_cmds{$cmd})) {
-#       print CONFIG $cmd . "\n";
-#       $unique_cmds{$cmd} = 1;
-#     }
-# }
-# foreach my $cmd (@main_commands) {
-#     unless(exists($unique_cmds{$cmd})) {
-#       print CONFIG $cmd . "\n";
-#       $unique_cmds{$cmd} = 1;
-#     }
-# }
-# foreach my $cmd (@post_commands) {
-#     unless(exists($unique_cmds{$cmd})) {
-#       print CONFIG $cmd . "\n";
-#       $unique_cmds{$cmd} = 1;
-#     }
-# }
+        ($ipp_btt, $remote_btt) = uri_to_outputs_raw($btt);
+
+        # Initialize the ppI command
+        my $ppImage_command = "ppImage -file $remote_uri";
+        $ppImage_command   .= " -burntool $remote_btt ";
+
+        foreach my $det_type (keys (%det_types)) {
+            if (( $video_cells) && ($det_type eq 'DARK')) { next; }
+            if ((!$video_cells) && ($det_type eq 'VIDEODARK')) { next; }
+
+            my $det_id   = $det_ids{$det_type};
+            my $det_iter = $det_iters{$det_type};
+
+            if (not defined $det_id) { next; }
+
+            if ($verbose) { print STDERR "det_type, det_id, det_iter: $det_type, $det_id, $det_iter\n"; }
+
+            # Add detrend information to the command line
+            my $detselect_command = "detselect -select ";
+            $detselect_command   .= " -det_id $det_id";
+            $detselect_command   .= " -iteration $det_iter";
+            $detselect_command   .= " -class_id $class_id";
+            $detselect_command   .= " -dbname $dbname " if defined($dbname);
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $detselect_command, verbose => $dbverbose);
+
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+                &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $error_code, $fail_state);
+            }
+
+            my $detMDlist = $mdcParser->parse(join "", @$stdout_buf) or
+                &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
+            my $detImfileMD = parse_md_list($detMDlist);
+            my $detImfile = $detImfileMD->[0];
+            my $duri = $detImfile->{uri};
+
+            my $det_code = $det_types{$det_type};
+            my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
+            $ppImage_command .= " $det_code $remote_det_uri ";
+        }
+
+        # Add output root
+        my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}";
+        my $remote_outroot = uri_local_to_remote($ipp_outroot);
+        $ppImage_command .= " $remote_outroot ";
+        print STDERR "$remote_outroot $ipp_outroot $class_id\n";
+        # Complete reduction information.
+        $reduction = 'DEFAULT' unless defined $reduction;
+        my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage
+        my $recipe_psphot  = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot
+
+        $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage ";
+        $ppImage_command .= " -recipe PSPHOT $recipe_psphot ";
+        $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats ";
+        $ppImage_command .= " -threads $threads_req ";
+        $ppImage_command .= " -image_id $chip_imfile_id ";
+        $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace ";
+        $ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
+
+        # Calculate pre and post commands
+        my $remote_outroot_dir = dirname($remote_outroot);
+        my $pre_command =  "mkdir -p $remote_outroot_dir";
+
+        my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
+        $post_commandA   .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot ";
+        $post_commandA   .= " -magicked 0 -hostname $remote_hostname -dtime_script 0 ";
+        $post_commandA   .= " -dbname $dbname " if defined $dbname;
+
+        my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
+        my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
+
+        $job_index++;
+
+        print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
+
+        # Determine which output files need to be returned.
+        foreach my $component (@return_component_list) {
+            my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
+            my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
+            my $remote_outroot_dir = dirname($ipp_disk);
+            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
+        }
+        print CONFIG "\n";
+    }
+}
 
 close(CONFIG);
@@ -386 +326 @@
 close(GENERATE);
 
-#
-# Construct the moab command last, so we can use the job_index counter to estimate resources.  Somehow.
-my $proc_need = $job_index * $threads;       # how many total processors do we need?
-my $node_need = $proc_need / $proc_per_node; # this equals how many nodes?
-my $time_need = $job_index * $job_cost;      # How many seconds will this take?
-
-my $fill_factor = 0.8;  # This is the factor of how much of the time allocation we'd like to fill
-my ($time_req,$node_req);
-if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) {
-    $time_req = $min_time;
-    $node_req = $min_nodes;
-}
-elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) {
-    $time_req = $max_time;
-    $node_req = $max_nodes;
-    print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job.  This exceeds the max limits ($max_nodes, $max_time).  Using those max values instead.  Good luck.\n";
-}
-else {
-    $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1;
-    $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1;
-}
-$time_req += 1; # Safety addition.
-
-open(COMMAND,  ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR);
-print COMMAND "#!/bin/tcsh\n";
-print COMMAND "##### Moab controll lines\n";
-print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES
-print COMMAND "#MSUB -j oe\n";
-print COMMAND "#MSUB -V\n";
-print COMMAND "#MSUB -o ${remote_root}chip.${remote_id}.out\n";
-print COMMAND "date\n";
-print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n";
-print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n";
-#print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n";
-print COMMAND "date\n";
-close(COMMAND);
-
-
 ## We're done here. The execution and handling are done elsewhere.
 # Quick review:
@@ -429 +331 @@
 # auth
 unless($no_update) {
-    my $command = "remotetool -updaterun -remote_id $remote_id ";
-    $command .= " -set_state pending ";
+    my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $chip_id ";
+    $command .= " -set_jobs $job_index";
+    $command .= " -set_path_base $path_base";
+    $command .= " -set_state prep_done";
     $command .= " -dbname $dbname " if defined $dbname;
 
     system($command);
 }
-
-
-    
-
-
+exit (0);
 
 sub uri_convert {
@@ -445 +345 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri );
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -461 +361 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -476 +376 @@
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
-    
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print TRANSFER "$ipp_disk\n";
-        print CHECK    "$remote_disk\n";
-    }
+
+    print TRANSFER "$ipp_disk\n";
+    print CHECK    "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -489 +386 @@
     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
     $remote_disk = $remote_raw . $ipp_disk;
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print TRANSFER "$ipp_disk\n";
-        print CHECK    "$remote_disk\n";
-    }
+
+    print TRANSFER "$ipp_disk\n";
+    print CHECK    "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -500 +395 @@
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri );
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print RETURN "$ipp_disk\n";
-        print GENERATE "$remote_disk\n";
-    }
+    print RETURN "$ipp_disk\n";
+    print GENERATE "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -518 +410 @@
     return($remote_uri);
 }
- 
+
 sub uri_remote_to_local {
     # This needs to replace the remote root directory with the nebulous tag.
@@ -524 +416 @@
     $remote_uri =~ s%${remote_root}%%;
     my $local_uri  = "neb:///" . $remote_uri;
-    
+
     return($local_uri);
 }
+
 sub my_die {
     my $msg = shift;
-    my $id  = shift;
+    my $remote_id  = shift;
+    my $stage_id  = shift;
     my $exit_code = shift;
     my $exit_state = shift;
@@ -536 +430 @@
 
     carp($msg);
-    
-    if (defined $id and not $no_update) {
-        my $command = "remotetool -updaterun -remote_id $id";
-        $command .= " -fault $exit_code " if defined $exit_code;
-        $command .= " -set_state $exit_state " if defined $exit_state;
-        $command .= " -dbname $dbname " if defined $dbname;
-
-        system($command);
+
+    if (defined $remote_id and defined $stage_id and not $no_update) {
+        my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id";
+        $command .= " -set_state $exit_state " if defined $exit_state;
+        $command .= " -dbname $dbname " if defined $dbname;
+
+        system($command);
     }
 

trunk/ippScripts/scripts/sc_prepare_stack.pl

@@ -1 +1 @@
 #! /usr/bin/env perl
+
+# generate the input & output files lists and commands for a single stackRun
 
 use Carp;
@@ -16 +18 @@
 use Pod::Usage qw( pod2usage );
 
-
 # Hard coded values
-#my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
-my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-$remote_root = '/scratch3/watersc1/';
-my $threads       = 2;                      # How many threads are we going to use?
-my $job_cost      = 1500 / 60 / 60;           # Estimate of how long a job runs, in hours.
-my $proc_per_node = 24;                     # processors per node
-my $min_nodes     = 1;                      # smallest allocation to ask for
-my $max_nodes     = 1000;                   # largest allocation to ask for
-my $min_time      = 1;                      # shortest allocation to ask for
-my $max_time      = 8;                      # longest allocation to ask for
-my $remote_hostname   = "LANL/Mustang";         # Name of the remote node.
-my @return_component_list = ("DBINFO.EXP", "PPSTACK.UNCONV","PPSTACK.UNCONV.MASK","PPSTACK.UNCONV.VARIANCE",
-                             "PPSTACK.UNCONV.EXP","PPSTACK.UNCONV.EXPWT","PPSTACK.UNCONV.EXPNUM","PSPHOT.OUTPUT",
-                             "LOG.EXP", "PSASTRO.STATS",
-                             "PPSTACK.CONFIG","PPSTACK.TARGET.PSF","PSPHOT.STACK.PSF.SAVE");
-#,"PPSTACK.CONV.KERNEL");
+# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
+my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
+my $threads_req     = 4;                      # How many threads are we going to use?
+
+my $fail_state = "prep_fail";
+
 # Look for programs we need
 my $missing_tools;
-
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
-my $stacktool    = can_run('stacktool') or (warn "Can't find warptool" and $missing_tools = 1);
+my $stacktool    = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -48 +39 @@
 GetOptions(
     'remote_id=s'    => \$remote_id,
+    'stack_id=s'     => \$stack_id,
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
@@ -56 +48 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --stack_id --camera --dbname --path_base", -exitval => 3) unless
     defined($remote_id) and
+    defined($stack_id) and
     defined($camera) and
     defined($path_base) and
     defined($dbname);
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
+
+my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $stack_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-#
-# Step 1: Get a list of the components that make up this remoteRun
-
-# SHould this call listrun to ensure we're in state new?
-my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id";
-$rt_cmd   .= " -dbname $dbname " if defined($dbname);
-
-my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-    run(command => $rt_cmd, verbose => $verbose);
-unless ($success) {
-    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to run remotetool to determine stage parameters.",
-            $remote_id,$error_code);
-}
-my $compData = $mdcParser->parse(join "", @$stdout_buf) or
-    &my_die("Unable to determine component information.",
-            $remote_id,$PS_EXIT_PROG_ERROR);
-my $compData2= parse_md_list($compData);
-
-#
-# Step 1b: Open output files
-my $uri_command = $path_base . ".cmd";
+my @return_component_list = ("DBINFO.EXP", "PPSTACK.UNCONV","PPSTACK.UNCONV.MASK","PPSTACK.UNCONV.VARIANCE",
+                             "PPSTACK.UNCONV.EXP","PPSTACK.UNCONV.EXPWT","PPSTACK.UNCONV.EXPNUM",
+                             "PSPHOT.OUT.CMF.MEF", # Use this output product to get the one that exists.
+                             "LOG.EXP", "PSASTRO.STATS",
+                             "PPSTACK.CONFIG","PPSTACK.TARGET.PSF","PSPHOT.STACK.PSF.SAVE");
+#,"PPSTACK.CONV.KERNEL");
+
+# STEP 0: Open output files
 my $uri_transfer= $path_base . ".transfer";
 my $uri_check   = $path_base . ".check";
@@ -93 +73 @@
 my $uri_return  = $path_base . ".return";
 
-my $disk_command = $ipprc->file_resolve($uri_command,1);
 my $disk_transfer= $ipprc->file_resolve($uri_transfer,1);
 my $disk_check   = $ipprc->file_resolve($uri_check,1);
@@ -100 +79 @@
 my $disk_return  = $ipprc->file_resolve($uri_return,1);
 
-my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
-
-open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$remote_id,$PS_EXIT_SYS_ERROR);
-open(CHECK,    ">$disk_check")  || &my_die("Couldn't open file? $disk_check",$remote_id,$PS_EXIT_SYS_ERROR);
-open(CONFIG,   ">$disk_config")  || &my_die("Couldn't open file? $disk_config",$remote_id,$PS_EXIT_SYS_ERROR);
-open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id,$PS_EXIT_SYS_ERROR);
-open(RETURN,   ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR);
-
-
-#
-# Step 2: Iterate over all componenets in this remote run.
+my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it.
+
+open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CHECK,    ">$disk_check")     || &my_die("Couldn't open file? $disk_check",   $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CONFIG,   ">$disk_config")    || &my_die("Couldn't open file? $disk_config",  $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(GENERATE, ">$disk_generate")  || &my_die("Couldn't open file? $disk_generate",$remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(RETURN,   ">$disk_return")    || &my_die("Couldn't open file? $disk_return",  $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state);
+
 my $job_index = 0;
-my @pre_commands = ();
-my @main_commands = ();
-my @post_commands = ();
-my %file_filter = ();
-
-
-foreach my $compEntry (@$compData2) {
-    my $stack_id = $compEntry->{stage_id};
-
-# Get exposure level information from the stackRun we're working from.
-    my $run_command = "$stacktool -tosum -stack_id $stack_id";
-    $run_command .= " -dbname $dbname " if defined($dbname);
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $run_command, verbose => 0);
+
+# STEP 1: Get exposure level information from the stackRun we're working from.
+my ($reduction, $workdir, $tess_id, $skycell_id);
+{
+    my $command = "$stacktool -tosum -stack_id $stack_id";
+    $command .= " -dbname $dbname " if defined($dbname);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
     unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to run stacktool -tosum ",
-                $stack_id,$error_code);
-    }
-
-    my $stackData = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to determine stack component information.",
-                $stack_id,$PS_EXIT_PROG_ERROR);
-    my $stackData2= parse_md_list($stackData);
-
-    my $stack = ${ $stackData2 }[0];
-    my $reduction = $stack->{reduction};
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to run stacktool -tosum ", $remote_id, $stack_id, $error_code, $fail_state);
+    }
+
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state);
+    my $metadata = parse_md_list($MDlist);
+
+    my $stack = $metadata->[0];
+
+    $reduction = $stack->{reduction};
     $reduction = 'DEFAULT' unless defined($reduction);
-    my $workdir   = $stack->{workdir};
-    my $tess_id   = $stack->{tess_id};
-    my $skycell_id= $stack->{skycell_id};
-
-    my $ipp_outroot    = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}";    
-    my $remote_outroot = uri_local_to_remote($ipp_outroot);    
-    my $remote_outdir  = uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}");
+    $workdir   = $stack->{workdir};
+    $tess_id   = $stack->{tess_id};
+    $skycell_id= $stack->{skycell_id};
+}
+my $ipp_outroot    = "${workdir}/${tess_id}/${skycell_id}/${tess_id}.${skycell_id}.stk.${stack_id}";
+my $remote_outroot = &uri_local_to_remote($ipp_outroot);
+my $remote_outdir  = &uri_local_to_remote("${workdir}/${tess_id}/${skycell_id}");
+
+# STEP 2: Get the list of input warps
+my ($warpData);
+{
     # This actually returns all the individual stack/skyfiles that comprise this run.  Because consistency.
     my $command = "$stacktool -inputskyfile -stack_id $stack_id ";
     $command .= " -dbname $dbname " if defined($dbname);
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 0);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
     unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to run stacktool -inputskyfile ",
-                $stack_id,$error_code);
-    }
-
-    my $warpData = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to determine stack component information.",
-                $stack_id,$PS_EXIT_PROG_ERROR);
-    my $warpData2= parse_md_list($warpData);
-
-#
-# Step 3: Iterate over the sub-components
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to run stacktool -inputskyfile ", $remote_id, $stack_id, $error_code, $fail_state);
+    }
+
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to determine stack component information.", $remote_id, $stack_id, $PS_EXIT_PROG_ERROR, $fail_state);
+    $warpData = parse_md_list($MDlist);
+}
+
+# STEP 3: Iterate over the sub-components
+{
+    # Generate a file mapping the symbolic and "disk" representations of each needed file.
+    my $uri_compmap   = "${ipp_outroot}.compmap";
+    my $component_map = $ipprc->file_resolve($uri_compmap,1);    
+    my ($ipp_compmap, $remote_compmap) = &uri_to_outputs($component_map);
+    open(COMPMAP,">$ipp_compmap") || &my_die("Couldn't open file? $ipp_compmap",  $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state);
+    
+    # Loop over all needed components for this stack.
     my $warp_path_base_string = "";
-    foreach my $warpEntry ( @{ $warpData2 } ) {
-        my $warp_path_base = $warpEntry->{path_base};
-        my $remote_path_base = uri_local_to_remote($warp_path_base);
-        $warp_path_base_string .= " $remote_path_base ";
-
-        # Append file names to transfer lists so we can recover if warps are deleted.
-        uri_to_outputs($warp_path_base . ".fits");
-        uri_to_outputs($warp_path_base . ".wt.fits");
-        uri_to_outputs($warp_path_base . ".mask.fits");
-        uri_to_outputs($warp_path_base . ".cmf");
-#       uri_to_outputs($warp_path_base . ".psf");
-    }
-    my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl $warp_path_base_string > ${remote_outroot}.in.mdc";
-    my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use                                                                          
+    foreach my $warpEntry ( @{ $warpData } ) {
+        my $warp_path_base = $warpEntry->{path_base};
+        my $remote_path_base = &uri_local_to_remote($warp_path_base);
+        $warp_path_base_string .= " $remote_path_base ";
+
+        if ($verbose) { print STDERR "$warp_path_base_string\n"; }
+
+        # Append file names to transfer lists so we can recover if warps are deleted.
+        my $remote_file;
+        my $ipp_disk;
+        #  ${remote_root}/tmp/${ipp_disk}
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".fits");
+        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".fits");
+        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".wt.fits");
+        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".wt.fits");
+        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".mask.fits");
+        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".mask.fits");
+        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".cmf");
+        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".cmf");
+        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+        (undef,$remote_file) = &uri_to_outputs($warp_path_base . ".psf");
+        ($ipp_disk,undef)    = &uri_convert($warp_path_base . ".psf");
+        print COMPMAP "$remote_file ${remote_root}/tmp/${ipp_disk}\n";
+    }
+
+    close(COMPMAP);
+
+    my $mk_mdc_command = "mkdir -p $remote_outdir && sc_mk_stack_mdc.pl --compmap ${remote_root}/tmp/${component_map} $warp_path_base_string > ${remote_outroot}.in.mdc";
+    my $recipe_ppstack = $ipprc->reduction($reduction, 'STACK_PPSTACK'); # Recipe to use
     my $ppstack_command  = " ppStack -input ${remote_outroot}.in.mdc ";
     $ppstack_command    .= " ${remote_outroot} -stats ${remote_outroot}.stats ";
@@ -191 +188 @@
     $ppstack_command    .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE COMP_STACK";
     $ppstack_command    .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log ";
-    $ppstack_command    .= " -threads 4 ";
+    $ppstack_command    .= " -threads $threads_req ";
     $ppstack_command    .= " -dumpconfig ${remote_outroot}.mdc ";
     $ppstack_command    .= " -stack_id $stack_id -skycell_id $skycell_id -tess_id $tess_id ";
@@ -199 +196 @@
     $post_cmd_echo   .=  " -dbname $dbname " if defined $dbname;
     $post_cmd_echo   .= " -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo ";
-        
+
     my $post_cmd_SfM  = " ppStatsFromMetadata ${remote_outroot}.stats - STACK_SKYCELL >> ${remote_outroot}.dbinfo ";
 
@@ -207 +204 @@
     # Determine which output files need to be returned
     foreach my $component(@return_component_list) {
-        my $filename = $ipprc->filename($component,$ipp_outroot);
-        my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename);
-        my $remote_outroot_dir = dirname($ipp_disk);
-        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
-    }
-
-#     my $cleanup_cmd   = " (rm -f /scratch3/watersc1/sys_temp/${tess_id}.${skycell_id}.stk.${stack_id}.*conv.*.fits || true) ";    
-#     print CONFIG " && ${cleanup_cmd} \n";
-
+        my $filename = $ipprc->filename($component,$ipp_outroot);
+        my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename);
+        my $remote_outroot_dir = dirname($ipp_disk);
+        print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
+    }
     print CONFIG "\n";
 }
@@ -224 +217 @@
 close(GENERATE);
 
-#
-# Construct the moab command last, so we can use the job_index counter to estimate resources.  Somehow.
-my $proc_need = $job_index * $threads;       # how many total processors do we need?
-my $node_need = $proc_need / $proc_per_node; # this equals how many nodes?
-my $time_need = $job_index * $job_cost;      # How many seconds will this take?
-
-my $fill_factor = 0.8;  # This is the factor of how much of the time allocation we'd like to fill
-my ($time_req,$node_req);
-if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) {
-    $time_req = $min_time;
-    $node_req = $min_nodes;
-}
-elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) {
-    $time_req = $max_time;
-    $node_req = $max_nodes;
-    print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job.  This exceeds the max limits ($max_nodes, $max_time).  Using those max values instead.  Good luck.\n";
-}
-else {
-    $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1;
-    $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1;
-}
-
-$time_req += 2;
-$node_req *= 3;
-open(COMMAND,  ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR);
-print COMMAND "#!/bin/tcsh\n";
-print COMMAND "##### Moab controll lines\n";
-print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES
-print COMMAND "#MSUB -j oe\n";
-print COMMAND "#MSUB -V\n";
-print COMMAND "#MSUB -o ${remote_root}stack.${remote_id}.out\n";
-print COMMAND "date\n";
-print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n";
-print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 3' . "\n";
-print COMMAND "date\n";
-close(COMMAND);
-
 unless($no_update) {
-    my $command = "remotetool -updaterun -remote_id $remote_id ";
-    $command .= " -set_state pending ";
+    my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stack_id ";
+    $command .= " -set_jobs $job_index";
+    $command .= " -set_path_base $path_base";
+    $command .= " -set_state prep_done";
     $command .= " -dbname $dbname " if defined $dbname;
 
@@ -269 +227 @@
 }
 
-
 ## Common SC routines
 
+# KEEP
 sub uri_convert { # (ipp_disk,remote_disk) = uri_convert(neb_uri);
     my $neb_uri = shift;
     my $ipp_disk= $ipprc->file_resolve( $neb_uri );
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -288 +246 @@
 }
 
+# KEEP
 sub uri_convert_and_create { # (ipp_disk,remote_disk) = uri_convert_and_create(neb_uri); ipp_disk is created if it doesn't exist
     my $neb_uri = shift;
     my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $stack_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -304 +263 @@
 }
 
+# KEEP
 sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
-    
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print TRANSFER "$ipp_disk\n";
-        print CHECK    "$remote_disk\n";
-    }
+
+    print TRANSFER "$ipp_disk\n";
+    print CHECK    "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
 
+# KEEP
 sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri );
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print RETURN "$ipp_disk\n";
-        print GENERATE "$remote_disk\n";
-    }
+
+    print RETURN "$ipp_disk\n";
+    print GENERATE "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
 
+# KEEP
 sub uri_local_to_remote { #(remote_uri) = uri_local_to_remote(local_neb_uri);
     # This needs to replace the nebulous tag with the remote root.
@@ -337 +294 @@
     return($remote_uri);
 }
- 
-sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri);
-    # This needs to replace the remote root directory with the nebulous tag.
-    my $remote_uri = shift;
-    $remote_uri =~ s%${remote_root}%%;
-    my $local_uri  = "neb:///" . $remote_uri;
-    
-    return($local_uri);
-}
-
 
 sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status);
     my $msg = shift;
-    my $id  = shift;
+    my $remote_id  = shift;
+    my $stage_id  = shift;
     my $exit_code = shift;
     my $exit_state = shift;
@@ -357 +305 @@
 
     carp($msg);
-    
-    if (defined $id and not $no_update) {
-        my $command = "remotetool -updaterun -remote_id $id";
-        $command .= " -fault $exit_code " if defined $exit_code;
+
+    if (defined $remote_id and defined $stage_id and not $no_update) {
+        my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id";
         $command .= " -set_state $exit_state " if defined $exit_state;
         $command .= " -dbname $dbname " if defined $dbname;

trunk/ippScripts/scripts/sc_prepare_warp.pl

@@ -1 +1 @@
 #! /usr/bin/env perl
+
+# generate the input & output files lists and commands for a single warpRun
 
 use Carp;
@@ -16 +18 @@
 use Pod::Usage qw( pod2usage );
 
-
 # Hard coded values
-# my $remote_root   = '/scratch3/watersc1/';  
-my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-$remote_root = '/scratch3/watersc1/';
-my $threads       = 2;                      # How many threads are we going to use?
-my $job_cost      = 110 / 60 / 60;          # Estimate of how long a job runs, in hours.
-my $proc_per_node = 24;                     # processors per node
-my $min_nodes     = 1;                      # smallest allocation to ask for
-my $max_nodes     = 1000;                   # largest allocation to ask for
-my $min_time      = 1;                      # shortest allocation to ask for
-my $max_time      = 8;                      # longest allocation to ask for
-my $remote_hostname= "LANL/Mustang";         # Name of the remote node.
-my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP");
+# my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = '/scratch3/watersc1/';
+my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
+my $threads_req     = 4;                      # How many threads are we going to use?
+
+my $fail_state = "prep_fail";
+
 # Look for programs we need
 my $missing_tools;
-
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
 my $warptool    = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
@@ -41 +36 @@
 }
 
-my ($remote_id,$warp_id,,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$warp_id,$camera,$dbname,$verbose,$path_base,$no_update);
 GetOptions(
     'remote_id=s'    => \$remote_id,
+    'warp_id=s'      => \$warp_id,
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
@@ -52 +48 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id -warp_id --camera --dbname --path_base", -exitval => 3) unless
     defined($remote_id) and
+    defined($warp_id) and
     defined($camera) and
     defined($path_base) and
     defined($dbname);
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id);
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id, $warp_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-#
-# Step 1: Get a list of the components that make up this remoteRun
-
-# SHould this call listrun to ensure we're in state new?
-my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id";
-$rt_cmd   .= " -dbname $dbname " if defined($dbname);
-
-my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-    run(command => $rt_cmd, verbose => $verbose);
-unless ($success) {
-    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("Unable to run remotetool to determine stage parameters.",
-            $remote_id,$error_code);
-}
-my $compData = $mdcParser->parse(join "", @$stdout_buf) or
-    &my_die("Unable to determine component information.",
-            $remote_id,$PS_EXIT_PROG_ERROR);
-my $compData2= parse_md_list($compData);
-
-#
-# Step 1b: Open output files
-my $uri_command = $path_base . ".cmd";
+my @return_component_list = ("DBINFO.EXP", "PSWARP.CONFIG", "PSWARP.OUTPUT", "PSWARP.OUTPUT.MASK", "PSWARP.OUTPUT.VARIANCE", "PSWARP.OUTPUT.SOURCES","PSPHOT.PSF.SKY.SAVE","LOG.EXP");
+
+# STEP 0: Open output files
 my $uri_transfer= $path_base . ".transfer";
 my $uri_check   = $path_base . ".check";
@@ -89 +68 @@
 my $uri_return  = $path_base . ".return";
 
-my $disk_command = $ipprc->file_resolve($uri_command,1);
 my $disk_transfer= $ipprc->file_resolve($uri_transfer,1);
 my $disk_check   = $ipprc->file_resolve($uri_check,1);
@@ -96 +74 @@
 my $disk_return  = $ipprc->file_resolve($uri_return,1);
 
-my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
-
-open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$remote_id,$PS_EXIT_SYS_ERROR);
-open(CHECK,    ">$disk_check")  || &my_die("Couldn't open file? $disk_check",$remote_id,$PS_EXIT_SYS_ERROR);
-open(CONFIG,   ">$disk_config")  || &my_die("Couldn't open file? $disk_config",$remote_id,$PS_EXIT_SYS_ERROR);
-open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$remote_id,$PS_EXIT_SYS_ERROR);
-open(RETURN,   ">$disk_return") || &my_die("Couldn't open file? $disk_return", $remote_id, $PS_EXIT_SYS_ERROR);
-
-
-#
-# Step 2: Iterate over all componenets in this remote run.
+my (undef, $remote_config) = &uri_convert($uri_config); # Needs to be done after we've created it.
+
+open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CHECK,    ">$disk_check")     || &my_die("Couldn't open file? $disk_check",   $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CONFIG,   ">$disk_config")    || &my_die("Couldn't open file? $disk_config",  $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(GENERATE, ">$disk_generate")  || &my_die("Couldn't open file? $disk_generate",$remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(RETURN,   ">$disk_return")    || &my_die("Couldn't open file? $disk_return",  $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state);
+
 my $job_index = 0;
-my @pre_commands = ();
-my @main_commands = ();
-my @post_commands = ();
-
-my %file_filter = ();
-
-foreach my $compEntry (@$compData2) {
-    my $warp_id = $compEntry->{stage_id};
-
-# Get exposure level information from the warpRun we're working from.
+
+# STEP 1: Get exposure level information from the warpRun we're working from.
+my ($warpData);
+{
     # This actually returns all the individual warp/skyfiles that comprise this run.  Because consistency.
     my $command = "$warptool -towarped -warp_id $warp_id ";
     $command   .= " -dbname $dbname " if defined($dbname);
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 0);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
     unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to run warptool -pendingimfile ",
-                $warp_id,$error_code);
-    }
-    my $warpData = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to determine warp component information.",
-                $warp_id,$PS_EXIT_PROG_ERROR);
-    my $warpData2= parse_md_list($warpData);
-
-#
-# Step 3: Iterate over the sub-components
-    foreach my $warpEntry ( @{ $warpData2 } ) {
-        my $workdir = $warpEntry->{workdir};
-        my $exp_tag = $warpEntry->{exp_tag};
-        my $skycell_id = $warpEntry->{skycell_id};
-        my $tess_id = $warpEntry->{tess_id};
-        my $reduction = 'DEFAULT' unless defined $warpEntry->{reduction};
-        my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use                                                                          
-        my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.wrp.${warp_id}.${skycell_id}";
-        print "$ipp_outroot\n";
-        my $remote_outroot = uri_local_to_remote($ipp_outroot);
-
-        my $reg_command = "$warptool -scmap -warp_id $warp_id -skycell_id ${skycell_id}";
-        $reg_command   .= " -dbname $dbname " if defined($dbname);
-        
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $reg_command, verbose => 0);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to run warptool -pendingimfile: $reg_command ",
-                    $warp_id,$error_code);
-        }
-    
-        my $inpData = $mdcParser->parse(join "", @$stdout_buf) or
-            &my_die("Unable to determine warp component information.",
-                    $warp_id, $PS_EXIT_PROG_ERROR);
-        my $inpData2=parse_md_list($inpData);
-        
-        my $pre_cmd_ims = "ls -1 ";
-        my $pre_cmd_masks= "ls -1 ";
-        my $pre_cmd_vars = "ls -1 ";
-        my $pre_cmd_astrom = "";
-
-        foreach my $imfile (@{ $inpData2 }) {
-            my $image = $ipprc->filename("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id});
-            my $mask  = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
-            my $var   = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id});
-
-            my $astrom= $ipprc->filename("PSASTRO.OUTPUT", $imfile->{cam_path_base});
-            
-            my $remote_image = uri_local_to_remote($image);
-            my $remote_mask  = uri_local_to_remote($mask);
-            my $remote_var   = uri_local_to_remote($var);
-            my $remote_astrom= uri_local_to_remote($astrom);
-
-            $pre_cmd_ims     .= " $remote_image ";
-            $pre_cmd_masks   .= " $remote_mask  "; 
-            $pre_cmd_vars    .= " $remote_var   "; 
-            $pre_cmd_astrom   = "echo $remote_astrom > ${remote_outroot}.astrom ";
-        }
-        $pre_cmd_ims   .= "  > ${remote_outroot}.imlist ";
-        $pre_cmd_masks .= "  > ${remote_outroot}.masklist ";
-        $pre_cmd_vars  .= "  > ${remote_outroot}.varlist ";
-
-        my $skycell_command = " dvoImageExtract -D CATDIR /turquoise/usr/projects/ps1/watersc1/tess/${tess_id} $skycell_id -o ${remote_outroot}.skyfile ";
-
-        my $pswarp_command  = " pswarp -list ${remote_outroot}.imlist ";
-        $pswarp_command    .= " -masklist ${remote_outroot}.masklist -variancelist ${remote_outroot}.varlist ";
-        $pswarp_command    .= " -astromlist ${remote_outroot}.astrom ";
-        $pswarp_command    .= " ${remote_outroot} ${remote_outroot}.skyfile ";
-        $pswarp_command    .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE ";
-        $pswarp_command    .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF ";
-        $pswarp_command    .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF ";
-        $pswarp_command    .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS ";
-        $pswarp_command    .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL ";
-        $pswarp_command    .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID ";
-        $pswarp_command    .= " -recipe PSWARP $recipe_pswarp ";
-        $pswarp_command    .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log ";
-        $pswarp_command    .= " -threads 4 "; # -image_id ${image_id} -source_id ${source_id} ";
-        $pswarp_command    .= " -recipe PPSTATS WARPSTATS ";
-        $pswarp_command    .= " -dumpconfig ${remote_outroot}.pswarp.mdc -stats ${remote_outroot}.stats ";
-
-#       print "$pswarp_command \n";
-        
-        my $post_cmd_echo = " echo -n \"warptool  -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id ";
-        $post_cmd_echo   .= " -path_base $remote_outroot -uri UNKNOWN ";
-        $post_cmd_echo   .=  " -dbname $dbname " if defined $dbname;
-        $post_cmd_echo   .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo ";
-        
-        my $post_cmd_SfM  = " ppStatsFromMetadata ${remote_outroot}.stats - WARP_SKYCELL >> ${remote_outroot}.dbinfo ";
-
-        print CONFIG "${pre_cmd_ims} && ${pre_cmd_masks} && ${pre_cmd_vars} && ${pre_cmd_astrom} && ${skycell_command} && ${pswarp_command} && ${post_cmd_echo} && ${post_cmd_SfM} ";
-        $job_index++;
-
-        # Determine which output files need to be returned
-        foreach my $component(@return_component_list) {
-            my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id);
-            my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename);
-            my $remote_outroot_dir = dirname($ipp_disk);
-            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
-        }
-
-        print CONFIG "\n";
-#       die();
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to run warptool -pendingimfile ", $remote_id, $warp_id, $error_code, $fail_state);
+    }
+
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state);
+    $warpData = parse_md_list($MDlist);
+}
+
+# STEP 2: Iterate over the sub-components
+{
+    foreach my $warpEntry ( @{ $warpData } ) {
+        my $workdir = $warpEntry->{workdir};
+        my $exp_tag = $warpEntry->{exp_tag};
+        my $skycell_id = $warpEntry->{skycell_id};
+        my $tess_id = $warpEntry->{tess_id};
+
+        my $reduction = $warpEntry->{reduction};
+        $reduction = 'DEFAULT' unless defined $warpEntry->{reduction};
+
+        my $recipe_pswarp = $ipprc->reduction($reduction, 'WARP_PSWARP'); # Recipe to use
+        if ($recipe_pswarp eq "") {
+            &my_die("Unable to determine PSWARP recipe for $reduction", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state);
+        }
+
+        my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.wrp.${warp_id}.${skycell_id}";
+        print "$ipp_outroot\n";
+        my $remote_outroot = &uri_local_to_remote($ipp_outroot);
+
+        my $command = "$warptool -scmap -warp_id $warp_id -skycell_id ${skycell_id}";
+        $command   .= " -dbname $dbname " if defined($dbname);
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to run warptool -pendingimfile: $command ", $remote_id, $warp_id, $error_code, $fail_state);
+        }
+
+        my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to determine warp component information.", $remote_id, $warp_id, $PS_EXIT_PROG_ERROR, $fail_state);
+        my $inData = parse_md_list($MDlist);
+
+        my $pre_cmd_ims = "ls -1 ";
+        my $pre_cmd_masks= "ls -1 ";
+        my $pre_cmd_vars = "ls -1 ";
+        my $pre_cmd_astrom = "";
+
+        foreach my $imfile (@{ $inData }) {
+            my $image = $ipprc->filename("PPIMAGE.CHIP", $imfile->{chip_path_base}, $imfile->{class_id});
+            my $mask  = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
+            my $var   = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id});
+
+            my $astrom= $ipprc->filename("PSASTRO.OUTPUT", $imfile->{cam_path_base});
+
+            my $remote_image = &uri_local_to_remote($image);
+            my $remote_mask  = &uri_local_to_remote($mask);
+            my $remote_var   = &uri_local_to_remote($var);
+            my $remote_astrom= &uri_local_to_remote($astrom);
+
+            $pre_cmd_ims     .= " $remote_image ";
+            $pre_cmd_masks   .= " $remote_mask  ";
+            $pre_cmd_vars    .= " $remote_var   ";
+            $pre_cmd_astrom   = "echo $remote_astrom > ${remote_outroot}.astrom ";
+        }
+        $pre_cmd_ims   .= "  > ${remote_outroot}.imlist ";
+        $pre_cmd_masks .= "  > ${remote_outroot}.masklist ";
+        $pre_cmd_vars  .= "  > ${remote_outroot}.varlist ";
+
+        my $skycell_command = " dvoImageExtract -D CATDIR /turquoise/usr/projects/ps1/watersc1/tess/${tess_id} $skycell_id -o ${remote_outroot}.skyfile ";
+
+        my $pswarp_command  = " pswarp -list ${remote_outroot}.imlist ";
+        $pswarp_command    .= " -masklist ${remote_outroot}.masklist -variancelist ${remote_outroot}.varlist ";
+        $pswarp_command    .= " -astromlist ${remote_outroot}.astrom ";
+        $pswarp_command    .= " ${remote_outroot} ${remote_outroot}.skyfile ";
+        $pswarp_command    .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE ";
+        $pswarp_command    .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF ";
+        $pswarp_command    .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF ";
+        $pswarp_command    .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS ";
+        $pswarp_command    .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL ";
+        $pswarp_command    .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID ";
+        $pswarp_command    .= " -recipe PSWARP $recipe_pswarp ";
+        $pswarp_command    .= " -tracedest ${remote_outroot}.trace -log ${remote_outroot}.log ";
+        $pswarp_command    .= " -threads $threads_req "; # -image_id ${image_id} -source_id ${source_id} ";
+        $pswarp_command    .= " -recipe PPSTATS WARPSTATS ";
+        $pswarp_command    .= " -dumpconfig ${remote_outroot}.pswarp.mdc -stats ${remote_outroot}.stats ";
+
+#       print "$pswarp_command \n";
+
+        my $post_cmd_echo = " echo -n \"warptool  -addwarped -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_id ";
+        $post_cmd_echo   .=  " -dbname $dbname " if defined $dbname;
+        $post_cmd_echo   .= " -uri ${ipp_outroot}.fits ";
+        $post_cmd_echo   .= " -path_base $ipp_outroot -hostname $remote_hostname -dtime_script 0 \" > ${remote_outroot}.dbinfo ";
+
+        my $post_cmd_SfM  = " ppStatsFromMetadata ${remote_outroot}.stats - WARP_SKYCELL >> ${remote_outroot}.dbinfo ";
+
+        print CONFIG "${pre_cmd_ims} && ${pre_cmd_masks} && ${pre_cmd_vars} && ${pre_cmd_astrom} && ${skycell_command} && ${pswarp_command} && ${post_cmd_echo} && ${post_cmd_SfM} ";
+        $job_index++;
+
+        # Determine which output files need to be returned
+        foreach my $component(@return_component_list) {
+            my $filename = $ipprc->filename($component,$ipp_outroot,$skycell_id);
+            my ($ipp_disk, $remote_disk) = &uri_to_outputs_for_return( $filename);
+            my $remote_outroot_dir = dirname($ipp_disk);
+            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
+        }
+        print CONFIG "\n";
+#       die();
     }
 }
@@ -236 +210 @@
 close(GENERATE);
 
-#
-# Construct the moab command last, so we can use the job_index counter to estimate resources.  Somehow.
-my $proc_need = $job_index * $threads;       # how many total processors do we need?
-my $node_need = $proc_need / $proc_per_node; # this equals how many nodes?
-my $time_need = $job_index * $job_cost;      # How many seconds will this take?
-
-my $fill_factor = 0.8;  # This is the factor of how much of the time allocation we'd like to fill
-my ($time_req,$node_req);
-if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) {
-    $time_req = $min_time;
-    $node_req = $min_nodes;
-}
-elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) {
-    $time_req = $max_time;
-    $node_req = $max_nodes;
-    print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job.  This exceeds the max limits ($max_nodes, $max_time).  Using those max values instead.  Good luck.\n";
-}
-else {
-    $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1;
-    $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1;
-}
-$time_req += 1; # Safety addition.
-
-open(COMMAND,  ">$disk_command") || &my_die("Couldn't open file? $disk_command",$remote_id,$PS_EXIT_SYS_ERROR);
-print COMMAND "#!/bin/tcsh\n";
-print COMMAND "##### Moab controll lines\n";
-print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES
-print COMMAND "#MSUB -j oe\n";
-print COMMAND "#MSUB -V\n";
-print COMMAND "#MSUB -o ${remote_root}warp.${remote_id}.out\n";
-print COMMAND "date\n";
-print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n";
-print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n";
-print COMMAND "date\n";
-close(COMMAND);
-
 unless($no_update) {
-    my $command = "remotetool -updaterun -remote_id $remote_id ";
-    $command .= " -set_state pending ";
+    my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $warp_id ";
+    $command .= " -set_jobs $job_index";
+    $command .= " -set_path_base $path_base";
+    $command .= " -set_state prep_done";
     $command .= " -dbname $dbname " if defined $dbname;
 
     system($command);
 }
-
+exit (0);
 
 ## Common SC routines
@@ -287 +227 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri );
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -303 +243 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $warp_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -317 +257 @@
 sub uri_to_outputs { # (ipp_disk,remote_disk) = uri_to_output(neb_uri); Appends to TRANSFER and CHECK filehandles
     my $neb_uri = shift;
-    my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
-    
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print TRANSFER "$ipp_disk\n";
-        print CHECK    "$remote_disk\n";
-    }
+    my ($ipp_disk, $remote_disk) = &uri_convert( $neb_uri );
+
+    print TRANSFER "$ipp_disk\n";
+    print CHECK    "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -329 +266 @@
 sub uri_to_outputs_for_return { # (ipp_disk,remote_disk) = uri_to_outputs_for_return(neb_uri); create ipp_disk, append to RETURN and GENERATE
     my $neb_uri = shift;
-    my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri );
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print RETURN "$ipp_disk\n";
-        print GENERATE "$remote_disk\n";
-    }
+    my ($ipp_disk, $remote_disk) = &uri_convert_and_create( $neb_uri );
+
+    print RETURN "$ipp_disk\n";
+    print GENERATE "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -348 +283 @@
     return($remote_uri);
 }
- 
+
 sub uri_remote_to_local { #(local_neb_uri) = uri_remote_to_local(remote_uri);
     # This needs to replace the remote root directory with the nebulous tag.
@@ -354 +289 @@
     $remote_uri =~ s%${remote_root}%%;
     my $local_uri  = "neb:///" . $remote_uri;
-    
+
     return($local_uri);
 }
@@ -361 +296 @@
 sub my_die { # exit with status; my_die(message,stage_id,exit_code,exit_status);
     my $msg = shift;
-    my $id  = shift;
+    my $remote_id  = shift;
+    my $stage_id  = shift;
     my $exit_code = shift;
     my $exit_state = shift;
@@ -368 +304 @@
 
     carp($msg);
-    
-    if (defined $id and not $no_update) {
-        my $command = "remotetool -updaterun -remote_id $id";
-        $command .= " -fault $exit_code " if defined $exit_code;
+
+    if (defined $remote_id and defined $stage_id and not $no_update) {
+        my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id";
         $command .= " -set_state $exit_state " if defined $exit_state;
         $command .= " -dbname $dbname " if defined $dbname;

trunk/ippScripts/scripts/sc_remote_exec.pl

@@ -1 @@
-#!/usr/bin/env perl 
+#!/usr/bin/env perl
 
 use Carp;
@@ -16 +16 @@
 # Hard coded values
 my $DMZ_HOST = 'wtrw';
-my $SEC_HOST = 'mu-fe';
+my @SEC_HOSTS= ('mu-fe1','mu-fe2','mu-fe3'
+                # ,'mu-fe4'
+    );
+my $SEC_HOST = $SEC_HOSTS[int(rand(@SEC_HOSTS))]; # 'mu-fe';
 my $IPP_PATH = '/turquoise/usr/projects/cosmo/mswarren/ipp/';
 my $remote_root  = '/scratch3/watersc1/';
@@ -32 +35 @@
 
 # Options
-my ($remote_id,$path_base,$policy,$poll,$force,$job_id,$dbname,$verbose,$no_update,$camera);
-$force = 0;
+my ($remote_id,$job_id,$path_base,$dbname,$verbose,$no_update,$camera);
 $verbose = 0;
 GetOptions(
     'remote_id=s'   => \$remote_id,
-    'job_id=s'      => \$job_id,
     'path_base=s'   => \$path_base,
-    'policy=s'      => \$policy,
-    'poll'          => \$poll,
-    'force-poll'    => \$force,
     'camera=s'      => \$camera,
     'dbname=s'      => \$dbname,
@@ -54 +52 @@
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $remote_id);
-my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files                                    
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 # Phase 1: See if we can actually do anything.
@@ -69 +67 @@
 unless ($rt_success) {
     $rt_error_code = (($rt_error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    &my_die("UNable to run remotetool to determine remote run status",
-            $remote_id,$rt_error_code);
-}
-my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or 
-    &my_die("UNable to run remotetool to determine remote run status",
-            $remote_id,$rt_error_code);
+    &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code);
+}
+my $rrData = $mdcParser->parse(join "", @$rt_stdout_buf) or
+    &my_die("Unable to run remotetool to determine remote run status", $remote_id, $rt_error_code);
 my $rrData2 = parse_md_list($rrData);
 my $runData = ${ $rrData2 }[0]; # There should be only one
 
-# These are operations we only need to do on the first call, not on subsequent poll operations.
-unless ($poll) {
+my ($remote_path) = &uri_local_to_remote($path_base);
+print "REMOTE PATH: $remote_path\n";
+
 # Phase 2: Ensure files are in place:
 # Copy command files
-    my @files = ();
-    
-    my $uri_command = $path_base . ".cmd";
-    my $uri_transfer= $path_base . ".transfer";
-    my $uri_check   = $path_base . ".check";
-    my $uri_config  = $path_base . ".config";
-    my $uri_generate= $path_base . ".generate";
-    my $uri_return  = $path_base . ".return";
-
-    my $disk_command = $ipprc->file_resolve($uri_command);
-    my $disk_transfer= $ipprc->file_resolve($uri_transfer);
-    my $disk_check   = $ipprc->file_resolve($uri_check);
-    my $disk_config  = $ipprc->file_resolve($uri_config);
-    my $disk_generate= $ipprc->file_resolve($uri_generate);
-    my $disk_return  = $ipprc->file_resolve($uri_return);
-    
-    scp_put($disk_command,uri_local_to_remote($uri_command));
-    scp_put($disk_transfer,uri_local_to_remote($uri_transfer));
-    scp_put($disk_check,uri_local_to_remote($uri_check));
-    scp_put($disk_config,uri_local_to_remote($uri_config));
-    scp_put($disk_generate,uri_local_to_remote($uri_generate));
-    scp_put($disk_return, uri_local_to_remote($uri_return));
-    
-# Run check command
-    my (undef,$remote_check) = uri_convert($uri_check);
-    my (undef,$remote_transfer)= uri_convert($uri_transfer);
-
-    my $ssh_check_stdout = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_check --fetch $remote_transfer");
-    # We no longer need to parse this output, as it retrieves files itself.
-    foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) {
-        print "$l\n";
-    }
+my @files = ();
+
+my $uri_command = $path_base . ".cmd";
+my $uri_transfer= $path_base . ".transfer";
+my $uri_check   = $path_base . ".check";
+my $uri_config  = $path_base . ".config";
+my $uri_generate= $path_base . ".generate";
+my $uri_return  = $path_base . ".return";
+
+my $disk_command = $ipprc->file_resolve($uri_command);
+my $disk_transfer= $ipprc->file_resolve($uri_transfer);
+my $disk_check   = $ipprc->file_resolve($uri_check);
+my $disk_config  = $ipprc->file_resolve($uri_config);
+my $disk_generate= $ipprc->file_resolve($uri_generate);
+my $disk_return  = $ipprc->file_resolve($uri_return);
+
+&scp_put($disk_command,  &uri_local_to_remote($uri_command));
+&scp_put($disk_transfer, &uri_local_to_remote($uri_transfer));
+&scp_put($disk_check,    &uri_local_to_remote($uri_check));
+&scp_put($disk_config,   &uri_local_to_remote($uri_config));
+&scp_put($disk_generate, &uri_local_to_remote($uri_generate));
+&scp_put($disk_return,   &uri_local_to_remote($uri_return));
+
+my $ssh_check_stdout = &ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_path --fetch");
+
+# We no longer need to parse this output, as it retrieves files itself.
+foreach my $l (split /\n/, (join '', @$ssh_check_stdout)) {
+    print "$l\n";
+}
 
 # Run real command
-    my (undef,$remote_command) = uri_convert($uri_command);
-    my $ssh_exec_stdout = ssh_exec_command("msub -V $remote_command");
-    if ($#{ $ssh_exec_stdout } != -1) { # Parse the output
-        $job_id = ${ $ssh_exec_stdout }[0];
-        chomp($job_id);
-        $job_id =~ s/\s+//g;
-    }
-
-    unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly.
-        &my_die("No job_id returned.  Sorry.",$remote_id,$PS_EXIT_PROG_ERROR);
-    }
-
-    # Notify the database that this entry is currently running.
-    my $command = "$remotetool -updaterun -remote_id $remote_id ";
-    $command .= " -set_state run ";
-    $command .= " -job_id $job_id ";
-    $command .= " -dbname $dbname " if defined $dbname;
-    
-    system($command);
-    
-} # End of exec phase
-
-# Check that we have a valid job_id, either from the command line, or from the msub command.
+my (undef,$remote_command) = &uri_convert($uri_command);
+my $ssh_exec_stdout = &ssh_exec_command("msub -V $remote_command");
+if ($#{ $ssh_exec_stdout } != -1) { # Parse the output
+    $job_id = ${ $ssh_exec_stdout }[0];
+    chomp($job_id);
+    $job_id =~ s/\s+//g;
+}
+
 unless(defined($job_id)) { # If we don't have a job_id from this command, it didn't run correctly.
-    &my_die("No job_id returned.  Sorry.",$remote_id,$PS_EXIT_PROG_ERROR);
-}
-
-
-# Poll the job status?
-my $poll_count = 0;
-my $poll_max   = 1; # If it isn't finished now, it's unlikely to magically finish in the next five minutes.
-my $poll_sleep = 60;
-my $poll_response = 0;
-my $poll_word  = "nothing";
-
-if ($force) {
-    $poll_response = 1;
-    $poll_word     = "Completed";
-}
-
-while (($poll_count < $poll_max)&&($poll_response != 1)&&($poll_response != -1)&&($force != 1)) {
-    if ($verbose) {
-        print "Polling code.  Sleeping.  $poll_count of $poll_max $poll_response\n";
-    }
-    unless ($poll_count == 0) {
-        sleep($poll_sleep);
-    }
-    ($poll_response,$poll_word) = poll_job($job_id);
-
-    if ($poll_response == 1) { # This job has completed
-        last;
-    }
-    elsif ($poll_response == -1) { # This job has an error
-        warn("The job exited incorrectly: $remote_id $job_id $poll_word.  Attempting to recover what we can.");
-        $poll_response = 1; # To fool the check later into proceding
-    }
-    $poll_count++;
-}
-if ($verbose) {
-    print "Stopped Polling code.  $poll_count of $poll_max $poll_response\n";
-}
-if ($poll_response != 1) {
-    &my_die("Job hasn't completed.  Ending polling.", $remote_id, 0, "run", $job_id);
-}
-
-
-# Initialize the remote side to sync things
-
-# Once sync completes, execute all the database commands
-# Grab the return list
-my $uri_return  = $path_base . ".return";
-my ($ipp_return,$remote_return) = uri_convert($uri_return);
-# We shouldn't need to get this file.  We made that file.
-#scp_get($remote_return,$ipp_return);
-
-# Construct list of dbinfo files we'll need to execute.
-my @returned_files = ();
-my @dbinfo_files   = ();
-my @dbindex        = ();
-my $iter           = 0;
-open(RETURN,   "$ipp_return") || &my_die("Couldn't open file? $ipp_return", $remote_id,$PS_EXIT_SYS_ERROR);
-while (<RETURN>) {
-    chomp;
-    push @returned_files, $_;
-    if ($_ =~ /dbinfo/) {
-        push @dbinfo_files, $_;
-        push @dbindex, $iter;
-    }
-    $iter++;
-
-}
-close(RETURN);
-print STDERR "Expect $#dbinfo_files dbinfo files\n";
-
-my $first = 0;
-my $last  = $#dbinfo_files;
-my $mid   = 0;
-my $range = 20;  # How wide of a chunk to look through to catch threading drops
-my $offset = 0;  # Offset to start from.
-
-# Feed the return list into the transfer tool
-# If this fails, we should go to an auth wait point.
-if ($poll_word eq 'Completed') {  
-    # We're complete.  All files should exist on disk
-    my $iter = 0;
-    my $transfer_success = 1;
-    do {
-        $transfer_success = 1;
-        $iter++;
-        check_ssh_connection(); 
-        my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
-        my $return_push_output_print = join "\n", @{ $return_push_output };
-        if ($return_push_output_print =~ /exited with value 255/) { $transfer_success = 0; }
-        print "$return_push_output_print\n";
-    } while (($transfer_success != 1) &&  ($iter < 10));  # Try harder;
-}
-else {
-    # Just try and fetch things, because we didn't complete correctly, it seems.
-    my $return_push_output = ssh_exec_command("${remote_root}/sc_transfer_tool.pl --input $remote_return --offset $offset");
-    my $return_push_output_print = join "\n", @{ $return_push_output };
-    print "$return_push_output_print\n";
-}
-
-foreach my $file (@dbinfo_files) {
-    open(DBF,"$file") || warn "Missing file $file\n";
-    my $cmd = <DBF>;
-    close(DBF);
-    chomp($cmd);
-#    print STDERR "$file $cmd\n";
-    $cmd =~ s/-/ -/g; # This is just a safety check for missing space.
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $cmd, verbose => $verbose);
-    unless ($success) {
-
-        # This shouldn't fail, but we also can't suddenly abort if something does fail.  
-        # Now we're going to drop the component that owns this dbinfo, which should force a retry.
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        warn("The command that shouldn't fail has failed: $file $cmd $remote_id $error_code");
-        if ($poll_word ne 'Completed') {  # If the job claims it completed, then we're fighting the network.
-            drop_component($remote_id,$cmd,$file);
-        }
-    }
-
-}
-
-# We're done, so set the state and exit.
-&my_die("Finished",$remote_id,0,"full");
-
+    &my_die("No job_id returned.  Sorry.", $remote_id, $PS_EXIT_PROG_ERROR);
+}
+
+# Notify the database that this entry is currently running.
+&my_die("Finished", $remote_id, 0, "run", $job_id);
 
 # END PROGRAM
-
 
 sub check_ssh_connection {
@@ -272 +131 @@
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $cmd, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Authorization check failed.",$remote_id,0,'auth');
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Authorization check failed.", $remote_id, 0, 'pending');
     }
 }
@@ -285 +144 @@
 
     my $directory = dirname($destination);
-    ssh_exec_command("mkdir -p $directory");
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $cmd, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die(); 
+    &ssh_exec_command("mkdir -p $directory");
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("failed to send file $file to LANL", $remote_id, $error_code);
     }
 }
@@ -302 +161 @@
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $cmd, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die(); 
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("failed to get file $file from LANL", $remote_id, $error_code);
     }
 }
@@ -311 +170 @@
 sub ssh_exec_command {
     my $cmd = shift;
-    $cmd = "$ssh  $DMZ_HOST ssh  ${SEC_HOST} $cmd";
+    $cmd = "$ssh -n $DMZ_HOST ssh  ${SEC_HOST} $cmd";
     print "EXEC: $cmd\n";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $cmd, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        # If we failed, see if we failed due to authorization.
-        check_ssh_connection(); 
-        &my_die("Failed to execute command: >>$cmd<<"); 
+        run(command => $cmd, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        # If we failed, see if we failed due to authorization.
+        check_ssh_connection();
+        &my_die("Failed to execute remote command: >>$cmd<<", $remote_id, $error_code);
     }
     return ($stdout_buf);
-}    
-
-sub poll_job {
-    my $job_id = shift;
-    my $response_array = ssh_exec_command("checkjob -v $job_id");
-    my $response = join "\n", @$response_array;
-    
-    if ($response =~ /State: Running/) { 
-        return(0,"Running");
-    }
-    elsif ($response =~ /State: Completed/) {
-        return(1,"Completed");
-    }
-    elsif ($response =~ /State: Idle/) {
-        return(-2,"Idle");
-    }
-    else {
-        my $response_word = $response;
-        $response_word =~ s/.*State: //;
-        $response_word =~ s/\n.*//;
-        return(-1,$response_word);
-    }
-}
-
-
+}
+
+# neb URI to (ipp_disk,remote_disk)
 sub uri_convert {
     my $neb_uri = shift;
     my $ipp_disk= $ipprc->file_resolve( $neb_uri );
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die("Failed to generate or find uri $neb_uri", $remote_id, $PS_EXIT_SYS_ERROR);
     }
 
@@ -363 +200 @@
 }
 
-sub uri_to_outputs {
-    my $neb_uri = shift;
-    my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
-
-#    print TRANSFER "$ipp_disk\n";
-#    print CHECK    "$remote_disk\n";
-    return($ipp_disk,$remote_disk);
-}
-
+# neb URI to remote URI-on-disk
 sub uri_local_to_remote {
     # This needs to replace the nebulous tag with the remote root.
@@ -382 +211 @@
     return($remote_uri);
 }
- 
-sub uri_remote_to_local {
-    # This needs to replace the remote root directory with the nebulous tag.
-    my $remote_uri = shift;
-    $remote_uri =~ s%${remote_root}%%;
-    my $local_uri  = "neb:///" . $remote_uri;
-    
-    return($local_uri);
-}
-
-sub drop_component {
-    my $remote_id = shift;
-    my $cmd = shift;
-    my $file = shift;
-
-    if ($cmd eq '') { 
-        # Zero byte file.  This dbinfo was never constructed properly.
-        
-        # Steal the stage_id from the filename.  This isn't something I'm happy about.
-        my $stage = $runData->{stage};
-        my $stage_id = $file;
-        if ($stage eq 'chip') { 
-            $stage_id =~ s/^.*ch\.//;
-        }
-        elsif ($stage eq 'camera') {
-            $stage_id =~ s/^.*cm\.//;
-        }
-        elsif ($stage eq 'warp') {
-            $stage_id =~ s/^.*wrp\.//;
-        }
-        elsif ($stage eq 'stack') {
-            $stage_id =~ s/^.*stk\.//;
-        }
-        $stage_id =~ s/\..*$//;
-        
-        # Drop this from the database
-        my $drop_cmd = "$remotetool -dropcomponent -remote_id $remote_id -stage_id $stage_id ";
-        $drop_cmd   .= " -dbname $dbname " if defined($dbname);
-        system($drop_cmd);
-    }
-
-}
 
 sub my_die {
@@ -434 +221 @@
 
     carp($msg);
-    
+
     if (defined $id and not $no_update) {
-        my $command = "$remotetool -updaterun -remote_id $id";
-        $command .= " -fault $exit_code " if defined $exit_code;
-        $command .= " -job_id $job_id " if defined $jobid;
-        $command .= " -set_state $exit_state " if defined $exit_state;
-        $command .= " -dbname $dbname " if defined $dbname;
-
-        system($command);
+        my $command = "$remotetool -updaterun -remote_id $id";
+        $command .= " -fault $exit_code " if defined $exit_code;
+        $command .= " -job_id $job_id " if defined $jobid;
+        $command .= " -set_state $exit_state " if defined $exit_state;
+        $command .= " -dbname $dbname " if defined $dbname;
+
+        system($command);
     }
 

trunk/ippScripts/scripts/sc_transfer_tool.pl

@@ -10 +10 @@
 use Sys::Hostname;
 
+my $scp_cmd = "/usr/projects/cosmo/amd6100/bin/scp";
+my $ssh_cmd = "/usr/projects/cosmo/amd6100/bin/ssh";
+
 my $remote_root = '/lustre/scratch1/turquoise/watersc1/ps1/';
-$remote_root = '/scratch3/watersc1/';
-my $local_raw = "${remote_root}/raw/";
+   $remote_root = '/scratch3/watersc1/';
+
+#my $local_raw = "${remote_root}/tmp/";
 my $local_tmp = "${remote_root}/tmp/";
-my $threads = 10;
-my @hosts = ('ippc20.ipp.ifa.hawaii.edu','ippc24.ipp.ifa.hawaii.edu','ippc28.ipp.ifa.hawaii.edu',
-             'ippc21.ipp.ifa.hawaii.edu','ippc25.ipp.ifa.hawaii.edu','ippc29.ipp.ifa.hawaii.edu',
-             'ippc22.ipp.ifa.hawaii.edu','ippc26.ipp.ifa.hawaii.edu',
-             'ippc23.ipp.ifa.hawaii.edu','ippc27.ipp.ifa.hawaii.edu'    );
+my $threads = 11;
+my @hosts = (
+    'ippc20.ipp.ifa.hawaii.edu',
+    'ippc21.ipp.ifa.hawaii.edu',
+    'ippc22.ipp.ifa.hawaii.edu',
+    'ippc23.ipp.ifa.hawaii.edu',
+    'ippc24.ipp.ifa.hawaii.edu',
+    'ippc25.ipp.ifa.hawaii.edu',
+    'ippc26.ipp.ifa.hawaii.edu',
+    'ippc27.ipp.ifa.hawaii.edu',
+    'ippc28.ipp.ifa.hawaii.edu',        
+#'ippc29.ipp.ifa.hawaii.edu', (mark's hi-mem test machine)
+#'ippc30.ipp.ifa.hawaii.edu', (postage stamp server)    
+    'ippc31.ipp.ifa.hawaii.edu',        
+    'ippc32.ipp.ifa.hawaii.edu'    
+    );
 @hosts = (@hosts, @hosts, @hosts, @hosts, @hosts);
-my $input_file;
+my %server_options = (
+    'ippc20.ipp.ifa.hawaii.edu' => '',
+    'ippc21.ipp.ifa.hawaii.edu' => '',
+    'ippc22.ipp.ifa.hawaii.edu' => '',
+    'ippc23.ipp.ifa.hawaii.edu' => '',
+    'ippc24.ipp.ifa.hawaii.edu' => '',
+    'ippc25.ipp.ifa.hawaii.edu' => '',
+    'ippc26.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc27.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc28.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc29.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc30.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc31.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes',
+    'ippc32.ipp.ifa.hawaii.edu' => '', # -o NoneSwitch=yes -o NoneEnabled=yes'
+    );
+
+my $input_path;
+my $local_file;
+my $remote_file;
+
 my $verbose = 0;
 my $fetch = 0;
 my $offset = 0;
+my $retry = 0;
+my $quickfetch = 0;
 
 GetOptions(
     'threads=s'   => \$threads,
-    'input=s'     => \$input_file,
-    'fetch=s'     => \$fetch,
+    'input=s'     => \$input_path,
     'offset=s'    => \$offset,
+    'fetch'       => \$fetch,
     'verbose'     => \$verbose,
+    'retry'       => \$retry,
+    'quickfetch'  => \$quickfetch,
     ) or pod2usage( 2 );
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2) if @ARGV;
 pod2usage( -msg => "Required options: --input", -exitval => 3) unless
-    defined($input_file);
+    defined($input_path);
 
 my $hostname = hostname;
@@ -41 +79 @@
 # split in input file list
 unless(-d $local_tmp) { system("mkdir -p $local_tmp"); }
-unless(-d $local_raw) { system("mkdir -p $local_raw"); }
-
-open(I,$input_file) || die "Couldn't find input file specified\n";
+#unless(-d $local_raw) { system("mkdir -p $local_raw"); }
+
+# we have two modes: fetch (-> LANL) and return (-> IPP)
+
 if ($fetch) {
-    open(F,$fetch) || die "Couldn't find fetch file specified\n";
-}
-my $input_base = basename($input_file);
-my @filehandles;
+    $local_file = "$input_path.check";
+    $remote_file = "$input_path.transfer";
+} else {
+    $local_file = "$input_path.generate";
+    $remote_file = "$input_path.return";
+}
+my $input_base = basename($local_file);
+
+unless ($retry) {
+  # open the input file list
+  open (LOCAL,$local_file) || die "Couldn't find input file specified $local_file\n";
+  open (REMOTE,$remote_file) || die "Couldn't find input file specified $remote_file\n";
+  
+  # generate N output files (to be fed to the N rsync / tar threads)
+  my @filehandles;
+  my $i;
+  my $line = 0;
+  for ($i = 0; $i < $threads; $i++) {
+      open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
+  }
+  
+  unless ($fetch) {
+      my $stat_file = "$input_path.stat";
+      print "STATFILE: $stat_file\n";
+      open (STATFILE, ">$stat_file");
+  }
+  
+  $i = 0;
+  while (my $Lname = <LOCAL>) {
+      my $Rname = <REMOTE>;
+  
+      chomp $Lname;
+      chomp $Rname;
+  
+      $line++;
+      if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
+  
+      if ($line % 250 == 0)  { print STDERR "line $line : $Lname\n"; }
+  
+      $i = int(rand($#filehandles + 1));
+      if ($i >= $threads) {
+        print STDERR "HUH: impossible file handle?\n";
+        die;
+      }
+  
+      # if we are fetching, and 'quickfetch' is requested, skip file if it exists
+      # NOTE: this does not check that the transfer was complete or the file is current
+  
+      if ($fetch) {
+        # We are fetching, and do not already have this file.
+        if ($quickfetch && (-e $Lname)) { next; }
+        print { $filehandles[$i] } "$Rname\n";
+      } else { 
+        # We are pushing, but only send back files we actually have
+        # we cannot have the remote file unless the local file exists 
+        # (it is a link to the local file and is generated after the local file)
+        my $haveRemote = -e "$local_tmp/$Rname";
+        if ($haveRemote) {
+            print { $filehandles[$i] } "$Rname\n";
+            print STATFILE "$Rname PASS\n";
+        } else {
+            my $haveLocal = -e $Lname;
+            if ($haveLocal) {
+                print STATFILE "$Rname PART\n";
+            } else {
+                print STATFILE "$Rname FAIL\n";
+            }
+        }
+      }
+  }
+  close(LOCAL);
+  close(REMOTE);
+  unless ($fetch) { close(STATFILE); }
+  
+  for ($i = 0; $i < $threads; $i++) {
+      close($filehandles[$i]);
+  }
+}
+
 my $i;
-my $line = 0;
-for ($i = 0; $i < $threads; $i++) {
-    open($filehandles[$i], ">${local_tmp}/${input_base}.${i}");
-}
-$i = 0;
-while(<I>) {
-    chomp;
-    $line++;
-    if ($line < $offset) { next; } # I think this is off-by-one in a safe direction
-
-    $i = int(rand($#filehandles + 1));
-
-    my $fline;
-    if ($fetch) {
-        $fline = <F>;
-    }
-    if (($fetch)&&(!(-e $_))) {  # We are fetching, and do not already have this file.
-        print { $filehandles[$i] } $fline;
-    }
-    elsif (!($fetch)) { # We are pushing
-        # The rsync call expects to find files of a given name in the directory specified.
-#       print { $filehandles[$i] } "${local_tmp}/$_" . "\n";
-        print { $filehandles[$i] } "$_" . "\n";
-    }
-}
-close(I);
-
-for ($i = 0; $i < $threads; $i++) {
-    close($filehandles[$i]);
-}
-
-# fork the tars
+# fork the rsync
 my @pids = ();
 for ($i = 0; $i < $threads; $i++) {
@@ -89 +172 @@
         my $code = 0;
         if ($fetch) { 
-            $code = fetch_task($host,"${input_base}.${i}", 0);
-        }
-        else {
+            $code = fetch_task($host,"${local_tmp}/${input_base}.${i}", 0);
+        } else {
             $code = transfer_task($host,"${local_tmp}/${input_base}.${i}", 0);
         }
+        print STDERR "$host $input_base $i $code\n";
         exit($code);
     }
 }
+
+my $global_status = 0;
 for ($i = 0; $i < $threads; $i++) {
     waitpid($pids[$i],0);
+    my $this_status = $?;
+    print "exit status $hosts[$i] : $this_status\n";
+    if ($this_status) { $global_status = ($this_status >> 8); }
 }
 
@@ -105 +193 @@
 }
 
+# my $scp_command = "$scp_cmd $option ${local_tmp}/${input_base}.stat $hosts[0]:/tmp/";
+# print "$scp_command\n";
+# system("$scp_command");
+# XXX check return status
+
+print "global status: $global_status\n";
+exit ($global_status);
 
 # distribute bundles to nodes
@@ -115 +210 @@
     my $error = shift;
 
-    # the transform bit is there because it looks like the ' gets dropped, so the * is interpreted, and why is our tar so out of date?
-#    my $command = "tar cf - --ignore-failed-read --dereference --files-from=${transfer_filelist} | /usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar xf - -C /data/ --transform '" . 's,^.\*/data/,,' . "' --dereference"; 
-    my $command = "rsync -Lpt -e '/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
+    my $option = $server_options{$destination_host};
+
+    my $command = "rsync -L --size-only --omit-dir-times -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${local_tmp} ${destination_host}:/";
     print STDERR "$command\n";
-
+    
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
+
+    my $return_value = 0;
     unless ($success) {
-        print STDERR "ERROR:  $error_code\n";
-#       my $std = join "\n", @{ $stdout_buf };
-#       print "STDOUT: $std\n";
-#       $std = join "\n", @{ $stderr_buf };
-#       print "STDERR: $std\n";
-
-#       foreach my $line (@{ $stderr_buf }) {  # This is a hack.  A messy ugly hack.
-#           if ($line =~ /No such file or directory/) { 
-#               my $file = (split /\s+/, $line)[1];
-#               $file =~ s/:$//;
-#               if ($file !~ /nebulous/) { next; }
-#               print STDERR "Touching $file so tar can continue.\n";
-#               system("touch $file");
-#           }
-#       }
-
-        warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
-        $error++;
-        if ($error < 4) {
-            $error_code = transfer_task($destination_host,$transfer_filelist,$error);
-        }
-        else {
-            die("Failed too many times $error $destination_host $transfer_filelist");
+        if ($error_code =~ /value 23/) {
+            print STDERR "Some files failed to transfer.  Partial run?\n";
+            $error = 23;
+            $return_value = 23;
+        } else {
+            print STDERR "ERROR:  $error_code\n";
+            warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
+            sleep(5);
+            $error++;
+            if ($error < 4) {
+                $error_code = transfer_task($destination_host,$transfer_filelist,$error);
+            }
+            else {
+                die("Failed too many times $error $destination_host $transfer_filelist");
+            }
+            $return_value = 1;
         }
     }
-    return($error_code);
+    return($return_value);
 }
     
@@ -156 +246 @@
     my $error = shift;
 
-    system("/usr/projects/cosmo/amd6100/bin/scp -o NoneSwitch=yes -o NoneEnabled=yes ${local_tmp}/${transfer_filelist} ${destination_host}:/tmp/");
-    my $command = "/usr/projects/cosmo/amd6100/bin/ssh -o NoneSwitch=yes -o NoneEnabled=yes $destination_host tar cf - --ignore-failed-read  --dereference --files-from=/tmp/${transfer_filelist} | tar xf - -C ${local_raw} --skip-old-files --warning=existing-file --dereference ";
+    my $option = $server_options{$destination_host};
+
+    my $command = "rsync --size-only -e '$ssh_cmd ${option}' --files-from=${transfer_filelist} ${destination_host}:/ ${local_tmp}";
     print STDERR "$command\n";
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    my $error_code = 0;
+    my ( $success, $error_msg, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        $error_code = (($error_code >> 8) or 4);
-        warn("Transfer of $transfer_filelist to $destination_host failed with error $error_code.");
-        $error++;
-        if ($error < 4) {
-            $error_code = fetch_task($destination_host,$transfer_filelist,$error);
-        }
-        else {
-            die("Failed too many times $error $destination_host $transfer_filelist");
+        print "raw error_msg: $error_msg...\n";
+        ($error_code) = $error_msg =~ m/exited with value (\d+)/;
+        warn("Transfer of files in $transfer_filelist from $destination_host failed with error ($error_code) $error_msg.");
+        print "*** stdout: *** \n";
+        foreach my $line (@$stdout_buf) {
+            print STDERR "stdout: $line\n";
+        }
+        print "*** stderr: *** \n";
+        foreach my $line (@$stderr_buf) {
+            print STDERR "stderr: $line\n";
+        }
+        if ($error_code == 23) {
+            print STDERR "Some files failed to transfer\n";
+        } else {
+            $error++;
+            if ($error < 4) {
+                $error_code = fetch_task($destination_host,$transfer_filelist,$error);
+            } else {
+                print STDERR "Failed too many times $error $destination_host $transfer_filelist\n";
+            }
         }
     }

trunk/ippScripts/scripts/stack_skycell.pl

@@ -225 +225 @@
     $num++;
 
-    my $image = $file->{uri};   # Image name
+#    my $image = $file->{uri};   # Image name
+    my $image = $ipprc->filename("PSWARP.OUTPUT", $file->{path_base} ); # Image name
     my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name
     my $weight = $ipprc->filename( "PSWARP.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name

trunk/ippTools/share/Makefile.am

@@ -524 +524 @@
         remotetool_definebyquery_warp.sql \
         remotetool_definebyquery_stack.sql \
+        remotetool_definebyquery_staticsky.sql \
         remotetool_dropcomponent.sql \
+        remotetool_listcomponent.sql \
+        remotetool_doneprep.sql \
         remotetool_updatepoll.sql

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -2385 +2385 @@
     remote_id BIGINT,
     stage_id  BIGINT,
+    jobs      INT
+    state     VARCHAR(64) NOT NULL,
+    path_base VARCHAR(255),
     KEY (stage_id),
     FOREIGN KEY (remote_id) REFERENCES remoteRun(remote_id)

trunk/ippTools/share/remotetool_definebyquery_cam.sql

@@ -11 +11 @@
 LEFT JOIN
 
-(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state 
-   FROM remoteRun 
+(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state
+   FROM remoteRun
    JOIN remoteComponent USING (remote_id)
 WHERE
-remoteRun.stage = 'camera'
+remoteRun.stage = 'camera' and remoteComponent.state != 'retry'
 -- where hook %s
 ) AS R

trunk/ippTools/share/remotetool_definebyquery_chip.sql

@@ -11 +11 @@
 LEFT JOIN
 
-(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state 
-   FROM remoteRun 
+(SELECT remote_id,remoteRun.stage,remoteRun.label,remoteComponent.stage_id,remoteRun.state
+   FROM remoteRun
    JOIN remoteComponent USING (remote_id)
 WHERE
-remoteRun.stage = 'chip'
+remoteRun.stage = 'chip' and remoteComponent.state != 'retry'
 -- where hook %s
 ) AS R
@@ -24 +24 @@
 
 
-       
+

trunk/ippTools/share/remotetool_definebyquery_stack.sql

@@ -15 +15 @@
    JOIN remoteComponent USING (remote_id)
 WHERE
-remoteRun.stage = 'stack'
+remoteRun.stage = 'stack' and remoteComponent.state != 'retry'
 -- where hook %s
 ) AS R

trunk/ippTools/share/remotetool_definebyquery_warp.sql

@@ -17 +17 @@
    JOIN remoteComponent USING (remote_id)
 WHERE
-remoteRun.stage = 'warp'
+remoteRun.stage = 'warp' and remoteComponent.state != 'retry'
 -- where hook %s
 ) AS R

trunk/ippTools/src/remotetool.c

@@ -21 +21 @@
 static bool listrunMode(pxConfig *config);
 static bool listcomponentMode(pxConfig *config);
+
+// list remoteRuns with all remoteComponents in state 'prep_done' or 'prep_fail'
+static bool doneprepMode(pxConfig *config);
 
 // Update
@@ -26 +29 @@
 static bool updaterunMode(pxConfig *config);
 static bool updatepollMode(pxConfig *config);
+static bool updatecomponentMode(pxConfig *config);
 static bool revertrunMode(pxConfig *config);
 static bool revertauthMode(pxConfig *config);
+static bool revertcomponentMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -47 +52 @@
 
   switch (config->mode) {
-    MODECASE(REMOTETOOL_MODE_DEFINEBYQUERY, definebyqueryMode);
-    MODECASE(REMOTETOOL_MODE_LISTRUN,       listrunMode);
-    MODECASE(REMOTETOOL_MODE_LISTCOMPONENT, listcomponentMode);
-    MODECASE(REMOTETOOL_MODE_DROPCOMPONENT, dropcomponentMode);
-    MODECASE(REMOTETOOL_MODE_UPDATERUN,     updaterunMode);
-    MODECASE(REMOTETOOL_MODE_UPDATEPOLL,    updatepollMode);
-    MODECASE(REMOTETOOL_MODE_REVERTRUN,     revertrunMode);
-    MODECASE(REMOTETOOL_MODE_REVERTAUTH,    revertauthMode);
+    MODECASE(REMOTETOOL_MODE_DEFINEBYQUERY,   definebyqueryMode);
+    MODECASE(REMOTETOOL_MODE_LISTRUN,         listrunMode);
+    MODECASE(REMOTETOOL_MODE_DONEPREP,         doneprepMode);
+    MODECASE(REMOTETOOL_MODE_LISTCOMPONENT,   listcomponentMode);
+    MODECASE(REMOTETOOL_MODE_DROPCOMPONENT,   dropcomponentMode);
+    MODECASE(REMOTETOOL_MODE_UPDATERUN,       updaterunMode);
+    MODECASE(REMOTETOOL_MODE_UPDATEPOLL,      updatepollMode);
+    MODECASE(REMOTETOOL_MODE_UPDATECOMPONENT, updatecomponentMode);
+    MODECASE(REMOTETOOL_MODE_REVERTRUN,       revertrunMode);
+    MODECASE(REMOTETOOL_MODE_REVERTAUTH,      revertauthMode);
+    MODECASE(REMOTETOOL_MODE_REVERTCOMPONENT, revertcomponentMode);
 
   default:
@@ -92 +100 @@
   PXOPT_LOOKUP_BOOL(simple,    config->args, "-simple",       false);
   PXOPT_LOOKUP_BOOL(pretend,   config->args, "-pretend",      false);
-  
+
   set_label = set_label ? set_label : label;
 
@@ -105 +113 @@
       return(false);
     }
-    
+
     if (label) {
       psStringAppend(&whereOption, "\n AND (label = '%s')", label);
@@ -116 +124 @@
       return(false);
     }
-    
+
     if (label) {
       psStringAppend(&whereOption, "\n AND (label = '%s')", label);
     }
-  }    
+  }
   else if (!strcmp(stage,"warp")) {
     query = pxDataGet("remotetool_definebyquery_warp.sql");
@@ -127 +135 @@
       return(false);
     }
-    
+
     if (label) {
       // Because warp has to check to see if something other than the state is set correctly, I have to use this wonky way to pass the label in.
@@ -133 +141 @@
       psStringAppend(&whereOption, "\n AND (label = '%s')", label);
     }
-  }    
+  }
   else if (!strcmp(stage,"stack")) {
     query = pxDataGet("remotetool_definebyquery_stack.sql");
@@ -140 +148 @@
       return(false);
     }
-    
+
     if (label) {
       psStringAppend(&whereOption, "\n AND (label = '%s')", label);
     }
-  }    
+  }
+  else if (!strcmp(stage,"staticsky")) {
+    query = pxDataGet("remotetool_definebyquery_staticsky.sql");
+    if (!query) {
+      psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement");
+      return(false);
+    }
+
+    if (label) {
+      psStringAppend(&whereOption, "\n AND (label = '%s')", label);
+    }
+  }
   else {
     psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage);
@@ -191 +210 @@
   // Insert the top level run object
   remoteRunRow *run = remoteRunRowAlloc(0, // remote_id
-                                        "new", // state
-                                        stage,
-                                        set_label,
-                                        path_base,
-                                        -1, // job_id
-                                        NULL, // last_poll
-                                        0   // fault
-                                          );
+                                        "new", // state
+                                        stage,
+                                        set_label,
+                                        path_base,
+                                        -1, // job_id
+                                        NULL, // last_poll
+                                        0   // fault
+                                          );
   if (!run) {
-    psError(PS_ERR_UNKNOWN, false, "failed to alloc lapRun object");
+    psError(PS_ERR_UNKNOWN, false, "failed to alloc remoteRun object");
     return(true);
   }
-  
+
   if (!remoteRunInsertObject(config->dbh, run)) {
     if (!psDBRollback(config->dbh)) {
@@ -213 +232 @@
   }
   psS64 remote_id = psDBLastInsertID(config->dbh);
-  
+
   psArray *list = psArrayAllocEmpty(limit);
   for (long i=0; i < psArrayLength(output); i++) {
@@ -219 +238 @@
     psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id");
 
-    remoteComponentRow *comp = remoteComponentRowAlloc(remote_id,
-                                                       stage_id);
+    remoteComponentRow *comp = remoteComponentRowAlloc(remote_id, stage_id, 0, "new", NULL);
     if (!comp) {
-      psError(PS_ERR_UNKNOWN, false, "failed to alloc lapRun object");
+      psError(PS_ERR_UNKNOWN, false, "failed to alloc remoteComponent object");
       return(true);
     }
-    
+
     if (!remoteComponentInsertObject(config->dbh, comp)) {
       if (!psDBRollback(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+        psError(PS_ERR_UNKNOWN, false, "database error");
       }
       psError(PS_ERR_UNKNOWN, false, "database error");
@@ -237 +255 @@
     psArrayAdd(list, list->n, comp);
     psFree(comp);
-    
+
   }
   psFree(run);
@@ -245 +263 @@
     return false;
   }
-  
+
   if (!remoteComponentPrintObjects(stdout, list, !simple)) {
     if (!psDBRollback(config->dbh)) {
@@ -254 +272 @@
     return false;
   }
-  
+
   psFree(list);
   psFree(output);
-                                        
+
   return true;
 }
@@ -265 +283 @@
 {
   psMetadata *where = psMetadataAlloc();
-  
+
   PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "==");
   PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
@@ -274 +292 @@
   PXOPT_COPY_TIME(config->args, where, "-poll_begin", "last_poll", ">=");
   PXOPT_COPY_TIME(config->args, where, "-poll_end", "last_poll", "<=");
-  
+
   PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
 
   PXOPT_LOOKUP_S16(limit,   config->args, "-limit",  false, false);
   PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-  
 
   psArray *remRuns = NULL;
   remRuns = psDBSelectRows(config->dbh, "remoteRun", where, limit);
-  
+
   if (!ippdbPrintMetadatas(stdout, remRuns, "remoteRun", !simple)) {
     psError(PS_ERR_UNKNOWN, false, "failed to print array");
@@ -299 +316 @@
 
   PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-stage_id",  "stage_id",  "==");
+  PXOPT_COPY_S64(config->args, where, "-jobs",      "jobs",      "==");
+
+  PXOPT_COPY_STR(config->args, where, "-state", "remoteComponent.state", "==");
+  PXOPT_COPY_STR(config->args, where, "-label", "remoteRun.label", "==");
+
   PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-  
-  psArray *remComps = NULL;
-  remComps = psDBSelectRows(config->dbh, "remoteComponent", where, 0);
-  
-  if (!ippdbPrintMetadatas(stdout, remComps, "remoteComponent", !simple)) {
+  PXOPT_LOOKUP_S16(limit,   config->args, "-limit",  false, false);
+
+  psString query = pxDataGet("remotetool_listcomponent.sql");
+  if (!query) {
+      psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement");
+      return(false);
+  }
+
+  if (psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+      psStringAppend(&query, " WHERE %s", whereClause);
+      psFree(whereClause);
+  }
+  psFree(where);
+
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, " %s", limitString);
+    psFree(limitString);
+  }
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("remotetool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (!ippdbPrintMetadatas(stdout, output, "remoteComponent", !simple)) {
     psError(PS_ERR_UNKNOWN, false, "failed to print array");
-    psFree(remComps);
-    return false;
-  }
-  psFree(remComps);
+    psFree(output);
+    return false;
+  }
+  psFree(output);
 
   return true;
 }
 
+// List
+static bool doneprepMode(pxConfig *config)
+{
+  psMetadata *where = psMetadataAlloc();
+
+  PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
+
+  PXOPT_COPY_STR(config->args, where, "-stage",  "stage",  "==");
+  PXOPT_COPY_STR(config->args, where, "-state", "remoteRun.state", "==");
+  PXOPT_COPY_STR(config->args, where, "-label",  "label",  "==");
+
+  PXOPT_LOOKUP_S16(limit,   config->args, "-limit",  false, false);
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+  psString query = pxDataGet("remotetool_doneprep.sql");
+  if (!query) {
+      psError(PXTOOLS_ERR_SYS,false, "failed to retreive SQL statement");
+      return(false);
+  }
+
+  psString whereDone = NULL;
+  psString whereJobs = NULL;
+  if (psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+      psStringAppend(&whereDone, " \n AND %s", whereClause);
+      psStringAppend(&whereJobs, " \n WHERE %s", whereClause);
+      psFree(whereClause);
+  }
+
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, " %s", limitString);
+    psFree(limitString);
+  }
+
+  // fprintf (stderr, "query: %s\n", query);
+  // fprintf (stderr, "whereDone: %s\n", whereDone);
+  // fprintf (stderr, "whereJobs: %s\n", whereJobs);
+
+  if (!p_psDBRunQueryF(config->dbh, query, whereDone, whereJobs)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(whereDone);
+      psFree(whereJobs);
+      psFree(query);
+      return false;
+  }
+  psFree(whereDone);
+  psFree(whereJobs);
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("remotetool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  if (!ippdbPrintMetadatas(stdout, output, "remoteRun", !simple)) {
+    psError(PS_ERR_UNKNOWN, false, "failed to print array");
+    psFree(output);
+    return false;
+  }
+  psFree(output);
+
+  return true;
+}
+
 // Update
 
 static bool dropcomponentMode(pxConfig *config)
 {
+  fprintf (stderr, "do we really want to use -dropcomponent?\n");
+  exit (1);
+
   // Assert that we have a unique component to operate on.
   PXOPT_LOOKUP_S64(remote_id,config->args, "-remote_id",true, false);
@@ -329 +462 @@
     return false;
   }
-  
+
   if (!psDBTransaction(config->dbh)) {
     psError(PS_ERR_UNKNOWN, false, "database error");
@@ -341 +474 @@
   }
   psFree(query);
-  
-  
 
   // point of no return
@@ -349 +480 @@
     return false;
   }
-  
+
   return true;
 }
@@ -364 +495 @@
   // Wheres
   PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "==");
-  
+
   // Values to set
   PXOPT_COPY_STR(config->args, values, "-set_label", "label", "==");
@@ -374 +505 @@
   psFree(values);
 
-  if (rows < 1) {
-    psError(PS_ERR_UNKNOWN,false, "no rows affected");
-    return(false);
-  }
-  
+  if (rows < 1) fprintf (stderr, "no rows changed (run may already be in desired state)\n");
   return(true);
 }
@@ -393 +520 @@
   psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
   psFree(where);
-  
+
   psString query = NULL;
   query = pxDataGet("remotetool_updatepoll.sql");
@@ -412 +539 @@
   return(true);
 }
-  
-
-  
+
+static bool updatecomponentMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+  PXOPT_LOOKUP_S64(remote_id,  config->args, "-remote_id",    true, false);
+  psAssert(remote_id,"This should have failed before this point.");
+
+  psMetadata *where = psMetadataAlloc();
+  psMetadata *values = psMetadataAlloc();
+
+  // Wheres
+  PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-stage_id",  "stage_id",  "==");
+  PXOPT_COPY_STR(config->args, where, "-state",     "state",     "==");
+
+  // Values to set
+  PXOPT_COPY_S32(config->args, values, "-set_jobs",      "jobs",      "==");
+  PXOPT_COPY_STR(config->args, values, "-set_path_base", "path_base", "==");
+  PXOPT_COPY_STR(config->args, values, "-set_state", "state", "==");
+
+  long rows = psDBUpdateRows(config->dbh, "remoteComponent", where, values);
+  psFree(values);
+
+  if (rows < 1) fprintf (stderr, "no rows changed (component may already be in desired state)\n");
+  return true;
+}
 
 static bool revertrunMode(pxConfig *config)
@@ -426 +576 @@
   PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "==");
   PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
-  
+  PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
   // Value to set
-  PXOPT_COPY_S16(config->args, values, "-fault", "fault", "==");
-  long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values);
-  psFree(values);
-  psFree(where);
-  if (rows < 1) {
-    psError(PS_ERR_UNKNOWN,false, "no rows affected");
-    return(false);
-  }
-
-  return(true);
-}
-
-static bool revertauthMode(pxConfig *config)
-{
-  PS_ASSERT_PTR_NON_NULL(config, false);
-
-  psMetadata *where = psMetadataAlloc();
-  psMetadata *values = psMetadataAlloc();
-
-  // required
-  PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "==");
-  PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
-
-  psMetadataAddStr(where, PS_LIST_TAIL, "state", 0, "==", "auth"); 
-
-  // Value to set
-  psMetadataAddStr(values, PS_LIST_TAIL, "state", 0, "==", "run");
+  psMetadataAddS16(values,PS_LIST_TAIL,"fault",0, "==", 0);
   
   long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values);
@@ -466 +590 @@
   return(true);
 }
-    
+
+static bool revertcomponentMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config,false);
+  psMetadata *where = psMetadataAlloc();
+  psMetadata *values = psMetadataAlloc();
+
+  // required
+  PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
+  psMetadataAddStr(where,PS_LIST_TAIL,"state",0, "LIKE", "%fail");
+  
+  // value to set
+  psMetadataAddStr(values, PS_LIST_TAIL, "state",0, "==", "retry");
+  
+
+  long rows = psDBUpdateRows(config->dbh, "remoteComponent", where, values);
+  psFree(values);
+  psFree(where);
+  if (rows < 1) {
+    psError(PS_ERR_UNKNOWN,false, "no rows affected");
+    return(false);
+  }
+
+  return(true);
+}
+  
+  
+
+static bool revertauthMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  psMetadata *where = psMetadataAlloc();
+  psMetadata *values = psMetadataAlloc();
+
+  // required
+  PXOPT_COPY_S64(config->args, where, "-remote_id", "remote_id", "==");
+  PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+
+  psMetadataAddStr(where, PS_LIST_TAIL, "state", 0, "==", "auth");
+
+  // Value to set
+  psMetadataAddStr(values, PS_LIST_TAIL, "state", 0, "==", "run");
+
+  long rows = psDBUpdateRows(config->dbh, "remoteRun", where, values);
+  psFree(values);
+  psFree(where);
+  if (rows < 1) {
+    psError(PS_ERR_UNKNOWN,false, "no rows affected");
+    return(false);
+  }
+
+  return(true);
+}
+

trunk/ippTools/src/remotetool.h

@@ -12 +12 @@
   REMOTETOOL_MODE_DEFINEBYQUERY,
   REMOTETOOL_MODE_LISTRUN,
+  REMOTETOOL_MODE_DONEPREP,
   REMOTETOOL_MODE_LISTCOMPONENT,
   REMOTETOOL_MODE_DROPCOMPONENT,
   REMOTETOOL_MODE_UPDATERUN,
   REMOTETOOL_MODE_UPDATEPOLL,
-  REMOTETOOL_MODE_REVERTRUN,    
-  REMOTETOOL_MODE_REVERTAUTH,   
+  REMOTETOOL_MODE_UPDATECOMPONENT,
+  REMOTETOOL_MODE_REVERTRUN,
+  REMOTETOOL_MODE_REVERTAUTH,
+  REMOTETOOL_MODE_REVERTCOMPONENT,
 } remotetoolMode;
 

trunk/ippTools/src/remotetoolConfig.c

@@ -40 +40 @@
   ADD_OPT(Bool, definebyqueryArgs, "-pretend",       "pretend to do this.", 0);
   ADD_OPT(Bool, definebyqueryArgs,"-simple",         "simple print format",0);
-  
+
   // -listrun
   psMetadata *listrunArgs = psMetadataAlloc();
@@ -59 +59 @@
   ADD_OPT(S16, listrunArgs,  "-limit",             "limit to number of runs to display", 0);
 
+  // -doneprep
+  psMetadata *doneprepArgs = psMetadataAlloc();
+  ADD_OPT(S64, doneprepArgs, "-remote_id",          "run to list", 0);
+  ADD_OPT(S64, doneprepArgs, "-job_id",             "job_id", 0);
+  ADD_OPT(Str, doneprepArgs, "-stage",              "stage to return", NULL);
+  ADD_OPT(Str, doneprepArgs, "-state",              "state to return", NULL);
+  ADD_OPT(Str, doneprepArgs, "-label",              "remRun label to use", NULL);
+
+  ADD_OPT(Bool, doneprepArgs, "-simple",            "simple print format", false);
+  ADD_OPT(S16, doneprepArgs,  "-limit",             "limit to number of runs to display", 0);
+
+  // -revertcomponent
+  psMetadata   *revertcomponentArgs = psMetadataAlloc();
+  ADD_OPT(S64,  revertcomponentArgs, "-remote_id",       "run to revert components from", 0);
+  ADD_OPT(S64,  revertcomponentArgs, "-stage_id",        "component stage_id to revert", 0);
+  
+  
   // -listcomponents
-  psMetadata *listcomponentArgs = psMetadataAlloc();
-  ADD_OPT(S64, listcomponentArgs, "-remote_id",             "run to list", 0);
-  ADD_OPT(S16, listcomponentArgs, "-limit",              "limit to number of runs to display", 0);
+  psMetadata   *listcomponentArgs = psMetadataAlloc();
+  ADD_OPT(S64,  listcomponentArgs, "-remote_id",         "run to list", 0);
+  ADD_OPT(S64,  listcomponentArgs, "-stage_id",          "run to list", 0);
+  ADD_OPT(S32,  listcomponentArgs, "-jobs",              "run to list", 0);
+  ADD_OPT(Str,  listcomponentArgs, "-state",             "run to list", NULL);
+  ADD_OPT(S16,  listcomponentArgs, "-limit",             "limit to number of runs to display", 0);
   ADD_OPT(Bool, listcomponentArgs, "-simple",            "simple print format", false);
+  ADD_OPT(Str,  listcomponentArgs, "-label",             "label to use", NULL);
 
   // -dropcomponent
@@ -69 +90 @@
   ADD_OPT(S64, dropcomponentArgs, "-remote_id",         "run to update", 0);
   ADD_OPT(S64, dropcomponentArgs, "-stage_id",          "stage_id to remote", 0);
-  
+
   // -updaterun
   psMetadata *updaterunArgs = psMetadataAlloc();
@@ -83 +104 @@
   psMetadata *updatepollArgs = psMetadataAlloc();
   ADD_OPT(S64, updatepollArgs, "-remote_id",            "run to update", 0);
-  
+
+  // -updatecomponent
+  psMetadata  *updatecomponentArgs = psMetadataAlloc();
+  ADD_OPT(S64, updatecomponentArgs, "-remote_id",     "remote_id to update", 0);
+  ADD_OPT(S64, updatecomponentArgs, "-stage_id",      "stage_id to update", 0);
+  ADD_OPT(Str, updatecomponentArgs, "-state",         "limit to this remoteComponent state", NULL);
+  ADD_OPT(Str, updatecomponentArgs, "-set_state",     "remoteComponent state to assign", NULL);
+  ADD_OPT(Str, updatecomponentArgs, "-set_path_base", "remoteRun path_base to assign", NULL);
+  ADD_OPT(S32, updatecomponentArgs, "-set_jobs",      "number of jobs for this stage_id", 0);
+
   // -revertrun
   psMetadata *revertrunArgs = psMetadataAlloc();
@@ -89 +119 @@
   ADD_OPT(Str, revertrunArgs, "-label",      "remRun label to use", NULL);
   ADD_OPT(S16, revertrunArgs, "-fault",              "fault code", 0);
-  
+
   // -revertauth
   psMetadata *revertauthArgs = psMetadataAlloc();
@@ -98 +128 @@
   psMetadata *modes   = psMetadataAlloc();
 
-  PXOPT_ADD_MODE("-definebyquery",     "", REMOTETOOL_MODE_DEFINEBYQUERY,  definebyqueryArgs);
-  PXOPT_ADD_MODE("-listrun",           "", REMOTETOOL_MODE_LISTRUN,        listrunArgs);
-  PXOPT_ADD_MODE("-listcomponent",     "", REMOTETOOL_MODE_LISTCOMPONENT,  listcomponentArgs);
-  PXOPT_ADD_MODE("-dropcomponent",     "", REMOTETOOL_MODE_DROPCOMPONENT,  dropcomponentArgs);
-  PXOPT_ADD_MODE("-updaterun",         "", REMOTETOOL_MODE_UPDATERUN,      updaterunArgs);
-  PXOPT_ADD_MODE("-updatepoll",        "", REMOTETOOL_MODE_UPDATEPOLL,     updatepollArgs);
-  PXOPT_ADD_MODE("-revertrun",         "", REMOTETOOL_MODE_REVERTRUN,      revertrunArgs);
-  PXOPT_ADD_MODE("-revertauth",        "", REMOTETOOL_MODE_REVERTAUTH,     revertauthArgs);
+  PXOPT_ADD_MODE("-definebyquery",     "", REMOTETOOL_MODE_DEFINEBYQUERY,   definebyqueryArgs);
+  PXOPT_ADD_MODE("-listrun",           "", REMOTETOOL_MODE_LISTRUN,         listrunArgs);
+  PXOPT_ADD_MODE("-doneprep",          "", REMOTETOOL_MODE_DONEPREP,        doneprepArgs);
+  PXOPT_ADD_MODE("-listcomponent",     "", REMOTETOOL_MODE_LISTCOMPONENT,   listcomponentArgs);
+  PXOPT_ADD_MODE("-revertcomponent",   "", REMOTETOOL_MODE_REVERTCOMPONENT, revertcomponentArgs);
+  PXOPT_ADD_MODE("-dropcomponent",     "", REMOTETOOL_MODE_DROPCOMPONENT,   dropcomponentArgs);
+  PXOPT_ADD_MODE("-updaterun",         "", REMOTETOOL_MODE_UPDATERUN,       updaterunArgs);
+  PXOPT_ADD_MODE("-updatepoll",        "", REMOTETOOL_MODE_UPDATEPOLL,      updatepollArgs);
+  PXOPT_ADD_MODE("-updatecomponent",   "", REMOTETOOL_MODE_UPDATECOMPONENT, updatecomponentArgs);
+  PXOPT_ADD_MODE("-revertrun",         "", REMOTETOOL_MODE_REVERTRUN,       revertrunArgs);
+  PXOPT_ADD_MODE("-revertauth",        "", REMOTETOOL_MODE_REVERTAUTH,      revertauthArgs);