Changeset 37833 for trunk/ippScripts/scripts/sc_prepare_chip.pl

Timestamp:

Jan 12, 2015, 12:53:36 PM (11 years ago)

Author:

eugene

Message:

merging changes from ipp-pv3-20140717 (via branches/eam_branches/ipp-pv3-20140717-merge)

Location:

trunk/ippScripts/scripts

Files:

• . (modified) (1 prop)
• sc_prepare_chip.pl (modified) (16 diffs)

trunk/ippScripts/scripts/sc_prepare_chip.pl

@@ -1 +1 @@
 #! /usr/bin/env perl
+
+# generate the input & output files lists and commands for a single chipRun
 
 use Carp;
@@ -16 +18 @@
 use Pod::Usage qw( pod2usage );
 
-
 # Hard coded values
-# my $remote_root   = '/scratch3/watersc1/';  # Far side destination base location
-my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
-$remote_root = '/scratch3/watersc1/';
-my $remote_raw    = "${remote_root}/raw/";  # Directory to find raw data in.
-my $threads       = 2;                      # How many threads are we going to use?
-my $job_cost      = 150 / 60 / 60;          # Estimate of how long a job runs, in hours.
-my $proc_per_node = 24;                     # processors per node
-my $min_nodes     = 1;                      # smallest allocation to ask for
-my $max_nodes     = 1000;                   # largest allocation to ask for
-my $min_time      = 1;                      # shortest allocation to ask for
-my $max_time      = 8;                      # longest allocation to ask for
-
-# We need to ensure we only ever try to transfer a file once.
-my %file_filter = ();
+# my $remote_root   = '/lustre/scratch1/turquoise/watersc1/ps1/'; # Far side destination base location
+my $remote_root     = '/scratch3/watersc1/';  # Far side destination base location
+my $remote_hostname = "LANL/Mustang";         # Name of the remote node.
+my $remote_raw      = "${remote_root}/tmp/";  # Directory to find raw data in.
+my $threads_req     = 4;
+
+my $fail_state = "prep_fail";
+my $hard_fail_state = "fail";
 
 # Look for programs we need
 my $missing_tools;
-my $regtool  = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
 my $detselect= can_run('detselect') or (warn "Can't find detselect" and $missing_tools = 1);
 my $remotetool = can_run('remotetool') or (warn "Can't find remotetool" and $missing_tools = 1);
+my $ipp_burntool_fix = can_run('ipp_apply_burntool_fix.pl') or (warn "Can't find ipp_apply_burntool_fix.pl" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -46 +41 @@
 
 # Options
-my ($exp_id,$exp_name,$remote_id,$chip_id,$detrends,$camera,$dbname,$verbose,$path_base,$no_update);
+my ($remote_id,$chip_id,$camera,$dbname,$path_base,$no_update,$verbose,$dbverbose);
 GetOptions(
     'remote_id=s'    => \$remote_id,
+    'chip_id=s'      => \$chip_id,
     'camera|c=s'     => \$camera,
     'dbname|d=s'     => \$dbname,
@@ -54 +50 @@
     'no_update'      => \$no_update,
     'verbose'        => \$verbose,
+    'dbverbose'      => \$dbverbose,
     ) or pod2usage( 2 );
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --remote_id --camera --dbname --path_base", -exitval => 3) unless
+pod2usage( -msg => "Required options: --remote_id --chip_id --camera --dbname --path_base", -exitval => 3) unless
     defined($remote_id) and
+    defined($chip_id) and
     defined($camera) and
     defined($path_base) and
     defined($dbname);
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id);
+
+my $ipprc = PS::IPP::Config->new( $camera ) or &my_die( "Unable to set up", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-#
-# Step 1: Get a list of the components that make up this remoteRun
-
-# SHould this call listrun to ensure we're in state new?
-my $rt_cmd = "$remotetool -listcomponent -remote_id $remote_id";
-$rt_cmd   .= " -dbname $dbname " if defined($dbname);
-
-my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-    run(command => $rt_cmd, verbose => $verbose);
-unless ($success) {
-    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-    
-    &my_die("Unable to run chiptool to determine stage parameters.",
-            $remote_id,$error_code);
-}
-my $compData = $mdcParser->parse(join "", @$stdout_buf) or
-    &my_die("Unable to determine component information.",
-            $remote_id,$PS_EXIT_PROG_ERROR);
-my $compData2= parse_md_list($compData);
-
-#
-# Step 1a: Load up the catalog of detrends 
-# Detrend concept holders
-my %detrends = ();
-my %det_types = ('MASK' => '-mask',
-                 'FLAT' => '-flat',
-                 'DARK' => '-dark',
-                 'VIDEODARK' => '-dark',
-                 'LINEARITY' => '-linearity',
-                 'FRINGE'    => '-fringe',
-                 'NOISEMAP'  => '-noisemap');
-
-foreach my $det_type (keys (%det_types)) {
-    my $dt_cmd = "$detselect -show -camera $camera -det_type $det_type ";
-    $dt_cmd   .= " -dbname $dbname " if defined($dbname);
-
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $dt_cmd, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("UNable to run detselect to determine detrend catalog",
-                $remote_id,$error_code);
-    }
-    my $detData = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to determine detrend information.",
-                $remote_id,$PS_EXIT_PROG_ERROR);
-    my $detData2= parse_md_list($detData);
-
-    # Data structure for this:
-    # $detrend{NAME} = \@list_of_detselect_shows
-    # $detrend{NAME}[0] = \%hash_of_detselect_constraint_results (first element)
-    # $detrend{NAME}[0]{class_id} = uri
-
-    # At this point, I have the list of detselect shows, assign it
-    $detrends{$det_type} = $detData2;
-
-    for (my $dd = 0; $dd <= $#{ @{ $detrends{$det_type} } }; $dd++) {
-        my $det_id = ${ $detrends{$det_type} }[$dd]->{det_id};
-        my $det_iter=${ $detrends{$det_type} }[$dd]->{iteration};
-
-        my $detselect_command2 = "detselect -select ";
-        $detselect_command2   .= " -det_id $det_id ";
-        $detselect_command2   .= " -iteration $det_iter ";
-        $detselect_command2   .= " -dbname $dbname " if defined($dbname);
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $detselect_command2, verbose => 0);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            
-            &my_die("No valid detrend available for this image: $det_type $det_id $det_iter",
-                    $chip_id,$error_code);
-        }
-        
-        my $detImfile = $mdcParser->parse(join "", @$stdout_buf) or 
-            &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter",
-                    $chip_id,$error_code);
-        my $detImfile2= parse_md_list($detImfile);
-        foreach $detImfile (@$detImfile2) {
-            my $class_id = $detImfile->{class_id};
-            my $duri = $detImfile->{uri};
-            
-            ${ $detrends{$det_type} }[$dd]->{$class_id} = $duri;
-        }
-    }
-}
-
-#print Dumper(%detrends);
+# Detrend concept holders (name, ppImage option)
+my %det_types = (
+    'MASK'      => '-mask',
+    'FLAT'      => '-flat',
+    'DARK'      => '-dark',
+    'VIDEODARK' => '-dark',
+    'LINEARITY' => '-linearity',
+    'FRINGE'    => '-fringe',
+    'NOISEMAP'  => '-noisemap'
+    );
 
 my @return_component_list = ("DBINFO.IMFILE","PPIMAGE.STATS","LOG.IMFILE","PPIMAGE.BACKMDL","PPIMAGE.PATTERN");
 
-#
-# Step 1b: Open output files
-my $uri_command = $path_base . ".cmd";
+# STEP 0: Open output files
 my $uri_transfer= $path_base . ".transfer";
 my $uri_check   = $path_base . ".check";
@@ -165 +85 @@
 my $uri_return  = $path_base . ".return";
 
-my $disk_command = $ipprc->file_resolve($uri_command,1);
 my $disk_transfer= $ipprc->file_resolve($uri_transfer,1);
 my $disk_check   = $ipprc->file_resolve($uri_check,1);
@@ -174 +93 @@
 my (undef, $remote_config) = uri_convert($uri_config); # Needs to be done after we've created it.
 
-open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer",$chip_id,$PS_EXIT_SYS_ERROR);
-open(CHECK,    ">$disk_check")  || &my_die("Couldn't open file? $disk_check",$chip_id,$PS_EXIT_SYS_ERROR);
-open(CONFIG,   ">$disk_config")  || &my_die("Couldn't open file? $disk_config",$chip_id,$PS_EXIT_SYS_ERROR);
-open(GENERATE, ">$disk_generate") || &my_die("Couldn't open file? $disk_generate",$chip_id,$PS_EXIT_SYS_ERROR);
-open(RETURN,   ">$disk_return") || &my_die("Couldn't open file? $disk_return", $chip_id, $PS_EXIT_SYS_ERROR);
-
-
-#
-# Step 2: Iterate over all componenets in this remote run.
+open(TRANSFER, ">$disk_transfer")  || &my_die("Couldn't open file? $disk_transfer", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CHECK,    ">$disk_check")     || &my_die("Couldn't open file? $disk_check",    $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(CONFIG,   ">$disk_config")    || &my_die("Couldn't open file? $disk_config",   $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(GENERATE, ">$disk_generate")  || &my_die("Couldn't open file? $disk_generate", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+open(RETURN,   ">$disk_return")    || &my_die("Couldn't open file? $disk_return",   $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
+
 my $job_index = 0;
-my @pre_commands = ();
-my @main_commands = ();
-my @post_commands = ();
-
-foreach my $compEntry (@$compData2) {
-
-    my $chip_id = $compEntry->{stage_id};
-    my $exp_id ;
-# Get exposure level information from the chipRun we're working from.
+
+# STEP 1 : Get exposure level information for this chipRun
+my ($exp_id, $filter, $altfilt, $dateobs);
+{
     my $command = "$chiptool -listrun -chip_id $chip_id ";
     $command   .= " -dbname $dbname " if defined($dbname);
-    
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 0);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $dbverbose);
     unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        
-        &my_die("Unable to run chiptool to determine stage parameters.",
-                $chip_id,$error_code);
-    }
-    
-# Parse chipRun level data to determine detrend information 
-    my %detrends_to_use = ();
-    {
-        my $chipData = $mdcParser->parse(join "", @$stdout_buf) or
-            &my_die("Unable to determine chip component information.",
-                    $chip_id,$PS_EXIT_PROG_ERROR);
-        my $chipData2= parse_md_list($chipData);
-        my $chipEntry = ${ $chipData2 }[0];
-        
-        $exp_id = $chipEntry->{exp_id};    
-        my $filter = $chipEntry->{filter};
-        my $altfilt= $filter;
-        $altfilt =~ s/.00000//;
-        my $dateobs= $chipEntry->{dateobs};
-        my ($date, $time) = split /T/, $dateobs;
-        my ($year,$month,$day) = split /-/, $date;
-        my ($hour,$minute,$second) = split /:/, $time; #/;
-        my $dateobs_obj = DateTime->new(year => $year, month => $month, day => $day, hour => $hour, minute => $minute, second => $second);
-
-        foreach my $det_type (%detrends) {
-            if (($filter !~ /y/)&&($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y
-
-            foreach my $det_obj (@{ $detrends{$det_type} }) { # Iterate over the available options
-                if (defined($det_obj->{filter})) { 
-                    if (($det_obj->{filter} ne $filter)&&($det_obj->{filter} ne $altfilt)) {
-                        next; # Skip detrends with a filter that we can't use.
-                    }
-                }
-                if (defined($det_obj->{time_begin})) {
-                    if (DateTime->compare($dateobs_obj,$det_obj->{time_begin}) == -1) {
-                        next; # Skip detrends that come into effect after this exposure (d1 < d2)
-                    }
-                }
-                if (defined($det_obj->{time_end})) {
-                    if (DateTime->compare($dateobs_obj,$det_obj->{time_end}) == 1) {
-                        next; # Skip detrends that stop working before this exposure (d1 > d2)
-                    }
-                }
-                $detrends_to_use{$det_type} = $det_obj;
-                last;
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+        &my_die("Unable to run chiptool to determine stage parameters.", $remote_id, $chip_id, $error_code, $fail_state);
+    }
+
+    # Parse chipRun level data to determine exposure information
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to determine chip component information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
+    my $metadata = parse_md_list($MDlist);
+    my $chipEntry = $metadata->[0];
+
+    $exp_id  = $chipEntry->{exp_id};
+    $filter  = $chipEntry->{filter};
+    $altfilt = $filter;
+    $altfilt =~ s/.00000//;
+    $dateobs = $chipEntry->{dateobs};
+}
+
+# STEP 2 : select the det_ids for each desired det_type (appropriate to this exposure)
+my %det_ids = ();
+my %det_iters = ();
+
+foreach my $det_type (keys (%det_types)) {
+    if (($filter !~ /y/) && ($det_type eq 'FRINGE')) { next; } # We can skip fringe for all but y
+
+    my $command = "detselect -search -det_type $det_type -time $dateobs";
+    $command   .= " -dbname $dbname " if defined($dbname);
+
+    if (($det_type eq 'FLAT') || ($det_type eq 'FRINGE')) {
+        $command .= " -filter $altfilt";
+    }
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $dbverbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+        &my_die("No valid detrend available for this image: $det_type", $remote_id, $chip_id, $error_code, $fail_state);
+    }
+
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Could not parse detrend information for this image: $det_type", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
+    my $metadata = parse_md_list($MDlist);
+    my $detEntry = $metadata->[0];
+
+    if ($verbose) {
+        print STDERR "det_id: $detEntry->{det_id}\n";
+        print STDERR "det_iter: $detEntry->{iteration}\n";
+    }
+
+    # save det_id and iteration
+    $det_ids{$det_type} = $detEntry->{det_id};
+    $det_iters{$det_type} = $detEntry->{iteration};
+}
+
+# Step 3: Iterate over the sub-components
+{
+    # select chipProcessedImfile entries
+    my $command = "$chiptool -pendingimfile -chip_id $chip_id";
+    $command   .= " -dbname $dbname "   if defined($dbname);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $dbverbose);
+
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+        &my_die("Unable to run chiptool -pendingimfile", $remote_id, $chip_id, $error_code, $fail_state);
+    }
+
+    my $MDlist = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse chiptool -pendingimfile information.", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
+    my $metadata = parse_md_list($MDlist);
+
+    # Iterate over the chipProcessedImfile level data.
+    foreach my $chipEntry (@$metadata) {
+        # Get information we need to pass to ppImage
+        my $uri            = $chipEntry->{uri};
+        my $class_id       = $chipEntry->{class_id};
+        my $video_cells    = $chipEntry->{video_cells};
+        my $reduction      = $chipEntry->{reduction};
+        my $exp_tag        = $chipEntry->{exp_tag};
+        my $workdir        = $chipEntry->{workdir};
+        my $chip_imfile_id = $chipEntry->{chip_imfile_id};
+
+        # Process the image and burntool table
+        # IMAGE CHECK: Look up the image location, prove it exists, and if not, abort the prep for this object.
+        my ($ipp_uri, $remote_uri) = uri_convert($uri);
+        unless (-e $ipp_uri) {
+            # Try to fix it.
+            system("neb-repair $uri");
+            ($ipp_uri, $remote_uri) = uri_convert($uri);
+            unless (-e $ipp_uri) {
+                # This image file is missing, so it will require manual intervention.
+                &my_die("Required image file $ipp_uri ($chip_id, $class_id) not found.  Stopping this prep.",
+                        $remote_id,$chip_id,$PS_EXIT_CONFIG_ERROR,$hard_fail_state);
             }
         }
-    }
-
-#
-# Step 3: Iterate over the sub-components
-# Iterate over the chipProcessedImfile level data.
-    my $reg_command = "$chiptool -pendingimfile ";
-    $reg_command   .= " -chip_id $chip_id " if defined($chip_id);
-    $reg_command   .= " -dbname $dbname "   if defined($dbname);
-    
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $reg_command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        
-        &my_die("Unable to run chiptool -pendingimfile",
-                $chip_id,$error_code);
-    }
-    
-    my $chipData = $mdcParser->parse(join "", @$stdout_buf) or 
-        next;
-#       &my_die("Unable to parse chiptool -pendingimfile information.",
-#               $chip_id,$error_code);
-    my $chipData2= parse_md_list($chipData);
-
-    
-    foreach my $chipEntry (@$chipData2) {
-        # Get information we need to pass to ppImage
-        my $uri            = $chipEntry->{uri};
-        my $class_id       = $chipEntry->{class_id};
-        my $video_cells    = $chipEntry->{video_cells};
-        my $reduction      = $chipEntry->{reduction};
-        my $exp_tag        = $chipEntry->{exp_tag};
-        my $workdir        = $chipEntry->{workdir};
-        my $chip_imfile_id = $chipEntry->{chip_imfile_id};
-        
-        # Process the image and burntool table
-        my ($ipp_uri,$remote_uri) = uri_to_outputs_raw($uri);
-        my $btt = $uri;
-        $btt =~ s/fits$/burn.tbl/;
-        my ($ipp_btt,$remote_btt) = uri_to_outputs_raw($btt);
-        
-        # Initialize the ppI command
-        my $ppImage_command = "ppImage -file $remote_uri";
-        $ppImage_command   .= " -burntool $remote_btt ";
-        
-        # Add detrend information to the command line
-        foreach my $det (keys %detrends_to_use) {
-            if ((($video_cells)&&($det eq 'DARK'))||
-                ((!$video_cells)&&($det eq 'VIDEODARK'))) {
-                next;
+
+        # The image exists, so continue.  This resets the variables, but also outputs the file locations.
+        ($ipp_uri, $remote_uri) = uri_to_outputs_raw($uri);
+        my $btt = $uri;
+        $btt =~ s/fits$/burn.tbl/;
+
+        # Check burntool table for existance, and if it doesn't, regenerate it.
+        my ($ipp_btt, $remote_btt) = uri_convert($btt);
+        unless (-e $ipp_btt) {
+            my $exp_name = $chipEntry->{exp_name};
+            my $burntool_command = "$ipp_burntool_fix --exp_name $exp_name --class_id $class_id";
+            $burntool_command   .= " -dbname $dbname" if defined($dbname);
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $burntool_command, verbose => $dbverbose);
+            # That command repairs the tables, but the instance we learned about may not be correct.  
+            # Look up the new one.
+            ($ipp_btt, $remote_btt) = uri_convert($btt);
+            
+            unless (-e $ipp_btt) {
+                &my_die("Attempted regeneration of burntool table ($chip_id, $class_id) has failed.  Stopping this prep.", 
+                        $remote_id,$chip_id, $PS_EXIT_CONFIG_ERROR, $hard_fail_state);
             }
-            my $det_code = $det_types{$det};
-            my $duri = $detrends_to_use{$det}->{$class_id};
-            my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
-            $ppImage_command .= " $det_code $remote_det_uri ";
         }
-            
-        
-        # Add output root
-        my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}";
-        my $remote_outroot = uri_local_to_remote($ipp_outroot);
-        $ppImage_command .= " $remote_outroot ";
-        print STDERR "$remote_outroot $ipp_outroot $class_id\n";
-        # Complete reduction information.
-        $reduction = 'DEFAULT' unless defined $reduction;
-        my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage
-        my $recipe_psphot  = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot
-        
-        $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage ";
-        $ppImage_command .= " -recipe PSPHOT $recipe_psphot ";
-        $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats ";
-        $ppImage_command .= " -threads 4 ";
-        $ppImage_command .= " -image_id $chip_imfile_id ";
-        $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace ";
-        $ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
-
-#       push @main_commands, $ppImage_command;
-
-        # Calculate pre and post commands
-        my $remote_outroot_dir = dirname($remote_outroot);
-        my $pre_command =  "mkdir -p $remote_outroot_dir";
-        
-        my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
-        $post_commandA   .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot ";
-        $post_commandA   .= " -magicked 0 -hostname REMOTE -dtime_script 0 ";
-        $post_commandA   .= " -dbname $dbname " if defined $dbname;
-
-        my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
-        my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
-        
-#       push @post_commands, "$post_commandB $post_commandC";
-
-        $job_index++;
-
-        print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
-
-
-        # Determine which output files need to be returned.
-        foreach my $component (@return_component_list) {
-#           uri_convert_and_create( $ipprc->filename($component, $ipp_outroot, $class_id) );
-            my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
-            my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
-            my $remote_outroot_dir = dirname($ipp_disk);
-            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
-#           print "  $filename $ipp_disk $remote_disk\n";
-#           die;
-        }
-
-        print CONFIG "\n";
-    }
-}
-
-# Actually put the commands into the output file.
-# my %unique_cmds = ();
-# foreach my $cmd (@pre_commands) {
-#     unless(exists($unique_cmds{$cmd})) {
-#       print CONFIG $cmd . "\n";
-#       $unique_cmds{$cmd} = 1;
-#     }
-# }
-# foreach my $cmd (@main_commands) {
-#     unless(exists($unique_cmds{$cmd})) {
-#       print CONFIG $cmd . "\n";
-#       $unique_cmds{$cmd} = 1;
-#     }
-# }
-# foreach my $cmd (@post_commands) {
-#     unless(exists($unique_cmds{$cmd})) {
-#       print CONFIG $cmd . "\n";
-#       $unique_cmds{$cmd} = 1;
-#     }
-# }
+        ($ipp_btt, $remote_btt) = uri_to_outputs_raw($btt);
+
+        # Initialize the ppI command
+        my $ppImage_command = "ppImage -file $remote_uri";
+        $ppImage_command   .= " -burntool $remote_btt ";
+
+        foreach my $det_type (keys (%det_types)) {
+            if (( $video_cells) && ($det_type eq 'DARK')) { next; }
+            if ((!$video_cells) && ($det_type eq 'VIDEODARK')) { next; }
+
+            my $det_id   = $det_ids{$det_type};
+            my $det_iter = $det_iters{$det_type};
+
+            if (not defined $det_id) { next; }
+
+            if ($verbose) { print STDERR "det_type, det_id, det_iter: $det_type, $det_id, $det_iter\n"; }
+
+            # Add detrend information to the command line
+            my $detselect_command = "detselect -select ";
+            $detselect_command   .= " -det_id $det_id";
+            $detselect_command   .= " -iteration $det_iter";
+            $detselect_command   .= " -class_id $class_id";
+            $detselect_command   .= " -dbname $dbname " if defined($dbname);
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $detselect_command, verbose => $dbverbose);
+
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+
+                &my_die("No valid detrend available for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $error_code, $fail_state);
+            }
+
+            my $detMDlist = $mdcParser->parse(join "", @$stdout_buf) or
+                &my_die("Could not parse detrend information for this image: $det_type $det_id $det_iter", $remote_id, $chip_id, $PS_EXIT_CONFIG_ERROR, $fail_state);
+            my $detImfileMD = parse_md_list($detMDlist);
+            my $detImfile = $detImfileMD->[0];
+            my $duri = $detImfile->{uri};
+
+            my $det_code = $det_types{$det_type};
+            my ($ipp_det_uri, $remote_det_uri) = uri_to_outputs_raw($duri);
+            $ppImage_command .= " $det_code $remote_det_uri ";
+        }
+
+        # Add output root
+        my $ipp_outroot = "${workdir}/${exp_tag}/${exp_tag}.ch.${chip_id}";
+        my $remote_outroot = uri_local_to_remote($ipp_outroot);
+        $ppImage_command .= " $remote_outroot ";
+        print STDERR "$remote_outroot $ipp_outroot $class_id\n";
+        # Complete reduction information.
+        $reduction = 'DEFAULT' unless defined $reduction;
+        my $recipe_ppImage = $ipprc->reduction($reduction, 'CHIP_PPIMAGE'); # Recipe to use for ppImage
+        my $recipe_psphot  = $ipprc->reduction($reduction, 'CHIP_PSPHOT'); # Recipe to use for psphot
+
+        $ppImage_command .= " -recipe PPIMAGE $recipe_ppImage ";
+        $ppImage_command .= " -recipe PSPHOT $recipe_psphot ";
+        $ppImage_command .= " -recipe PPSTATS CHIPSTATS -stats ${remote_outroot}.${class_id}.stats ";
+        $ppImage_command .= " -threads $threads_req ";
+        $ppImage_command .= " -image_id $chip_imfile_id ";
+        $ppImage_command .= " -tracedest ${remote_outroot}.${class_id}.trace ";
+        $ppImage_command .= " -log ${remote_outroot}.${class_id}.log ";
+
+        # Calculate pre and post commands
+        my $remote_outroot_dir = dirname($remote_outroot);
+        my $pre_command =  "mkdir -p $remote_outroot_dir";
+
+        my $post_commandA = "chiptool -addprocessedimfile -exp_id $exp_id -chip_id $chip_id -class_id $class_id ";
+        $post_commandA   .= " -uri ${ipp_outroot}.ch.${class_id}.ch.fits -path_base $ipp_outroot ";
+        $post_commandA   .= " -magicked 0 -hostname $remote_hostname -dtime_script 0 ";
+        $post_commandA   .= " -dbname $dbname " if defined $dbname;
+
+        my $post_commandB = "echo -n \"$post_commandA\" > ${remote_outroot}.${class_id}.dbinfo  ";
+        my $post_commandC = "ppStatsFromMetadata ${remote_outroot}.${class_id}.stats - CHIP_IMFILE >> ${remote_outroot}.${class_id}.dbinfo";
+
+        $job_index++;
+
+        print CONFIG "${pre_command} && ${ppImage_command} && ${post_commandB} && ${post_commandC}";
+
+        # Determine which output files need to be returned.
+        foreach my $component (@return_component_list) {
+            my $filename = $ipprc->filename($component, $ipp_outroot, $class_id);
+            my ($ipp_disk, $remote_disk) = uri_to_outputs_for_return( $filename );
+            my $remote_outroot_dir = dirname($ipp_disk);
+            print CONFIG " && mkdir -p ${remote_root}/tmp/${remote_outroot_dir} && ln -sf $remote_disk ${remote_root}/tmp/${ipp_disk} && touch $remote_disk ";
+        }
+        print CONFIG "\n";
+    }
+}
 
 close(CONFIG);
@@ -386 +326 @@
 close(GENERATE);
 
-#
-# Construct the moab command last, so we can use the job_index counter to estimate resources.  Somehow.
-my $proc_need = $job_index * $threads;       # how many total processors do we need?
-my $node_need = $proc_need / $proc_per_node; # this equals how many nodes?
-my $time_need = $job_index * $job_cost;      # How many seconds will this take?
-
-my $fill_factor = 0.8;  # This is the factor of how much of the time allocation we'd like to fill
-my ($time_req,$node_req);
-if ($node_need * $job_cost < $fill_factor * $min_nodes * $min_time) {
-    $time_req = $min_time;
-    $node_req = $min_nodes;
-}
-elsif ($node_need * $job_cost > $fill_factor * $max_nodes * $max_time) {
-    $time_req = $max_time;
-    $node_req = $max_nodes;
-    print STDERR "You've requested the construction of a bundle that appears to need $node_need nodes and $job_cost time per job.  This exceeds the max limits ($max_nodes, $max_time).  Using those max values instead.  Good luck.\n";
-}
-else {
-    $time_req = int(($node_need * $job_cost) / ($fill_factor * $max_nodes)) + 1;
-    $node_req = int(($node_need * $job_cost) / ($fill_factor * $time_req)) + 1;
-}
-$time_req += 1; # Safety addition.
-
-open(COMMAND,  ">$disk_command") || &my_die("Couldn't open file? $disk_command",$chip_id,$PS_EXIT_SYS_ERROR);
-print COMMAND "#!/bin/tcsh\n";
-print COMMAND "##### Moab controll lines\n";
-print COMMAND "#MSUB -l nodes=${node_req}:ppn=${proc_per_node},walltime=${time_req}:00:00\n"; ## CHECK RESOURCES
-print COMMAND "#MSUB -j oe\n";
-print COMMAND "#MSUB -V\n";
-print COMMAND "#MSUB -o ${remote_root}chip.${remote_id}.out\n";
-print COMMAND "date\n";
-print COMMAND 'srun -n $SLURM_JOB_NUM_NODES -m cyclic -l /bin/hostname | sort -n | awk \'{printf "%s\n", $2}\' > hosts.${SLURM_JOB_ID}' . "\n";
-print COMMAND "${remote_root}/stask_chip.py $remote_config " . 'hosts.${SLURM_JOB_ID} 1' . "\n";
-#print COMMAND "validate_processing.pl ${remote_root}chip.${remote_id}.out\n";
-print COMMAND "date\n";
-close(COMMAND);
-
-
 ## We're done here. The execution and handling are done elsewhere.
 # Quick review:
@@ -429 +331 @@
 # auth
 unless($no_update) {
-    my $command = "remotetool -updaterun -remote_id $remote_id ";
-    $command .= " -set_state pending ";
+    my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $chip_id ";
+    $command .= " -set_jobs $job_index";
+    $command .= " -set_path_base $path_base";
+    $command .= " -set_state prep_done";
     $command .= " -dbname $dbname " if defined $dbname;
 
     system($command);
 }
-
-
-    
-
-
+exit (0);
 
 sub uri_convert {
@@ -445 +345 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri );
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -461 +361 @@
     my $ipp_disk= $ipprc->file_resolve( $neb_uri , 1);
     my $remote_disk = $ipp_disk;
-    
+
     unless(defined($ipp_disk)) {
-        my_die();
+        &my_die( "Unable to generate file for $neb_uri ", $remote_id, $chip_id, $PS_EXIT_SYS_ERROR, $fail_state);
     }
 
@@ -476 +376 @@
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
-    
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print TRANSFER "$ipp_disk\n";
-        print CHECK    "$remote_disk\n";
-    }
+
+    print TRANSFER "$ipp_disk\n";
+    print CHECK    "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -489 +386 @@
     my ($ipp_disk, $remote_disk) = uri_convert( $neb_uri );
     $remote_disk = $remote_raw . $ipp_disk;
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print TRANSFER "$ipp_disk\n";
-        print CHECK    "$remote_disk\n";
-    }
+
+    print TRANSFER "$ipp_disk\n";
+    print CHECK    "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -500 +395 @@
     my $neb_uri = shift;
     my ($ipp_disk, $remote_disk) = uri_convert_and_create( $neb_uri );
-    unless (exists($file_filter{$neb_uri})) {
-        $file_filter{$neb_uri} = 1;
-        print RETURN "$ipp_disk\n";
-        print GENERATE "$remote_disk\n";
-    }
+    print RETURN "$ipp_disk\n";
+    print GENERATE "$remote_disk\n";
     return($ipp_disk,$remote_disk);
 }
@@ -518 +410 @@
     return($remote_uri);
 }
- 
+
 sub uri_remote_to_local {
     # This needs to replace the remote root directory with the nebulous tag.
@@ -524 +416 @@
     $remote_uri =~ s%${remote_root}%%;
     my $local_uri  = "neb:///" . $remote_uri;
-    
+
     return($local_uri);
 }
+
 sub my_die {
     my $msg = shift;
-    my $id  = shift;
+    my $remote_id  = shift;
+    my $stage_id  = shift;
     my $exit_code = shift;
     my $exit_state = shift;
@@ -536 +430 @@
 
     carp($msg);
-    
-    if (defined $id and not $no_update) {
-        my $command = "remotetool -updaterun -remote_id $id";
-        $command .= " -fault $exit_code " if defined $exit_code;
-        $command .= " -set_state $exit_state " if defined $exit_state;
-        $command .= " -dbname $dbname " if defined $dbname;
-
-        system($command);
+
+    if (defined $remote_id and defined $stage_id and not $no_update) {
+        my $command = "remotetool -updatecomponent -remote_id $remote_id -stage_id $stage_id";
+        $command .= " -set_state $exit_state " if defined $exit_state;
+        $command .= " -dbname $dbname " if defined $dbname;
+
+        system($command);
     }