Index: trunk/ippScripts/scripts/ipp_cleanup.pl
===================================================================
--- trunk/ippScripts/scripts/ipp_cleanup.pl	(revision 26549)
+++ trunk/ippScripts/scripts/ipp_cleanup.pl	(revision 26939)
@@ -61,10 +61,13 @@
 
 my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff"  => 1,
-	       "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1,
-	       "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1,
-	       "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 );
-unless ($stages{$stage}) {
+	       "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0,
+	       "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 );
+unless (exists($stages{$stage})) {
     die "unknown stage $stage for ipp_cleanup.pl\n";
 }
+unless (($stages{$stage})) {
+    die "unimplemented stage $stage for ipp_cleanup.pl\n";
+}
+
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -888,8 +891,11 @@
 } 
 # Detrend stages
-if ($stage eq "detrend.process.imfile") {
+if ($stage eq "detrend.processed") {
 
     die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id;
     ### select the imfiles for this entry
+
+    # Neither det_id nor exp_id uniquely determine a det exposure, so we pack them.
+    my ($det_id,$exp_id) = split /\./, $stage_id;       #/ trailing slash for emacs;
 
     # this stage uses 'dettool'
@@ -899,59 +905,45 @@
     # XXX may need a different my_die for each stage
     my $imfiles;                      # Array of component files
-    my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run
+    my $metadata;
+    my $command = "$dettool -pendingcleanup_processedimfile -det_id $det_id -exp_id $exp_id"; # Command to run
     $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
+        &my_die("Unable to perform dettool: $error_code", "detrend.processed", $stage_id, $error_code);
     }
 
     # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
-    if (@$stdout_buf == 0)  {
-	exit 0; # Silently exit if there's nothing to do.  I don't know how we'd ever get here, but let's be safe.
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+    if (@$stdout_buf != 0)  {
+#	exit 0; # Silently exit if there's nothing to do.  I don't know how we'd ever get here, but let's be safe.
+
+
+	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
-
-    # extract the metadata for the files into a hash list
-    $imfiles = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
-
+	
+	# extract the metadata for the files into a hash list
+	$imfiles = parse_md_list($metadata) or
+	    &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+    }
     # loop over all of the imfiles, determine the path_base and class_id for each
     foreach my $imfile (@$imfiles) {
-	my $exp_id   = $imfile->{exp_id};
+	my $iexp_id   = $imfile->{exp_id};
         my $class_id = $imfile->{class_id};
         my $path_base = $imfile->{path_base};
         my $status = 1;
 
-        # don't clean up unless the data needed to update is available
-        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
-	# goto_scrubbed now requires the config file to not exist.
-	
-	# Possibly not the correct config file, but simtest doesn't leave any around to check.
-        if ($mode eq "goto_cleaned") {
-            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
-
-	    unless ($ipprc->file_exists($config_file)) {
-                print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id "
-                    . " because config file is missing\n";
-                $status = 0;
-            }
-        }
-	elsif ($mode eq "goto_scrubbed") {
-	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
-
-	    if ($ipprc->file_exists($config_file)) {
-		print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id "
-		    . " because config file is present\n";
-		$status = 0;
-	    }
-	}
-
+	unless (defined($path_base)) {
+	    print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n";
+	    $status = 0;
+	}
+	unless (defined($class_id)) {
+	    print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n";
+	    $status = 0;
+	}
+	# Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, 
+	# and so there is no check for config files.
         if ($status) {
             # array of actual filenames to delete
             my @files = ();
-
             # delete the temporary image datafiles
             addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
@@ -964,5 +956,4 @@
                 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
             }
-
             # actual command to delete the files
             $status = &delete_files (\@files);
@@ -970,5 +961,5 @@
 
         if ($status)  {
-            my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile";
+            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile";
             if ($mode eq "goto_purged") {
                 $command .= " -data_state purged";
@@ -980,5 +971,4 @@
 		$command .= " -data_state scrubbed";
 	    }
-
             $command .= " -dbname $dbname" if defined $dbname;
 
@@ -991,7 +981,7 @@
 
         } else {
-
-	    # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
-	    my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state";
+	    # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* 
+	    my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile ";
+	    $command .= " -data_state $error_state";
 	    $command .= " -dbname $dbname" if defined $dbname;
 
@@ -1002,48 +992,55 @@
                 &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
             }
-        }
-    }
-
-    # Check to see if we can mark the whole detRunSummary object as cleaned.
-
-    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+	}
+    }
+
+    # Flag the detProcessedExp as clean now as well, if it is marked to be cleaned (this is clunky, but works for now).
+
+    $command = "$dettool -pendingcleanup_processedexp -det_id $det_id -exp_id $exp_id";
     $command .= " -dbname $dbname" if defined $dbname;
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     unless ($success) {
 	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+	&my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code);
     }
     if (@$stdout_buf != 0) {
 	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
-	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to parse metadata config doc", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR);
 	my $exps = parse_md_list($metadata) or
-	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to parse metadata list", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR);
 	
 	foreach my $exp (@$exps) {
-	    my $iteration = $exp->{iteration};
-	    my $command;
+	    my $exp_id = $exp->{exp_id};
+	    my $command = "$dettool -updateprocessedexp -det_id $det_id -exp_id $exp_id ";
 	    if ($mode eq "goto_cleaned") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+		$command .= " -data_state cleaned ";
 	    }
 	    if ($mode eq "goto_scrubbed") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+		$command .= " -data_state scrubbed ";
 	    }
 	    if ($mode eq "goto_purged") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
-	    }
-	    $command .= " -dbname $dbname" if defined $dbname;
+		$command .= " -data_state purged ";
+	    }
+	    $command .= " -dbname $dbname" if defined $dbname;
+#	    print "$command\n";
 	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 		run(command => $command, verbose => $verbose);
 	    unless ($success) {
 		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
-	    }
-	}
-    }
-    
+		&my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code);
+	    }
+	}
+    }
+     
     exit 0;
 }
-if ($stage eq "detrend.process.exp") {
+if ($stage eq "detrend.resid") {
+
     die "--stage_id required for stage $stage\n" if !$stage_id;
+    ### select the imfiles for this entry
+
+    # Neither det_id nor exp_id uniquely determine the det exposure, so we pack them
+    my ($det_id,$exp_id) = split /\./, $stage_id;          #/ trailing slash for emacs;
+
     # this stage uses 'dettool'
     my $dettool = can_run('dettool') or die "Can't find dettool";
@@ -1051,118 +1048,120 @@
     # Get list of component imfiles
     # XXX may need a different my_die for each stage
-    my $exps;                      # Array of component files
-    my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run
+    my $imfiles;                      # Array of component files
+    my $metadata;
+    my $command = "$dettool -pendingcleanup_residimfile -det_id $det_id -exp_id $exp_id"; # Command to run
     $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
     }
     
-    if (@$stdout_buf == 0) {
-	exit 0; #silently abort. I need to fix this for propers
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
-
-    $exps = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
-
-
-    foreach my $exp (@$exps) {
-	my $path_base = $exp->{path_base};
-	my $exp_id    = $exp->{exp_id};
-
-	my $status = 1;
-	# don't clean up unless the data needed to update is available
-	# goto_scrubbed now requires the config file to not be present
-	if ($mode eq "goto_cleaned") {
-	    my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
-	    
-	    unless ($ipprc->file_exists($config_file)) {
-		print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
-		$status = 0;
-	    }
-	}
-	elsif ($mode eq "goto_scrubbed") {
-	    my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base);
-	    
-	    if ($ipprc->file_exists($config_file)) {
-		print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
-		$status = 0;
-	    }
-	}
-	if ($status) {
-	    my @files = ();
-	    # delete the temporary image datafiles
-	    # I can't find anything to put here
-	    if ($mode eq "goto_purged") {
-		# additional files to remove for 'purge' mode
-		addFilename (\@files, "PPIMAGE.JPEG1", $path_base);
-		addFilename (\@files, "PPIMAGE.JPEG2", $path_base);
-	    }
-	    # actual command to delete the files
-	    $status = &delete_files (\@files);
-	}
+    # if there are no detResidImfiles (@$stdout_buf == 0), then silently exit.
+    if (@$stdout_buf != 0) {
+	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
+	    &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
 	
-	if ($status)  {
-	    my $command;
-	    if ($mode eq "goto_cleaned") {
-		$command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned";
-	    }
-	    if ($mode eq "goto_scrubbed") {
-		$command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed";
-	    }
-	    if ($mode eq "goto_purged") {
-		$command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged";
-	    }
-	    $command .= " -dbname $dbname" if defined $dbname;
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run(command => $command, verbose => $verbose);
-	    unless ($success) {
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
-	    }
-	} else {
-	    # since 'camera' has only a single imfile, we can just update the run
-	    my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state";
-	    $command .= " -dbname $dbname" if defined $dbname;
-	    
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run(command => $command, verbose => $verbose);
-	    unless ($success) {
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
-	    }
-	    exit $PS_EXIT_UNKNOWN_ERROR;
-	}
-    }
-    # Check to see if we can mark the whole detRunSummary object as cleaned.
-
-    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
+	# extract the metadata for the files into a hash list
+	$imfiles = parse_md_list($metadata) or
+	    &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
+    }
+    # loop over all of the imfiles, determine the path_base and class_id for each
+    foreach my $imfile (@$imfiles) {
+	my $iexp_id = $imfile->{exp_id};
+        my $class_id = $imfile->{class_id};
+        my $path_base = $imfile->{path_base};
+	my $iteration = $imfile->{iteration};
+
+        my $status = 1;
+
+	# Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent,
+	# and so there is no check for config files.
+	unless (defined($path_base)) {
+	    print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n";
+	    $status = 0;
+	}
+	unless (defined($class_id)) {
+	    print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n";
+	    $status = 0;
+	}
+        if ($status) {
+            # array of actual filenames to delete
+            my @files = ();
+
+            # delete the temporary image datafiles
+            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
+            if ($mode eq "goto_purged") {
+                # additional files to remove for 'purge' mode
+		addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
+		addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
+            }
+#	    foreach my $f (@files) {
+#		print "RESID: $f\n";
+#	    }
+            # actual command to delete the files
+            $status = &delete_files (\@files);
+        }
+
+        if ($status)  {
+            my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile ";
+            if ($mode eq "goto_purged") {
+                $command .= " -data_state purged";
+            }
+	    elsif ($mode eq "goto_cleaned") {
+                $command .= " -data_state cleaned";
+            }
+	    elsif ($mode eq "goto_scrubbed") {
+		$command .= " -data_state scrubbed";
+	    }
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        } else {
+	    my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile ";
+	    $command .= " -data_state $error_state ";
+	    $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
+            }
+        }
+    }
+    # Flag the detResidExp as clean now as well, if it is marked tobe cleaned (this is still clunky).
+
+    $command = "$dettool -pendingcleanup_residexp -det_id $det_id -exp_id $exp_id"; 
     $command .= " -dbname $dbname" if defined $dbname;
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
     unless ($success) {
 	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+	&my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code);
     }
     if (@$stdout_buf != 0) {
 	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
-	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
-	$exps = parse_md_list($metadata) or
-	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
+	    &my_die("Unable to parse metadata config doc", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR);
+	my $exps = parse_md_list($metadata) or
+	    &my_die("Unable to parse metadata list", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR);
 	
 	foreach my $exp (@$exps) {
 	    my $iteration = $exp->{iteration};
-	    my $command;
+	    my $command = "$dettool -updateresidexp -det_id $det_id -exp_id $exp_id ";
 	    if ($mode eq "goto_cleaned") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
+		$command .= " -data_state cleaned";
 	    }
 	    if ($mode eq "goto_scrubbed") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
+		$command .= " -data_state scrubbed";
 	    }
 	    if ($mode eq "goto_purged") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
+		$command .= " -data_state purged";
 	    }
 	    $command .= " -dbname $dbname" if defined $dbname;
@@ -1171,5 +1170,5 @@
 	    unless ($success) {
 		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
+		&my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code);
 	    }
 	}
@@ -1178,4 +1177,5 @@
     exit 0;
 }
+
 if ($stage eq "detrend.stack.imfile") {
     
@@ -1210,25 +1210,4 @@
 
 	my $status = 1;
-
-# 	if ($mode eq "goto_cleaned") {
-# 	    my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
-
-# 	    $config_file =~ s%^file://%%;
-# 	    if (!$config_file or ! -e $config_file) {
-# 		print STDERR "skipping cleanup for $stage $stage_id $path_base" .
-# 		    " because config file is missing\n";
-# 		$status = 0;
-# 	    }
-# 	    $config_file = 'file://' . $config_file;
-# 	}
-# 	elsif ($mode eq "goto_scrubbed") {
-# 	    my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id);
-# 	    $config_file =~ s%^file://%%;
-# 	    if ($config_file and -e $config_file) {
-# 		print STDERR "skipping scrubbed for $stage $stage_id $path_base" .
-# 		    " because config file is present\n";
-# 		$status = 0;
-# 	    }
-# 	}
 
 	if ($status) {
@@ -1684,285 +1663,4 @@
     exit 0;
 }
-if ($stage eq "detrend.resid.imfile") {
-
-    die "--stage_id required for stage $stage\n" if !$stage_id;
-    ### select the imfiles for this entry
-
-    # this stage uses 'dettool'
-    my $dettool = can_run('dettool') or die "Can't find dettool";
-
-    # Get list of component imfiles
-    # XXX may need a different my_die for each stage
-    my $imfiles;                      # Array of component files
-    my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code);
-    }
-
-    if (@$stdout_buf == 0) {
-	exit 0;
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
-
-    # extract the metadata for the files into a hash list
-    $imfiles = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
-
-    # loop over all of the imfiles, determine the path_base and class_id for each
-    foreach my $imfile (@$imfiles) {
-        my $class_id = $imfile->{class_id};
-	my $iteration = $imfile->{iteration};
-	my $exp_id = $imfile->{exp_id};
-        my $path_base = $imfile->{path_base};
-        my $status = 1;
-
-        # don't clean up unless the data needed to update is available
-        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
-	# goto_scrubbed now requires the config file to not exist.
-	
-	# Possibly not the correct config file, but simtest doesn't leave any around to check.
-        if ($mode eq "goto_cleaned") {
-            my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
-
-	    unless ($ipprc->file_exists($config_file)) {
-                print STDERR "skipping cleanup for $stage $stage_id $class_id "
-                    . " because config file is missing\n";
-                $status = 0;
-            }
-        }
-	elsif ($mode eq "goto_scrubbed") {
-	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
-
-	    if ($ipprc->file_exists($config_file)) {
-		print STDERR "skipping scrubbed for $stage $stage_id $class_id "
-		    . " because config file is present\n";
-		$status = 0;
-	    }
-	}
-
-        if ($status) {
-            # array of actual filenames to delete
-            my @files = ();
-
-            # delete the temporary image datafiles
-            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
-            if ($mode eq "goto_purged") {
-                # additional files to remove for 'purge' mode
-		addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
-		addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
-                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
-            }
-
-            # actual command to delete the files
-            $status = &delete_files (\@files);
-        }
-
-        if ($status)  {
-            my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id";
-            if ($mode eq "goto_purged") {
-                $command .= " -data_state purged";
-            }
-	    elsif ($mode eq "goto_cleaned") {
-                $command .= " -data_state cleaned";
-            }
-	    elsif ($mode eq "goto_scrubbed") {
-		$command .= " -data_state scrubbed";
-	    }
-
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
-            }
-        } else {
-	    my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state";
-	    $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
-            }
-        }
-    }
-    # Check to see if we can mark the whole detRunSummary object as cleaned.
-
-    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
-    $command .= " -dbname $dbname" if defined $dbname;
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-    unless ($success) {
-	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
-    }
-    if (@$stdout_buf != 0) {
-	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
-	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
-	my $exps = parse_md_list($metadata) or
-	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
-	
-	foreach my $exp (@$exps) {
-	    my $iteration = $exp->{iteration};
-	    my $command;
-	    if ($mode eq "goto_cleaned") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
-	    }
-	    if ($mode eq "goto_scrubbed") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
-	    }
-	    if ($mode eq "goto_purged") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
-	    }
-	    $command .= " -dbname $dbname" if defined $dbname;
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run(command => $command, verbose => $verbose);
-	    unless ($success) {
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
-	    }
-	}
-    }
-
-    exit 0;
-}
-if ($stage eq "detrend.resid.exp") {
-    die "--stage_id required for stage $stage\n" if !$stage_id;
-    # this stage uses 'camtool'
-    my $dettool = can_run('dettool') or die "Can't find dettool";
-
-    # Get list of component imfiles
-    # XXX may need a different my_die for each stage
-    my $exps;                      # Array of component files
-    my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
-    }
-    # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this
-    if (@$stdout_buf == 0) {
-	exit 0;
-    }
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
-
-    $exps = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR);
-
-    foreach my $exp (@$exps) {
-	my $exp_id = $exp->{exp_id};
-	my $iteration = $exp->{iteration};
-	my $path_base = $exp->{path_base};
-#	my $class_id  = $exp->{class_id} 
-	my $status = 1;
-	# don't clean up unless the data needed to update is available
-	# goto_scrubbed now requires the config file to not be present
-	if ($mode eq "goto_cleaned") {
-	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
-	    
-	    unless ($ipprc->file_exists($config_file)) {
-		print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n";
-		$status = 0;
-	    }
-	}
-	elsif ($mode eq "goto_scrubbed") {
-	    my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base);
-	    
-	    if ($ipprc->file_exists($config_file)) {
-		print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n";
-		$status = 0;
-	    }
-	}
-	if ($status) {
-	    my @files = ();
-	    # delete the temporary image datafiles
-	    if ($mode eq "goto_purged") {
-		# additional files to remove for 'purge' mode
-                addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id);
-                addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id);
-	    }
-	    # actual command to delete the files
-	    $status = &delete_files (\@files);
-	}
-	
-	if ($status)  {
-	    my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration";
-	    if ($mode eq "goto_cleaned") {
-		$command .= " -data_state cleaned";
-	    }
-	    if ($mode eq "goto_scrubbed") {
-		$command .= " -data_state scrubbed";
-	    }
-	    if ($mode eq "goto_purged") {
-		$command .= " -data_state purged";
-	    }
-	    $command .= " -dbname $dbname" if defined $dbname;
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run(command => $command, verbose => $verbose);
-	    unless ($success) {
-		print STDERR " residexp had an issue setting the state:? $success $error_code\n";
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
-	    }
-	} else {
-	    my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state";
-	    $command .= " -dbname $dbname" if defined $dbname;
-	    
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run(command => $command, verbose => $verbose);
-	    unless ($success) {
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		&my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code);
-	    }
-	    exit $PS_EXIT_UNKNOWN_ERROR;
-	}
-    }
-    # Check to see if we can mark the whole detRunSummary object as cleaned.
-
-    $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; 
-    $command .= " -dbname $dbname" if defined $dbname;
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-    unless ($success) {
-	$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-	&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
-    }
-    if (@$stdout_buf != 0) {
-	$metadata = $mdcParser->parse(join "", @$stdout_buf) or
-	    &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
-	$exps = parse_md_list($metadata) or
-	    &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR);
-	
-	foreach my $exp (@$exps) {
-	    my $iteration = $exp->{iteration};
-	    my $command;
-	    if ($mode eq "goto_cleaned") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned";
-	    }
-	    if ($mode eq "goto_scrubbed") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed";
-	    }
-	    if ($mode eq "goto_purged") {
-		$command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged";
-	    }
-	    $command .= " -dbname $dbname" if defined $dbname;
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run(command => $command, verbose => $verbose);
-	    unless ($success) {
-		$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		&my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code);
-	    }
-	}
-    }
-    exit 0;
-}