Index: trunk/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- trunk/ippScripts/scripts/ipp_cleanup.pl (revision 24642) +++ trunk/ippScripts/scripts/ipp_cleanup.pl (revision 25299) @@ -60,8 +60,8 @@ -my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff => 1, - detrend.process.imfile => 1, detrend.process.exp => 1, detrend.stack.imfile => 1, - detrend.normstat.imfile => 1, detrend.norm.imfile => 1, detrend.norm.exp => 1, - detrend.resid.imfile => 1, detrend.resid.exp => 1 ); +my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, + "detrend.process.imfile" => 1, "detrend.process.exp" => 1, "detrend.stack.imfile" => 1, + "detrend.normstat.imfile" => 1, "detrend.norm.imfile" => 1, "detrend.norm.exp" => 1, + "detrend.resid.imfile" => 1, "detrend.resid.exp" => 1 ); unless ($stages{$stage}) { die "unknown stage $stage for ipp_cleanup.pl\n"; @@ -119,7 +119,8 @@ # don't clean up unless the data needed to update is available # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - print STDERR "CHIP: CONFIG_FILE : $config_file\n"; + if (!$config_file or ! -e $config_file) { print STDERR "skipping cleanup for chipRun $stage_id $class_id " @@ -128,4 +129,13 @@ } } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for chipRun $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } if ($status) { @@ -160,7 +170,12 @@ if ($mode eq "goto_purged") { $command .= " -topurgedimfile"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedimfile"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } + $command .= " -dbname $dbname" if defined $dbname; @@ -218,4 +233,5 @@ my $status = 1; # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); @@ -225,4 +241,12 @@ $status = 0; } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; + $status = 0; + } } if ($status) { @@ -246,5 +270,5 @@ } if ($mode eq "goto_scrubbed") { - $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned"; + $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed"; } if ($mode eq "goto_purged") { @@ -311,4 +335,13 @@ } } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } + } if ($status) { # delete the temporary image datafiles @@ -337,7 +370,11 @@ if ($mode eq "goto_purged") { $command .= " -topurgedskyfile"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedskyfile"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedskyfile"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -395,6 +432,5 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - print STDERR "MY CONFIG FILE = $config_file\n"; - printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); + $config_file =~ s%^file://%%; if (!$config_file or ! -e $config_file) { @@ -404,4 +440,13 @@ } $config_file = 'file://' . $config_file; + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); + $config_file =~ s%^file://%%; + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } } if ($status) { @@ -428,6 +473,10 @@ if ($mode eq "goto_purged") { $command .= " -updaterun -state purged"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -updaterun -state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updaterun -state scrubbed"; } $command .= " -dbname $dbname" if defined $dbname; @@ -485,6 +534,5 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - print STDERR "MY CONFIG FILE = $config_file\n"; - printf(STDERR "BOOLS: %d %d %d %s\n",!$config_file, ! -e $config_file, -e $config_file,$config_file); + $config_file =~ s%^file://%%; if (!$config_file or ! -e $config_file) { @@ -494,4 +542,13 @@ } $config_file = 'file://' . $config_file; + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + $config_file =~ s%^file://%%; + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . + " because config file ($config_file) is present\n"; + $status = 0; + } } if ($status) { @@ -529,10 +586,15 @@ if ($status) { my $command = "$difftool -diff_id $stage_id"; -# my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; + if ($mode eq "goto_purged") { $command .= " -updaterun -state purged"; - } else { + } + elsif ($mode eq "goto_cleaned") { $command .= " -updaterun -state cleaned"; } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updaterun -state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; @@ -545,5 +607,5 @@ } else { my $command = "$difftool -updaterun -diff_id $stage_id -state $error_state"; -# my $command = "$difftool -updaterun -diff_id $stage_id -skycell_id $skycell_id -state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; @@ -560,7 +622,1208 @@ } if ($stage eq 'fake') { - die "ipp_cleanup.pl -stage fake not yet implemented. Probably will just mark database cleaned.\n"; + print STDERR "This does not seem to work at present, as no files exist. Terminating quietly.\n"; + exit(0); + die "--stage_id required for stage fake\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'chiptool' + my $faketool = can_run('faketool') or die "Can't find faketool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$faketool -pendingcleanupimfile -fake_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + + # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' + if (@$stdout_buf == 0) { + my $command = "$faketool -fake_id $stage_id -updaterun -set_state new"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + exit 0; + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "fake", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "fake", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for fakeRun $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPSIM.OUTPUT.MEF", $path_base, $class_id); + addFilename (\@files, "PPSIM.OUTPUT.SPL", $path_base, $class_id); + addFilename (\@files, "PPSIM.FAKE.CHIP", $path_base, $class_id); + addFilename (\@files, "PPSIM.FORCE.CHIP", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$faketool -fake_id $stage_id -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -topurgedimfile"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -tocleanedimfile"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + } else { + + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + } + } + exit 0; + } -# fake : faketool : -pendingcleanupimfile (loop over imfiles) +# Detrend stages +if ($stage eq "detrend.process.imfile") { + + die "--stage_id required for stage detrend.process.imfile\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find chiptool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$dettool -pendingcleanup_processedimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); + } + + # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' + if (@$stdout_buf == 0) { + exit 0; # Silently exit if there's nothing to do. I don't know how we'd ever get here, but let's be safe. + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $exp_id = $imfile->{exp_id}; + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + + # Possibly not the correct config file, but simtest doesn't leave any around to check. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for detrend.process.imfile $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for detrend.process.imfile $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -det_id $stage_id -exp_id $exp_id -class_id $class_id -updateprocessedimfile"; + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + } else { + + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$dettool -updateprocessedimfile -det_id $stage_id -exp_id $exp_id -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } + } + + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.process.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_processedexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; #silently abort. I need to fix this for propers + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + + foreach my $exp (@$exps) { + my $path_base = $exp->{path_base}; + my $exp_id = $exp->{exp_id}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + # I can't find anything to put here + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + # since 'camera' has only a single imfile, we can just update the run + my $command = "$dettool -updateprocessedexp -det_id $stage_id -exp_id $exp_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.stack.imfile") { + + die "--stage_id required for stage $stage\n" if !$stage_id; + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + my $stacks; # Array reference of component files + my $command = "$dettool -pendingcleanup_stacked -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $stacks = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + my @files = (); + foreach my $stack (@{ $stacks }) { + # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. + my $path_base = $stack->{uri}; + my $iteration = $stack->{iteration}; + my $class_id = $stack->{class_id}; + + $path_base =~ s/\.fits$//; # That should do it? + + my $status = 1; + +# if ($mode eq "goto_cleaned") { +# my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); + +# $config_file =~ s%^file://%%; +# if (!$config_file or ! -e $config_file) { +# print STDERR "skipping cleanup for $stage $stage_id $path_base" . +# " because config file is missing\n"; +# $status = 0; +# } +# $config_file = 'file://' . $config_file; +# } +# elsif ($mode eq "goto_scrubbed") { +# my $config_file = $ipprc->filename("PPMERGE.CONFIG", $path_base, $stage_id); +# $config_file =~ s%^file://%%; +# if ($config_file and -e $config_file) { +# print STDERR "skipping scrubbed for $stage $stage_id $path_base" . +# " because config file is present\n"; +# $status = 0; +# } +# } + + if ($status) { + # delete the temporary image datafiles + # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. + # I understand that they all point to the same filename right now, but that may not be true in + # the future. + addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id); + + + addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode +# addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id); + } + + $status = &delete_files(\@files); + } + + if ($status) { + my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -updatestacked -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -updatestacked -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updatestacked -data_state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} +if ($stage eq "detrend.normstat.imfile") { + print STDERR "I'm not convinced there's anything to clean up from stage $stage\n"; + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'camtool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + my $command = "$dettool -pendingcleanup_normalizedstat -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { +# my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + if ($status) { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR "WTF: $success TTT $error_code QQQ\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; + +} +if ($stage eq "detrend.norm.imfile") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_normalizedimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); + addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); + + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); + addFilename (\@files, "PPIMAGE.STATS", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.norm.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_normalizedexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $exp_id = $exp->{exp_id}; + my $iteration = $exp->{iteration}; + my $path_base = $exp->{path_base}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR " residexp had an issue setting the state:? $success $error_code\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} +if ($stage eq "detrend.resid.imfile") { + + die "--stage_id required for stage $stage\n" if !$stage_id; + ### select the imfiles for this entry + + # this stage uses 'dettool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$dettool -pendingcleanup_residimfile -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + exit 0; + } + + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $iteration = $imfile->{iteration}; + my $exp_id = $imfile->{exp_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + + # Possibly not the correct config file, but simtest doesn't leave any around to check. + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($config_file and -e $config_file) { + print STDERR "skipping scrubbed for $stage $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updateresidimfile -det_id $stage_id -exp_id $exp_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + + exit 0; +} +if ($stage eq "detrend.resid.exp") { + die "--stage_id required for stage $stage\n" if !$stage_id; + # this stage uses 'camtool' + my $dettool = can_run('dettool') or die "Can't find dettool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $exps; # Array of component files + my $command = "$dettool -pendingcleanup_residexp -det_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + # This is a hack to bomb out until I can diagnose why pantasks wants to keep running this + if (@$stdout_buf == 0) { + exit 0; + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $exp_id = $exp->{exp_id}; + my $iteration = $exp->{iteration}; + my $path_base = $exp->{path_base}; +# my $class_id = $exp->{class_id} + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($config_file and -e $config_file) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base);#, $class_id); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base);#, $class_id); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR " residexp had an issue setting the state:? $success $error_code\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updateresidexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + # Check to see if we can mark the whole detRunSummary object as cleaned. + + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + if (@$stdout_buf != 0) { + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } + } + exit 0; +} + die "ipp_cleanup.pl -stage $stage not yet implemented\n"; @@ -570,9 +1833,26 @@ my $files = shift; # reference to a list of files to unlink +# open(TMPLOG,">>/tmp/czw.cleanup.log"); +# flock(TMPLOG,2); + # this script is, of course, very dangerous. foreach my $file (@$files) { print STDERR "unlinking $file\n"; +# print TMPLOG "$stage $stage_id $file"; +# my $ff = $file; +# $ff =~ s%^file://%%; +# unless (-e $ff) { +# print TMPLOG "\t File not found\n"; +# } +# else { +# print TMPLOG "\n"; +# } + $ipprc->file_delete($file); } + +# flock(TMPLOG,8); +# close(TMPLOG); + return 1; }