Index: trunk/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- trunk/ippScripts/scripts/ipp_cleanup.pl (revision 27144) +++ trunk/ippScripts/scripts/ipp_cleanup.pl (revision 27638) @@ -61,6 +61,6 @@ my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, - "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0, - "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 ); + "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0, + "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 ); unless (exists($stages{$stage})) { die "unknown stage $stage for ipp_cleanup.pl\n"; @@ -96,14 +96,14 @@ # XXX Why? This could just mean there's nothing to cleanup, or that we're trying to rerun an errored run. if (@$stdout_buf == 0) { - my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); - } - exit 0; + my $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + exit 0; } @@ -120,26 +120,29 @@ my $path_base = $imfile->{path_base}; my $status = 1; + $status = 0 unless defined $path_base and $path_base ne "NULL"; # don't clean up unless the data needed to update is available # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent - # goto_scrubbed now requires the config file to not exist. - if ($mode eq "goto_cleaned") { - my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - - unless ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for chipRun $stage_id $class_id " - . " because config file ($config_file) is missing\n"; - $status = 0; - } - } - elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping scrubbed for chipRun $stage_id $class_id " - . " because config file ($config_file) is present\n"; - $status = 0; - } - } + # goto_scrubbed now requires the config file to not exist. + if ($status) { + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for chipRun $stage_id $class_id " + . " because config file ($config_file) is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for chipRun $stage_id $class_id " + . " because config file ($config_file) is present\n"; + $status = 0; + } + } + } if ($status) { @@ -175,10 +178,10 @@ $command .= " -topurgedimfile"; } - elsif ($mode eq "goto_cleaned") { + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedimfile"; } - elsif ($mode eq "goto_scrubbed") { - $command .= " -toscrubbedimfile"; - } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -191,20 +194,20 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } } else { - # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) - my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -214,15 +217,15 @@ &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); } - - # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. - $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); - } + + # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. + $command = "$chiptool -chip_id $stage_id -updaterun -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } } @@ -264,6 +267,6 @@ if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); - - unless ($ipprc->file_exists($config_file)) { + + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n"; $status = 0; @@ -271,10 +274,10 @@ } elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; - $status = 0; - } + my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for camRun $stage_id because config file ($config_file) is present\n"; + $status = 0; + } } if ($status) { @@ -296,8 +299,8 @@ if ($mode eq "goto_cleaned") { $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned"; - } + } if ($mode eq "goto_scrubbed") { $command = "$camtool -updaterun -cam_id $stage_id -set_state scrubbed"; - } + } if ($mode eq "goto_purged") { $command = "$camtool -updaterun -cam_id $stage_id -set_state purged"; @@ -311,17 +314,17 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } } else { - # since 'camera' has only a single imfile, we can just update the run + # since 'camera' has only a single imfile, we can just update the run my $command = "$camtool -updaterun -cam_id $stage_id -set_state $error_state"; $command .= " -dbname $dbname" if defined $dbname; @@ -356,19 +359,19 @@ &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); } - + if (@$stdout_buf == 0) { - # No skycells were found for some reason. - # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." - my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); - } - - exit(0); + # No skycells were found for some reason. + # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." + my $command = "$warptool -updaterun -warp_id $stage_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + + exit(0); } my $metadata = $mdcParser->parse(join "", @$stdout_buf) or @@ -384,45 +387,50 @@ my $status = 1; - if ($mode eq "goto_cleaned") { - my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); - - unless ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . - " because config file is missing\n"; - $status = 0; - } - } - elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . - " because config file is present\n"; - $status = 0; - } - } - if ($status) { - if ($skyfile->{quality} != 8007) { - my @files = (); - - # delete the temporary image datafiles - addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); - addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); - addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); + $status = 0 unless defined $path_base and $path_base ne "NULL"; + + if ($status) { + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . + " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for warpRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } + } + } + + if ($status) { + if ($skyfile->{quality} != 8007) { + my @files = (); + + # delete the temporary image datafiles + addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id); # addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); - if ($mode eq "goto_purged") { - # additional files to remove for 'purge' mode - addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); - addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); - addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); - # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); - - # XXX: do we want to delete these? - # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); - # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); - } + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); + addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); + # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); + + # XXX: do we want to delete these? + # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); + # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); + } # actual command to delete the files - $status = &delete_files (\@files); - } + $status = &delete_files (\@files); + } } @@ -431,11 +439,11 @@ if ($mode eq "goto_purged") { $command .= " -topurgedskyfile"; - } - elsif ($mode eq "goto_cleaned") { + } + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedskyfile"; } - elsif ($mode eq "goto_scrubbed") { - $command .= " -toscrubbedskyfile"; - } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedskyfile"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -447,18 +455,18 @@ } - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } } else { - my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; + my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -469,5 +477,5 @@ } - # exit $PS_EXIT_UNKNOWN_ERROR; + # exit $PS_EXIT_UNKNOWN_ERROR; } } @@ -487,124 +495,124 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); } if (@$stdout_buf == 0) { - # No skycells were found for some reason. - # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." - my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); - } - - exit(0); - } - - my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or - &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR); - - $skyfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "stack", $stage_id, $PS_EXIT_PROG_ERROR); - + # No skycells were found for some reason. + # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." + my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); + } + + exit(0); + } + + my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + &my_die("Unable to parse metadata config doc", "stack", $stage_id, $PS_EXIT_PROG_ERROR); + + $skyfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "stack", $stage_id, $PS_EXIT_PROG_ERROR); + my @files = (); foreach my $skyfile (@{ $skyfiles }) { - my $path_base = $skyfile->{path_base}; - my $skycell_id = $skyfile->{skycell_id}; - - my $status = 1; - if ((!exists($skyfile->{path_base}))|| - (!defined($path_base))) { - $status = 0; - } - if ($status) { - if ($mode eq "goto_cleaned") { - my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - - unless ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . - " because config file is missing\n"; - $status = 0; - } - } - elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . - " because config file is present\n"; - $status = 0; - } - } - } - - if ($status) { - my @files = (); - # delete the temporary image datafiles - addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id); - addFilename(\@files, "PPSTACK.OUTPUT.MASK", $path_base, $skycell_id); - addFilename(\@files, "PPSTACK.OUTPUT.VARIANCE", $path_base, $skycell_id); - - if ($mode eq "goto_purged") { - # additional files to remove for 'purge' mode - addFilename(\@files, "PPSTACK.CONV.KERNEL", $path_base, $skycell_id); - addFilename(\@files, "PPSTACK.OUTPUT.JPEG1", $path_base, $skycell_id); - addFilename(\@files, "PPSTACK.OUTPUT.JPEG2", $path_base, $skycell_id); - # Commented out to match warp files. - #addFilename(\@files, "PPSTACK.TARGET.PSF", $path_base, $skycell_id); - #addFilename(\@files, "PPSTACK.CONFIG", $path_base, $skycell_id); - } - - $status = &delete_files(\@files); - } - - if ($status) { - my $command = "$stacktool -stack_id $stage_id"; - if ($mode eq "goto_purged") { - $command .= " -updaterun -set_state purged"; - } - elsif ($mode eq "goto_cleaned") { - $command .= " -updaterun -set_state cleaned"; - } - elsif ($mode eq "goto_scrubbed") { - $command .= " -updaterun -set_state scrubbed"; - } - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); - } - - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } - - } else { - my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); - } -# exit $PS_EXIT_UNKNOWN_ERROR; - } + my $path_base = $skyfile->{path_base}; + my $skycell_id = $skyfile->{skycell_id}; + + my $status = 1; + if ((!exists($skyfile->{path_base}))|| + (!defined($path_base))) { + $status = 0; + } + if ($status) { + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for stackRun $stage_id $skycell_id" . + " because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSTACK.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for stackRun $stage_id $skycell_id" . + " because config file is present\n"; + $status = 0; + } + } + } + + if ($status) { + my @files = (); + # delete the temporary image datafiles + addFilename(\@files, "PPSTACK.OUTPUT", $path_base, $skycell_id); + addFilename(\@files, "PPSTACK.OUTPUT.MASK", $path_base, $skycell_id); + addFilename(\@files, "PPSTACK.OUTPUT.VARIANCE", $path_base, $skycell_id); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename(\@files, "PPSTACK.CONV.KERNEL", $path_base, $skycell_id); + addFilename(\@files, "PPSTACK.OUTPUT.JPEG1", $path_base, $skycell_id); + addFilename(\@files, "PPSTACK.OUTPUT.JPEG2", $path_base, $skycell_id); + # Commented out to match warp files. + #addFilename(\@files, "PPSTACK.TARGET.PSF", $path_base, $skycell_id); + #addFilename(\@files, "PPSTACK.CONFIG", $path_base, $skycell_id); + } + + $status = &delete_files(\@files); + } + + if ($status) { + my $command = "$stacktool -stack_id $stage_id"; + if ($mode eq "goto_purged") { + $command .= " -updaterun -set_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -updaterun -set_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updaterun -set_state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); + } + + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } + + } else { + my $command = "$stacktool -updaterun -stack_id $stage_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); + } +# exit $PS_EXIT_UNKNOWN_ERROR; + } } exit 0; @@ -612,5 +620,5 @@ if ($stage eq 'diff') { - + die "--stage_id required for stage diff\n" if !$stage_id; @@ -624,141 +632,141 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); } if (@$stdout_buf == 0) { - # No skycells were found for some reason. - # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." - my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); - } - - exit(0); - } - - my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or - &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); - - $skyfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "diff", $stage_id, $PS_EXIT_PROG_ERROR); - + # No skycells were found for some reason. + # Not technically an "error," but a "you told me to do X, and I can't. Please fix this yourself." + my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); + } + + exit(0); + } + + my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); + + $skyfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "diff", $stage_id, $PS_EXIT_PROG_ERROR); + my @files = (); foreach my $skyfile (@{ $skyfiles }) { - my $path_base = $skyfile->{path_base}; - my $skycell_id = $skyfile->{skycell_id}; - - my $status = 1; - if ((!exists($skyfile->{path_base}))|| - (!defined($path_base))) { - $status = 0; - } - if ($status) { - - if ($mode eq "goto_cleaned") { - my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - - unless ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . - " because config file ($config_file) is missing\n"; - $status = 0; - } - } - elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . - " because config file ($config_file) is present\n"; - $status = 0; - } - } - } - if ($status) { - my @files = (); - # delete the temporary image datafiles - addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id); - - addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id); - - addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.INPUT.CONV.VARIANCE", $path_base, $skycell_id); - - addFilename(\@files, "PPSUB.REF.CONV", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.REF.CONV.MASK", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id); - - if ($mode eq "goto_purged") { - # additional files to remove for 'purge' mode - addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id); - # Commented out to match warp files. - #addFilename(\@files, "PPSUB.CONFIG", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.OUTPUT.SOURCES", $path_base, $skycell_id); - addFilename(\@files, "PPSUB.INVERSE.SOURCES", $path_base, $skycell_id); - - } -# print STDERR "MY FILES: @files\n"; - $status = &delete_files(\@files); - } -# print STDERR "MY STATUS: $status\n"; - if ($status) { - my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; - - if ($mode eq "goto_purged") { - $command .= " -topurgedskyfile"; - } - elsif ($mode eq "goto_cleaned") { - $command .= " -tocleanedskyfile"; - } - elsif ($mode eq "goto_scrubbed") { - $command .= " -toscrubbedskyfile"; - } - - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); - } - - # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. - my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; - $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; - $command .= " -dbname $dbname" if defined $dbname; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); - } - - } else { - my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; - - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); - } -# exit $PS_EXIT_UNKNOWN_ERROR; - } + my $path_base = $skyfile->{path_base}; + my $skycell_id = $skyfile->{skycell_id}; + + my $status = 1; + if ((!exists($skyfile->{path_base}))|| + (!defined($path_base))) { + $status = 0; + } + if ($status) { + + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for diffRun $stage_id $skycell_id" . + " because config file ($config_file) is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for diffRun $stage_id $skycell_id" . + " because config file ($config_file) is present\n"; + $status = 0; + } + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + addFilename(\@files, "PPSUB.OUTPUT", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.OUTPUT.MASK", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.OUTPUT.VARIANCE", $path_base, $skycell_id); + + addFilename(\@files, "PPSUB.INVERSE", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.INVERSE.MASK", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.INVERSE.VARIANCE", $path_base, $skycell_id); + + addFilename(\@files, "PPSUB.INPUT.CONV", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.INPUT.CONV.MASK", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.INPUT.CONV.VARIANCE", $path_base, $skycell_id); + + addFilename(\@files, "PPSUB.REF.CONV", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.REF.CONV.MASK", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.REF.CONV.VARIANCE", $path_base, $skycell_id); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename(\@files, "PPSUB.OUTPUT.KERNELS", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.OUTPUT.JPEG1", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.OUTPUT.JPEG2", $path_base, $skycell_id); + # Commented out to match warp files. + #addFilename(\@files, "PPSUB.CONFIG", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.OUTPUT.SOURCES", $path_base, $skycell_id); + addFilename(\@files, "PPSUB.INVERSE.SOURCES", $path_base, $skycell_id); + + } +# print STDERR "MY FILES: @files\n"; + $status = &delete_files(\@files); + } +# print STDERR "MY STATUS: $status\n"; + if ($status) { + my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; + + if ($mode eq "goto_purged") { + $command .= " -topurgedskyfile"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -tocleanedskyfile"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedskyfile"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); + } + + # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same. + my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool"; + $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -state full"; + $command .= " -dbname $dbname" if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code); + } + + } else { + my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); + } +# exit $PS_EXIT_UNKNOWN_ERROR; + } } exit 0; @@ -786,14 +794,14 @@ # if there are no fakeProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' if (@$stdout_buf == 0) { - my $command = "$faketool -fake_id $stage_id -updaterun -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); - } - exit 0; + my $command = "$faketool -fake_id $stage_id -updaterun -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform faketool: $error_code", "fake", $stage_id, $error_code); + } + exit 0; } @@ -813,9 +821,9 @@ # don't clean up unless the data needed to update is available # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent - # goto_scrubbed now requires the config file to not exist. + # goto_scrubbed now requires the config file to not exist. if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); - unless ($ipprc->file_exists($config_file)) { + unless ($ipprc->file_exists($config_file)) { print STDERR "skipping cleanup for fakeRun $stage_id $class_id " . " because config file is missing\n"; @@ -823,13 +831,13 @@ } } - elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " - . " because config file is present\n"; - $status = 0; - } - } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPSIM.CONFIG", $path_base, $class_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for fakeRun $stage_id $class_id " + . " because config file is present\n"; + $status = 0; + } + } if ($status) { @@ -844,7 +852,7 @@ if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode - addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); - addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); - addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FAKE.SOURCES", $path_base, $class_id); + addFilename (\@files, "PPSIM.FORCE.SOURCES", $path_base, $class_id); } @@ -858,10 +866,10 @@ $command .= " -topurgedimfile"; } - elsif ($mode eq "goto_cleaned") { + elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedimfile"; } - elsif ($mode eq "goto_scrubbed") { - $command .= " -toscrubbedimfile"; - } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedimfile"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -875,7 +883,7 @@ } else { - # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) - my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$faketool -updateprocessedimfile -fake_id $stage_id -class_id $class_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -889,5 +897,5 @@ exit 0; -} +} # Detrend stages if ($stage eq "detrend.processed") { @@ -916,31 +924,31 @@ # if there are no detProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new' if (@$stdout_buf != 0) { -# exit 0; # Silently exit if there's nothing to do. I don't know how we'd ever get here, but let's be safe. - - - $metadata = $mdcParser->parse(join "", @$stdout_buf) or +# exit 0; # Silently exit if there's nothing to do. I don't know how we'd ever get here, but let's be safe. + + + $metadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); } # loop over all of the imfiles, determine the path_base and class_id for each foreach my $imfile (@$imfiles) { - my $iexp_id = $imfile->{exp_id}; + my $iexp_id = $imfile->{exp_id}; my $class_id = $imfile->{class_id}; my $path_base = $imfile->{path_base}; my $status = 1; - unless (defined($path_base)) { - print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n"; - $status = 0; - } - unless (defined($class_id)) { - print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n"; - $status = 0; - } - # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, - # and so there is no check for config files. + unless (defined($path_base)) { + print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n"; + $status = 0; + } + unless (defined($class_id)) { + print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n"; + $status = 0; + } + # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, + # and so there is no check for config files. if ($status) { # array of actual filenames to delete @@ -952,6 +960,6 @@ if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode - addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); } @@ -965,10 +973,10 @@ $command .= " -data_state purged"; } - elsif ($mode eq "goto_cleaned") { + elsif ($mode eq "goto_cleaned") { $command .= " -data_state cleaned"; } - elsif ($mode eq "goto_scrubbed") { - $command .= " -data_state scrubbed"; - } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -981,8 +989,8 @@ } else { - # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* - my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile "; - $command .= " -data_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; + # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* + my $command = "$dettool -det_id $det_id -exp_id $iexp_id -class_id $class_id -updateprocessedimfile "; + $command .= " -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -992,5 +1000,5 @@ &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); } - } + } } @@ -1001,36 +1009,36 @@ ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code); } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); - - foreach my $exp (@$exps) { - my $exp_id = $exp->{exp_id}; - my $command = "$dettool -updateprocessedexp -det_id $det_id -exp_id $exp_id "; - if ($mode eq "goto_cleaned") { - $command .= " -data_state cleaned "; - } - if ($mode eq "goto_scrubbed") { - $command .= " -data_state scrubbed "; - } - if ($mode eq "goto_purged") { - $command .= " -data_state purged "; - } - $command .= " -dbname $dbname" if defined $dbname; -# print "$command\n"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code); - } - } - } - + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detProcessedExp)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $exp_id = $exp->{exp_id}; + my $command = "$dettool -updateprocessedexp -det_id $det_id -exp_id $exp_id "; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned "; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed "; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged "; + } + $command .= " -dbname $dbname" if defined $dbname; +# print "$command\n"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detProcessedExp)", $stage_id, $error_code); + } + } + } + exit 0; } @@ -1057,33 +1065,33 @@ &my_die("Unable to perform dettool: $error_code", "detrend.process.imfile", $stage_id, $error_code); } - + # if there are no detResidImfiles (@$stdout_buf == 0), then silently exit. if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - # extract the metadata for the files into a hash list - $imfiles = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + # extract the metadata for the files into a hash list + $imfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); } # loop over all of the imfiles, determine the path_base and class_id for each foreach my $imfile (@$imfiles) { - my $iexp_id = $imfile->{exp_id}; + my $iexp_id = $imfile->{exp_id}; my $class_id = $imfile->{class_id}; my $path_base = $imfile->{path_base}; - my $iteration = $imfile->{iteration}; + my $iteration = $imfile->{iteration}; my $status = 1; - # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, - # and so there is no check for config files. - unless (defined($path_base)) { - print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n"; - $status = 0; - } - unless (defined($class_id)) { - print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n"; - $status = 0; - } + # Detrends cannot be updated, so goto_cleaned and goto_scrubbed are treated as equivalent, + # and so there is no check for config files. + unless (defined($path_base)) { + print STDERR "PATH BASE: >>$path_base<< didn't get defined for $iexp_id $exp_id $det_id $class_id\n"; + $status = 0; + } + unless (defined($class_id)) { + print STDERR "CLASS_ID: >>$class_id<< didn't get defined for $iexp_id $exp_id $det_id $path_base\n"; + $status = 0; + } if ($status) { # array of actual filenames to delete @@ -1094,11 +1102,11 @@ if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode - addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); } -# foreach my $f (@files) { -# print "RESID: $f\n"; -# } +# foreach my $f (@files) { +# print "RESID: $f\n"; +# } # actual command to delete the files $status = &delete_files (\@files); @@ -1110,10 +1118,10 @@ $command .= " -data_state purged"; } - elsif ($mode eq "goto_cleaned") { + elsif ($mode eq "goto_cleaned") { $command .= " -data_state cleaned"; } - elsif ($mode eq "goto_scrubbed") { - $command .= " -data_state scrubbed"; - } + elsif ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } $command .= " -dbname $dbname" if defined $dbname; @@ -1126,7 +1134,7 @@ } } else { - my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile "; - $command .= " -data_state $error_state "; - $command .= " -dbname $dbname" if defined $dbname; + my $command = "$dettool -det_id $det_id -exp_id $iexp_id -iteration $iteration -class_id $class_id -updateresidimfile "; + $command .= " -data_state $error_state "; + $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -1140,37 +1148,37 @@ # Flag the detResidExp as clean now as well, if it is marked tobe cleaned (this is still clunky). - $command = "$dettool -pendingcleanup_residexp -det_id $det_id -exp_id $exp_id"; + $command = "$dettool -pendingcleanup_residexp -det_id $det_id -exp_id $exp_id"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code); } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); - - foreach my $exp (@$exps) { - my $iteration = $exp->{iteration}; - my $command = "$dettool -updateresidexp -det_id $det_id -exp_id $exp_id "; - if ($mode eq "goto_cleaned") { - $command .= " -data_state cleaned"; - } - if ($mode eq "goto_scrubbed") { - $command .= " -data_state scrubbed"; - } - if ($mode eq "goto_purged") { - $command .= " -data_state purged"; - } - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code); - } - } + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detResidExp)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command = "$dettool -updateresidexp -det_id $det_id -exp_id $exp_id "; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detResidExp)", $stage_id, $error_code); + } + } } @@ -1179,5 +1187,5 @@ if ($stage eq "detrend.stack.imfile") { - + die "--stage_id required for stage $stage\n" if !$stage_id; @@ -1191,116 +1199,116 @@ my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); - } - my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or - &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - - $stacks = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); - + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @{ $stdout_buf }) or + &my_die("Unable to parse metadata config doc", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + + $stacks = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage", $stage_id, $PS_EXIT_PROG_ERROR); + my @files = (); foreach my $stack (@{ $stacks }) { - # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. - my $path_base = $stack->{uri}; - my $iteration = $stack->{iteration}; - my $class_id = $stack->{class_id}; - - $path_base =~ s/\.fits$//; # That should do it? - - my $status = 1; - - if ($status) { - my @files = (); - # delete the temporary image datafiles - # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. - # I understand that they all point to the same filename right now, but that may not be true in - # the future. - addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id); - addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id); - addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id); - addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id); - addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id); - addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id); - - - addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id); - addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id); - - if ($mode eq "goto_purged") { - # additional files to remove for 'purge' mode -# addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id); - } - - $status = &delete_files(\@files); - } - - if ($status) { - my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; - if ($mode eq "goto_purged") { - $command .= " -updatestacked -data_state purged"; - } - elsif ($mode eq "goto_cleaned") { - $command .= " -updatestacked -data_state cleaned"; - } - elsif ($mode eq "goto_scrubbed") { - $command .= " -updatestacked -data_state scrubbed"; - } - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); - } - } else { - my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); - } - exit $PS_EXIT_UNKNOWN_ERROR; - } + # detStackedImfile does not have a path_base column. This is inconvenient, as it means we need to calculate it. + my $path_base = $stack->{uri}; + my $iteration = $stack->{iteration}; + my $class_id = $stack->{class_id}; + + $path_base =~ s/\.fits$//; # That should do it? + + my $status = 1; + + if ($status) { + my @files = (); + # delete the temporary image datafiles + # There's no convenient way to get the detrend type, so I'm queueing all of them for deletion. + # I understand that they all point to the same filename right now, but that may not be true in + # the future. + addFilename(\@files, "PPMERGE.OUTPUT.MASK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.BIAS", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.DARK", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.SHUTTER", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FLAT", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.FRINGE", $path_base, $stage_id); + + + addFilename(\@files, "PPMERGE.OUTPUT.SIGMA", $path_base, $stage_id); + addFilename(\@files, "PPMERGE.OUTPUT.COUNT", $path_base, $stage_id); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode +# addFilename(\@files, "PPMERGE.OUTPUT", $path_base, $stage_id); + } + + $status = &delete_files(\@files); + } + + if ($status) { + my $command = "$dettool -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_purged") { + $command .= " -updatestacked -data_state purged"; + } + elsif ($mode eq "goto_cleaned") { + $command .= " -updatestacked -data_state cleaned"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -updatestacked -data_state scrubbed"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatestacked -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } } # Check to see if we can mark the whole detRunSummary object as cleaned. - $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - my $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - - foreach my $exp (@$exps) { - my $iteration = $exp->{iteration}; - my $command; - if ($mode eq "goto_cleaned") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; - } - if ($mode eq "goto_scrubbed") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; - } - if ($mode eq "goto_purged") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; - } - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); - } - } + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + my $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } } exit 0; @@ -1326,75 +1334,75 @@ foreach my $exp (@$exps) { -# my $path_base = $exp->{path_base}; - my $iteration = $exp->{iteration}; - my $class_id = $exp->{class_id}; - - my $status = 1; - if ($status) { - my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; - if ($mode eq "goto_cleaned") { - $command .= " -data_state cleaned"; - } - if ($mode eq "goto_scrubbed") { - $command .= " -data_state scrubbed"; - } - if ($mode eq "goto_purged") { - $command .= " -data_state purged"; - } - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); - } - } else { - my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); - } - exit $PS_EXIT_UNKNOWN_ERROR; - } +# my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + if ($status) { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedstat -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } } # Check to see if we can mark the whole detRunSummary object as cleaned. - $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - - foreach my $exp (@$exps) { - my $iteration = $exp->{iteration}; - my $command; - if ($mode eq "goto_cleaned") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; - } - if ($mode eq "goto_scrubbed") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; - } - if ($mode eq "goto_purged") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; - } - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); - } - } + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } } @@ -1424,108 +1432,108 @@ foreach my $exp (@$exps) { - my $path_base = $exp->{path_base}; - my $iteration = $exp->{iteration}; - my $class_id = $exp->{class_id}; - - my $status = 1; - # don't clean up unless the data needed to update is available - # goto_scrubbed now requires the config file to not be present - if ($mode eq "goto_cleaned") { - my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - - unless ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; - $status = 0; - } - } - elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; - $status = 0; - } - } - if ($status) { - my @files = (); - - if ($mode eq "goto_purged") { - # additional files to remove for 'purge' mode - addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); - addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); - - addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); - addFilename (\@files, "PPIMAGE.STATS", $path_base); - } - # actual command to delete the files - $status = &delete_files (\@files); - } - - if ($status) { - my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; - if ($mode eq "goto_cleaned") { - $command .= " -data_state cleaned"; - } - if ($mode eq "goto_scrubbed") { - $command .= " -data_state scrubbed"; - } - if ($mode eq "goto_purged") { - $command .= " -data_state purged"; - } - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); - } - } else { - my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); - } - exit $PS_EXIT_UNKNOWN_ERROR; - } + my $path_base = $exp->{path_base}; + my $iteration = $exp->{iteration}; + my $class_id = $exp->{class_id}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base); + addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base); + + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base); + addFilename (\@files, "PPIMAGE.STATS", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedimfile -det_id $stage_id -iteration $iteration -class_id $class_id -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } } # Check to see if we can mark the whole detRunSummary object as cleaned. - $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - - foreach my $exp (@$exps) { - my $iteration = $exp->{iteration}; - my $command; - if ($mode eq "goto_cleaned") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; - } - if ($mode eq "goto_scrubbed") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; - } - if ($mode eq "goto_purged") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; - } - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); - } - } + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } } @@ -1549,5 +1557,5 @@ if (@$stdout_buf == 0) { - exit 0; + exit 0; } my $metadata = $mdcParser->parse(join "", @$stdout_buf) or @@ -1558,106 +1566,106 @@ foreach my $exp (@$exps) { - my $exp_id = $exp->{exp_id}; - my $iteration = $exp->{iteration}; - my $path_base = $exp->{path_base}; - - my $status = 1; - # don't clean up unless the data needed to update is available - # goto_scrubbed now requires the config file to not be present - if ($mode eq "goto_cleaned") { - my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - - unless ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; - $status = 0; - } - } - elsif ($mode eq "goto_scrubbed") { - my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); - - if ($ipprc->file_exists($config_file)) { - print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; - $status = 0; - } - } - if ($status) { - my @files = (); - # delete the temporary image datafiles - if ($mode eq "goto_purged") { - # additional files to remove for 'purge' mode - addFilename (\@files, "PPIMAGE.JPEG1", $path_base); - addFilename (\@files, "PPIMAGE.JPEG2", $path_base); - } - # actual command to delete the files - $status = &delete_files (\@files); - } - - if ($status) { - my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration"; - if ($mode eq "goto_cleaned") { - $command .= " -data_state cleaned"; - } - if ($mode eq "goto_scrubbed") { - $command .= " -data_state scrubbed"; - } - if ($mode eq "goto_purged") { - $command .= " -data_state purged"; - } - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - print STDERR " residexp had an issue setting the state:? $success $error_code\n"; - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); - } - } else { - my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); - } - exit $PS_EXIT_UNKNOWN_ERROR; - } + my $exp_id = $exp->{exp_id}; + my $iteration = $exp->{iteration}; + my $path_base = $exp->{path_base}; + + my $status = 1; + # don't clean up unless the data needed to update is available + # goto_scrubbed now requires the config file to not be present + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + unless ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file is missing\n"; + $status = 0; + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping cleanup for $stage $stage_id because config file ($config_file) is present\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -iteration $iteration"; + if ($mode eq "goto_cleaned") { + $command .= " -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command .= " -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command .= " -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + print STDERR " residexp had an issue setting the state:? $success $error_code\n"; + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + } else { + my $command = "$dettool -updatenormalizedexp -det_id $stage_id -exp_id $exp_id -iteration $iteration -data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } } # Check to see if we can mark the whole detRunSummary object as cleaned. - $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; + $command = "$dettool -pendingcleanup_detrunsummary -det_id $stage_id"; $command .= " -dbname $dbname" if defined $dbname; ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); } if (@$stdout_buf != 0) { - $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - $exps = parse_md_list($metadata) or - &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); - - foreach my $exp (@$exps) { - my $iteration = $exp->{iteration}; - my $command; - if ($mode eq "goto_cleaned") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; - } - if ($mode eq "goto_scrubbed") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; - } - if ($mode eq "goto_purged") { - $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; - } - $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); - } - } + $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "$stage (detRunSummary)", $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $exp (@$exps) { + my $iteration = $exp->{iteration}; + my $command; + if ($mode eq "goto_cleaned") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state cleaned"; + } + if ($mode eq "goto_scrubbed") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state scrubbed"; + } + if ($mode eq "goto_purged") { + $command = "$dettool -updatedetrunsummary -det_id $stage_id -iteration $iteration -data_state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform dettool: $error_code", "$stage (detRunSummary)", $stage_id, $error_code); + } + } } exit 0; @@ -1671,8 +1679,8 @@ my $files = shift; # reference to a list of files to unlink # my $test_verbose = 1; - + # if ($test_verbose == 1) { -# open(TMPLOG,">>/tmp/czw.cleanup.log"); -# flock(TMPLOG,2); +# open(TMPLOG,">>/tmp/czw.cleanup.log"); +# flock(TMPLOG,2); # } @@ -1680,18 +1688,18 @@ foreach my $file (@$files) { print STDERR "unlinking $stage $stage_id $file"; - unless ($ipprc->file_exists($file)) { - print STDERR "\t File not found\n"; - } - else { - print STDERR "\n"; - } - -# if ($test_verbose == 1) { -# print TMPLOG "$stage $stage_id $file"; - -# else { -# print TMPLOG "\n"; -# } -# } + unless ($ipprc->file_exists($file)) { + print STDERR "\t File not found\n"; + } + else { + print STDERR "\n"; + } + +# if ($test_verbose == 1) { +# print TMPLOG "$stage $stage_id $file"; + +# else { +# print TMPLOG "\n"; +# } +# } $ipprc->file_delete($file); @@ -1699,6 +1707,6 @@ # if ($test_verbose == 1) { -# flock(TMPLOG,8); -# close(TMPLOG); +# flock(TMPLOG,8); +# close(TMPLOG); # }