Index: trunk/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- trunk/ippScripts/scripts/ipp_cleanup.pl (revision 19397) +++ trunk/ippScripts/scripts/ipp_cleanup.pl (revision 19561) @@ -50,13 +50,17 @@ } +my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff => 1); +unless ($stages{$stage}) { + die "unknown stage $stage for ipp_cleanup.pl\n"; +} + $ipprc->define_camera($camera); my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files - -my $fault = 0; # choice of files to delete depends on the stage if ($stage eq "chip") { + die "--stage_id required for stage chip\n" if !$stage_id; ### select the imfiles for this entry @@ -85,28 +89,43 @@ my $class_id = $imfile->{class_id}; my $path_base = $imfile->{path_base}; - - # array of actual filenames to delete - my @files = (); - - # delete the temporary image datafiles - addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.OUTPUT.WEIGHT", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.CHIP.WEIGHT", $path_base, $class_id); - if ($mode eq "goto_purged") { - # additional files to remove for 'purge' mode - addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.JPEG", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); - } + my $status = 1; + + # don't clean up unless the data needed to update is available + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for chipRun $stage_id $class_id " + . " because config file is missing\n"; + $status = 0; + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the temporary image datafiles + addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.WEIGHT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CHIP.WEIGHT", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.JPEG", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); + } - # actual command to delete the files - my $status = &delete_files (\@files); + # actual command to delete the files + $status = &delete_files (\@files); + } if ($status) { @@ -119,5 +138,6 @@ $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -125,43 +145,18 @@ } } else { - my $command = "$chiptool -updateimfile -chip_id $stage_id -class_id $class_id -code 1"; + my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -code 1"; $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); } - $fault = 1; } } -if (0) { - # This block is not necessary - # chiptool -tocleanedimfile and -topurgedimfile automatically update chipRun when the - # last image gets set to purged or cleaned - - if (! $fault) { - # set the state of the chipRun entry to "cleaned" or "purged" as needed - my $command = "$chiptool -updaterun -chip_id $stage_id"; - if ($mode eq "goto_purged") { - $command .= " -state purged"; - } else { - $command .= " -state cleaned"; - } - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); - } - } -} - exit 0; -} - -# choice of files to delete depends on the stage -if ($stage eq "camera") { +} elsif ($stage eq "camera") { + die "--stage_id required for stage camera\n" if !$stage_id; # this stage uses 'camtool' my $camtool = can_run('camtool') or die "Can't find camtool"; @@ -169,5 +164,5 @@ # Get list of component imfiles # XXX may need a different my_die for each stage - my $imfiles; # Array of component files + my $exps; # Array of component files my $command = "$camtool -pendingcleanupexp -cam_id $stage_id"; # Command to run $command .= " -dbname $dbname" if defined $dbname; @@ -175,17 +170,167 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform camtool: $error_code", "cam", $stage_id, $error_code); + &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); } my $metadata = $mdcParser->parse(join "", @$stdout_buf) or - &my_die("Unable to parse metadata config doc", "cam", $stage_id, $PS_EXIT_PROG_ERROR); - + &my_die("Unable to parse metadata config doc", "camera", $stage_id, $PS_EXIT_PROG_ERROR); + + $exps = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "camera", $stage_id, $PS_EXIT_PROG_ERROR); + + my $n_exps = @$exps; + &my_die("unexpected number of exposures $n_exps", "camera", $stage_id, $PS_EXIT_PROG_ERROR) + if $n_exps != 1; + + my $exp = $exps->[0]; + my $path_base = $exp->{path_base}; + + my $status = 1; + # don't clean up unless the data needed to update is available + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSASTRO.CONFIG", $path_base); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for camRun $stage_id because config file is missing\n"; + $status = 0; + } + } + if ($status) { + my @files = (); + # delete the temporary image datafiles + addFilename (\@files, "PSASTRO.OUTPUT", $path_base); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPIMAGE.JPEG1", $path_base); + addFilename (\@files, "PPIMAGE.JPEG2", $path_base); + addFilename (\@files, "PSASTRO.STATS", $path_base); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$camtool -cam_id $stage_id -updaterun"; + if ($mode eq "goto_cleaned") { + $command .= " -state cleaned"; + } else { + $command .= " -state purged"; + } + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); + } + } else { + my $command = "$camtool -updateprocessedexp -cam_id $stage_id -code 1"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform camtool: $error_code", "camera", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } exit 0; -} - +} elsif ($stage eq "warp") { + die "--stage_id required for stage warp\n" if !$stage_id; + # this stage uses 'warptool' + my $warptool = can_run('warptool') or die "Can't find warptool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $skyfiles; # Array of component files + my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR); + + $skyfiles = parse_md_list($metadata) or + &my_die("Unable to parse metadata list", "warp", $stage_id, $PS_EXIT_PROG_ERROR); + + my @files = (); + foreach my $skyfile (@$skyfiles) { + my $path_base = $skyfile->{path_base}; + my $skycell_id = $skyfile->{skycell_id}; + + my $status = 1; + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + if (!$config_file or ! -e $config_file) { + print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . + " because config file is missing\n"; + $status = 0; + } + } + if ($status) { + # delete the temporary image datafiles + addFilename(\@files, "PSWARP.OUTPUT", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + addFilename(\@files, "PSWARP.OUTPUT.WEIGHT", $path_base, $skycell_id); + addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); + + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename(\@files, "PSWARP.BIN1", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); + addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); + # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); + + # XXX: do we want to delete these? + # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); + # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); + } + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id"; + if ($mode eq "goto_purged") { + $command .= " -topurgedskyfile"; + } else { + $command .= " -tocleanedskyfile"; + } + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + } else { + # XXX: -updateskyfile mode does not exist, need to add it + my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -code 1"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + exit $PS_EXIT_UNKNOWN_ERROR; + } + } + exit 0; +} + +# left TODO # fake : faketool : -pendingcleanupimfile (loop over imfiles) -# warp : warptool : -pendingcleanupskyfile (loop over skyfiles) - -die "unknown stage for ipp_cleanup.pl\n"; -exit 1; +# stack: stacktool : -pendingcleanupskyfile (loop over skyfiles) +# diff: difftool : -pendingcleanupskyfile + +die "ipp_cleanup.pl -stage $stage not yet implemented\n"; sub delete_files @@ -196,4 +341,5 @@ foreach my $file (@$files) { print STDERR "unlinking $file\n"; + # XXX: need to handle files in nebulous unlink $file; } @@ -217,8 +363,8 @@ { my $msg = shift; # Warning message on die - my $stage = shift; # Chiptool identifier - my $stage_id = shift; # Chiptool identifier - my $exit_code = shift; # Exit code to add - # outputImage and outroot are globals + my $stage = shift; # stage name + my $stage_id = shift; # identifier + my $exit_code = shift; # Exit code + # outputImage and path_base are globals carp($msg);