Index: trunk/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- trunk/ippScripts/scripts/ipp_cleanup.pl (revision 29420) +++ trunk/ippScripts/scripts/ipp_cleanup.pl (revision 29671) @@ -18,5 +18,5 @@ # Parse the command-line arguments -my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist); +my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile); GetOptions('stage=s' => \$stage, # which analysis stage to clean? 'camera|i=s' => \$camera, # user-supplied camera name @@ -27,5 +27,6 @@ 'verbose' => \$verbose, # Print to stdout 'no-op' => \$no_op, # pretend but don't actually inject - 'helplist' => \$helplist # give help listing + 'helplist' => \$helplist, # give help listing + 'logfile=s' => \$logfile # destination for stdout and stderr ) or pod2usage( 2 ); @@ -53,8 +54,11 @@ } +$ipprc->redirect_output($logfile) or + &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile; + # set this to 1 to enable checking for files on dead nodes # it is off for now because the implementation is a hack # See comments below. -my $check_for_gone = 0; +my $check_for_gone = 1; my $error_state; @@ -66,4 +70,5 @@ my %stages = ( "chip" => 1, "camera" => 1, "fake" => 1, "warp" => 1, "stack" => 1, "diff" => 1, + "chip_bg" => 1, "warp_bg" => 1, "detrend.processed" => 1, "detrend.resid" => 1, "detrend.process.exp" => 0, "detrend.stack.imfile" => 0, "detrend.normstat.imfile" => 0, "detrend.norm.imfile" => 0, "detrend.norm.exp" => 0 ); @@ -1648,4 +1653,290 @@ } +if ($stage eq "chip_bg") { + + die "--stage_id required for stage chip_bg\n" if !$stage_id; + + &my_die("only mode goto_cleaned is supported for stage chip_bg", "chip_bg", $stage_id, $PS_EXIT_PROG_ERROR) + if $mode ne 'goto_cleaned'; + + ### select the imfiles for this entry + + # this stage uses 'bgtool' + my $bgtool = can_run('bgtool') or die "Can't find bgtool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$bgtool -pendingcleanupchipimfile -chip_bg_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code); + } + + # XXX: Is not having any files to process really a bug? + if (@$stdout_buf == 0) { + my $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code); + } + exit 0; + } + + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "chip_bg", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and class_id for each + foreach my $imfile (@$imfiles) { + my $class_id = $imfile->{class_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + $status = 0 unless defined $path_base and $path_base ne "NULL"; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + if ($status) { + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PPBACKGROUND.CONFIG", $path_base, $class_id); + + unless ($ipprc->file_exists($config_file)) { + if (file_gone($config_file)) { + print STDERR "forcing cleanup for chipRun $stage_id $class_id " + . " because config file ($config_file) is gone\n"; + } else { + print STDERR "skipping cleanup for chipRun $stage_id $class_id " + . " because config file ($config_file) is missing\n"; + $status = 0; + } + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PPBACKGROUND.CONFIG", $path_base, $class_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for chipBackgroundRun $stage_id $class_id " + . " because config file ($config_file) is present\n"; + $status = 0; + } + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the image datafiles + addFilename (\@files, "PPBACKGROUND.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPBACKGROUND.OUTPUT.MASK", $path_base, $class_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PPBACKGROUND.STATS", $path_base, $class_id); #clean? + addFilename (\@files, "PPBACKGOROUND.CONFIG", $path_base, $class_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$bgtool -chip_bg_id $stage_id -class_id $class_id"; + if ($mode eq "goto_cleaned") { + $command .= " -tocleanedchipimfile"; + } + elsif ($mode eq "goto_purged") { + $command .= " -topurgedchipimfile"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedchipimfile"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + + } else { + + # if an error happens for one chip, the chipBackgroundRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) + my $command = "$bgtool -updatechipimfile -chip_bg_id $stage_id -class_id $class_id -set_data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "chip_bg", $stage_id, $error_code); + } + + # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. + $command = "$bgtool -chip_bg_id $stage_id -updatechip -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + + } + } + exit 0; +} +if ($stage eq "warp_bg") { + + die "--stage_id required for stage warp_bg\n" if !$stage_id; + + &my_die("only mode goto_cleaned is supported for stage warp_bg", "warp_bg", $stage_id, $PS_EXIT_PROG_ERROR) + if $mode ne 'goto_cleaned'; + + ### select the imfiles for this entry + + # this stage uses 'bgtool' + my $bgtool = can_run('bgtool') or die "Can't find bgtool"; + + # Get list of component imfiles + # XXX may need a different my_die for each stage + my $imfiles; # Array of component files + my $command = "$bgtool -pendingcleanupwarpskyfile -warp_bg_id $stage_id"; # Command to run + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code); + } + + # XXX: Is not having any files to process really a bug? + if (@$stdout_buf == 0) { + my $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code); + } + exit 0; + } + + # extract the metadata for the files into a hash list + $imfiles = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "warp_bg", $stage_id, $PS_EXIT_PROG_ERROR); + + # loop over all of the imfiles, determine the path_base and skycell_id for each + foreach my $imfile (@$imfiles) { + my $skycell_id = $imfile->{skycell_id}; + my $path_base = $imfile->{path_base}; + my $status = 1; + $status = 0 unless defined $path_base and $path_base ne "NULL"; + + # don't clean up unless the data needed to update is available + # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent + # goto_scrubbed now requires the config file to not exist. + if ($status) { + if ($mode eq "goto_cleaned") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + unless ($ipprc->file_exists($config_file)) { + if (file_gone($config_file)) { + print STDERR "forcing cleanup for warpBackgroundSkyfile $stage_id $skycell_id " + . " because config file ($config_file) is gone\n"; + } else { + print STDERR "skipping cleanup for warpBackgroundSkyfile $stage_id $skycell_id " + . " because config file ($config_file) is missing\n"; + $status = 0; + } + } + } + elsif ($mode eq "goto_scrubbed") { + my $config_file = $ipprc->filename("PSWARP.CONFIG", $path_base, $skycell_id); + + if ($ipprc->file_exists($config_file)) { + print STDERR "skipping scrubbed for chipBackgroundRun $stage_id $skycell_id " + . " because config file ($config_file) is present\n"; + $status = 0; + } + } + } + + if ($status) { + # array of actual filenames to delete + my @files = (); + + # delete the image datafiles + addFilename (\@files, "PSWARP.OUTPUT", $path_base, $skycell_id); + addFilename (\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id); + if ($mode eq "goto_purged") { + # additional files to remove for 'purge' mode + addFilename (\@files, "PSWARP.STATS", $path_base, $skycell_id); #clean? + addFilename (\@files, "PSWARP.CONFIG", $path_base, $skycell_id); + } + + # actual command to delete the files + $status = &delete_files (\@files); + } + + if ($status) { + my $command = "$bgtool -warp_bg_id $stage_id -skycell_id $skycell_id"; + # recall that only goto_cleaned is supported (currently) + if ($mode eq "goto_cleaned") { + $command .= " -tocleanedwarpskyfile"; + } + elsif ($mode eq "goto_purged") { + $command .= " -topurgedwarpskyfile"; + } + elsif ($mode eq "goto_scrubbed") { + $command .= " -toscrubbedwarpskyfile"; + } + + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code); + } + + } else { + + # if an error happens for one skycell, the warpBackgroundRun will stay in goto_*, but the skycells will go to error_* (matching the goto_*) + my $command = "$bgtool -updatewarpskyfile -warp_bg_id $stage_id -skycell_id $skycell_id -set_data_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp_bg", $stage_id, $error_code); + } + + # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. + $command = "$bgtool -warp_bg_id $stage_id -updatewarp -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform bgtool: $error_code", "warp", $stage_id, $error_code); + } + } + } + exit 0; +} die "ipp_cleanup.pl -stage $stage not yet implemented\n";