Index: branches/eam_branches/ipp-20121130/ippScripts/Build.PL =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/Build.PL (revision 34755) +++ branches/eam_branches/ipp-20121130/ippScripts/Build.PL (revision 34839) @@ -128,4 +128,5 @@ scripts/listvideocells.pl scripts/skycalibration.pl + scripts/regenerate_background.pl )], dist_abstract => 'Scripts for running the Pan-STARRS IPP', Index: branches/eam_branches/ipp-20121130/ippScripts/MANIFEST =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/MANIFEST (revision 34755) +++ branches/eam_branches/ipp-20121130/ippScripts/MANIFEST (revision 34839) @@ -46,3 +46,4 @@ scripts/skycell_jpeg.pl scripts/lap_science.pl +scripts/regenerate_background.pl t/00_distribution.t Index: branches/eam_branches/ipp-20121130/ippScripts/scripts/chip_imfile.pl =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/scripts/chip_imfile.pl (revision 34755) +++ branches/eam_branches/ipp-20121130/ippScripts/scripts/chip_imfile.pl (revision 34839) @@ -150,4 +150,5 @@ $outputStats = prepare_output("PPIMAGE.STATS", $outroot, $class_id, 1); my $do_binned_images = 1; + if ($run_state eq 'new') { # prepare the files that are only created for a new run @@ -586,5 +587,7 @@ # check whether psphot outputs should be regenerated. -# Whether we need to or not is a somewhat complicated question. +# Whether we need to or not was a somewhat complicated question. Since we clean cmfs now we only rerun +# photometry if the psf file has gone missing since it is needed to run warps + sub rerun_photometry { @@ -594,4 +597,6 @@ or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + my $update_sources_if_gone = 0; # set this to regenerate sources if gone. + # We no longer do this as of 2012-12 my $make_sources = 0; my $sources_available = 0; @@ -599,15 +604,17 @@ $sources_available = 1; } else { - carp "WARNING: photometry sources file $outputSources is not available"; - my $gone; - if (storage_object_exists($outputSources, \$gone)) { - # check whether the file is permanantely or temporarily gone - if ($gone) { - carp "WARNING: photometry sources storage object exists but all instances are permanently gone"; + if ($update_sources_if_gone) { + carp "WARNING: photometry sources file $outputSources is not available"; + my $gone; + if (storage_object_exists($outputSources, \$gone)) { + # check whether the file is permanantely or temporarily gone + if ($gone) { + carp "WARNING: photometry sources storage object exists but all instances are permanently gone"; + $make_sources = 1; + } + } else { + # storage object must have been deleted $make_sources = 1; } - } else { - # storage object must have been deleted - $make_sources = 1; } } @@ -616,5 +623,7 @@ my $psf_available = 0; my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE", $outroot, $class_id) - or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", + $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR); + if ($ipprc->file_exists($outputPsf)) { $psf_available = 1; @@ -638,10 +647,12 @@ } - # if either of the files are gone rerun photometry unless the other file is temporarily not available - - if (!$sources_available && !$make_sources) { - # destreak will die if the sources is not available - &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); - } + if ($update_sources_if_gone) { + if (!$sources_available && !$make_sources) { + # destreak will die if the sources is not available + # but magic is dead.... + &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR); + } + } + if (!$psf_available && !$make_psf) { # warp updates need the psf file Index: branches/eam_branches/ipp-20121130/ippScripts/scripts/detrend_process_imfile.pl =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/scripts/detrend_process_imfile.pl (revision 34755) +++ branches/eam_branches/ipp-20121130/ippScripts/scripts/detrend_process_imfile.pl (revision 34839) @@ -27,4 +27,6 @@ my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1); my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1); +my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1); + if ($missing_tools) { warn("Can't find required tools."); @@ -124,4 +126,11 @@ # Run ppImage unless ($no_op) { + my $repair_cmd = "$nebrepair $input_uri"; + my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose); + unless ($repair_success) { + &my_die("Unable to attempt repair: $input_uri $repair_error_code", $det_id, $exp_id, $class_id $PS_EXIT_SYS_ERROR); + } + + my $command = "$ppImage -file $input_uri $outroot"; $command .= " -recipe PPIMAGE $ppimage_recipe"; Index: branches/eam_branches/ipp-20121130/ippScripts/scripts/ipp_cleanup.pl =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/scripts/ipp_cleanup.pl (revision 34755) +++ branches/eam_branches/ipp-20121130/ippScripts/scripts/ipp_cleanup.pl (revision 34839) @@ -18,5 +18,12 @@ # Parse the command-line arguments -my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile); +my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile, $check_all); +my $very_verbose = 0; + +# this gets set to 1 the first time we set the corresponding destreak run to be cleaned +#my $ds_done = 0; +# magic is dead +my $ds_done = 1; + GetOptions('stage=s' => \$stage, # which analysis stage to clean? 'camera|i=s' => \$camera, # user-supplied camera name @@ -24,4 +31,5 @@ 'mode|m=s' => \$mode, # cleanup mode (clean / purge) 'path_base=s' => \$path_base, # basename for files + 'check-all' => \$check_all, # if set clean all chips regardless of data_state 'dbname|d=s' => \$dbname, # Database name 'verbose' => \$verbose, # Print to stdout @@ -56,4 +64,7 @@ $ipprc->redirect_output($logfile) or &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile; + + +my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n'; # set this to 1 to enable checking for files on dead nodes @@ -91,4 +102,5 @@ # this stage uses 'chiptool' my $chiptool = can_run('chiptool') or die "Can't find chiptool"; + my $censorObjects = can_run('censorObjects') or die "Can't find censorObjects"; # Get list of component imfiles @@ -96,6 +108,9 @@ my $imfiles; # Array of component files my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run + $command .= ' -all' if ($check_all); $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) + = run(command => $command, verbose => $very_verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -114,5 +129,5 @@ unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + &my_die("Unable to perform chiptool -processedimfile: $error_code", "chip", $stage_id, $error_code); } exit 0; @@ -123,28 +138,104 @@ &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); + my $numchips = scalar @$imfiles; + print "Found $numchips to clean\n"; + + my $clean_sources = 0; + if ((scalar @$imfiles > 0) and ($mode eq 'goto_cleaned')) { + # go and find the smf file(s) for the associated camRun and check the status of the file + # if a good one is found we have the sources for this chipRun and thus can clean the cmfs + my $command = "$chiptool -listrun -chip_id $stage_id"; + $command .= " -dbname $dbname" if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool -listrun: $error_code", "chip", $stage_id, $error_code); + } + my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR); + my $good_smf = 0; + foreach my $entry (@$entries) { + my $camRun_state = $entry->{camRun_state}; + next if $camRun_state ne 'full'; + my $cam_id = $entry->{cam_id}; + if (!$cam_id) { + carp('no cam_id for listrun entry'); + next; + } + my $cam_path_base = $entry->{cam_path_base}; + if ( !defined $cam_path_base ) { + carp("no path_base for $cam_id\n"); + next; + } + + # XXX: This assumes that the filerules are filerules-split + my $smf = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base); + if (!$ipprc->file_exists($smf)) { + carp("smf for $cam_path_base not found"); + next; + } + # we run the program censorObjects in the check mode + # If this program succeeds the smf is a valid fits file and each of the + # extensions was succesfully read. + # XXX: create a new program outside of magic that performs this check + + my $command = "$censorObjects -checkinputonly -file $smf"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $very_verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + print STDERR "censorObjects failed:\n"; + print STDERR "\nSTDOUT:\n" . join "", @$stdout_buf; + print STDERR "\nSTDERR:\n" . join "", @$stderr_buf; + &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code); + } + $good_smf++; + } + if ($good_smf) { + # we have a good one so we can clean the sources + $clean_sources = 1; + print "Found $good_smf good smf files will clean sources\n"; + } else { + print "Unable to find good smf file will NOT clean sources\n"; + } + } + # loop over all of the imfiles, determine the path_base and class_id for each + my $num_errors = 0; foreach my $imfile (@$imfiles) { my $class_id = $imfile->{class_id}; my $path_base = $imfile->{path_base}; + my $data_state = $imfile->{data_state}; my $status = 1; $status = 0 unless defined $path_base and $path_base ne "NULL"; - my $poor_quality = $imfile->{quality} > 0; + my $quality = $imfile->{quality}; + my $good_quality = ($quality == 0); + + print "Starting cleanup for $class_id\n"; # don't clean up unless the data needed to update is available # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent # goto_scrubbed now requires the config file to not exist. - if ($status and !$poor_quality) { + if ($status and $good_quality) { if ($mode eq "goto_cleaned") { my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id); unless ($ipprc->file_exists($config_file)) { + my $fault = $imfile->{fault}; + if (file_gone($config_file)) { - print STDERR "forcing cleanup for chipRun $stage_id $class_id " + # config file was lost. Clean up. If the chip is ever updated a new config + # file will be created + print STDERR "forcing cleanup chip $stage_id $class_id fault: $fault quality: $quality" . " because config file ($config_file) is gone\n"; + } elsif ($fault == 0 and $quality == 0) { + print STDERR "skipping cleaning up chip $stage_id $class_id fault: $fault quality: $quality" + . " because config file ($config_file) is missing\n"; + $status = 0; } else { - print STDERR "skipping cleanup for chipRun $stage_id $class_id " - . " because config file ($config_file) is missing\n"; - $status = 0; + # config file is missing but this is a bad chip anyways so clean it + print STDERR "cleaning up chip $stage_id $class_id fault: $fault quality: $quality" + . " even though config file ($config_file) is missing\n"; } } @@ -154,5 +245,5 @@ if ($ipprc->file_exists($config_file)) { - print STDERR "skipping scrubbed for chipRun $stage_id $class_id " + print STDERR "skipping scrubbed for chip $stage_id $class_id " . " because config file ($config_file) is present\n"; $status = 0; @@ -165,21 +256,30 @@ my @files = (); - # delete the temporary image datafiles -# addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id); -# addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id); -# addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1); addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1); addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1); addFilename (\@files, "PPIMAGE.PATTERN", $path_base, $class_id, 0); + if ($clean_sources) { + addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + } if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode + if (!$clean_sources) { + # these weren't added above but we do want to clean it + addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id); + addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); + } + + # background model is needed to build stack background images so we do not remove it + # addFilename (\@files, "PSPHOT.BACKMDL", $path_base, $class_id); + + addFilename (\@files, "PSPHOT.PSF.SAVE", $path_base, $class_id); addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id); addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id); # clean? - addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); # clean? + addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id); addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id); addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id); - addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); #clean? + addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id); addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id); } @@ -188,29 +288,42 @@ $status = &delete_files (\@files); } + bzip2_file("LOG.IMFILE", $path_base, $class_id); + bzip2_file("LOG.IMFILE.UPDATE", $path_base, $class_id); if ($status) { + my $update_chip = 1; my $command = "$chiptool -chip_id $stage_id -class_id $class_id"; if ($mode eq "goto_purged") { $command .= " -topurgedimfile"; + if ($data_state eq 'purged') { + $update_chip = 0; + } } elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedimfile"; + if ($data_state eq 'cleaned') { + $update_chip = 0; + } } elsif ($mode eq "goto_scrubbed") { $command .= " -toscrubbedimfile"; - } - - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); - } - - set_destreak_goto_cleaned(); - + if ($data_state eq 'scrubbed') { + $update_chip = 0; + } + } + + if ($update_chip) { + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); + } + set_destreak_goto_cleaned(); + } } else { + $num_errors++; # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*) @@ -218,6 +331,4 @@ $command .= " -dbname $dbname" if defined $dbname; -if (0) { - # XXX Don't set components to error cleaned anymore my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); @@ -226,5 +337,4 @@ &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code); } -} # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. @@ -241,4 +351,7 @@ } } + print "Cleanup completed for chip_id $stage_id."; + print " num_errors: $num_errors" if $num_errors; + print "\n"; exit 0; } @@ -350,7 +463,8 @@ my $skyfiles; # Array of component files my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run + $command .= ' -all' if $check_all; $command .= " -dbname $dbname" if defined $dbname; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + run(command => $command, verbose => $very_verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -378,8 +492,13 @@ &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR); + my $numskycells = scalar @$skyfiles; + print "Found $numskycells to clean\n"; + my @files = (); + my $num_errors = 0; foreach my $skyfile (@$skyfiles) { my $path_base = $skyfile->{path_base}; my $skycell_id = $skyfile->{skycell_id}; + my $data_state = $skyfile->{data_state}; my $status = 1; @@ -391,11 +510,17 @@ unless ($ipprc->file_exists($config_file)) { + my $fault = $skyfile->{fault}; + my $quality = $skyfile->{quality}; if (file_gone($config_file)) { - print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" . - " because config file is gone\n"; + print STDERR "forcing cleanup warp $stage_id $skycell_id fault: $fault quality: $quality" + . " because config file ($config_file) is gone\n"; + } elsif ($fault == 0 and $quality == 0) { + print STDERR "skipping cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality" + . " because config file ($config_file) is missing\n"; + $status = 0; } else { - print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" . - " because config file is missing\n"; - $status = 0; + # config file is missing but this is a bad warp anyways so clean it + print STDERR "cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality" + . " even though config file ($config_file) is missing\n"; } } @@ -413,5 +538,6 @@ if ($status) { - if ($skyfile->{quality} != 8007) { + # XXX: what is special about quality == 8007? + if ($skyfile->{quality} != 8007 || $check_all) { my @files = (); @@ -420,5 +546,7 @@ addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1); addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1); -# addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); + # these are rebuilt during update so we can delete them here + addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id); + addFilename(\@files, "SKYCELL.TEMPLATE", $path_base, $skycell_id ); if ($mode eq "goto_purged") { # additional files to remove for 'purge' mode @@ -426,30 +554,58 @@ addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id ); addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id ); - # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base); - - # XXX: do we want to delete these? + addFilename(\@files, "SKYCELL.STATS.UPDATE", $path_base, $skycell_id ); + addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); + + # XXX: do we want to delete these? trace file is empty # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id); - # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id); - } - # actual command to delete the files + } + # actual command to delete the files $status = &delete_files (\@files); } } + bzip2_file("LOG.EXP", $path_base, $skycell_id); + bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id); if ($status) { + my $update_skyfile = 1; my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id"; if ($mode eq "goto_purged") { $command .= " -topurgedskyfile"; + if ($data_state eq 'purged') { + $update_skyfile = 0; + } } elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedskyfile"; + if ($data_state eq 'cleaned') { + $update_skyfile = 0; + } } elsif ($mode eq "goto_scrubbed") { $command .= " -toscrubbedskyfile"; - } - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => $verbose); + if ($data_state eq 'scrubbed') { + $update_skyfile = 0; + } + } + $command .= " -dbname $dbname" if defined $dbname; + + if ($update_skyfile) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); + } + + set_destreak_goto_cleaned(); + } + + } else { + $num_errors++; + my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -457,11 +613,9 @@ } - set_destreak_goto_cleaned(); - - } else { - my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state"; - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again. + $command = "$warptool -warp_id $stage_id -updaterun -set_state $error_state"; + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { @@ -469,8 +623,9 @@ &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code); } - - # exit $PS_EXIT_UNKNOWN_ERROR; - } - } + } + } + print "Cleanup completed for warp_id $stage_id."; + print " num_errors: $num_errors" if $num_errors; + print "\n"; exit 0; } @@ -603,5 +758,4 @@ &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code); } -# exit $PS_EXIT_UNKNOWN_ERROR; } } @@ -619,6 +773,7 @@ my $skyfiles; # Array reference of component files my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run + $command .= ' -all' if $check_all; $command .= " -dbname $dbname" if defined $dbname; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose); unless ($success) { $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); @@ -646,8 +801,10 @@ &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR); + my $num_errors = 0; my @files = (); foreach my $skyfile (@{ $skyfiles }) { my $path_base = $skyfile->{path_base}; my $skycell_id = $skyfile->{skycell_id}; + my $data_state = $skyfile->{data_state}; my $status = 1; @@ -661,4 +818,5 @@ my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); + if (0) { unless ($ipprc->file_exists($config_file)) { if (file_gone($config_file)) { @@ -672,4 +830,21 @@ } } + unless ($ipprc->file_exists($config_file)) { + my $fault = $skyfile->{fault}; + my $quality = $skyfile->{quality}; + if (file_gone($config_file)) { + print STDERR "forcing cleanup diff $stage_id $skycell_id fault: $fault quality: $quality" + . " because config file ($config_file) is gone\n"; + } elsif ($fault == 0 and $quality == 0) { + print STDERR "skipping cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality" + . " because config file ($config_file) is missing\n"; + $status = 0; + } else { + # config file is missing but this is a bad diff anyways so clean it + print STDERR "cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality" + . " even though config file ($config_file) is missing\n"; + } + } + } elsif ($mode eq "goto_scrubbed") { my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id); @@ -712,21 +887,49 @@ } -# print STDERR "MY FILES: @files\n"; $status = &delete_files(\@files); } -# print STDERR "MY STATUS: $status\n"; + + bzip2_file("LOG.EXP", $path_base, $skycell_id); + bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id); + if ($status) { my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id"; + my $update_skyfile = 1; if ($mode eq "goto_purged") { $command .= " -topurgedskyfile"; + if ($data_state eq 'purged') { + $update_skyfile = 0; + } } elsif ($mode eq "goto_cleaned") { $command .= " -tocleanedskyfile"; + if ($data_state eq 'cleaned') { + $update_skyfile = 0; + } } elsif ($mode eq "goto_scrubbed") { $command .= " -toscrubbedskyfile"; - } - + if ($data_state eq 'scrubbed') { + $update_skyfile = 0; + } + } + + $command .= " -dbname $dbname" if defined $dbname; + + if ($update_skyfile) { + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); + } + set_destreak_goto_cleaned(); + } + + + } else { + $num_errors++; + my $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $skycell_id -set_state $error_state"; $command .= " -dbname $dbname" if defined $dbname; @@ -738,12 +941,9 @@ } - set_destreak_goto_cleaned(); - - } else { - my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; - - $command .= " -dbname $dbname" if defined $dbname; - - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state"; + + $command .= " -dbname $dbname" if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); unless ($success) { @@ -751,7 +951,9 @@ &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code); } -# exit $PS_EXIT_UNKNOWN_ERROR; - } - } + } + } + print "Cleanup completed for diff_id $stage_id."; + print " num_errors: $num_errors" if $num_errors; + print "\n"; exit 0; } @@ -1917,5 +2119,5 @@ foreach my $file (@$files) { - print STDERR "unlinking $stage $stage_id $file\n"; + print STDERR "unlinking $stage $stage_id $file\n" if $very_verbose; my $error_code = $ipprc->kill_file($file); @@ -2008,6 +2210,4 @@ } -# this gets set to 1 the first time we set the corresponding destreak run to be cleaned -my $ds_done = 0; sub set_destreak_goto_cleaned { @@ -2027,4 +2227,37 @@ } +sub bzip2_file { + my $filerule = shift; + my $path_base = shift; + my $component = shift; + + my $filename = $ipprc->filename($filerule, $path_base, $component); + if (!$ipprc->file_exists($filename)) { + return 1; + } + if (my $resolved = $ipprc->file_resolve($filename)) { + my $bzip2_filename = $filename . '.bz2'; + if ($ipprc->file_exists($bzip2_filename)) { + $ipprc->kill_file($bzip2_filename); + } + my $bzip2_file = $ipprc->file_create($bzip2_filename); + my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file; + + my $command = "$bzip2 < $resolved > $bzip2_file"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $very_verbose); + if ($success) { + # success delete the original file + my $error_code = $ipprc->kill_file($filename); + } else { + # if bzip2 failed. Carry on but don't delete the existing file + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + print STDERR "Failed to bzip2 $filename: $error_code\n"; + return 0; + } + } + return 1; +} + # XXX we currently do not set the error state in the db on my_die sub my_die Index: branches/eam_branches/ipp-20121130/ippScripts/scripts/regenerate_background.pl =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/scripts/regenerate_background.pl (revision 34839) +++ branches/eam_branches/ipp-20121130/ippScripts/scripts/regenerate_background.pl (revision 34839) @@ -0,0 +1,358 @@ +#! /usr/bin/env perl + +# Script to call the appropriate commands to regenerate the background models +# for stages subsequent to the chip stage. + +use warnings; +use strict; +use Carp; + +use IPC::Cmd 0.36 qw( can_run run ); +use File::Spec; +use File::Temp qw( tempfile ); +use PS::IPP::Config 1.01 qw( :standard ); +use Getopt::Long qw (GetOptions :config auto_help auto_version gnu_getopt ); +use Pod::Usage qw( pod2usage ); +use PS::IPP::Metadata::Config; +use PS::IPP::Metadata::List qw( parse_md_list ); + + + +my $ppConfigDump = can_run('ppConfigDump'); +my $pswarp = can_run('pswarp'); + +# Parse command line options. +my ($stage, $camera, $stage_id, $skycell_id, $dbname, $verbose, $logfile, $save_temps); +GetOptions('stage=s' => \$stage, # which analysis stage to generate bkg model + 'camera|i=s' => \$camera, # user-supplied camera name + 'stage_id=s' => \$stage_id, # id for this stage + 'skycell_id=s' => \$skycell_id, # skycell_id + 'verbose' => \$verbose, # verbose commands + 'logfile=s' => \$logfile, # destination for stdout and stderr + 'save_temps' => \$save_temps, # save temporary files. + 'dbname=s' => \$dbname # database name + ) or pod2usage ( 2 ); + +pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id (stage_id) [--dbname (dbname)]", + -exitval => 2 ) if scalar @ARGV; + +pod2usage( -msg => "Required options: --camera (name) --stage (stage) --stage_id (stage_id)", + -exitval => 3 ) unless + defined $camera and + defined $stage and + defined $stage_id; + +pod2usage( -msg => "Required options: --camera (name) --stage (stage) --stage_id (stage_id) --skycell_id (skycell_id)", + -exitval => 3) if (($stage eq 'warp')&&(!defined($skycell_id))); + + +my $ipprc = PS::IPP::Config->new ( $camera ) or + &my_die("Unable to set up", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR); # used for path/neb info + +# We can only regenerate a background for certain stages. +unless (($stage eq "warp") || ($stage eq "stack") || ($stage eq "warpstack")) { + &my_die("Invalid stage to regenerate", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR); +} + +$ipprc->redirect_output($logfile) or + &my_die("Unable to redirect output to logfile", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile; + +my $mdcParser = PS::IPP::Metadata::Config->new; # parser for metadata config files + + +## warp stage +if ($stage eq 'warp') { + my $warp_id = $stage_id; # Because I think I used the wrong id in places. + &my_die("--stage_id required for stage warp", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if !$stage_id; + &my_die("--skycell_id required for stage warp", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if !$skycell_id; + + # Configuration stuff + my $warptool = can_run('warptool') or + &my_die("Can't find warptool",$stage,$stage_id, $PS_EXIT_UNKNOWN_ERROR); + + my $astromSource; # The astrometry source + { + my $command = "$ppConfigDump -camera $camera -recipe PSWARP BACKGROUND -dump-recipe PSWARP -"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppConfigDump: $error_code", + $stage, $stage_id, $error_code); + } + my $metadata = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", + $stage, $stage_id, $PS_EXIT_PROG_ERROR); + $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); + } + + + my $imfiles; # Array of component files + my $command = "$warptool -warped -warp_id $stage_id -skycell_id $skycell_id"; + $command .= " -dbname $dbname " if defined $dbname; + + my ( $success, $error_code, $full_buf, $stdout_buf, $stderrr_buf ) = + run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", $stage, $stage_id, $error_code); + } + + if (@$stdout_buf == 0) { + # No results + } + + my $in_md = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $stage, $stage_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($in_md) or + &my_die("Unable to parse metadata", $stage, $stage_id, $PS_EXIT_PROG_ERROR); + + foreach my $imfile (@$imfiles) { + my $skycell_id = $imfile->{skycell_id}; + my $path_base = $imfile->{path_base}; + my $tess_id = $imfile->{tess_id}; + + # Parse the results to determine if there are any problems that prevent this from + # being processed. I think I might need to think about this a bit more. + my $status = 1; + $status = 0 unless defined $path_base and $path_base ne "NULL"; + my $poor_quality = $imfile->{quality} > 0; + +# my $bkg_file = $ipprc->filename("PSWARP.OUTPUT.BKGMODEL",$path_base,$skycell_id); +# my $bkg_exists = $ipprc->file_exists($bkg_file); + my $bkg_exists = $imfile->{background_model}; + if ($bkg_exists) { $status = 0; } # We do not need to remake this. + + # Construct temp files + my ($bkgList_file,$bkgList_name) = tempfile("/tmp/bkgreg.warp.bkg.${warp_id}.${skycell_id}.XXXX", + UNLINK => !$save_temps); + my ($astromList_file,$astromList_name) = tempfile("/tmp/bkgreg.warp.astrom.${warp_id}.${skycell_id}.XXXX", + UNLINK => !$save_temps); + my $chipFiles; + { + my $inputs_command = "$warptool -scmap -warp_id $warp_id "; + $inputs_command .= " -skycell_id $skycell_id "; + $inputs_command .= " -dbname $dbname " if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run (command => $inputs_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool -scmap: $error_code", + $skycell_id,$warp_id,$PS_EXIT_SYS_ERROR); + } + my $md = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse warptool -scmap: $error_code", + $skycell_id,$warp_id,$PS_EXIT_SYS_ERROR); + $chipFiles = parse_md_list($md) or + &my_die("Unable to parse warptool -scmap: $error_code", + $skycell_id,$warp_id,$PS_EXIT_SYS_ERROR); + } + my $wrote_astrom = 0; + foreach my $chipFile (@$chipFiles) { + my $bkgModel = $ipprc->filename("PSPHOT.BACKMDL", + $chipFile->{chip_path_base}, + $chipFile->{class_id}); + my $astrom = $ipprc->filename($astromSource,$chipFile->{cam_path_base}); + print $bkgList_file "$bkgModel\n"; + if (!$wrote_astrom) { + print $astromList_file "$astrom\n"; + $wrote_astrom = 1; + } + } + close ($bkgList_file); + close ($astromList_file); + + # Construct skyfile if needed + my $skyCell_name = $ipprc->filename("SKYCELL.TEMPLATE",$path_base,$skycell_id); + my $skyCell_exists = $ipprc->file_exists($skyCell_name); + if (!$skyCell_exists) { # Generate it + my $skyFile = prepare_output("SKYCELL.TEMPLATE", $path_base, $skycell_id, 1); + $ipprc->skycell_file( $tess_id, $skycell_id, $skyFile, $verbose ) or + &my_die("Unable to generate template skycell",$skycell_id,$warp_id,$PS_EXIT_SYS_ERROR); + } + my $bkgOut_log = prepare_output("LOG.EXP",$path_base . ".BKG_REG",$skycell_id,1); + + # Construct pswarp command + my $pswarp_command = "$pswarp "; + $pswarp_command .= " -list $bkgList_name -astromlist $astromList_name -bkglist $bkgList_name "; + $pswarp_command .= " ${path_base} "; + $pswarp_command .= " $skyCell_name "; + $pswarp_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF "; + $pswarp_command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; + $pswarp_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS "; + $pswarp_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL "; + $pswarp_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID "; + $pswarp_command .= " -recipe PSWARP WARP -Db PSF FALSE -Db BACKGROUND.MODEL TRUE "; + $pswarp_command .= " -Db SOURCES FALSE "; + $pswarp_command .= " -log $bkgOut_log -threads 1 "; + $pswarp_command .= " -dbname $dbname " if defined $dbname; + + # Execute + my ( $success, $error_code, $full_buf, $stdout_buf, $stderrr_buf ) = + run(command => $pswarp_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform pswarp: $error_code", $stage, $stage_id, $error_code); + } + + my $update_command = "$warptool "; + $update_command .= " -updateskyfile -warp_id $warp_id -skycell_id $skycell_id "; + $update_command .= " -set_background_model 1 "; + $update_command .= " -dbname $dbname " if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderrr_buf ) = + run(command => $update_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform warptool: $error_code", $stage, $stage_id, $error_code); + } + + + } +} +## stack stage +elsif ($stage eq 'stack') { + &my_die("--stage_id required for stage stack", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if !$stage_id; + my $stack_id = $stage_id; # Same as above. Alias this so I don't make mistakes. + + # Configuration stuff + my $stacktool = can_run('stacktool') or + &my_die("Can't find stacktool",$stage,$stage_id,$PS_EXIT_UNKNOWN_ERROR); + my $ppStackMedian = can_run('ppStackMedian') or + &my_die("Can't find ppStackMedian",$stage,$stage_id,$PS_EXIT_UNKNOWN_ERROR); + + # Get the information about this run + my $st_command = "$stacktool -sumskyfile -stack_id $stack_id"; + $st_command .= " -dbname $dbname " if defined $dbname; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $st_command, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -sumskyfile",$stage,$stage_id,$error_code); + } + if (@$stdout_buf == 0) { + # Nothing to do; + } + my $in_md = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $stage, $stage_id, $PS_EXIT_PROG_ERROR); + my $stacks = parse_md_list($in_md) or + &my_die("Unable to parse metadata", $stage, $stage_id, $PS_EXIT_PROG_ERROR); + my $stack = shift(@$stacks); + my $path_base = $stack->{path_base}; + + # Get inputs + my $imfiles; # Array of component files + my $command = "$stacktool -inputskyfile -stack_id $stack_id"; + $command .= " -dbname $dbname " if defined $dbname; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool -inputskyfile",$stage,$stage_id,$error_code); + } + if (@$stdout_buf == 0) { + # Nothing to do; + } + $in_md = $mdcParser->parse(join "", @$stdout_buf) or + &my_die("Unable to parse metadata config doc", $stage, $stage_id, $PS_EXIT_PROG_ERROR); + $imfiles = parse_md_list($in_md) or + &my_die("Unable to parse metadata", $stage, $stage_id, $PS_EXIT_PROG_ERROR); + + my $num = 0; + my $expTime; + my ($inputMDC_file, $inputMDC_name) = tempfile("/tmp/bkgreg.stack.mdc.${stage_id}.XXXX", + UNLINK => !$save_temps); + foreach my $imfile (@$imfiles) { + if ($imfile->{ignored}) {next; } + unless ($imfile->{background_model}) { + &my_die("Not all inputs have valid background models", + $stage,$stage_id,$PS_EXIT_PROG_ERROR); + } + print $inputMDC_file "INPUT${num}\tMETADATA\n"; + + print $inputMDC_file "\tIMAGE\tSTR\t" . $ipprc->filename( "PSWARP.OUTPUT.BKGMODEL", + $imfile->{path_base} ) . "\n"; + print $inputMDC_file "\tSOURCES\tSTR\t" . $ipprc->filename( "PSWARP.OUTPUT.SOURCES", + $imfile->{path_base} ) . "\n"; + print $inputMDC_file "END\n\n"; + $num++; + + $expTime = $imfile->{exp_time}; # I need something here. + } + close($inputMDC_file); + + my $bkgOut_log = prepare_output("LOG.EXP",$path_base . ".BKG_REG",1); + + my $ppStack_command = "$ppStackMedian -input $inputMDC_name ${path_base}.mdl "; + $ppStack_command .= " -recipe PPSTACK STACK -recipe PPSUB STACK -recipe PSPHOT STACK "; + $ppStack_command .= " -recipe PPSTATS STACKSTATS -stack-type DEEP_STACK "; +# $ppStack_command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE "; +# $ppStack_command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF "; +# $ppStack_command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS "; +# $ppStack_command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL "; +# $ppStack_command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID "; + $ppStack_command .= " -Db PHOTOMETRY F -Db VARIANCE F -Db CONVOLVE F "; + $ppStack_command .= " -Df DEFAULT.EXPTIME $expTime "; + $ppStack_command .= " -threads 1 -log $bkgOut_log "; + $ppStack_command .= " -dbname $dbname " if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $ppStack_command, verbose => $verbose); + unless($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform ppStackMedian",$stage,$stage_id,$error_code); + } + + my $update_command = "$stacktool "; + $update_command .= " -updatesumskyfile -stack_id $stack_id "; + $update_command .= " -set_background_model 1 -fault 0 "; + $update_command .= " -dbname $dbname " if defined $dbname; + + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $update_command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform stacktool: $error_code", $stage, $stage_id, $error_code); + } + + + +} +## do both warp and stack +elsif ($stage eq 'warpstack') { + +} + +sub prepare_output +{ + my $filerule = shift; + my $outroot = shift; + my $skycell_id = shift; + my $delete = shift; + $delete = 0 if !defined $delete; + + my $error; + my $output = $ipprc->prepare_output($filerule, $outroot, $skycell_id, $delete, \$error) + or &my_die("failed to prepare output file for: $filerule", $skycell_id, $stage_id, $error); + return $output; +} + + +sub my_die { + my $msg = shift; # Warning message on die + my $stage = shift; # stage name + my $stage_id = shift; # identifier + my $exit_code = shift; # exit code + + carp($msg); + exit $exit_code; +} + + + + + + + + + Index: branches/eam_branches/ipp-20121130/ippScripts/scripts/skycalibration.pl =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/scripts/skycalibration.pl (revision 34755) +++ branches/eam_branches/ipp-20121130/ippScripts/scripts/skycalibration.pl (revision 34839) @@ -134,60 +134,63 @@ # First check the expected file name where the stack_id is the FILE_ID my $file; -if (!$singlefilter) { +# if (!$singlefilter) { +if (1) { + $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $stack_id); if (! $ipprc->file_resolve($file)) { - # XXX: Beginning of section that can be removed eventually - - # no file with the expected name found - # assume that the input is from an early staticsky run that did not use stack_id as the FILE_ID - # but instead used FILE_ID = [0 .. num_filters-1] print "\nfailed to resolve $file\n"; - if (!$filter) { - &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); - } - print "Trying old style FILE_ID\n"; - my $max_filters = 5; - for (my $i=0; $i < $max_filters; $i++) { - my $file_id = sprintf "%03d", $i; - $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id); - my $resolved = $ipprc->file_resolve($file); - if (!$resolved) { - # we fail here assumming that if file_id N doesn't exist, neither to N+1 - print "\nfailed to resolve $file\n\n"; - &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); + # file with proper file rule not found. Try older systems + if ($singlefilter) { + # input is from a single filter static sky run which used psphot instead of psphotStack + # The file rule is different for psphot + $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base); + &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) + unless $ipprc->file_exists($file); + } else { + # XXX: Beginning of section that can be removed eventually + # no file with the expected name found + # assume that the input is from an early multifilter staticsky run that did not use stack_id as the FILE_ID + # but instead used FILE_ID = [0 .. num_filters-1] + if (!$filter) { + &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); } - # Check the filter id for this file - my $command = "$fhead $resolved | grep FILTERID"; - my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR); + print "Trying old style FILE_ID\n"; + my $max_filters = 5; + for (my $i=0; $i < $max_filters; $i++) { + my $file_id = sprintf "%03d", $i; + $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id); + my $resolved = $ipprc->file_resolve($file); + if (!$resolved) { + # we fail here assumming that if file_id N doesn't exist, neither to N+1 + print "\nfailed to resolve $file\n\n"; + &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR); + } + # Check the filter id for this file + my $command = "$fhead $resolved | grep FILTERID"; + my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = + run(command => $command, verbose => 0); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR); + } + # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name) + my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf); + my $this_filter = substr $filt, 1, 7; + + # if it matches we're done + last if $this_filter eq $filter; + + # nope loop around to try the next one + print "Input file for $filter is not $file ($this_filter)\n"; + $file = undef; } - # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name) - my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf); - my $this_filter = substr $filt, 1, 7; - - # if it matches we're done - last if $this_filter eq $filter; - - # nope loop around to try the next one - print "Input file for $filter is not $file ($this_filter)\n"; - $file = undef; - } + } + # XXX: End of section that can be removed eventually + } + if ($file) { print "\nInput file for $stack_id filter: $filter: $file\n"; - - # XXX: End of section that can be removed eventually - } -} else { - # input is from a single filter static sky run use a different file rule - # XXX: can't we just make staticsky.pl use the same file rule? - $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base); - &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) - unless $ipprc->file_exists($file); -} - -if (!$file) { - &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR); + } else { + &my_die("Unable to find input file for stack_id $stack_id.", $skycal_id, $PS_EXIT_SYS_ERROR); + } } @@ -197,9 +200,11 @@ } -# read the input header to find the number of detections and the number of detections -# with extended model +# XXX: The following is for compatability with psphotStack cmfs created prior to adding NDET +# read the input header to find the number of detections +my $compatability_flags = ""; my $n_detections = 0; -my $n_extended = 0; { + # NSTARS is buggy. It only counts sources with a model, and counts them twice. + # We want the actual number of sources (real + matched). Use the length of the psf table my $command = "echo $resolved | $fields -n SkyChip.psf NAXIS2"; my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = @@ -211,11 +216,6 @@ unless defined $n_detections; - $command = "echo $resolved | $fields -n SkyChip.hdr NDET_EXT"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = - run(command => $command, verbose => 0); - (undef, $n_extended) = split " ", join "", @$stdout_buf; - chomp $n_extended; - &my_die("Unable to find number of extended objects from $file: ", $skycal_id, $PS_EXIT_SYS_ERROR) - unless defined $n_extended; + $compatability_flags .= " -n_detections $n_detections"; + } @@ -257,4 +257,7 @@ } + # if ppStats didn't get n_detections use the compatability version extracted above + $cmdflags .= $compatability_flags unless ($cmdflags =~ /-n_detections/); + my $quality; ($quality) = $cmdflags =~ /-quality (\d+)/; @@ -276,6 +279,4 @@ { my $command = "$staticskytool -skycal_id $skycal_id"; - $command .= " -n_detections $n_detections"; - $command .= " -n_extended $n_extended"; $command .= " -addskycalresult -path_base $outroot"; $command .= " $cmdflags"; Index: branches/eam_branches/ipp-20121130/ippScripts/scripts/skycell_jpeg.pl =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/scripts/skycell_jpeg.pl (revision 34755) +++ branches/eam_branches/ipp-20121130/ippScripts/scripts/skycell_jpeg.pl (revision 34839) @@ -268,4 +268,12 @@ my %tangents = (); + + my %products = ('image' => "PPSTACK.UNCONV", + 'mask' => "PPSTACK.UNCONV.MASK", + 'variance' => "PPSTACK.UNCONV.VARIANCE", + 'exp' => "PPSTACK.UNCONV.EXP", + 'num' => "PPSTACK.UNCONV.EXPNUM", + 'bkg' => "PPSTACK.OUTPUT.BKGMODEL" + ); foreach my $imfile (@$imfiles) { @@ -281,42 +289,51 @@ $projection_cell =~ s/^(.*)\..*$/$1/; - + unless (exists($tangents{$projection_cell})) { # Make a temp file and fill, but be sure to save - ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX", - UNLINK => !$save_temps); - $tangents{$projection_cell}{FILE} = $tempFile; - $tangents{$projection_cell}{NAME} = $tempName; - if ($masks) { - my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX", - UNLINK => !$save_temps); - $tangents{$projection_cell}{MFILE} = $maskFile; - $tangents{$projection_cell}{MNAME} = $maskName; - } - } - print "$skycell_id $projection_cell\n"; - my $file = $ipprc->filename("PPSTACK.OUTPUT", $path_base, $skycell_id); - print "$file $state $quality\n"; - my $f_fh = $tangents{$projection_cell}{FILE}; - print $f_fh "$file\n"; - if ($masks) { - my $mask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base, $skycell_id); - print "$mask\n"; - my $m_fh = $tangents{$projection_cell}{MFILE}; - print $m_fh "$mask\n"; + foreach my $key (keys %products) { + ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.$key.XXXX", + UNLINK => !$save_temps); + $tangents{$projection_cell}{$key}{FILE} = $tempFile; + $tangents{$projection_cell}{$key}{NAME} = $tempName; + } + } + foreach my $key (keys %products) { + print "$skycell_id $projection_cell\n"; + my $file = $ipprc->filename($products{$key}, $path_base, $skycell_id); + print "$file $state $quality\n"; + my $f_fh = $tangents{$projection_cell}{$key}{FILE}; + print $f_fh "$file\n"; } } foreach my $projection_cell (keys %tangents) { - $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}"; - if ($masks) { - $command .= " -masks $tangents{$projection_cell}{MNAME} "; - } - $command .= " ${outroot}.${projection_cell} "; - print "$command\n"; - ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); - unless ($success) { - $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); - &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code); - } + ## Loop over results here. + # Images + # Masks + # Variances + # Nexptime + # Nexp + # Backgrounds + + foreach my $key (keys %products) { + $command = "$ppSkycell -images $tangents{$projection_cell}{$key}{NAME}"; + $command .= " ${outroot}.${projection_cell}.${key} "; + if ($key eq 'bkg') { + $command .= " -Di BIN1 1 -Di BIN2 1 "; + } + elsif ($key eq 'image') { + $command .= " -masks $tangents{$projection_cell}{mask}{NAME} "; + } + elsif ($key eq 'mask') { + next; # This should be made with the images. + } + print "$command\n"; + ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose); + unless ($success) { + $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); + &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code); + } + } + # Update database: $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot"; Index: branches/eam_branches/ipp-20121130/ippScripts/scripts/stack_skycell.pl =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/scripts/stack_skycell.pl (revision 34755) +++ branches/eam_branches/ipp-20121130/ippScripts/scripts/stack_skycell.pl (revision 34839) @@ -229,4 +229,6 @@ my $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name + my $bkgmodel = $ipprc->filename("PSWARP.OUTPUT.BKGMODEL", $file->{path_base}); + &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image ); &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask ); @@ -240,4 +242,5 @@ print $listFile "\tPSF\tSTR\t" . $psf . "\n" if $convolve; print $listFile "\tSOURCES\tSTR\t" . $sources . "\n"; + print $listFile "\tBKGMODEL\tSTR\t" . $bkgmodel . "\n" if $ipprc->file_exists( $bkgmodel ); print $listFile "END\n\n"; Index: branches/eam_branches/ipp-20121130/ippScripts/scripts/warp_skycell.pl =================================================================== --- branches/eam_branches/ipp-20121130/ippScripts/scripts/warp_skycell.pl (revision 34755) +++ branches/eam_branches/ipp-20121130/ippScripts/scripts/warp_skycell.pl (revision 34839) @@ -128,4 +128,5 @@ # Where do we get the astrometry source from? my $astromSource; # The astrometry source +my $doBackground; # Do we want to make background models? { my $command = "$ppConfigDump -camera $camera -recipe PSWARP $recipe_pswarp -dump-recipe PSWARP -"; @@ -139,4 +140,5 @@ &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR); $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE'); + $doBackground = metadataLookupBool($metadata, 'BACKGROUND.MODEL'); } @@ -168,4 +170,8 @@ if ($do_stats) { $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1) if $do_stats; +} +my $outputBKGs; +if ($doBackground) { + $outputBKGs = prepare_output ("PSWARP.OUTPUT.BKGMODEL", $outroot, $skycell_id, 1); } my $configuration; @@ -209,5 +215,8 @@ my ($weightFile, $weightName) = tempfile( "$tempOutRoot.weight.list.XXXX", UNLINK => !$save_temps); my ($astromFile, $astromName) = tempfile( "$tempOutRoot.astrom.list.XXXX", UNLINK => !$save_temps); - +my ($bkgFile, $bkgName); +if ($doBackground) { + ($bkgFile, $bkgName) = tempfile( "$tempOutRoot.bkg.list.XXXX", UNLINK => !$save_temps); +} my $wrote_astrom = 0; foreach my $imfile (@$imfiles) { @@ -235,4 +244,9 @@ print $maskFile "$mask\n"; print $weightFile "$weight\n"; + my $bkg; + if ($doBackground) { + $bkg = $ipprc->filename("PSPHOT.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id}); + print $bkgFile "$bkg\n"; + } if (!$wrote_astrom) { @@ -245,5 +259,7 @@ close $weightFile; close $astromFile; - +if ($doBackground) { + close($bkgFile); +} # We need the recipe to determine if we care whether the PSF is generated or not my $recipe; @@ -269,4 +285,5 @@ $command .= " -variancelist $weightName"; $command .= " -astromlist $astromName"; + $command .= " -bkglist $bkgName" if ($doBackground); $command .= " $outroot $skyFile"; $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE";