Changeset 34839 for branches/eam_branches/ipp-20121130/ippScripts

Timestamp:

Dec 18, 2012, 5:57:19 AM (14 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20121130/ippScripts

Files:

• Build.PL (modified) (1 diff)
• MANIFEST (modified) (1 diff)
• scripts/chip_imfile.pl (modified) (6 diffs)
• scripts/destreak_restore_camera.pl (modified) (1 prop)
• scripts/detrend_process_imfile.pl (modified) (2 diffs)
• scripts/ipp_apply_burntool_single.pl (modified) (1 prop)
• scripts/ipp_cleanup.pl (modified) (32 diffs)
• scripts/magic_destreak.pl (modified) (1 prop)
• scripts/regenerate_background.pl (copied) (copied from trunk/ippScripts/scripts/regenerate_background.pl )
• scripts/skycalibration.pl (modified) (5 diffs)
• scripts/skycell_jpeg.pl (modified) (2 diffs)
• scripts/stack_skycell.pl (modified) (2 diffs)
• scripts/warp_skycell.pl (modified) (7 diffs)

branches/eam_branches/ipp-20121130/ippScripts/Build.PL

@@ -128 +128 @@
         scripts/listvideocells.pl
         scripts/skycalibration.pl
+        scripts/regenerate_background.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',

branches/eam_branches/ipp-20121130/ippScripts/MANIFEST

@@ -46 +46 @@
 scripts/skycell_jpeg.pl
 scripts/lap_science.pl
+scripts/regenerate_background.pl
 t/00_distribution.t

branches/eam_branches/ipp-20121130/ippScripts/scripts/chip_imfile.pl

@@ -150 +150 @@
 $outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
 my $do_binned_images = 1;
+
 if ($run_state eq 'new') {
     # prepare the files that are only created for a new run
@@ -586 +587 @@
 
 # check whether psphot outputs should be regenerated.
-# Whether we need to or not is a somewhat complicated question.
+# Whether we need to or not was a somewhat complicated question. Since we clean cmfs now we only rerun
+# photometry if the psf file has gone missing since it is needed to run warps
+
 sub rerun_photometry
 {
@@ -594 +597 @@
                     or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
 
+    my $update_sources_if_gone = 0; # set this to regenerate sources if gone. 
+                                    # We no longer do this as of 2012-12
     my $make_sources = 0;
     my $sources_available = 0;
@@ -599 +604 @@
         $sources_available = 1;
     } else {
-        carp "WARNING: photometry sources file $outputSources is not available";
-        my $gone;
-        if (storage_object_exists($outputSources, \$gone)) {
-            # check whether the file is permanantely or temporarily gone
-            if ($gone) {
-                carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
+        if ($update_sources_if_gone) {
+            carp "WARNING: photometry sources file $outputSources is not available";
+            my $gone;
+            if (storage_object_exists($outputSources, \$gone)) {
+                # check whether the file is permanantely or temporarily gone
+                if ($gone) {
+                    carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
+                    $make_sources = 1;
+                }
+            } else {
+                # storage object must have been deleted
                 $make_sources = 1;
             }
-        } else {
-            # storage object must have been deleted
-            $make_sources = 1;
         }
     }
@@ -616 +623 @@
     my $psf_available = 0;
     my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE",       $outroot, $class_id)
-                    or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+                    or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE",
+                                $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+
     if ($ipprc->file_exists($outputPsf)) {
         $psf_available = 1;
@@ -638 +647 @@
     }
 
-    # if either of the files are gone rerun photometry unless the other file is temporarily not available
-
-    if (!$sources_available && !$make_sources) {
-        # destreak will die if the sources is not available
-        &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-    }
+    if ($update_sources_if_gone) {
+        if (!$sources_available && !$make_sources) {
+            # destreak will die if the sources is not available
+            # but magic is dead....
+            &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+    }
+
     if (!$psf_available && !$make_psf) {
         # warp updates need the psf file

branches/eam_branches/ipp-20121130/ippScripts/scripts/detrend_process_imfile.pl

@@ -27 +27 @@
 my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $nebrepair = can_run('neb-repair') or (warn "Can't find neb-repair" and $missing_tools = 1);
+
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -124 +126 @@
 # Run ppImage
 unless ($no_op) {
+    my $repair_cmd = "$nebrepair $input_uri";
+    my ($repair_success, $repair_error_code, $repair_full_buf, $repair_stdout_buf, $repair_stderr_buf ) = run(command => $repair_cmd, verbose => $verbose);
+    unless ($repair_success) {
+        &my_die("Unable to attempt repair: $input_uri $repair_error_code", $det_id, $exp_id, $class_id $PS_EXIT_SYS_ERROR);
+    }
+
+
     my $command = "$ppImage -file $input_uri $outroot";
     $command .= " -recipe PPIMAGE $ppimage_recipe";

branches/eam_branches/ipp-20121130/ippScripts/scripts/ipp_cleanup.pl

@@ -18 +18 @@
 
 # Parse the command-line arguments
-my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile);
+my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist, $logfile, $check_all);
+my $very_verbose = 0;
+
+# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
+#my $ds_done = 0;
+# magic is dead
+my $ds_done = 1;
+
 GetOptions('stage=s'        => \$stage,     # which analysis stage to clean?
            'camera|i=s'     => \$camera,    # user-supplied camera name
@@ -24 +31 @@
            'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
            'path_base=s'    => \$path_base, # basename for files
+           'check-all'      => \$check_all, # if set clean all chips regardless of data_state
            'dbname|d=s'     => \$dbname,    # Database name
            'verbose'        => \$verbose,   # Print to stdout
@@ -56 +64 @@
 $ipprc->redirect_output($logfile) or 
         &my_die("Unable to redirect ouput", $stage, $stage_id, $PS_EXIT_UNKNOWN_ERROR) if $logfile;
+
+
+my $bzip2 = can_run('bzip2') or die 'cannot find bzip2\n';
 
 # set this to 1 to enable checking for files on dead nodes
@@ -91 +102 @@
     # this stage uses 'chiptool'
     my $chiptool = can_run('chiptool') or die "Can't find chiptool";
+    my $censorObjects = can_run('censorObjects') or die "Can't find censorObjects";
 
     # Get list of component imfiles
@@ -96 +108 @@
     my $imfiles;                      # Array of component files
     my $command = "$chiptool -pendingcleanupimfile -chip_id $stage_id"; # Command to run
+    $command .= ' -all' if ($check_all);
     $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) 
+        = run(command => $command, verbose => $very_verbose);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -114 +129 @@
         unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+            &my_die("Unable to perform chiptool -processedimfile: $error_code", "chip", $stage_id, $error_code);
         }
         exit 0;
@@ -123 +138 @@
         &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
 
+    my $numchips = scalar @$imfiles;
+    print "Found $numchips to clean\n";
+
+    my $clean_sources = 0;
+    if ((scalar @$imfiles > 0) and ($mode eq 'goto_cleaned')) {
+        # go and find the smf file(s) for the associated camRun and check the status of the file
+        # if a good one is found we have the sources for this chipRun and thus can clean the cmfs
+        my $command = "$chiptool -listrun -chip_id $stage_id";
+        $command .= " -dbname $dbname" if defined $dbname;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform chiptool -listrun: $error_code", "chip", $stage_id, $error_code);
+        }
+        my $entries = $mdcParser->parse_list(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
+        my $good_smf = 0;
+        foreach my $entry (@$entries) {
+            my $camRun_state = $entry->{camRun_state};
+            next if $camRun_state  ne 'full';
+            my $cam_id = $entry->{cam_id};
+            if (!$cam_id) {
+                carp('no cam_id for listrun entry');
+                next;
+            }
+            my $cam_path_base = $entry->{cam_path_base};
+            if ( !defined $cam_path_base ) {
+                carp("no path_base for $cam_id\n");
+                next;
+            }
+
+            # XXX: This assumes that the filerules are filerules-split
+            my $smf =  $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
+            if (!$ipprc->file_exists($smf)) {
+                carp("smf for $cam_path_base not found");
+                next;
+            }
+            # we run the program censorObjects in the check mode
+            # If this program succeeds the smf is a valid fits file and each of the
+            # extensions was succesfully read.
+            # XXX: create a new program outside of magic that performs this check
+
+            my $command = "$censorObjects -checkinputonly -file $smf";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 
+                                        run(command => $command, verbose => $very_verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                print STDERR "censorObjects failed:\n";
+                print STDERR "\nSTDOUT:\n" . join "", @$stdout_buf;
+                print STDERR "\nSTDERR:\n" . join "", @$stderr_buf;
+                &my_die("Unable to perform censorObjects -checkinputonly: $error_code", "chip", $stage_id, $error_code);
+            }
+            $good_smf++;
+        }
+        if ($good_smf) {
+            # we have a good one so we can clean the sources
+            $clean_sources = 1;
+            print "Found $good_smf good smf files will clean sources\n";
+        } else {
+            print "Unable to find good smf file will NOT clean sources\n";
+        }
+    }
+
     # loop over all of the imfiles, determine the path_base and class_id for each
+    my $num_errors = 0;
     foreach my $imfile (@$imfiles) {
         my $class_id = $imfile->{class_id};
         my $path_base = $imfile->{path_base};
+        my $data_state = $imfile->{data_state};
         my $status = 1;
         $status = 0 unless defined $path_base and $path_base ne "NULL";
 
-        my $poor_quality = $imfile->{quality} > 0;
+        my $quality = $imfile->{quality};
+        my $good_quality = ($quality == 0);
+
+        print "Starting cleanup for $class_id\n";
 
         # don't clean up unless the data needed to update is available
         # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
         # goto_scrubbed now requires the config file to not exist.
-        if ($status and !$poor_quality) {
+        if ($status and $good_quality) {
             if ($mode eq "goto_cleaned") {
                 my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
 
                 unless ($ipprc->file_exists($config_file)) {
+                    my $fault = $imfile->{fault};
+
                     if (file_gone($config_file)) {
-                        print STDERR "forcing cleanup for chipRun $stage_id $class_id "
+                        # config file was lost. Clean up. If the chip is ever updated a new config
+                        # file will be created
+                        print STDERR "forcing cleanup chip $stage_id $class_id fault: $fault quality: $quality"
                             . " because config file ($config_file) is gone\n";
+                    } elsif ($fault == 0 and $quality == 0) {
+                            print STDERR "skipping cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
+                                . " because config file ($config_file) is missing\n";
+                            $status = 0;
                     } else {
-                        print STDERR "skipping cleanup for chipRun $stage_id $class_id "
-                            . " because config file ($config_file) is missing\n";
-                        $status = 0;
+                            # config file is missing but this is a bad chip anyways so clean it
+                            print STDERR "cleaning up chip $stage_id $class_id fault: $fault quality: $quality"
+                                . " even though config file ($config_file) is missing\n";
                     }
                 }
@@ -154 +245 @@
 
                 if ($ipprc->file_exists($config_file)) {
-                    print STDERR "skipping scrubbed for chipRun $stage_id $class_id "
+                    print STDERR "skipping scrubbed for chip $stage_id $class_id "
                         . " because config file ($config_file) is present\n";
                     $status = 0;
@@ -165 +256 @@
             my @files = ();
 
-            # delete the temporary image datafiles
-#            addFilename (\@files, "PPIMAGE.OUTPUT", $path_base, $class_id);
-#            addFilename (\@files, "PPIMAGE.OUTPUT.MASK", $path_base, $class_id);
-#            addFilename (\@files, "PPIMAGE.OUTPUT.VARIANCE", $path_base, $class_id);
             addFilename (\@files, "PPIMAGE.CHIP", $path_base, $class_id, 1);
             addFilename (\@files, "PPIMAGE.CHIP.MASK", $path_base, $class_id, 1);
             addFilename (\@files, "PPIMAGE.CHIP.VARIANCE", $path_base, $class_id, 1);
             addFilename (\@files, "PPIMAGE.PATTERN", $path_base, $class_id, 0);
+            if ($clean_sources) {
+                addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
+            }
             if ($mode eq "goto_purged") {
                 # additional files to remove for 'purge' mode
+                if (!$clean_sources) {
+                    # these weren't added above but we do want to clean it
+                    addFilename (\@files, "PSPHOT.OUTPUT", $path_base, $class_id);
+                    addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);
+                }
+                
+                # background model is needed to build stack background images so we do not remove it
+                # addFilename (\@files, "PSPHOT.BACKMDL", $path_base, $class_id);
+
+                addFilename (\@files, "PSPHOT.PSF.SAVE", $path_base, $class_id);
                 addFilename (\@files, "PPIMAGE.OUTPUT.FPA1", $path_base, $class_id);
                 addFilename (\@files, "PPIMAGE.OUTPUT.FPA2", $path_base, $class_id);
-                addFilename (\@files, "PPIMAGE.BIN1", $path_base, $class_id);          # clean?
-                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);          # clean?
+                addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
                 addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
                 addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
-                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);         #clean?
+                addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
                 addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
             }
@@ -188 +288 @@
             $status = &delete_files (\@files);
         }
+        bzip2_file("LOG.IMFILE", $path_base, $class_id);
+        bzip2_file("LOG.IMFILE.UPDATE", $path_base, $class_id);
 
         if ($status)  {
+            my $update_chip = 1;
             my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
             if ($mode eq "goto_purged") {
                 $command .= " -topurgedimfile";
+                if ($data_state eq 'purged') {
+                    $update_chip = 0;
+                }
             }
             elsif ($mode eq "goto_cleaned") {
                 $command .= " -tocleanedimfile";
+                if ($data_state eq 'cleaned') {
+                    $update_chip = 0;
+                }
             }
             elsif ($mode eq "goto_scrubbed") {
                 $command .= " -toscrubbedimfile";
-            }
-
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
-            }
-
-            set_destreak_goto_cleaned();
-
+                if ($data_state eq 'scrubbed') {
+                    $update_chip = 0;
+                }
+            }
+
+            if ($update_chip) {
+                $command .= " -dbname $dbname" if defined $dbname;
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+                }
+                set_destreak_goto_cleaned();
+            }
         } else {
+            $num_errors++;
 
             # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
@@ -218 +331 @@
             $command .= " -dbname $dbname" if defined $dbname;
 
-if (0) {
-        # XXX Don't set components to error cleaned anymore
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                     run(command => $command, verbose => $verbose);
@@ -226 +337 @@
                 &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
             }
-}
 
             # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
@@ -241 +351 @@
         }
     }
+    print "Cleanup completed for chip_id $stage_id.";
+    print " num_errors: $num_errors" if $num_errors;
+    print "\n";
     exit 0;
 }
@@ -350 +463 @@
     my $skyfiles;                      # Array of component files
     my $command = "$warptool -pendingcleanupskyfile -warp_id $stage_id"; # Command to run
+    $command .= ' -all' if $check_all;
     $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
+            run(command => $command, verbose => $very_verbose);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -378 +492 @@
         &my_die("Unable to parse metadata config doc", "warp", $stage_id, $PS_EXIT_PROG_ERROR);
 
+    my $numskycells = scalar @$skyfiles;
+    print "Found $numskycells to clean\n";
+
     my @files = ();
+    my $num_errors = 0;
     foreach my $skyfile (@$skyfiles) {
         my $path_base = $skyfile->{path_base};
         my $skycell_id = $skyfile->{skycell_id};
+        my $data_state = $skyfile->{data_state};
 
         my $status = 1;
@@ -391 +510 @@
 
                 unless ($ipprc->file_exists($config_file)) {
+                    my $fault = $skyfile->{fault};
+                    my $quality = $skyfile->{quality};
                     if (file_gone($config_file)) {
-                        print STDERR "forcing cleanup for warpRun $stage_id $skycell_id" .
-                            " because config file is gone\n";
+                        print STDERR "forcing cleanup warp $stage_id $skycell_id fault: $fault quality: $quality"
+                            . " because config file ($config_file) is gone\n";
+                    } elsif ($fault == 0 and $quality == 0) {
+                            print STDERR "skipping cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
+                                . " because config file ($config_file) is missing\n";
+                            $status = 0;
                     } else {
-                        print STDERR "skipping cleanup for warpRun $stage_id $skycell_id" .
-                            " because config file is missing\n";
-                        $status = 0;
+                            # config file is missing but this is a bad warp anyways so clean it
+                            print STDERR "cleaning up warp $stage_id $skycell_id fault: $fault quality: $quality"
+                                . " even though config file ($config_file) is missing\n";
                     }
                 }
@@ -413 +538 @@
 
         if ($status) {
-            if ($skyfile->{quality} != 8007) {
+            # XXX: what is special about quality == 8007?
+            if ($skyfile->{quality} != 8007 || $check_all) {
                 my @files = ();
 
@@ -420 +546 @@
                 addFilename(\@files, "PSWARP.OUTPUT.MASK", $path_base, $skycell_id, 1);
                 addFilename(\@files, "PSWARP.OUTPUT.VARIANCE", $path_base, $skycell_id, 1);
-#            addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
+                # these are rebuilt during update so we can delete them here
+                addFilename(\@files, "PSWARP.OUTPUT.SOURCES", $path_base, $skycell_id);
+                addFilename(\@files, "SKYCELL.TEMPLATE", $path_base, $skycell_id );
                 if ($mode eq "goto_purged") {
                     # additional files to remove for 'purge' mode
@@ -426 +554 @@
                     addFilename(\@files, "PSWARP.BIN2", $path_base, $skycell_id );
                     addFilename(\@files, "SKYCELL.STATS", $path_base, $skycell_id );
-                    # addFilename(\@files, "PSPHOT.PSF.SKY.SAVE", $path_base);
-
-                    # XXX: do we want to delete these?
+                    addFilename(\@files, "SKYCELL.STATS.UPDATE", $path_base, $skycell_id );
+                    addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
+
+                    # XXX: do we want to delete these? trace file is empty
                     # addFilename(\@files, "TRACE.EXP", $path_base, $skycell_id);
-                    # addFilename(\@files, "PSWARP.CONFIG", $path_base, $skycell_id);
-                }
-            # actual command to delete the files
+                }
+                # actual command to delete the files
                 $status = &delete_files (\@files);
             }
         }
+        bzip2_file("LOG.EXP", $path_base, $skycell_id);
+        bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
 
         if ($status)  {
+            my $update_skyfile = 1;
             my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
             if ($mode eq "goto_purged") {
                 $command .= " -topurgedskyfile";
+                if ($data_state eq 'purged') {
+                    $update_skyfile = 0;
+                }
             }
             elsif ($mode eq "goto_cleaned") {
                 $command .= " -tocleanedskyfile";
+                if ($data_state eq 'cleaned') {
+                    $update_skyfile = 0;
+                }
             }
             elsif ($mode eq "goto_scrubbed") {
                 $command .= " -toscrubbedskyfile";
-            }
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
+                if ($data_state eq 'scrubbed') {
+                    $update_skyfile = 0;
+                }
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            if ($update_skyfile) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                        run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+                }
+
+                set_destreak_goto_cleaned();
+            }
+
+         } else {
+            $num_errors++;
+            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -457 +613 @@
             }
 
-            set_destreak_goto_cleaned();
-
-         } else {
-            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            # We want to flag the run as well, to avoid attempting to reprocess the same data over and over again.
+            $command = "$warptool -warp_id $stage_id -updaterun -set_state $error_state";
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $command, verbose => $verbose);
             unless ($success) {
@@ -469 +623 @@
                 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
             }
-
-            #            exit $PS_EXIT_UNKNOWN_ERROR;
-        }
-    }
+        }
+    }
+    print "Cleanup completed for warp_id $stage_id.";
+    print " num_errors: $num_errors" if $num_errors;
+    print "\n";
     exit 0;
 }
@@ -603 +758 @@
                 &my_die("Unable to perform stacktool: $error_code", "stack", $stage_id, $error_code);
             }
-#           exit $PS_EXIT_UNKNOWN_ERROR;
         }
     }
@@ -619 +773 @@
     my $skyfiles;                  # Array reference of component files
     my $command = "difftool -pendingcleanupskyfile -diff_id $stage_id"; # Command to run
+    $command .= ' -all' if $check_all;
     $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $very_verbose);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -646 +801 @@
         &my_die("Unable to parse metadata config doc", "diff", $stage_id, $PS_EXIT_PROG_ERROR);
 
+    my $num_errors = 0;
     my @files = ();
     foreach my $skyfile (@{ $skyfiles }) {
         my $path_base = $skyfile->{path_base};
         my $skycell_id = $skyfile->{skycell_id};
+        my $data_state = $skyfile->{data_state};
 
         my $status = 1;
@@ -661 +818 @@
                 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
 
+            if (0) {
                 unless ($ipprc->file_exists($config_file)) {
                     if (file_gone($config_file)) {
@@ -672 +830 @@
                 }
             }
+                unless ($ipprc->file_exists($config_file)) {
+                    my $fault = $skyfile->{fault};
+                    my $quality = $skyfile->{quality};
+                    if (file_gone($config_file)) {
+                        print STDERR "forcing cleanup diff $stage_id $skycell_id fault: $fault quality: $quality"
+                            . " because config file ($config_file) is gone\n";
+                    } elsif ($fault == 0 and $quality == 0) {
+                            print STDERR "skipping cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
+                                . " because config file ($config_file) is missing\n";
+                            $status = 0;
+                    } else {
+                            # config file is missing but this is a bad diff anyways so clean it
+                            print STDERR "cleaning up diff $stage_id $skycell_id fault: $fault quality: $quality"
+                                . " even though config file ($config_file) is missing\n";
+                    }
+                }
+            }
             elsif ($mode eq "goto_scrubbed") {
                 my $config_file = $ipprc->filename("PPSUB.CONFIG", $path_base, $skycell_id);
@@ -712 +887 @@
 
             }
-#           print STDERR "MY FILES: @files\n";
             $status = &delete_files(\@files);
         }
-#       print STDERR "MY STATUS: $status\n";
+
+        bzip2_file("LOG.EXP", $path_base, $skycell_id);
+        bzip2_file("LOG.EXP.UPDATE", $path_base, $skycell_id);
+
         if ($status) {
             my $command = "$difftool -diff_id $stage_id -skycell_id $skycell_id";
+            my $update_skyfile = 1;
 
             if ($mode eq "goto_purged") {
                 $command .= " -topurgedskyfile";
+                if ($data_state eq 'purged') {
+                    $update_skyfile = 0;
+                }
             }
             elsif ($mode eq "goto_cleaned") {
                 $command .= " -tocleanedskyfile";
+                if ($data_state eq 'cleaned') {
+                    $update_skyfile = 0;
+                }
             }
             elsif ($mode eq "goto_scrubbed") {
                 $command .= " -toscrubbedskyfile";
-            }
-
+                if ($data_state eq 'scrubbed') {
+                    $update_skyfile = 0;
+                }
+            }
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            if ($update_skyfile) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
+                }
+                set_destreak_goto_cleaned();
+            }
+
+
+        } else {
+            $num_errors++;
+            my $command = "$difftool -updatediffskyfile -diff_id $stage_id -skycell_id $skycell_id -set_state $error_state";
             $command .= " -dbname $dbname" if defined $dbname;
 
@@ -738 +941 @@
             }
 
-            set_destreak_goto_cleaned();
-
-        } else {
-            my $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
-
-            $command .= " -dbname $dbname" if defined $dbname;
-
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            $command = "$difftool -updaterun -diff_id $stage_id -set_state $error_state";
+
+            $command .= " -dbname $dbname" if defined $dbname;
+
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $command, verbose => $verbose);
             unless ($success) {
@@ -751 +951 @@
                 &my_die("Unable to perform difftool: $error_code", "diff", $stage_id, $error_code);
             }
-#           exit $PS_EXIT_UNKNOWN_ERROR;
-        }
-    }
+        }
+    }
+    print "Cleanup completed for diff_id $stage_id.";
+    print " num_errors: $num_errors" if $num_errors;
+    print "\n";
     exit 0;
 }
@@ -1917 +2119 @@
 
     foreach my $file (@$files) {
-        print STDERR "unlinking $stage $stage_id $file\n";
+        print STDERR "unlinking $stage $stage_id $file\n" if $very_verbose;
 
         my $error_code = $ipprc->kill_file($file);
@@ -2008 +2210 @@
 }
 
-# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
-my $ds_done = 0;
 sub set_destreak_goto_cleaned {
 
@@ -2027 +2227 @@
 }
 
+sub bzip2_file {
+    my $filerule = shift;
+    my $path_base = shift;
+    my $component = shift;
+
+    my $filename = $ipprc->filename($filerule, $path_base, $component);
+    if (!$ipprc->file_exists($filename)) {
+        return 1;
+    }
+    if (my $resolved = $ipprc->file_resolve($filename)) {
+        my $bzip2_filename = $filename . '.bz2';
+        if ($ipprc->file_exists($bzip2_filename)) {
+            $ipprc->kill_file($bzip2_filename);
+        }
+        my $bzip2_file = $ipprc->file_create($bzip2_filename);
+        my_die("Unable to create $bzip2_filename", $stage_id, $PS_EXIT_SYS_ERROR) unless $bzip2_file;
+
+        my $command = "$bzip2 < $resolved > $bzip2_file";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $very_verbose);
+        if ($success) {
+            # success delete the original file
+            my $error_code = $ipprc->kill_file($filename);
+        } else {
+            # if bzip2 failed. Carry on but don't delete the existing file
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            print STDERR "Failed to bzip2 $filename: $error_code\n";
+            return 0;
+        }
+    }
+    return 1;
+}
+
 # XXX we currently do not set the error state in the db on my_die
 sub my_die

branches/eam_branches/ipp-20121130/ippScripts/scripts/skycalibration.pl

@@ -134 +134 @@
 # First check the expected file name where the stack_id is the FILE_ID
 my $file;
-if (!$singlefilter) {
+# if (!$singlefilter) {
+if (1) {
+
     $file = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $stack_id);
     if (! $ipprc->file_resolve($file)) {
-        # XXX: Beginning of section that can be removed eventually
-
-        # no file with the expected name found
-        # assume that the input is from an early staticsky run that did not use stack_id as the FILE_ID
-        # but instead used FILE_ID = [0 .. num_filters-1]
         print "\nfailed to resolve $file\n";
-        if (!$filter) {
-            &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
-        }
-        print "Trying old style FILE_ID\n";
-        my $max_filters = 5;
-        for (my $i=0; $i < $max_filters; $i++) {
-            my $file_id = sprintf "%03d", $i;
-            $file  = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id);
-            my $resolved = $ipprc->file_resolve($file);
-            if (!$resolved) {
-                # we fail here assumming that if file_id N doesn't exist, neither to N+1
-                print "\nfailed to resolve $file\n\n";
-                &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
+        # file with proper file rule not found. Try older systems
+        if ($singlefilter) {
+            # input is from a single filter static sky run which used psphot instead of psphotStack
+            # The file rule is different for psphot
+            $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base);
+            &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) 
+                unless $ipprc->file_exists($file);
+        } else {
+            # XXX: Beginning of section that can be removed eventually
+            # no file with the expected name found
+            # assume that the input is from an early multifilter staticsky run that did not use stack_id as the FILE_ID
+            # but instead used FILE_ID = [0 .. num_filters-1]
+            if (!$filter) {
+                &my_die("filter not supplied unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
             }
-            # Check the filter id for this file
-            my $command = "$fhead $resolved | grep FILTERID";
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => 0);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR);
+            print "Trying old style FILE_ID\n";
+            my $max_filters = 5;
+            for (my $i=0; $i < $max_filters; $i++) {
+                my $file_id = sprintf "%03d", $i;
+                $file  = $ipprc->filename('PSPHOT.STACK.OUTPUT', $path_base, $file_id);
+                my $resolved = $ipprc->file_resolve($file);
+                if (!$resolved) {
+                    # we fail here assumming that if file_id N doesn't exist, neither to N+1
+                    print "\nfailed to resolve $file\n\n";
+                    &my_die("unable to identify appropriate staticsky input.", $skycal_id, $PS_EXIT_SYS_ERROR);
+                }
+                # Check the filter id for this file
+                my $command = "$fhead $resolved | grep FILTERID";
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => 0);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform $command: $error_code", $skycal_id, $PS_EXIT_SYS_ERROR);
+                }
+                # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name)
+                my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf);
+                my $this_filter = substr $filt, 1, 7;
+
+                # if it matches we're done
+                last if $this_filter eq $filter;
+
+                # nope loop around to try the next one
+                print "Input file for $filter is not $file ($this_filter)\n";
+                $file = undef;
             }
-            # Expected output: HIERARCH FPA.FILTERID = 'r.00000 ' / Filter used (parsed, abstract name)
-            my ($undef, undef, undef, $filt) = split " ", join("", @$stdout_buf);
-            my $this_filter = substr $filt, 1, 7;
-
-            # if it matches we're done
-            last if $this_filter eq $filter;
-
-            # nope loop around to try the next one
-            print "Input file for $filter is not $file ($this_filter)\n";
-            $file = undef;
-        }
+        }
+        # XXX: End of section that can be removed eventually
+    }
+    if ($file) {
         print "\nInput file for $stack_id filter: $filter: $file\n";
-
-        # XXX: End of section that can be removed eventually
-    }
-} else {
-    # input is from a single filter static sky run use a different file rule
-    # XXX: can't we just make staticsky.pl use the same file rule?
-    $file = $ipprc->filename('PSPHOT.OUT.CMF.MEF', $path_base);
-    &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR) 
-        unless $ipprc->file_exists($file);
-}
-
-if (!$file) {
-    &my_die("Unable to find input for $stack_id $filter", $skycal_id, $PS_EXIT_SYS_ERROR);
+    } else {
+        &my_die("Unable to find input file for stack_id $stack_id.", $skycal_id, $PS_EXIT_SYS_ERROR);
+    }
 }
 
@@ -197 +200 @@
 }
 
-# read the input header to find the number of detections and the number of detections
-# with extended model
+# XXX: The following is for compatability with psphotStack cmfs created prior to adding NDET
+# read the input header to find the number of detections
+my $compatability_flags = "";
 my $n_detections = 0;
-my $n_extended = 0;
 {
+    # NSTARS is buggy. It only counts sources with a model, and counts them twice.
+    # We want the actual number of sources (real + matched). Use the length of the psf table
     my $command = "echo $resolved | $fields -n SkyChip.psf NAXIS2";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -211 +216 @@
         unless defined $n_detections;
 
-    $command = "echo $resolved | $fields -n SkyChip.hdr NDET_EXT";
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 0);
-    (undef, $n_extended)  = split " ", join "", @$stdout_buf;
-    chomp $n_extended;
-    &my_die("Unable to find number of extended objects from $file: ", $skycal_id, $PS_EXIT_SYS_ERROR)
-        unless defined $n_extended;
+    $compatability_flags .= " -n_detections $n_detections";
+
 }
 
@@ -257 +257 @@
         }
 
+        # if ppStats didn't get n_detections use the compatability version extracted above
+        $cmdflags .= $compatability_flags unless ($cmdflags =~ /-n_detections/);
+
         my $quality;
         ($quality) =  $cmdflags =~ /-quality (\d+)/;
@@ -276 +279 @@
 {
     my $command = "$staticskytool -skycal_id $skycal_id";
-    $command .= " -n_detections $n_detections";
-    $command .= " -n_extended $n_extended";
     $command .= " -addskycalresult -path_base $outroot";
     $command .= " $cmdflags";

branches/eam_branches/ipp-20121130/ippScripts/scripts/skycell_jpeg.pl

@@ -268 +268 @@
 
     my %tangents = ();
+
+    my %products = ('image' => "PPSTACK.UNCONV",
+                    'mask'  => "PPSTACK.UNCONV.MASK",
+                    'variance' => "PPSTACK.UNCONV.VARIANCE",
+                    'exp'   => "PPSTACK.UNCONV.EXP",
+                    'num'   => "PPSTACK.UNCONV.EXPNUM",
+                    'bkg'   => "PPSTACK.OUTPUT.BKGMODEL"
+        );
     
     foreach my $imfile (@$imfiles) {
@@ -281 +289 @@
 
         $projection_cell =~ s/^(.*)\..*$/$1/;
-        
+
         unless (exists($tangents{$projection_cell})) {
             # Make a temp file and fill, but be sure to save 
-            ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.XXXX",
-                                                 UNLINK => !$save_temps);
-            $tangents{$projection_cell}{FILE} = $tempFile;
-            $tangents{$projection_cell}{NAME} = $tempName;
-            if ($masks) {
-                my ($maskFile, $maskName) = tempfile("/tmp/skycell.$projection_cell.masks.XXXX",
-                                                     UNLINK => !$save_temps);
-                $tangents{$projection_cell}{MFILE} = $maskFile;
-                $tangents{$projection_cell}{MNAME} = $maskName;
-            }           
-        }
-        print "$skycell_id $projection_cell\n"; 
-        my $file = $ipprc->filename("PPSTACK.OUTPUT", $path_base, $skycell_id);
-        print "$file $state $quality\n";
-        my $f_fh = $tangents{$projection_cell}{FILE};
-        print $f_fh "$file\n";
-        if ($masks) {
-            my $mask = $ipprc->filename("PPSTACK.OUTPUT.MASK", $path_base, $skycell_id);
-            print "$mask\n";
-            my $m_fh = $tangents{$projection_cell}{MFILE};
-            print $m_fh "$mask\n";
+            foreach my $key (keys %products) {
+                ($tempFile, $tempName) = tempfile("/tmp/skycell.$projection_cell.$key.XXXX",
+                                                  UNLINK => !$save_temps);
+                $tangents{$projection_cell}{$key}{FILE} = $tempFile;
+                $tangents{$projection_cell}{$key}{NAME} = $tempName;
+            }
+        }
+        foreach my $key (keys %products) {
+            print "$skycell_id $projection_cell\n";     
+            my $file = $ipprc->filename($products{$key}, $path_base, $skycell_id);
+            print "$file $state $quality\n";
+            my $f_fh = $tangents{$projection_cell}{$key}{FILE};
+            print $f_fh "$file\n";
         }
     }
     foreach my $projection_cell (keys %tangents) {
-        $command = "$ppSkycell -images $tangents{$projection_cell}{NAME}";
-        if ($masks) {
-            $command .= " -masks $tangents{$projection_cell}{MNAME} ";
-        }
-        $command .= " ${outroot}.${projection_cell} ";
-        print "$command\n";
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code);
-        }
+        ## Loop over results here.
+        # Images
+        # Masks
+        # Variances
+        # Nexptime
+        # Nexp
+        # Backgrounds
+        
+        foreach my $key (keys %products) {
+            $command = "$ppSkycell -images $tangents{$projection_cell}{$key}{NAME}";
+            $command .= " ${outroot}.${projection_cell}.${key} ";
+            if ($key eq 'bkg') {
+                $command .= " -Di BIN1 1 -Di BIN2 1 ";
+            }
+            elsif ($key eq 'image') {
+                $command .= " -masks $tangents{$projection_cell}{mask}{NAME} ";
+            }
+            elsif ($key eq 'mask') { 
+                next; # This should be made with the images.
+            }
+            print "$command\n";
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("unable to perform ppSkycell: $error_code", $stage_id, $error_code);
+            }
+        }
+
         # Update database:
         $command = "$stacktool -addsummary -sass_id $stage_id -projection_cell $projection_cell -path_base $outroot";

branches/eam_branches/ipp-20121130/ippScripts/scripts/stack_skycell.pl

@@ -229 +229 @@
     my $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name
 
+    my $bkgmodel = $ipprc->filename("PSWARP.OUTPUT.BKGMODEL", $file->{path_base});
+
     &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image );
     &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask );
@@ -240 +242 @@
     print $listFile "\tPSF\tSTR\t" . $psf . "\n" if $convolve;
     print $listFile "\tSOURCES\tSTR\t" . $sources . "\n";
+    print $listFile "\tBKGMODEL\tSTR\t" . $bkgmodel . "\n" if $ipprc->file_exists( $bkgmodel );
 
     print $listFile "END\n\n";

branches/eam_branches/ipp-20121130/ippScripts/scripts/warp_skycell.pl

@@ -128 +128 @@
 # Where do we get the astrometry source from?
 my $astromSource;               # The astrometry source
+my $doBackground;               # Do we want to make background models?
 {
     my $command = "$ppConfigDump -camera $camera -recipe PSWARP $recipe_pswarp -dump-recipe PSWARP -";
@@ -139 +140 @@
         &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
     $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
+    $doBackground = metadataLookupBool($metadata, 'BACKGROUND.MODEL');    
 }
 
@@ -168 +170 @@
 if ($do_stats) {
     $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1) if $do_stats;
+}
+my $outputBKGs;
+if ($doBackground) {
+    $outputBKGs = prepare_output ("PSWARP.OUTPUT.BKGMODEL", $outroot, $skycell_id, 1);
 }
 my $configuration;
@@ -209 +215 @@
 my ($weightFile, $weightName) = tempfile( "$tempOutRoot.weight.list.XXXX", UNLINK => !$save_temps);
 my ($astromFile, $astromName) = tempfile( "$tempOutRoot.astrom.list.XXXX", UNLINK => !$save_temps);
-
+my ($bkgFile, $bkgName);
+if ($doBackground) {
+    ($bkgFile, $bkgName) = tempfile( "$tempOutRoot.bkg.list.XXXX", UNLINK => !$save_temps);
+}
 my $wrote_astrom = 0;
 foreach my $imfile (@$imfiles) {
@@ -235 +244 @@
     print $maskFile   "$mask\n";
     print $weightFile "$weight\n";
+    my $bkg;
+    if ($doBackground) {
+        $bkg    = $ipprc->filename("PSPHOT.BACKMDL", $imfile->{chip_path_base}, $imfile->{class_id});
+        print $bkgFile "$bkg\n";
+    }
 
     if (!$wrote_astrom) {
@@ -245 +259 @@
 close $weightFile;
 close $astromFile;
-
+if ($doBackground) {
+    close($bkgFile);
+}
 # We need the recipe to determine if we care whether the PSF is generated or not
 my $recipe;
@@ -269 +285 @@
     $command .= " -variancelist $weightName";
     $command .= " -astromlist $astromName";
+    $command .= " -bkglist $bkgName" if ($doBackground);
     $command .= " $outroot $skyFile";
     $command .= " -F PSPHOT.PSF.SAVE PSPHOT.PSF.SKY.SAVE";