Index: trunk/ippScripts/scripts/chip_imfile.pl
===================================================================
--- trunk/ippScripts/scripts/chip_imfile.pl	(revision 28899)
+++ trunk/ippScripts/scripts/chip_imfile.pl	(revision 29893)
@@ -66,5 +66,5 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state",
+pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state --dbname",
            -exitval => 3) unless
     defined $exp_id and
@@ -75,12 +75,24 @@
     defined $camera and
     defined $outroot and
+    defined $dbname and
     defined $run_state;
 
 my_die ("$run_state is an invalid value for run-state", $exp_id, $chip_id, $class_id, $PS_EXIT_PROG_ERROR) unless ($run_state eq 'new' or $run_state eq 'update');
 
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-
-my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-$logDest .= ".update" if $run_state eq "update";
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR );
+
+my $neb;
+if (file_scheme($outroot) eq 'neb') {
+    $neb = $ipprc->nebulous();
+}
+
+my ($logDest, $traceDest);
+if ($run_state eq 'new') {
+    $logDest = prepare_output("LOG.IMFILE", $outroot, $class_id, 0);
+    $traceDest = prepare_output("TRACE.IMFILE",  $outroot, $class_id, 1);
+} else {
+    $logDest = prepare_output("LOG.IMFILE.UPDATE", $outroot, $class_id, 1);
+    $traceDest = prepare_output("TRACE.IMFILE.UPDATE",  $outroot, $class_id, 1);
+}
 
 if ($redirect) {
@@ -90,4 +102,5 @@
     print STDOUT "FULL COMMAND: $0 @ARGS\n\n";
 }
+
 
 # Recipes to use based on reduction class
@@ -114,16 +127,46 @@
 
 ## these names are used in ppImage, and thus may be URIs
-my $outputImage   = $ipprc->filename("PPIMAGE.CHIP",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputMask    = $ipprc->filename("PPIMAGE.CHIP.MASK",   $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputWeight  = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputBin1    = $ipprc->filename("PPIMAGE.BIN1",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputBin2    = $ipprc->filename("PPIMAGE.BIN2",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $outputStats   = $ipprc->filename("PPIMAGE.STATS",       $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $traceDest     = $ipprc->filename("TRACE.IMFILE",        $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-my $configuration = $ipprc->filename("PPIMAGE.CONFIG",      $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-
-if ($run_state eq 'update') {
-    $outputStats .= '.update';
-    $traceDest .= '.update';
+my $outputImage   = prepare_output("PPIMAGE.CHIP",          $outroot, $class_id, 1 );
+my $outputMask    = prepare_output("PPIMAGE.CHIP.MASK",     $outroot, $class_id, 1);
+my $outputWeight  = prepare_output("PPIMAGE.CHIP.VARIANCE", $outroot, $class_id, 1);
+my $pattern       = prepare_output("PPIMAGE.PATTERN",       $outroot, $class_id, 1);
+my $backmdl       = prepare_output("PSPHOT.BACKMDL",        $outroot, $class_id, 1);
+
+my $configuration;
+my $outputSources;
+my $outputPsf;
+my $outputStats;
+my $outputBin1;
+my $outputBin2;
+my $dump_config = 1;
+my $do_binned_images = 1;
+if ($run_state eq 'new') {
+    # prepare the files that are only created for a new run
+    $configuration = prepare_output("PPIMAGE.CONFIG",        $outroot, $class_id, 1);
+    $outputStats   = prepare_output("PPIMAGE.STATS",         $outroot, $class_id, 1);
+} else {
+    $configuration = $ipprc->filename('PPIMAGE.CONFIG', $outroot, $class_id) 
+            or &my_die("Missing entry from camera config: PPIMAGE.CONFIG", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+    if ($ipprc->file_exists($configuration)) {
+        $dump_config = 0;
+    } else {
+        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
+
+        # XXX: should we create a new config dump file?
+        # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
+        my $gone = 0;
+        if (storage_object_exists($configuration, \$gone)) {
+            if ($gone) {
+                rename_gone_file($configuration);
+                $configuration = prepare_output('PPIMAGE.CONFIG', $outroot, $class_id, 1);
+                # if we dump the config we need to insure that the config dump represents
+                # the full processing
+            } else {
+                # file is temporarily not available. Don't dump config.
+                $dump_config = 0;
+            }
+        }
+    }
+
     # make sure that any lingering destreak backup files are gone
     $ipprc->delete_destreak_backup_file($outputImage)
@@ -134,4 +177,8 @@
         or &my_die("failed to delete existing destreak backup weight file", $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR);
 }
+if ($do_binned_images) {
+    $outputBin1    = prepare_output("PPIMAGE.BIN1",          $outroot, $class_id, 1);
+    $outputBin2    = prepare_output("PPIMAGE.BIN2",          $outroot, $class_id, 1);
+}
 
 my $cmdflags;
@@ -152,4 +199,19 @@
     my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+
+    my $do_photom = metadataLookupBool($recipeData, 'PHOTOM');
+    if ($do_photom and ($run_state eq 'update')) {
+        # If previous photometry outputs are ok skip running photometry
+        if ($dump_config || rerun_photometry($outroot, $class_id)) {
+            carp "Will rerun photometry\n";
+        } else {
+            $do_photom = 0;
+        }
+    }
+    if ($do_photom) {
+        $outputSources = prepare_output("PSPHOT.OUTPUT",   $outroot, $class_id, 1);
+        $outputPsf     = prepare_output("PSPHOT.PSF.SAVE", $outroot, $class_id, 1);
+    }
+
 
     ## XXX make the feature more general?
@@ -357,29 +419,29 @@
     }
 
+    $command  = "$ppImage -file $uri $outroot";
+    if ($dump_config) {
+        $command .= " -recipe PPIMAGE $recipe_ppImage";
+        $command .= " -dumpconfig $configuration";
+    } else {
+        $command .= " -ipprc $configuration";
+    }
+    if ($do_photom) {
+        $command .= " -recipe PSPHOT $recipe_psphot";
+    } else {
+        $command .= " -Db PPIMAGE:PHOTOM FALSE";
+    }
     if ($run_state eq "new") {
-        $command  = "$ppImage -file $uri $outroot";
-        $command .= " -recipe PPIMAGE $recipe_ppImage";
-        $command .= " -recipe PSPHOT $recipe_psphot";
-        $command .= " -threads $threads" if defined $threads;
-        $command .= " -dbname $dbname" if defined $dbname;
-        $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
-        $command .= " -source_id $source_id" if defined $source_id;
-        $command .= " -dumpconfig $configuration";
-        $command .= " -tracedest $traceDest -log $logDest";
-        $do_stats = 1;
-    } else {
-        $command  = "$ppImage -file $uri $outroot";
-        $command .= " -ipprc $configuration";
-        $command .= " -threads $threads" if defined $threads;
-        $command .= " -dbname $dbname" if defined $dbname;
-        $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
-        $command .= " -source_id $source_id" if defined $source_id;
-        $command .= " -tracedest $traceDest -log $logDest";
-        $command .= " -Db PPIMAGE:PHOTOM FALSE";
-    }
-    if ($do_stats) {
         $command .= " -recipe PPSTATS CHIPSTATS";
         $command .= " -stats $outputStats";
-    }
+        $do_stats = 1;
+    }
+    if (!$do_binned_images) {
+        $command .= " -Db PPIMAGE:BIN1.FITS FALSE -Db PPIMAGE:BIN2.FITS FALSE";
+    }
+    $command .= " -threads $threads" if defined $threads;
+    $command .= " -image_id $chip_imfile_id" if defined $chip_imfile_id;
+    $command .= " -source_id $source_id" if defined $source_id;
+    $command .= " -tracedest $traceDest -log $logDest";
+    $command .= " -dbname $dbname" if defined $dbname;
 
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -394,4 +456,6 @@
     my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS');
     my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.VARIANCE.FITS');
+    my $outputBackmdlExpect = metadataLookupBool($recipeData, 'BACKGROUND');
+    my $outputPatternExpect = (metadataLookupBool($recipeData, 'PATTERN.ROW') or metadataLookupBool($recipeData, 'PATTERN.CELL')) ;
 
     my $quality;                # Quality flag
@@ -416,14 +480,21 @@
 
     if (!$quality) {
-        &my_die("Couldn't find expected output file: $outputImage\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputImageExpect or $ipprc->file_exists($outputImage);
-        &my_die("Couldn't find expected output file: $outputMask\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputMaskExpect or $ipprc->file_exists($outputMask);
-        &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputWeightExpect or $ipprc->file_exists($outputWeight);
-        &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
-        &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
-        if ($run_state eq 'new') {
-            &my_die("Couldn't find expected output file: $configuration", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
-        }
-    }
-
+        my $replicateImages = 0;
+        check_output($outputImage, $replicateImages) if $outputImageExpect;
+        check_output($outputMask, $replicateImages) if $outputMaskExpect;
+        check_output($outputWeight, $replicateImages) if $outputWeightExpect;
+        if ($do_binned_images) {
+            check_output($outputBin1, 1);
+            check_output($outputBin2, 1);
+        }
+        check_output($configuration, 1) if $dump_config;
+        check_output($backmdl, 1) if $outputBackmdlExpect;
+        check_output($pattern, 1) if $outputPatternExpect;
+        if ($do_photom) {
+            check_output($outputSources, 1);
+            check_output($outputPsf, 1);
+        }
+        # XXX: Do we want to replicate the stats, logs, trace file?
+    }
 }
 
@@ -462,4 +533,192 @@
 }
 
+exit 0;
+
+# check whether psphot outputs should be regenerated.
+# Whether we need to or not is a somewhat complicated question.
+sub rerun_photometry
+{
+    my $outroot = shift;
+    my $class_id = shift;
+    my $outputSources = $ipprc->filename('PSPHOT.OUTPUT', $outroot, $class_id) 
+                    or &my_die("Missing entry from camera config: PSPHOT.OUTPUT", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+
+    my $make_sources = 0;
+    my $sources_available = 0;
+    if ($ipprc->file_exists($outputSources)) {
+        $sources_available = 1;
+    } else {
+        carp "WARNING: photometry sources file $outputSources is not available";
+        my $gone;
+        if (storage_object_exists($outputSources, \$gone)) {
+            # check whether the file is permanantely or temporarily gone
+            if ($gone) {
+                carp "WARNING: photometry sources storage object exists but all instances are permanently gone";
+                rename_gone_file($outputSources);
+                $make_sources = 1;
+            }
+        } else {
+            # storage object must have been deleted
+            $make_sources = 1;
+        }
+    }
+
+    my $make_psf = 0;
+    my $psf_available = 0;
+    my $outputPsf = $ipprc->filename("PSPHOT.PSF.SAVE",       $outroot, $class_id)
+                    or &my_die("Missing entry from camera config: PSPHOT.PSF.SAVE", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+    if ($ipprc->file_exists($outputPsf)) {
+        $psf_available = 1;
+    } else {
+        carp "PSF file $outputPsf is missing";
+        my $gone = 0;
+        if (storage_object_exists($outputPsf, \$gone)) {
+            # object exists, but no instances are available. If they are permanently gone
+            # rename the storage object
+            if ($gone) {
+                carp "WARNING: PSF storage object exists but all instances are permanently gone";
+                rename_gone_file($outputPsf);
+                $make_psf = 1;
+            }
+        } else {
+            $make_psf = 1;
+        }
+    }
+
+    if ($sources_available && $psf_available) {
+        return 0;
+    }
+
+    # if either of the files are gone rerun photometry unless the other file is temporarily not available
+
+    if (!$sources_available && !$make_sources) {
+        # destreak will die if the sources is not available
+        &my_die("PSPHOT.SOURCES is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+    if (!$psf_available && !$make_psf) {
+        # warp updates need the psf file
+        &my_die("PSPHOT.PSF.SAVE is missing but we cannot regenerate it", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+
+    return $make_psf || $make_sources;
+}
+
+# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that
+# does not exist and one that all of the instances have been lost.
+# XXXX This should be implemented properly in Nebulous
+# For now uses Bill's script 'whichnode' which queries the nebulous database directly
+
+my $whichnode;
+sub storage_object_exists
+{
+    my $file = shift;
+    my $ref_all_gone = shift;
+
+    my $exists = $neb->storage_object_exists($file);
+    if (!$exists) {
+        return 0;
+    }
+
+    if (!$whichnode) {
+        $whichnode = can_run('whichnode') or
+            &my_die("Can't find whichnode",  $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my $command = "$whichnode $file";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform whichnode: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my @lines = split "\n", (join "", @$stdout_buf);
+
+    if (scalar @lines == 0) {
+        # no output the file is really and truely gone
+        # XXX: this is now caught above
+        print STDERR "storage object for $file does not exist\n";
+        return 0;
+    }
+
+    my $numGone = 0;
+    my $numNotGone = 0;
+    foreach my $line (@lines) {
+        chomp $line;
+
+        # output lines are either
+        #   "volume available"
+        # or 
+        #   "volume not available"
+
+        my ($volume, $answer, undef) = split " ", $line;
+        # our hack is if the volume has an X in the name it's gone
+        if ($volume =~ /X/) {
+            print STDERR "$file is on $volume which is gone\n";
+            $numGone++;
+        } elsif ($answer eq 'available') {
+            $numNotGone++;
+        } elsif ($answer eq 'not') {
+            print STDERR "$file is on $volume which is not available\n";
+            $numNotGone++;
+        } else {
+            print STDERR "unexpected output from whichnode: $line\n";
+        }
+    }
+    # if there are any instances that are not on a gone volume set all_gone to 0
+    if ($numNotGone == 0 and $numGone > 0) {
+        $$ref_all_gone = 1;
+    } else {
+        $$ref_all_gone = 0;
+    }
+
+    # storage object exists so return true
+    return 1;
+}
+sub rename_gone_file
+{
+    # check whether the only instance of file is on a lost volume
+    # XXX: we don't have a proper interface for this. 
+    # For now try to use Bill's hack: the script 'whichnode'
+
+    my $file = shift;
+    $neb->move($file, "$file.gone") or 
+                    &my_die("failed to rename: $file", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+}
+
+# Prepare to write to an output file
+#   Lookup the filename in the rules.
+#   Make sure that if file exists and is a nebulous file that there is only one instance
+#   Deal with files that have been lost.
+sub prepare_output
+{
+    my $filerule = shift;
+    my $outroot  = shift;
+    my $class_id = shift;
+    my $delete = shift;
+    $delete = 0 if !defined $delete;
+
+    my $error;
+    my $output = $ipprc->prepare_output($filerule, $outroot, $class_id, $delete, \$error)
+                    or &my_die("failed to prepare output file for: $filerule", $exp_id, $chip_id, $class_id, $error);
+    return $output;
+}
+
+sub check_output
+{
+    my $file = shift;
+    my $replicate = shift;
+
+    if (!defined $file) {
+        return;
+    }
+
+    &my_die("Couldn't find expected output file: $file",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
+
+    if ($replicate and (file_scheme($file) eq 'neb')) {
+        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+}
 
 sub my_die