Index: trunk/ippScripts/scripts/warp_skycell.pl
===================================================================
--- trunk/ippScripts/scripts/warp_skycell.pl	(revision 27754)
+++ trunk/ippScripts/scripts/warp_skycell.pl	(revision 30573)
@@ -71,6 +71,20 @@
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 
-my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR );
-$logDest .= ".update" if ($run_state eq 'update');
+my ($logDest, $traceDest);
+if ($run_state eq 'new') {
+    $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0);
+    $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1);
+} elsif ($run_state eq 'update')  {
+    $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 0);
+    $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1);
+} else {
+    &my_die( "invalid run_state: $run_state", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR );
+}
+
+my $neb;
+my $scheme = file_scheme($outroot);
+if ($scheme and $scheme eq 'neb') {
+    $neb = $ipprc->nebulous();
+}
 
 $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect;
@@ -117,8 +131,8 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code);
+        &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code);
     }
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
     $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
 }
@@ -141,21 +155,43 @@
 
 
-my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id );
-my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id);
-my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id);
-my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id);
-my $outputPSF = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $outroot);
-my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outroot, $skycell_id );
-my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outroot, $skycell_id );
-my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id );
-my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id);
-my $configuration =  $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id);
-
-if ($run_state eq 'update') {
-    $traceDest .= ".update";
-    $outputStats .= ".update";
-}
-
-my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
+my $outputImage = prepare_output ("PSWARP.OUTPUT", $outroot, $skycell_id, 1);
+my $outputMask = prepare_output ("PSWARP.OUTPUT.MASK", $outroot, $skycell_id, 1);
+my $outputWeight = prepare_output ("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id, 1);
+my $outputSources = prepare_output ("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id, 1);
+my $outputPSF = prepare_output ("PSPHOT.PSF.SKY.SAVE", $outroot, 1);
+my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1);
+my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1);
+my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1);
+my $configuration;
+
+my $dump_config = 1;
+if ($run_state eq 'new') {
+    $configuration =  prepare_output ("PSWARP.CONFIG", $outroot, $skycell_id, 1);
+} else {
+    $configuration =  $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id) or
+        &my_die("Missing entry from camera config PSWARP.CONFIG", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
+    if ($ipprc->file_exists($configuration)) {
+        $dump_config = 0;
+    } else {
+        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
+
+        # XXX: should we create a new config dump file?
+        # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
+        my $gone = 0;
+        if (storage_object_exists($configuration, \$gone)) {
+            if ($gone) {
+                $configuration = prepare_output('PSWARP.CONFIG', $outroot, $skycell_id, 1);
+                # if we dump the config we need to insure that the config dump represents
+                # the full processing
+            } else {
+                # file is temporarily not available. Don't dump config.
+                $dump_config = 0;
+            }
+        }
+    }
+}
+
+
+my $skyFile = prepare_output ("SKYCELL.TEMPLATE", $outroot, $skycell_id, 1);
 $ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
 ## XXX this seems to have insufficient error checking: dvoImageExtract can fail to write and still return a valid exit status
@@ -171,11 +207,11 @@
 foreach my $imfile (@$imfiles) {
     my $image = $imfile->{uri}; # Image name
-    my $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
+    my $weight = $ipprc->filename ("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
 
     my $mask;                   # Mask name
     if ($dynamicMasks) {
-        $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
+        $mask = $ipprc->filename ("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
     } else {
-        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id});
+        $mask = $ipprc->filename ("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id});
     }
 
@@ -186,5 +222,5 @@
     # Astrometry file: astrometry is done at the camera stage, and always results in a MEF file
     # XXX allow an option to use the image header astrometry?
-    my $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base});
+    my $astrom = $ipprc->filename ($astromSource, $imfile->{cam_path_base});
 
     &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless defined $astrom and $ipprc->file_exists($astrom);
@@ -245,7 +281,5 @@
         $do_stats = 1;
     } else {
-        #$command .= " -ipprc $configuration";
-        my $resolved = $ipprc->file_resolve($configuration);
-        $command .= " -ipprc $resolved";
+        $command .= " -ipprc $configuration";
     }
     if ($do_stats) {
@@ -263,9 +297,10 @@
     if ($do_stats) {
         # Check first for the stats file
+        check_output($outputStats, 0);
         my $outputStatsReal = $ipprc->file_resolve($outputStats);
-        &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
+#        &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
         &my_die("Stats file has zero size: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless -s $outputStatsReal;
 
-        # measure chip stats
+        # measure skycell stats
         $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL";
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -283,11 +318,18 @@
 
         if (!$quality) {
-            &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
-            &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
-            &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
-            &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
-            &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF);
-            if ($run_state eq 'new') {
-                &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
+        #    &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
+        #    &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
+        #    &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
+        #    &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
+        #    &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF);
+
+            check_output($outputImage, 0);
+            check_output($outputMask, 0);
+            check_output($outputWeight, 0);
+            check_output($outputSources, 1);
+            check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF')  ;
+            if ($dump_config)  {
+                check_output($configuration, 1);
+#                &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
             }
         }
@@ -335,4 +377,117 @@
     }
 }
+
+exit 0;
+
+# Prepare to write to an output file
+#   Lookup the filename in the rules.
+#   Make sure that if file exists and is a nebulous file that there is only one instance
+#   Deal with files that have been lost.
+sub prepare_output
+{
+    my $filerule = shift;
+    my $outroot  = shift;
+    my $skycell_id = shift;
+    my $delete = shift;
+    $delete = 0 if !defined $delete;
+
+    my $error;
+    my $output = $ipprc->prepare_output($filerule, $outroot, $skycell_id, $delete, \$error)
+                    or &my_die("failed to prepare output file for: $filerule", $warp_id, $skycell_id, $tess_dir, $error);
+    return $output;
+}
+
+sub check_output
+{
+    my $file = shift;
+    my $replicate = shift;
+
+    if (!defined $file) {
+        return;
+    }
+
+    &my_die("Couldn't find expected output file: $file",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
+
+    if ($replicate and $neb) {
+        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
+    }
+}
+
+# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that
+# does not exist and one that all of the instances have been lost.
+# XXXX This should be implemented properly in Nebulous
+# For now uses Bill's script 'whichnode' which queries the nebulous database directly
+
+my $whichnode;
+sub storage_object_exists
+{
+    return 0 if !$neb;
+
+    my $file = shift;
+    my $ref_all_gone = shift;
+
+    my $exists = $neb->storage_object_exists($file);
+    if (!$exists) {
+        return 0;
+    }
+
+    if (!$whichnode) {
+        $whichnode = can_run('whichnode') or
+            &my_die("Can't find whichnode",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my $command = "$whichnode $file";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform whichnode: $error_code", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my @lines = split "\n", (join "", @$stdout_buf);
+
+    if (scalar @lines == 0) {
+        # no output the file is really and truely gone
+        # XXX: this is now caught above
+        print STDERR "storage object for $file does not exist\n";
+        return 0;
+    }
+
+    my $numGone = 0;
+    my $numNotGone = 0;
+    foreach my $line (@lines) {
+        chomp $line;
+
+        # output lines are either
+        #   "volume available"
+        # or 
+        #   "volume not available"
+
+        my ($volume, $answer, undef) = split " ", $line;
+        # our hack is if the volume has an X in the name it's gone
+        if ($volume =~ /X/) {
+            print STDERR "$file is on $volume which is gone\n";
+            $numGone++;
+        } elsif ($answer eq 'available') {
+            $numNotGone++;
+        } elsif ($answer eq 'not') {
+            print STDERR "$file is on $volume which is not available\n";
+            $numNotGone++;
+        } else {
+            print STDERR "unexpected output from whichnode: $line\n";
+        }
+    }
+    # if there are any instances that are not on a gone volume set all_gone to 0
+    if ($numNotGone == 0 and $numGone > 0) {
+        $$ref_all_gone = 1;
+    } else {
+        $$ref_all_gone = 0;
+    }
+
+    # storage object exists so return true
+    return 1;
+}
+
 
 sub my_die