Changeset 30579 for branches/eam_branches/ipp-20101205/ippScripts/scripts

Timestamp:

Feb 11, 2011, 2:49:34 PM (15 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20101205/ippScripts/scripts

Files:

• chip_imfile.pl (modified) (1 diff)
• dist_advancerun.pl (modified) (8 diffs)
• dist_bundle.pl (modified) (8 diffs)
• dist_component.pl (modified) (5 diffs)
• dist_defineruns.pl (modified) (4 diffs)
• dist_make_fileset.pl (modified) (3 diffs)
• ipp_cleanup.pl (modified) (6 diffs)
• magic_destreak_cleanup.pl (modified) (4 diffs)
• nightly_science.pl (modified) (3 diffs)
• register_exp.pl (modified) (3 diffs)
• warp_skycell.pl (modified) (9 diffs)

branches/eam_branches/ipp-20101205/ippScripts/scripts/chip_imfile.pl

@@ -325 +325 @@
         # get the UNIX version of the (possible) neb: or path: filename
         my $uriReal = $ipprc->file_resolve( $uri );
+        &my_die("Unable to resolve $uri on $host", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) if !$uriReal;
 
         # funpack into the temp file.

branches/eam_branches/ipp-20101205/ippScripts/scripts/dist_advancerun.pl

@@ -23 +23 @@
 use PS::IPP::Config 1.01 qw( :standard );
 
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -29 +28 @@
 
 # Parse the command-line arguments
-my ($dist_id, $stage, $stage_id, $outdir, $clean);
+my ($dist_id, $stage, $stage_id, $outdir, $clean, $camera);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
 GetOptions(
-           'dist_id=s'  => \$dist_id,# Magic destreak run identifier
+           'dist_id=s'      => \$dist_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
@@ -53 +53 @@
     defined $stage_id and
     defined $outdir;
+
+my $ipprc = PS::IPP::Config->new($camera); # IPP configuration
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -134 +136 @@
 }
 
-# XXX should we create a file rule for this?
-my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc";
-
-{
-    my $command = "$tool_cmd $exportarg -outfile $dbinfo_file";
+
+# work around the fact that $ipprc->file_create does not actually create a file on disk
+# unless the scheme is nebulous
+sub create_file {
+    my $rule = shift;
+    my $path_base = shift;
+    my $ref = shift;
+
+    my $file;
+
+    my $error;
+    $file = $ipprc->prepare_output($rule, $path_base, undef, 1, \$error)
+        or &my_die("Unable to prepare outut for $rule", $dist_id, $PS_EXIT_SYS_ERROR);
+
+    my $scheme = file_scheme($file);
+    $scheme = "" if !$scheme;
+
+    my $resolved;
+    if ($scheme) {
+        $ipprc->file_create($file) 
+            or &my_die("Unable to create $file", $dist_id, $PS_EXIT_SYS_ERROR);
+
+        $resolved = $ipprc->file_resolve($file) 
+            or &my_die("Unable to resolve $file", $dist_id, $PS_EXIT_SYS_ERROR);
+
+        if ($scheme eq 'neb') {
+            &my_die("$resolved not found", $dist_id, $PS_EXIT_SYS_ERROR) unless ($resolved and -e $resolved);
+    }
+    } else {
+        $resolved = $file;
+    }
+    $$ref = $resolved;
+
+    return $file
+}
+
+my $dbinfo_root = "$outdir/dbinfo.$stage.$stage_id";
+my $resolved;
+my $dbinfo_file = create_file("DIST.OUTPUT.DBINFO", $dbinfo_root, \$resolved);
+
+{
+    my $command = "$tool_cmd $exportarg -outfile $resolved";
     $command .= " -clean" if ((defined $clean) and ($stage ne "raw"));
     $command .= " -dbname $dbname" if defined $dbname;
@@ -149 +188 @@
     }
 }
-my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc";
+
+my $dirinfo_root = "$outdir/dirinfo.$stage.$stage_id";
+my $dirinfo = create_file("DIST.OUTPUT.DIRINFO", $dirinfo_root, \$resolved);
+
 {
     my $command = "$tool_cmd $list_mode";
@@ -168 +210 @@
         &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
 
-    open MANIFEST, ">$dirinfo" or
-        &my_die("Unable to open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+    open MANIFEST, ">$resolved" or
+        &my_die("Unable to open dirinfo file $resolved",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
 
     my $destdir;
@@ -197 +239 @@
         &my_die("unable to find path",  $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path;
         my $component_dir = find_componentdir($destdir, $path);
-#        print MANIFEST "$component METADATA\n";
         print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n";
     }
@@ -217 +258 @@
     }
 }
-
 
 exit 0;

branches/eam_branches/ipp-20101205/ippScripts/scripts/dist_bundle.pl

@@ -16 +16 @@
 
 use IPC::Cmd 0.36 qw( can_run run );
-use File::Temp qw( tempfile );
+use File::Temp qw( tempfile tempdir );
 use File::Basename qw( basename );
 use Digest::MD5::File qw( file_md5_hex );
@@ -71 +71 @@
 # Parse the command-line arguments
 my ($camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $alt_path_base, $clean);
-my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix);
+my ($outroot, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 
@@ -88 +88 @@
            'magicked'       => \$magicked,   # magicked state for this component
            'alt_path_base=s'=> \$alt_path_base,  # path to alternate inputs
-           'outdir=s'       => \$outdir,     # "directory" for outputs
+           'outroot=s'      => \$outroot,    # outroot
            'prefix=s'       => \$prefix,     # "prefix" to apply to filenames
            'clean'          => \$clean,      # create clean distribution
@@ -99 +99 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outdir",
+pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outroot",
            -exitval => 3) unless
     defined $camera and
@@ -106 +106 @@
     defined $component and
     defined $path_base and
-    defined $outdir;
+    defined $outroot;
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -116 +116 @@
 $ipprc->define_camera($camera);
 
-# create the output directories if it is not a nebulous path and it doesn't exist
-if (index($outdir, "neb://") != 0) {
-    if (! -e $outdir ) {
-        my $code = system "mkdir -p $outdir";
-        &my_die("cannot create output directory $outdir", $component,
-                $code >> 8) if $code;
-    }
-}
+$ipprc->outroot_prepare($outroot);
 
 # Get the list of data products for this component
@@ -138 +131 @@
 # set up directory for temporary files
 
-my $tmpdir  = "$outdir/tmpdir.$component.$$";
-if (-e $tmpdir) {
-    if (-d $tmpdir) {
-        my $rc = system "rm -r $tmpdir";
-        &my_die("cannot rm $tmpdir return code: $rc", $component, $PS_EXIT_UNKNOWN_ERROR) if $rc;
-    } else {
-        unlink $tmpdir or &my_die("cannot delete $tmpdir", $component, $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
-mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $component,
-                       $PS_EXIT_UNKNOWN_ERROR);
+my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR");
+$temproot = "/tmp" if !defined $temproot;
+&my_die("directory for TEMP.DIR $temproot does not exist", $component, $PS_EXIT_CONFIG_ERROR) if ! -e $temproot;
+
+my $tmpdir  = tempdir("$temproot/dist.XXXX", CLEANUP => !$save_temps);
 
 #
@@ -349 +336 @@
 if ($num_files) {
     # create the tarfile
-    # XXX TODO: create a file rule for the tar file name
-    my $tbase = basename($path_base);
-    $tbase .= ".$component" if $component;
-    $file_name = ($prefix ? $prefix : "") . "$tbase.tgz";
-    my $tarfile = "$outdir/$file_name";
+#    my $tbase = basename($path_base);
+#    $tbase .= ".$component" if $component;
+#    $file_name = ($prefix ? $prefix : "") . "$tbase.tgz";
+    my $error;
+    my $output_tarfile;
+    my $rule;
+    if ($stage eq 'chip' or $stage eq 'chip_bg' or $stage eq 'raw') {
+        $rule = "DIST.OUTPUT.CHIP.BUNDLE";
+    } else {
+        $rule = "DIST.OUTPUT.BUNDLE";
+    }
+    $output_tarfile = $ipprc->prepare_output($rule, $outroot, $component, 1, \$error)
+            or &my_die("Failed to prepare output tarfile: $error", $component, $error);
+    $file_name = basename($output_tarfile);
+
+
+    my $scheme = file_scheme($output_tarfile);
+
+    my $tarfile;
+    if ($scheme) {
+        $ipprc->file_create($output_tarfile) 
+            or &my_die("Failed to create $output_tarfile: $error", $component, $error);
+        $tarfile = $ipprc->file_resolve($output_tarfile);
+
+        if ($scheme eq 'neb') {
+            &my_die("output file $output_tarfile not found", $component, $PS_EXIT_SYS_ERROR)
+                unless ($tarfile and -e $tarfile);
+        }
+    } else {
+        # no scheme the filename and the resolved filename are the same
+        $tarfile = $output_tarfile;
+    }
 
     my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";

branches/eam_branches/ipp-20101205/ippScripts/scripts/dist_component.pl

@@ -21 +21 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
-
 use PS::IPP::Config 1.01 qw( :standard );
-
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -77 +74 @@
     defined $outdir;
 
+my $ipprc = PS::IPP::Config->new($camera); # IPP configuration
 $ipprc->redirect_output($logfile) if $logfile;
+
+my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR");
+
+$temproot = "/tmp" if !defined $temproot;
 
 if (($stage eq 'raw') and !$clean and !defined $chip_path_base) {
@@ -85 +87 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my ($rf, $rf_name) = tempfile("/tmp/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
+my ($rf, $rf_name) = tempfile("$temproot/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
 close $rf;
+
+my $basename = basename($path_base);
+my $outroot = "$outdir/$basename";
 
 my ($file_name, $bytes, $md5sum);
@@ -92 +97 @@
     my $command = "$dist_make_bundle --camera $camera --stage $stage --stage_id $stage_id";
     $command .= " --results_file $rf_name";
-    $command .= " --component $component --path_base $path_base --outdir $outdir";
+    $command .= " --component $component --path_base $path_base --outroot $outroot";
     $command .= " --chip_path_base $chip_path_base" if $chip_path_base;
     $command .= " --state $run_state" if defined $run_state;
@@ -136 +141 @@
     &my_die("undefined file md5sum from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $md5sum;
 }
+
 
 {

branches/eam_branches/ipp-20101205/ippScripts/scripts/dist_defineruns.pl

@@ -42 +42 @@
 
 # Parse the command-line arguments
-my ($stage, $stage_limit, $dist_root, $no_magic);
+my ($stage, $stage_limit, $dist_root, $workdir, $no_magic);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 my @labels;
@@ -49 +49 @@
            'stage=s'        => \$stage,      # stage to queue
            'label=s'        => \@labels,     # labels
+           'workdir=s'      => \$workdir,     # workdir
            'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
-           'dist_root=s'    => \$dist_root,  # root of distribution work area
+#           'dist_root=s'    => \$dist_root,  # root of distribution work area
            'no_magic'       => \$no_magic,   # queue runs without requiring magic (for testing only)
            'dbname=s'       => \$dbname,     # Database name
@@ -66 +67 @@
 $ipprc->redirect_output($logfile) if $logfile;
 
-if (!$dist_root) {
-    $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT");
-    &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root;
+if (!$workdir) {
+    print "workdir not supplied will use DISTRIBUTION_ROOT\n";
+    # old method where we set workdir based on a config file
+    if (!$dist_root) {
+        $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT");
+        &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root;
+    }
+
+    my ($day, $month, $year) = (gmtime)[3,4,5];
+    my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
+
+    $workdir = $dist_root . "/$datestr";
+
+    print "workdir is $workdir\n";
 }
-
-my ($day, $month, $year) = (gmtime)[3,4,5];
-my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
-
-my $workdir = "$dist_root/$datestr";
-
-print "workdir is $workdir\n";
 
 # if stage is not supplied as an argument, loop over all stages
@@ -90 +95 @@
         my $command = "$disttool -definebyquery -stage $stage -workdir $workdir -label $label";
         $command .= " -no_magic" if $no_magic;
-        $command .= " -dry_run" if $no_update;
+        $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;
         $command .= " -set_label $label";

branches/eam_branches/ipp-20101205/ippScripts/scripts/dist_make_fileset.pl

@@ -96 +96 @@
 # make sure that the database info file for this run exists
 my $dbinfo_file = "$dist_dir/dbinfo.$stage.$stage_id.mdc";
-if (! -e "$dbinfo_file" ) {
+if (! $ipprc->file_exists($dbinfo_file) ) {
     &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
@@ -103 +103 @@
 # make sure that the dirinfo file for this run exists
 my $dirinfo_file = "$dist_dir/dirinfo.$stage.$stage_id.mdc";
-if (! -e "$dirinfo_file" ) {
+if (!$ipprc->file_exists($dirinfo_file)) {
     &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
@@ -277 +277 @@
     $command .= " -dbname $dbname" if $dbname;
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        print STDERR "Unable to perform $command error_code: $error_code\n";
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            print STDERR "Unable to perform $command error_code: $error_code\n";
+        }
+    } else {
+        print STDERR "skipping $command\n";
     }
     exit $fault;

branches/eam_branches/ipp-20101205/ippScripts/scripts/ipp_cleanup.pl

@@ -207 +207 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
         } else {
@@ -324 +315 @@
         }
 
-        # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-        my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-        $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-        $command .= " -dbname $dbname" if defined $dbname;
-        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-        }
+        set_destreak_goto_cleaned();
 
     } else {
@@ -469 +451 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
          } else {
@@ -603 +576 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
         } else {
@@ -759 +723 @@
             }
 
-            # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
-            my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
-            $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
-            $command .= " -dbname $dbname" if defined $dbname;
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
-            }
+            set_destreak_goto_cleaned();
 
         } else {
@@ -2062 +2017 @@
 }
 
+# this gets set to 1 the first time we set the corresponding destreak run to be cleaned
+my $ds_done = 0;
+sub set_destreak_goto_cleaned {
+
+    return if $ds_done;
+
+    # Tell magicdstool that we've cleaned up this data, so it needs to do the same if it needs to do the same.
+    my $magicdstool = can_run('magicdstool') or die "Can't find magicdstool";
+    my $command = "$magicdstool -stage $stage -stage_id $stage_id -updaterun -set_state goto_cleaned -set_label goto_cleaned";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magicdstool: $error_code", "$stage", $stage_id, $error_code);
+    }
+    $ds_done = 1;
+}
+
 # XXX we currently do not set the error state in the db on my_die
 sub my_die

branches/eam_branches/ipp-20101205/ippScripts/scripts/magic_destreak_cleanup.pl

@@ -159 +159 @@
         my $component = $comp->{component};
         my $backup_path_base = $comp->{backup_path_base};
-        my $recovery_path_base = $comp->{recovery_path_base};
         my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
         my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
@@ -197 +196 @@
                 $bsources = $ipprc->filename("PSPHOT.OUTPUT", $backup_path_base, $component);
             }
-            if ($recovery_path_base) {
-                $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $component);
-
-                if ($dynamicMasks) {
-                    my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $component);
-                    # This is kludgey but correct
-                    $rmask = dirname($recovery_path_base) . "/" . basename($mask);
-                    $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
-                } else {
-                    $rmask = $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $component);
-                }
-                $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $component);
-            }
         } elsif ($stage eq "camera") {
             if ($backup_path_base) {
@@ -221 +207 @@
                 $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
             }
-            if ($recovery_path_base) {
-                $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
-                $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
-                $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
-                $rsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $recovery_path_base);
-            }
         } elsif ($stage eq "diff") {
             my $name = "PPSUB.OUTPUT";
+            if ($backup_path_base) {
+                $bimage  = $ipprc->filename($name, $backup_path_base);
+                $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
+                $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
+                # bills 2011-01-24
+                # don't clean up the uncensored Diff sources file
+                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
+            }
+        }
+
+        delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
+
+        if ($stage eq "diff" and $warp_warp) {
+            my $name = "PPSUB.INVERSE";
             if ($backup_path_base) {
                 $bimage  = $ipprc->filename($name, $backup_path_base);
@@ -237 +231 @@
                 # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
             }
-            if ($recovery_path_base) {
-                $rimage  = $ipprc->filename($name, $recovery_path_base);
-                $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
-                $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
-                $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
-            }
-        }
-
-        delete_files($rimage, $rmask, $rweight, $rsources, $rastrom, $bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
-
-        if ($stage eq "diff" and $warp_warp) {
-            my $name = "PPSUB.INVERSE";
-            if ($backup_path_base) {
-                $bimage  = $ipprc->filename($name, $backup_path_base);
-                $bmask   = $ipprc->filename("$name.MASK", $backup_path_base);
-                $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
-                # bills 2011-01-24
-                # don't clean up the uncensored sources file
-                # $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
-            }
-            if ($recovery_path_base) {
-                $rimage  = $ipprc->filename($name, $recovery_path_base);
-                $rmask   = $ipprc->filename("$name.MASK", $recovery_path_base);
-                $rweight = $ipprc->filename("$name.VARIANCE", $recovery_path_base);
-                $rsources = $ipprc->filename("$name.SOURCES", $recovery_path_base);
-            }
-            # undef is not necessary, it's just to keep the lists in the same order.
-            delete_files($rimage, $rmask, $rweight, $rsources, undef, $bimage, $bmask, $bweight, $bsources);
+            delete_files($bimage, $bmask, $bweight, $bsources);
         }
         my $command = "$magicdstool -tocleanedfile -magic_ds_id $magic_ds_id -component $component";

branches/eam_branches/ipp-20101205/ippScripts/scripts/nightly_science.pl

@@ -1266 +1266 @@
 }
 
+sub multi_date_verify_uniqueness_diff {
+    my $warp_id_1 = shift;
+    my $warp_id_2 = shift;
+    my $date = shift;
+    my $target = shift;
+
+    my $db = init_gpc_db();
+    $date =~ s/-//g;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $count = 0;
+    
+    my $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_1 AND warp2 = $warp_id_2";
+    my $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    $sth = "SELECT DISTINCT diff_id from diffRun JOIN diffInputSkyfile USING(diff_id) where (label = '$label' OR label = 'goto_cleaned') AND warp1 = $warp_id_2 AND warp2 = $warp_id_1";
+    $data_ref = $db->selectall_arrayref( $sth );
+    $count += $#{ $data_ref } + 1;
+
+    return($count);
+}
+
 sub pre_diff_queue {
     my $date = shift;
@@ -1375 +1398 @@
         $metadata_out{nsDiffState} = 'DIFFING';
     }
-
+    if ($metadata_out{nsDiffState} eq 'FINISHED_DIFFS') {
+        foreach my $target (sort (keys %science_config)) {
+            if ($science_config{$target}{DIFFABLE} == 1) {
+                foreach my $filter (@filter_list) {
+                    multi_date_diff_queue($date,$target,$filter,$pretend);
+                }
+            }
+        }
+    }
 #     if (($Npotential == $Nnoexp)&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')&&($is_processing == 0)) {
 #       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
@@ -1511 +1542 @@
 
 }
+
+sub multi_date_diff_queue {
+    my $date = shift;
+    my $target = shift;
+    my $filter = shift;
+    my $pretend = shift;
+    my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target);
+
+    my $db = init_gpc_db();
+
+    my $obj_sth = "select DISTINCT rawExp.object from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+    $obj_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND rawExp.filter = '$filter' ORDER BY rawExp.object";
+
+    my $object_ref = $db->selectall_arrayref( $obj_sth );
+
+    my $Npotential = 0;
+    my $Nqueued = 0;
+    
+    foreach my $object_row (@{ $object_ref }) {
+        my $this_object = shift @{ $object_row };
+#       my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ";
+#       $input_sth .= " WHERE warpRun.state = 'full' AND warpRun.label = '$label' AND warpRun.data_group = '$data_group' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+#       $input_sth .= " ORDER BY dateobs ";
+        
+        my $input_sth = "select exp_id,warp_id,dateobs,rawExp.comment,warpRun.state AS warp_state FROM ";
+        $input_sth .=   " rawExp LEFT JOIN chipRun USING (exp_id) LEFT JOIN camRun USING (chip_id) LEFT JOIN fakeRun USING (cam_id) LEFT JOIN warpRun USING (fake_id) ";
+        $input_sth .=   " WHERE warpRun.label = '$label' AND rawExp.filter = '$filter' AND rawExp.object = '$this_object' ";
+        $input_sth .=   " ORDER BY dateobs ";
+
+        my $warps = $db->selectall_arrayref( $input_sth );
+
+        # Each comment should only appear once. Therefore, if we see it more than once, we assume the first is extra.
+        my %comment_hash = ();
+        foreach my $this_warp (@{ $warps }) {
+            my $this_comment = ${ $this_warp }[3];
+            my $this_exp_id  = ${ $this_warp }[0];
+            $comment_hash{$this_comment} = $this_exp_id;
+        }
+        
+        if (($#{ $warps } + 1) % 2 != 0) {
+            print STDERR "md_diff_queue: Number of input warps to make diffs is not even for target $target and object $this_object! $#{ $warps } ";
+            if ($#{ $warps} + 1 == 1) {
+                print STDERR ": I can do no diffs with only one exposure.\n";
+                next;
+            }
+            else {
+                print STDERR ": I should declare an exposure to be faulty.\n";
+                my @keep_warps = ();
+#               print "@{ $warps }\n";
+                foreach my $this_warp (@{ $warps }) {
+                    my $this_comment = ${ $this_warp }[3];
+                    my $this_exp_id  = ${ $this_warp }[0];
+                    if ($comment_hash{$this_comment} == $this_exp_id) {
+                        push @keep_warps, $this_warp;
+                    }
+                    else {
+                        print STDERR "md_diff_queue: excluding $this_exp_id for $this_object\n";
+                    }
+                }
+                @{ $warps } = @keep_warps;
+#               print "@{ $warps }\n";
+            }
+        }
+        
+        while ($#{ $warps } > -1) {
+            my $input_warp = shift @{ $warps };
+            my $input_exp_id = ${ $input_warp }[0];
+            my $input_comment = ${ $input_warp }[3];
+
+            
+            my $template_warp = shift @{ $warps };
+
+            my $template_exp_id = ${ $template_warp }[0];
+            
+            my $input_warp_id = ${ $input_warp }[1];
+            my $template_warp_id = ${ $template_warp }[1];
+
+            my $input_warp_state = ${ $input_warp }[4];
+            my $template_warp_state = ${ $template_warp }[4];
+            
+            unless(defined($template_warp)&& defined($template_exp_id)) {
+                print STDERR "md_diff received an undef! $input_exp_id $input_comment $this_object T: $template_warp V: @$template_warp\n";
+                next;
+            }
+            $Npotential++;
+            
+            unless (defined($input_warp_id) && defined($template_warp_id) &&
+                    ($input_warp_state eq 'full')&&($template_warp_state eq 'full')) {
+                print STDERR "md_Diff for this $date $target $input_exp_id ($input_warp_id $input_warp_state) $template_exp_id ($template_warp_id $template_warp_state) not fully processed\n";
+                next;
+            }
+
+            if (multi_date_verify_uniqueness_diff($input_warp_id,$template_warp_id,$date,$target) != 0) {
+                $Nqueued++;
+                print STDERR "md_Diffs already queued for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
+                next;
+            }
+
+            my $new_data_group = "${data_group}.multi";
+            my $cmd = "$difftool -dbname $dbname  -definewarpwarp ";
+            $cmd .= "-input_label $label  -template_label $label ";
+            $cmd .= "-backwards "; # Needed because difftool assumes a different date sorting.
+            $cmd .= "-set_workdir $workdir  -set_dist_group $dist_group  -set_data_group $new_data_group ";
+            $cmd .= " -simple  -set_label $label -exp_id $input_exp_id -template_exp_id $template_exp_id ";
+#               $cmd .= " -pretend ";
+            if (defined($pretend)) {
+                $cmd .= ' -pretend ';
+            }
+            if ($debug == 1) {
+                $cmd .= ' -pretend ';
+                print STDERR "md_Diffs would like to queue for this $date $target $input_exp_id $template_exp_id ($input_warp_id $template_warp_id) $this_object $input_comment\n";
+                print STDERR "md_diff_queue: $cmd\n";
+                print STDERR " $input_warp_id $template_warp_id\n";
+            }
+            
+            if (($debug == 0)&&(!defined($pretend))) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run ( command => $cmd, verbose => $verbose );
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to perform difftool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+                }
+                $Nqueued++;
+            }
+        }
+    }
+    $metadata_out{nsDiffPotential} += $Npotential;
+    $metadata_out{nsDiffQueued}    += $Nqueued;
+#      if (($metadata_out{nsDiffPotential} == $metadata_out{nsDiffQueued})&&($metadata_out{nsObservingState} eq 'END_OF_NIGHT')) {
+#       $metadata_out{nsDiffState} = 'FINISHED_DIFFS';
+#      }        
+
+}
             
 

branches/eam_branches/ipp-20101205/ippScripts/scripts/register_exp.pl

@@ -255 +255 @@
         if (exists($nightlyscience_config{$target}{OBSMODE})) {
             $possible++;
-            if ($obsmode =~ /$nightlyscience_config{$target}{OBSMODE}/) {
+            if ($obsmode =~ /^$nightlyscience_config{$target}{OBSMODE}$/) {
                 $match++;
             }
@@ -261 +261 @@
         if (exists($nightlyscience_config{$target}{OBJECT})) {
             $possible++;
-            if ($object =~ /$nightlyscience_config{$target}{OBJECT}/) {
+            if ($object =~ /^$nightlyscience_config{$target}{OBJECT}$/) {
                 $match++;
             }
@@ -267 +267 @@
         if (exists($nightlyscience_config{$target}{COMMENT})) {
             $possible++;
-            if ($comment =~ /$nightlyscience_config{$target}{COMMENT}/) {
+            if ($comment =~ /^$nightlyscience_config{$target}{COMMENT}$/) {
                 $match++;
             }

branches/eam_branches/ipp-20101205/ippScripts/scripts/warp_skycell.pl

@@ -71 +71 @@
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 
-my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR );
-$logDest .= ".update" if ($run_state eq 'update');
+my ($logDest, $traceDest);
+if ($run_state eq 'new') {
+    $logDest = prepare_output("LOG.EXP", $outroot, $skycell_id, 0);
+    $traceDest = prepare_output("TRACE.EXP", $outroot, $skycell_id, 1);
+} elsif ($run_state eq 'update')  {
+    $logDest = prepare_output("LOG.EXP.UPDATE", $outroot, $skycell_id, 0);
+    $traceDest = prepare_output("TRACE.EXP.UPDATE", $outroot, $skycell_id, 1);
+} else {
+    &my_die( "invalid run_state: $run_state", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR );
+}
+
+my $neb;
+my $scheme = file_scheme($outroot);
+if ($scheme and $scheme eq 'neb') {
+    $neb = $ipprc->nebulous();
+}
 
 $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect;
@@ -117 +131 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $error_code);
+        &my_die("Unable to perform ppConfigDump: $error_code", $warp_id, $skycell_id, $tess_dir, $error_code);
     }
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $warp_id, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to parse metadata config doc", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_PROG_ERROR);
     $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
 }
@@ -141 +155 @@
 
 
-my $outputImage = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id );
-my $outputMask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id);
-my $outputWeight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id);
-my $outputSources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id);
-my $outputPSF = $ipprc->filename("PSPHOT.PSF.SKY.SAVE", $outroot);
-my $outputBin1 = $ipprc->filename("PSWARP.BIN1", $outroot, $skycell_id );
-my $outputBin2 = $ipprc->filename("PSWARP.BIN2", $outroot, $skycell_id );
-my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id );
-my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id);
-my $configuration =  $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id);
-
-if ($run_state eq 'update') {
-    $traceDest .= ".update";
-    $outputStats .= ".update";
-}
-
-my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
+my $outputImage = prepare_output ("PSWARP.OUTPUT", $outroot, $skycell_id, 1);
+my $outputMask = prepare_output ("PSWARP.OUTPUT.MASK", $outroot, $skycell_id, 1);
+my $outputWeight = prepare_output ("PSWARP.OUTPUT.VARIANCE", $outroot, $skycell_id, 1);
+my $outputSources = prepare_output ("PSWARP.OUTPUT.SOURCES", $outroot, $skycell_id, 1);
+my $outputPSF = prepare_output ("PSPHOT.PSF.SKY.SAVE", $outroot, 1);
+my $outputBin1 = prepare_output ("PSWARP.BIN1", $outroot, $skycell_id, 1);
+my $outputBin2 = prepare_output ("PSWARP.BIN2", $outroot, $skycell_id, 1);
+my $outputStats = prepare_output ("SKYCELL.STATS", $outroot, $skycell_id, 1);
+my $configuration;
+
+my $dump_config = 1;
+if ($run_state eq 'new') {
+    $configuration =  prepare_output ("PSWARP.CONFIG", $outroot, $skycell_id, 1);
+} else {
+    $configuration =  $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id) or
+        &my_die("Missing entry from camera config PSWARP.CONFIG", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
+    if ($ipprc->file_exists($configuration)) {
+        $dump_config = 0;
+    } else {
+        print STDERR "WARNING: Config dump file $configuration is missing. Using current recipes and file rules.\n";
+
+        # XXX: should we create a new config dump file?
+        # I vote yes but only if we can distingusing between temporarily unavailable and GONE.
+        my $gone = 0;
+        if (storage_object_exists($configuration, \$gone)) {
+            if ($gone) {
+                $configuration = prepare_output('PSWARP.CONFIG', $outroot, $skycell_id, 1);
+                # if we dump the config we need to insure that the config dump represents
+                # the full processing
+            } else {
+                # file is temporarily not available. Don't dump config.
+                $dump_config = 0;
+            }
+        }
+    }
+}
+
+
+my $skyFile = prepare_output ("SKYCELL.TEMPLATE", $outroot, $skycell_id, 1);
 $ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
 ## XXX this seems to have insufficient error checking: dvoImageExtract can fail to write and still return a valid exit status
@@ -171 +207 @@
 foreach my $imfile (@$imfiles) {
     my $image = $imfile->{uri}; # Image name
-    my $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
+    my $weight = $ipprc->filename ("PPIMAGE.CHIP.VARIANCE", $imfile->{chip_path_base}, $imfile->{class_id}); # Mask name
 
     my $mask;                   # Mask name
     if ($dynamicMasks) {
-        $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
+        $mask = $ipprc->filename ("PSASTRO.OUTPUT.MASK", $imfile->{cam_path_base}, $imfile->{class_id});
     } else {
-        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id});
+        $mask = $ipprc->filename ("PPIMAGE.CHIP.MASK", $imfile->{chip_path_base}, $imfile->{class_id});
     }
 
@@ -186 +222 @@
     # Astrometry file: astrometry is done at the camera stage, and always results in a MEF file
     # XXX allow an option to use the image header astrometry?
-    my $astrom = $ipprc->filename($astromSource, $imfile->{cam_path_base});
+    my $astrom = $ipprc->filename ($astromSource, $imfile->{cam_path_base});
 
     &my_die("Couldn't find input file: $astrom", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless defined $astrom and $ipprc->file_exists($astrom);
@@ -245 +281 @@
         $do_stats = 1;
     } else {
-        #$command .= " -ipprc $configuration";
-        my $resolved = $ipprc->file_resolve($configuration);
-        $command .= " -ipprc $resolved";
+        $command .= " -ipprc $configuration";
     }
     if ($do_stats) {
@@ -263 +297 @@
     if ($do_stats) {
         # Check first for the stats file
+        check_output($outputStats, 0);
         my $outputStatsReal = $ipprc->file_resolve($outputStats);
-        &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
+#        &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
         &my_die("Stats file has zero size: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless -s $outputStatsReal;
 
-        # measure chip stats
+        # measure skycell stats
         $command = "$ppStatsFromMetadata $outputStatsReal - WARP_SKYCELL";
         ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -283 +318 @@
 
         if (!$quality) {
-            &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
-            &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
-            &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
-            &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
-            &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF);
-            if ($run_state eq 'new') {
-                &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
+        #    &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
+        #    &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
+        #    &my_die("Couldn't find expected output file: $outputWeight", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
+        #    &my_die("Couldn't find expected output file: $outputSources", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
+        #    &my_die("Couldn't find expected output file: $outputPSF", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) if metadataLookupBool($recipe, 'PSF') and not $ipprc->file_exists($outputPSF);
+
+            check_output($outputImage, 0);
+            check_output($outputMask, 0);
+            check_output($outputWeight, 0);
+            check_output($outputSources, 1);
+            check_output($outputPSF, 1) if metadataLookupBool($recipe, 'PSF')  ;
+            if ($dump_config)  {
+                check_output($configuration, 1);
+#                &my_die("Couldn't find expected output file: $configuration", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
             }
         }
@@ -335 +377 @@
     }
 }
+
+exit 0;
+
+# Prepare to write to an output file
+#   Lookup the filename in the rules.
+#   Make sure that if file exists and is a nebulous file that there is only one instance
+#   Deal with files that have been lost.
+sub prepare_output
+{
+    my $filerule = shift;
+    my $outroot  = shift;
+    my $skycell_id = shift;
+    my $delete = shift;
+    $delete = 0 if !defined $delete;
+
+    my $error;
+    my $output = $ipprc->prepare_output($filerule, $outroot, $skycell_id, $delete, \$error)
+                    or &my_die("failed to prepare output file for: $filerule", $warp_id, $skycell_id, $tess_dir, $error);
+    return $output;
+}
+
+sub check_output
+{
+    my $file = shift;
+    my $replicate = shift;
+
+    if (!defined $file) {
+        return;
+    }
+
+    &my_die("Couldn't find expected output file: $file",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($file);
+
+    if ($replicate and $neb) {
+        $ipprc->replicate_file($file) or &my_die("failed to replicate: $file\n",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR);
+    }
+}
+
+# subroutine to check the status of a nebulous file. Used to distinguish between a storage object that
+# does not exist and one that all of the instances have been lost.
+# XXXX This should be implemented properly in Nebulous
+# For now uses Bill's script 'whichnode' which queries the nebulous database directly
+
+my $whichnode;
+sub storage_object_exists
+{
+    return 0 if !$neb;
+
+    my $file = shift;
+    my $ref_all_gone = shift;
+
+    my $exists = $neb->storage_object_exists($file);
+    if (!$exists) {
+        return 0;
+    }
+
+    if (!$whichnode) {
+        $whichnode = can_run('whichnode') or
+            &my_die("Can't find whichnode",  $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my $command = "$whichnode $file";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform whichnode: $error_code", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR);
+    }
+
+    my @lines = split "\n", (join "", @$stdout_buf);
+
+    if (scalar @lines == 0) {
+        # no output the file is really and truely gone
+        # XXX: this is now caught above
+        print STDERR "storage object for $file does not exist\n";
+        return 0;
+    }
+
+    my $numGone = 0;
+    my $numNotGone = 0;
+    foreach my $line (@lines) {
+        chomp $line;
+
+        # output lines are either
+        #   "volume available"
+        # or 
+        #   "volume not available"
+
+        my ($volume, $answer, undef) = split " ", $line;
+        # our hack is if the volume has an X in the name it's gone
+        if ($volume =~ /X/) {
+            print STDERR "$file is on $volume which is gone\n";
+            $numGone++;
+        } elsif ($answer eq 'available') {
+            $numNotGone++;
+        } elsif ($answer eq 'not') {
+            print STDERR "$file is on $volume which is not available\n";
+            $numNotGone++;
+        } else {
+            print STDERR "unexpected output from whichnode: $line\n";
+        }
+    }
+    # if there are any instances that are not on a gone volume set all_gone to 0
+    if ($numNotGone == 0 and $numGone > 0) {
+        $$ref_all_gone = 1;
+    } else {
+        $$ref_all_gone = 0;
+    }
+
+    # storage object exists so return true
+    return 1;
+}
+
 
 sub my_die