Index: branches/pap/ippScripts/scripts/magic_destreak.pl
===================================================================
--- branches/pap/ippScripts/scripts/magic_destreak.pl	(revision 23948)
+++ branches/pap/ippScripts/scripts/magic_destreak.pl	(revision 25027)
@@ -16,9 +16,10 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
-use File::Basename qw( basename );
+use File::Basename qw( basename dirname );
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
 
 use PS::IPP::Config 1.01 qw( :standard );
+use Nebulous::Client;
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -29,4 +30,6 @@
 my $magicdstool   = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
 my $streaksremove = can_run('streaksremove') or (warn "Can't find streaksremove" and $missing_tools = 1);
+my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -35,7 +38,7 @@
 
 # Parse the command-line arguments
-my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
+my ($magic_ds_id, $camera, $streaks, $inv_streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
 my ($outroot, $recoveryroot);
-my ($replace, $remove, $release);
+my ($replace, $release);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -44,4 +47,5 @@
            'camera=s'       => \$camera,     # camera for evaluating file rules
            'streaks=s'      => \$streaks,    # file containing the list of streaks
+           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
@@ -49,9 +53,8 @@
            'uri=s'          => \$uri,        # uri of the input image
            'path_base=s'    => \$path_base,  # path_base of the input
-           'cam_path_base=s'=> \$cam_path_base,  # path_base of the associated camera run
+           'cam_path_base=s'=> \$cam_path_base,  # path_base from camera stage (for chip and raw)
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
            'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
            'replace=s'      => \$replace,    # replace the input images with the results.
-           'remove=s'       => \$remove,     # remove the original images when done YIKES!
            'release'        => \$release,    # NAN masked pixels for release
            'save-temps'     => \$save_temps, # Save temporary files?
@@ -89,16 +92,33 @@
 } elsif ($stage eq "diff") {
     $skycell_id = $component;
-} else {
+} elsif ($stage ne "camera") {
     &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
-}
+}    
+$inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL");
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
+my $dirname = dirname($path_base);
 my $basename = basename($path_base);
-
-#
-# parse the replace and remove arguments check for errors and set up
+my $nebulousInput = inNebulous($dirname);
+my $nebulousOutput = inNebulous($outroot);
+
+# parse replace arguments check for errors and set up
 # the appropriate paths
+my $nebulous;
 if (defined($replace) and ($replace eq "T")) {
+    # for camera stage we need a handle to the nebulous server
+    if ($stage eq 'camera') {
+        my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
+        &my_die("cannot find NEB_SERVER in site configuration", 
+                                        $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) 
+            if !$ nebulousServer;
+
+        $nebulous = eval { Nebulous::Client->new( proxy => $nebulousServer ); };
+        if ($@ or not defined $nebulous) {
+            &my_die ("Unable to create a Nebulous::Client object with proxy $nebulousServer",
+                                    $magic_ds_id, $component , $PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
     $replace = 1;
 } else {
@@ -106,6 +126,7 @@
 }
 
+
 # default value is "NULL" do not use
-if (defined($recoveryroot) and ($recoveryroot = "NULL")) {
+if (defined($recoveryroot) and ($recoveryroot eq "NULL")) {
     $recoveryroot = undef;
 }
@@ -116,16 +137,6 @@
 }
 
-if (defined($remove) and ($remove eq "T")) {
-    $remove = 1;
-} else {
-    $remove = 0;
-}
-
-if ($remove and !$replace) {
-    &my_die("cannot remove without replace", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
-}
-
 # create the output directories if it is not a nebulous path and it doesn't exist
-if (index($outroot, "neb://") != 0) {
+if (! $nebulousOutput) {
     if (! -e $outroot ) {
         my $code = system "mkdir -p $outroot";
@@ -136,11 +147,21 @@
 
 my $backup_path_base;
-if (! $remove) {
-    $backup_path_base = "$outroot/$basename";
+my $tmproot;
+if ($replace) {
+    # in replace mode, we place the output files in the same "directory" as the inputs
+    # Nebulous requires this for the two inputs to nebSwap which we use
+    # We prepend the path with SR_ This causes the filenames for instances of the swapped files to
+    # have SR in them.
+    $tmproot = $dirname . "/SR_";
+    $backup_path_base = $tmproot . "$basename";
+} else {
+    # note: trailing / is necessary
+    $tmproot = "$outroot/";
+    $backup_path_base = $tmproot . $basename;
 }
 
 my $recovery_path_base;
 if ($recoveryroot) {
-    if (index($recoveryroot, "neb://") != 0) {
+    if (!inNebulous($recoveryroot)) {
         if (! -e $recoveryroot ) {
             my $code = system "mkdir -p $recoveryroot";
@@ -149,81 +170,186 @@
         }
     }
-    $recovery_path_base = "$recoveryroot/$basename";
-}
-
-# get skycell list if needed
-my ($sfh, $skycell_list);
-if ($skycell_args) {
-    my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args";
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
-    }
-
-    my $getskycells_output = join "", @$stdout_buf;
-    if ($getskycells_output) {
-        my $metadata = $mdcParser->parse($getskycells_output) or
-            &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-        my $skycells = parse_md_list($metadata) or
-                &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
-
-        ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
-
-        foreach my $skycell (@$skycells) {
-            print $sfh "$skycell->{uri}\n"
-        }
-        close $sfh;
-    }
-}
-
-my $image = $uri;
-
-my ($mask, $ch_mask, $weight, $astrom);
-
-if ($stage eq "raw") {
-    $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
-    $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id) if $release ;
-} elsif ($stage eq "chip") {
-    # we use the mask output from the camera stage for input and replace
-    # the output of the chip stage with that mask as well.
-    $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
-    $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
-    $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
-    $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
-} elsif ($stage eq "warp") {
-    $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
-    $weight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base);
-} elsif ($stage eq "diff") {
-    $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
-    $weight = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
-}
-
-{
-    my $command = "$streaksremove -stage $stage -tmproot $outroot -streaks $streaks -image $image";
-
-    $command .= " -class_id $class_id" if defined $class_id;
-    $command .= " -recovery $recoveryroot" if defined $recoveryroot;
-    $command .= " -astrom $astrom" if defined $astrom;
-    $command .= " -mask $mask" if defined $mask;
-    $command .= " -chip_mask $ch_mask" if defined $ch_mask;
-    $command .= " -weight $weight" if defined $weight;
-    $command .= " -skycelllist $skycell_list" if defined $skycell_list;
-    $command .= " -replace" if $replace;
-    $command .= " -remove" if $remove;
-    $command .= " -release" if $release;
-    $command .= " -dbname $dbname" if defined $dbname;
-    unless (defined $no_op) {
+    # note: trailing / is necessary
+    $recovery_path_base = "$recoveryroot/$basename/";
+}
+
+if ($stage ne "camera") {
+    # get skycell list if needed
+    my ($sfh, $skycell_list);
+    if ($skycell_args) {
+        my $command = "$magicdstool -magic_ds_id $magic_ds_id -getskycells $skycell_args";
+        $command .= " -dbname $dbname" if defined $dbname;
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
         unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code);
-        }
-    } else {
-        print "skipping command $command\n";
+            &my_die("Unable to perform magicdstool -diffskyfile $skycell_args: $error_code", $magic_ds_id, $component, $error_code);
+        }
+
+        my $getskycells_output = join "", @$stdout_buf;
+        if ($getskycells_output) {
+            my $metadata = $mdcParser->parse($getskycells_output) or
+                &my_die("Unable to parse metadata config doc", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+            my $skycells = parse_md_list($metadata) or
+                    &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+
+            ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
+
+            foreach my $skycell (@$skycells) {
+                my $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                print $sfh "$skycell_uri\n";
+            }
+            close $sfh;
+        }
+    }
+
+    my ($image, $mask, $ch_mask, $weight, $astrom, $sources);
+
+    if ($stage eq "raw") {
+        $image = $uri;
+        $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
+        $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id) if $release ;
+    } elsif ($stage eq "chip") {
+        # we use the mask output from the camera stage for input and replace
+        # the output of the chip stage with that mask as well.
+        $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
+        $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
+        $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
+        $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+        $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
+        if (!$ipprc->file_exists($mask)) {
+            carp("camera mask file $mask for $component not found. Continuing using mask from chip stage.");
+            $mask = $ch_mask;
+            $ch_mask = undef;
+        }
+        # XXX: should we censor the sources as well? For now we're leaving them out of the distribution bundles
+        # because they confuse people
+        # XXX: make sure that this is the right file rule
+        # $sources = $ipprc->filename("PSPHOT.OUT.CMF.SPL",  $path_base);
+
+    } elsif ($stage eq "warp") {
+        $image  = $ipprc->filename("PSWARP.OUTPUT", $path_base);
+        $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
+        $weight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base);
+        $sources    = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $path_base);
+    } elsif ($stage eq "diff") {
+        $image  = $ipprc->filename("PPSUB.OUTPUT", $path_base);
+        $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
+        $weight = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
+        $sources    = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base);
+    }
+
+    {
+        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+
+        $command .= " -class_id $class_id" if defined $class_id;
+        $command .= " -recovery $recoveryroot" if defined $recoveryroot;
+        $command .= " -astrom $astrom" if defined $astrom;
+        $command .= " -mask $mask" if defined $mask;
+        $command .= " -chip_mask $ch_mask" if defined $ch_mask;
+        $command .= " -weight $weight" if defined $weight;
+        $command .= " -sources $sources" if defined $sources;
+        $command .= " -skycelllist $skycell_list" if defined $skycell_list;
+        $command .= " -replace" if $replace;
+        $command .= " -release" if $release;
+        $command .= " -dbname $dbname" if defined $dbname;
+        unless (defined $no_op) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code);
+            }
+        } else {
+            print "skipping command $command\n";
+        }
+    }
+    if (($stage eq "diff") and $inv_streaks) {
+        $image   = $ipprc->filename("PPSUB.INVERSE", $path_base);
+        $mask    = $ipprc->filename("PPSUB.INVERSE.MASK", $path_base);
+        $weight  = $ipprc->filename("PPSUB.INVERSE.VARIANCE", $path_base);
+        $sources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $path_base);
+
+        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $inv_streaks -image $image";
+
+        $command .= " -recovery $recoveryroot" if defined $recoveryroot;
+        $command .= " -mask $mask" if defined $mask;
+        $command .= " -weight $weight" if defined $weight;
+        $command .= " -sources $sources" if defined $sources;
+        $command .= " -replace" if $replace;
+        $command .= " -release" if $release;
+        $command .= " -dbname $dbname" if defined $dbname;
+        unless (defined $no_op) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform streaksremove: $error_code", $magic_ds_id, $component, $error_code);
+            }
+        } else {
+            print "skipping command $command\n";
+        }
+    }
+} else {
+    # camera stage. The only work to do is to censor the detections file
+
+    my $tempOutRoot = "/tmp/destreak";
+    my ($maskListFile,   $maskListName)   = tempfile( "$tempOutRoot.mask.list.XXXX",   UNLINK => !$save_temps);
+    my $files;
+    {
+        my $cam_id = $stage_id;
+        my $command = "$camtool -pendingimfile -cam_id $cam_id"; # Command to run
+        $command .= " -dbname $dbname" if defined $dbname;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform camtool: $error_code", $cam_id, $error_code);
+        }
+        my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", $cam_id, $PS_EXIT_PROG_ERROR);
+
+        # extract the metadata for the files into a hash list
+        $files = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", $cam_id, $PS_EXIT_PROG_ERROR);
+    }
+    foreach my $file (@$files) {
+        my $class_id = $file->{class_id};
+        my $quality = $file->{quality};
+        if (!$quality) {
+            my $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $path_base, $class_id); 
+            if (! $ipprc->file_exists($mask)) {
+                # camera mask doesn't exist for this chip. Fall back to the chip mask
+                $mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $file->{path_base}, $class_id);
+            }
+            print $maskListFile "$mask\n";
+        }
+    }
+    close $maskListFile;
+
+    my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $path_base);
+    {
+        my $command = "$censorObjects -file $astrom -masklist $maskListName $backup_path_base";
+        # $command .= " -replace" if $replace;
+        $command .= " -dbname $dbname" if defined $dbname;
+        unless (defined $no_op) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform censorObjects: $error_code", $magic_ds_id, $component, $error_code);
+            }
+            my $output = $ipprc->filename("CENSOR.OUTPUT", $backup_path_base); 
+            &my_die("expected output file $output not found ", $magic_ds_id, $component, $PS_EXIT_DATA_ERROR)
+                unless $ipprc->file_exists($output);
+
+            if ($replace) {
+                $nebulous->swap($astrom, $output) or
+                    &my_die("nebulous swap failed $astrom $output", $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+            }
+        } else {
+            print "skipping command $command\n";
+        }
     }
 }
@@ -267,4 +393,14 @@
 }
 
+sub inNebulous
+{
+    my $path = shift;
+
+    my $scheme = file_scheme($path);
+
+    return $scheme and ($scheme eq "neb");
+}
+
+
 sub my_die
 {