Index: trunk/ippScripts/scripts/magic_process.pl
===================================================================
--- trunk/ippScripts/scripts/magic_process.pl	(revision 20502)
+++ trunk/ippScripts/scripts/magic_process.pl	(revision 20675)
@@ -30,6 +30,7 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $redirect);
-my $skycellroot;
+my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my $hack_mask;
+
 GetOptions(
            'magic_id=s'      => \$magic_id,   # Magic identifier
@@ -42,6 +43,6 @@
            'no-update'       => \$no_update,  # Don't update the database?
            'no-op'           => \$no_op,      # Don't do any operations?
-           'redirect-output' => \$redirect,   # Redirect output?
-           'skycellroot=s'   => \$skycellroot, # root of the warps to find skycells (temporary)
+           'logfile=s'       => \$logfile,
+           'hack-mask'       => \$hack_mask   # enable extra special mask substitution
            ) or pod2usage( 2 );
 
@@ -56,5 +57,5 @@
 $ipprc->define_camera($camera);
 
-# Remove streaks doesn't know about nebulous. It expects to be able to append strings to outroot
+# RemoveStreaks doesn't know about nebulous. It expects to be able to append strings to outroot
 # to form valid file names.
 # So forbid nebulous path in outroot. We could relax this by change RemoveStreaks to take all
@@ -68,13 +69,10 @@
 $outroot = $ipprc->file_resolve($outroot);
     
-
-my $logDest = $ipprc->filename("LOG.EXP", $outroot, $magic_id) or
-    &my_die("Missing entry from camera config", $magic_id, $PS_EXIT_CONFIG_ERROR);
-$ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logfile) if $logfile;
 
 # Look for programs we need
 my $missing_tools;
-my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
-my $streaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1);
+my $magictool      = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $removestreaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -104,10 +102,14 @@
 
 
-my @outputs = ("${outroot}.clusters", "${outroot}_hough.fits", "${outroot}.streaks");
+my @outputs;
 ### Do the heavy lifting
 {
-    my $mode;
     my $command;                # Command to execute
-    $command = "$streaks --outroot $outroot";
+    $command = "$removestreaks --outroot $outroot";
+    $command .= " --verbose" if $verbose;
+
+#    $command .= " -u 6";
+#    $command .= " --test";
+
     if (scalar @$inputs == 1 and $node ne "root") {
         #
@@ -115,14 +117,26 @@
         #
         # Leaf node: 'detect' stage
-        $mode = 'detect';
         my $innode = $$inputs[0];     # Input node
 
-        my ($image, $mask, $weight) = resolve_inputs($innode);
+        # expected outputs for detect stage
+        @outputs = ("${outroot}.clusters", "${outroot}_hough.fits", "${outroot}.streaks");
+
+        my $diff_id;
+        $diff_id = $innode->{diff_id} if $hack_mask;
+        my ($image, $mask, $weight) = resolve_inputs($innode, $diff_id);
 
         $command .= " --detect --image $image --mask $mask --weight $weight";
 
+        # set threshold to 4 sigma
+        $command .= ' -t 4';
+
         # create the list of inputs used at this stage. At higher levels the
-        # thes files will get catenated together to create the file for the subsquent stage
+        # these files will get catenated together to create the file for the subsquent stage
+        # this causes major file pollution, but avoids multi-level queries
+        # at higher level nodes.
         
+        my ($in_fh, $input_list)  = open_list_file($outroot, "input.list");
+        print $in_fh "$outroot\n";
+        close $in_fh;
         my ($ifh, $image_list)  = open_list_file($outroot, "image.list");
         print $ifh "$image\n";
@@ -134,15 +148,4 @@
         print $wfh "$weight\n";
         close $wfh;
-
-        # work around missing wcs in difference images. Use the skycells from warp stage
-        my ($sfh, $skycell_list);
-        if ($skycellroot) {
-           ($sfh, $skycell_list) = open_list_file($outroot, "wcs.list");
-            # at this level the skycell_id is equal to the node
-            my $skycell_id = $node;
-            my $skycell = $ipprc->file_resolve("${skycellroot}.${skycell_id}.skycell");
-            print $sfh "$skycell\n";
-            close $sfh;
-        }
     } else {
         #
@@ -152,43 +155,42 @@
 
         # Branch node: 'merge' stage
-        $mode = 'merge';
-
-        my ($infh, $input_list) = open_list_file($outroot, "input.list");
-        my ($ifh, $image_list)  = open_list_file($outroot, "image.list");
-        my ($mfh, $mask_list)   = open_list_file($outroot, "mask.list");
-        my ($wfh, $weight_list) = open_list_file($outroot, "weight.list");
-        my ($sfh, $skycell_list);
-        if ($skycellroot) {
-           ($sfh, $skycell_list) = open_list_file($outroot, "wcs.list");
-        }
+
+        # expected outputs from merge stage
+        @outputs = ("${outroot}.streaks");
+
+        my ($in_fh, $input_list) = open_list_file($outroot, "input.list");
+        my ($ifh, $image_list)   = open_list_file($outroot, "image.list");
+        my ($mfh, $mask_list)    = open_list_file($outroot, "mask.list");
+        my ($wfh, $weight_list)  = open_list_file($outroot, "weight.list");
+        my ($sfh, $streaks_list) = open_list_file($outroot, "streaks.list");
 
         # do this in eval so we can fault the exposure without
         # passing the $magic_id and $node everywhere
+        # XXX  (they're global dummy fix this)
         eval {
             foreach my $innode (@$inputs) {
                 # root for inputs from previous stage
                 my $in_uri = $innode->{uri};
-                print $infh "$in_uri\n";
-
-                # build image lists by combining the lists from
+                print $sfh "$in_uri\n";
+
+                cat_list_to_list($in_fh, $in_uri, "input.list");
+                # build input lists by combining the lists from
                 # previous stages
                 cat_list_to_list($ifh, $in_uri, "image.list");
                 cat_list_to_list($mfh, $in_uri, "mask.list");
                 cat_list_to_list($wfh, $in_uri, "weight.list");
-                cat_list_to_list($sfh, $in_uri, "wcs.list") if $skycell_list;
             }
-            close $infh;
+            close $in_fh;
             close $ifh;
             close $mfh;
             close $wfh;
-            close $sfh if $skycell_list;
+            close $sfh;
 
             $command .= " --merge --inputs $input_list";
             $command .= " --images $image_list --masks $mask_list --weights $weight_list" ;
-            $command .= " --wcsList $skycell_list" if $skycell_list;
+            $command .= " --inputstreaks $streaks_list";
         };
         if ($@) {
-            &my_die("failed to create file lists: $@", $PS_EXIT_UNKNOWN_ERROR, $magic_id, $node,
-                $PS_EXIT_UNKNOWN_ERROR);
+            &my_die("failed to create file lists: $@", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
         }
     }
@@ -214,5 +216,4 @@
     }
 }
-
 
 
@@ -254,8 +255,9 @@
 }
 
-sub cat_list_to_list   { # ($infh, $in_uri, "input.list");
+sub cat_list_to_list   {
     my $out = shift;        # output file handle
     my $uri = shift;        # uri to append ...
-    my $extension = shift;  # extension to
+    my $extension = shift;  # ... the extension to
+
     my $filename = "$uri.$extension";
 
@@ -267,8 +269,41 @@
 
 sub resolve_inputs {
+    my %substitute_mask = (
+        30 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_30_mask_extra.fits",
+        31 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_31_mask_extra.fits",
+        34 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_34_mask_extra.fits",
+        35 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_35_mask_extra.fits",
+        36 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_36_mask_extra.fits",
+        39 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_39_mask_extra.fits",
+        40 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_40_mask_extra.fits",
+        41 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_41_mask_extra.fits",
+        44 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_44_mask_extra.fits",
+        45 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_45_mask_extra.fits",
+        49 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_49_mask_extra.fits",
+        77 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_77_mask_extra.fits",
+        81 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_81_mask_extra.fits",
+        86 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_86_mask_extra.fits",
+        89 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_89_mask_extra.fits",
+        90 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_90_mask_extra.fits",
+        94 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_94_mask_extra.fits",
+        95 => "/data/ipp036.0/price/MD_Nov2008/masking/diff_95_mask_extra.fits"
+    );
+
     my $node = shift;
     my $input_base = $node->{path_base};
+
+    my $diff_id = shift;
+
     my $image = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT", $input_base));
-    my $mask  = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
+    my $mask;
+
+    # if a diff_id is provided look up the extra special difference image mask from the table
+    # XXX: perhaps if the mask doesn't exist, we should fall through and use the original
+    if ($diff_id) {
+        $mask = $substitute_mask{$diff_id};
+        &my_die("substitute mask not found for $diff_id", $magic_id, $node->{node}, $PS_EXIT_SYS_ERROR) unless $mask;
+    } else {
+        $mask = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
+    }
     my $weight= $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.WEIGHT", $input_base));