Index: trunk/ippScripts/scripts/magic_process.pl
===================================================================
--- trunk/ippScripts/scripts/magic_process.pl	(revision 18784)
+++ trunk/ippScripts/scripts/magic_process.pl	(revision 20495)
@@ -15,4 +15,5 @@
 
 use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -30,4 +31,5 @@
 # Parse the command-line arguments
 my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $redirect);
+my $skycellroot;
 GetOptions(
            'magic_id=s'      => \$magic_id,   # Magic identifier
@@ -41,4 +43,5 @@
            'no-op'           => \$no_op,      # Don't do any operations?
            'redirect-output' => \$redirect,   # Redirect output?
+           'skycellroot=s'   => \$skycellroot, # root of the warps to find skycells (temporary)
            ) or pod2usage( 2 );
 
@@ -53,4 +56,17 @@
 $ipprc->define_camera($camera);
 
+# Remove streaks doesn't know about nebulous. It expects to be able to append strings to outroot
+# to form valid file names.
+# So forbid nebulous path in outroot. We could relax this by change RemoveStreaks to take all
+# of the file names as arguments
+if ($outroot =~ 'neb:/') {
+    &my_die("RemoveStreaks does not support nebulous paths in outroot", $magic_id, $node,
+        $PS_EXIT_CONFIG_ERROR);
+}
+
+# resolve any path:// or file:// in outroot
+$outroot = $ipprc->file_resolve($outroot);
+    
+
 my $logDest = $ipprc->filename("LOG.EXP", $outroot, $magic_id) or
     &my_die("Missing entry from camera config", $magic_id, $PS_EXIT_CONFIG_ERROR);
@@ -88,27 +104,98 @@
 
 
+my @outputs = ("${outroot}.clusters", "${outroot}_hough.fits", "${outroot}.streaks");
 ### Do the heavy lifting
-my @basenames;                  # Base names of inputs
-{
+{
+    my $mode;
     my $command;                # Command to execute
+    $command = "$streaks --outroot $outroot";
     if (scalar @$inputs == 1 and $node ne "root") {
+        #
+        #  RemoveStreak --detect --image filename --mask maskname --weight weightname --outroot path_base
+        #
         # Leaf node: 'detect' stage
-        $command = "$streaks --detect";
-
-        my $node = $$inputs[0];     # Input node
-        push @basenames, $ipprc->file_resolve( $node->{path_base} );
+        $mode = 'detect';
+        my $innode = $$inputs[0];     # Input node
+
+        my ($image, $mask, $weight) = resolve_inputs($innode);
+
+        $command .= " --detect --image $image --mask $mask --weight $weight";
+
+        # create the list of inputs used at this stage. At higher levels the
+        # thes files will get catenated together to create the file for the subsquent stage
+        
+        my ($ifh, $image_list)  = open_list_file($outroot, "image.list");
+        print $ifh "$image\n";
+        close $ifh;
+        my ($mfh, $mask_list)   = open_list_file($outroot, "mask.list");
+        print $mfh "$mask\n";
+        close $mfh;
+        my ($wfh, $weight_list) = open_list_file($outroot, "weight.list");
+        print $wfh "$weight\n";
+        close $wfh;
+
+        # work around missing wcs in difference images. Use the skycells from warp stage
+        my ($sfh, $skycell_list);
+        if ($skycellroot) {
+           ($sfh, $skycell_list) = open_list_file($outroot, "wcs.list");
+            # at this level the skycell_id is equal to the node
+            my $skycell_id = $node;
+            my $skycell = $ipprc->file_resolve("${skycellroot}.${skycell_id}.skycell");
+            print $sfh "$skycell\n";
+            close $sfh;
+        }
     } else {
+        #
+        # RemoveStreak --merge --inputs input.list --images image0_1.list \
+        #                      --masks mask0_1.list --weight weights0_1.list
+        #                      --outroot outroot
+
         # Branch node: 'merge' stage
-        $command = "$streaks --merge";
-
-        # Concatenate the names
-        foreach my $node (@$inputs) {
-            push @basenames, $ipprc->file_resolve( $node->{path_base} );
-        }
-    }
-
-    $command .= ' ' . join(' ', @basenames);
+        $mode = 'merge';
+
+        my ($infh, $input_list) = open_list_file($outroot, "input.list");
+        my ($ifh, $image_list)  = open_list_file($outroot, "image.list");
+        my ($mfh, $mask_list)   = open_list_file($outroot, "mask.list");
+        my ($wfh, $weight_list) = open_list_file($outroot, "weight.list");
+        my ($sfh, $skycell_list);
+        if ($skycellroot) {
+           ($sfh, $skycell_list) = open_list_file($outroot, "wcs.list");
+        }
+
+        eval {
+            foreach my $innode (@$inputs) {
+                my ($image, $mask, $weight) = resolve_inputs($innode);
+
+                my $in_uri = $innode->{uri};
+
+                print $infh "$in_uri\n";
+                cat_list_to_list($ifh, $in_uri, "image.list");
+                cat_list_to_list($mfh, $in_uri, "mask.list");
+                cat_list_to_list($wfh, $in_uri, "weight.list");
+                cat_list_to_list($sfh, $in_uri, "wcs.list") if $skycell_list;
+
+            }
+            close $infh;
+            close $ifh;
+            close $mfh;
+            close $wfh;
+            close $sfh if $skycell_list;
+
+            $command .= " --merge --inputs $input_list";
+            $command .= " --images $image_list --masks $mask_list --weights $weight_list" ;
+            $command .= " --wcsList $skycell_list" if $skycell_list;
+        };
+        if ($@) {
+            &my_die("failed to create file lists: $@", $PS_EXIT_UNKNOWN_ERROR, $magic_id, $node,
+                $PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
 
     unless ($no_op) {
+        # RemoveStreaks fails if the output file already exists
+        foreach my $output (@outputs) {
+            unlink($output) if -e $output;
+        }
+
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
@@ -118,8 +205,6 @@
         }
 
-        foreach my $basename (@basenames) {
-            file_check( $basename . '.clusters' );
-            file_check( $basename . '.streaks' );
-            file_check( $basename . '_hough.fits' );
+        foreach my $output (@outputs) {
+            file_check( $output );
         }
     } else {
@@ -154,4 +239,38 @@
 
 ### Pau.
+
+sub open_list_file {
+    my $outroot = shift;
+    my $extension = shift;
+
+    my $filename = "$outroot.$extension";
+
+    my $fh;
+    open $fh, ">$filename" or die "failed to open list file $filename";
+
+    return ($fh, $filename);
+}
+
+sub cat_list_to_list   { # ($infh, $in_uri, "input.list");
+    my $out = shift;        # output file handle
+    my $uri = shift;        # uri to append ...
+    my $extension = shift;  # extension to
+    my $filename = "$uri.$extension";
+
+    my $in;
+    open $in, "<$filename" or die "failed to open list file: $filename";
+    my $contents = <$in>;
+    print $out $contents;
+}
+
+sub resolve_inputs {
+    my $node = shift;
+    my $input_base = $node->{path_base};
+    my $image = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT", $input_base));
+    my $mask  = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
+    my $weight= $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.WEIGHT", $input_base));
+
+    return ($image, $mask, $weight);
+}
 
 sub file_check