Index: /branches/pap_magic/dbconfig/changes.txt
===================================================================
--- /branches/pap_magic/dbconfig/changes.txt	(revision 24148)
+++ /branches/pap_magic/dbconfig/changes.txt	(revision 24149)
@@ -1113,2 +1113,10 @@
 
 ALTER TABLE magicRun ADD COLUMN inverse TINYINT NOT NULL DEFAULT 0 AFTER diff_id;
+ALTER TABLE diffSkyfile DROP COLUMN uri;
+
+DROP TABLE IF EXISTS magicSkyfileMask;
+ALTER TABLE magicInputSkyfile DROP FOREIGN KEY magicInputSkyfile_ibfk_2;
+ALTER TABLE magicInputSkyfile DROP KEY diff_id;
+ALTER TABLE magicInputSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY(magic_id,node);
+ALTER TABLE magicInputSkyfile DROP COLUMN diff_id;
+ALTER TABLE magicNodeResult CHANGE COLUMN uri path_base VARCHAR(255);
Index: /branches/pap_magic/dbconfig/diff.md
===================================================================
--- /branches/pap_magic/dbconfig/diff.md	(revision 24148)
+++ /branches/pap_magic/dbconfig/diff.md	(revision 24149)
@@ -33,5 +33,4 @@
     diff_id      S64        0       # Primary Key fkey(diff_id) ref diffRun(diff_id)
     skycell_id   STR        64      # 
-    uri          STR        255
     path_base    STR        255
     data_state   STR        64
Index: /branches/pap_magic/dbconfig/magic.md
===================================================================
--- /branches/pap_magic/dbconfig/magic.md	(revision 24148)
+++ /branches/pap_magic/dbconfig/magic.md	(revision 24149)
@@ -16,7 +16,9 @@
 END
 
+### This is left over from when diffs were composed of a single skycell
+### When we're not too busy, it should be deleted in favour of:
+### magicRun JOIN diffSkyfile USING(diff_id) WHERE diffSkyfile.fault = 0 AND diffSkyfile.quality = 0
 magicInputSkyfile METADATA
     magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
-    diff_id     S64         0       # Primary Key fkey(diff_id) ref diffRun(diff_id)
     node        STR         64      #
 END
@@ -31,5 +33,5 @@
     magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
     node        STR         64      # Primary Key fkey(magic_id, node) ref magicTree(magic_id, node)
-    uri         STR         255
+    path_base   STR         255
     fault       S16         0       # Key
 END
Index: /branches/pap_magic/ippScripts/Build.PL
===================================================================
--- /branches/pap_magic/ippScripts/Build.PL	(revision 24148)
+++ /branches/pap_magic/ippScripts/Build.PL	(revision 24149)
@@ -73,4 +73,5 @@
         scripts/ipp_serial_stack.pl
         scripts/ipp_serial_mops.pl
+        scripts/ipp_serial_magic.pl
         scripts/ipp_mops_translate.pl
         scripts/ipp_simulation_data.pl
Index: /branches/pap_magic/ippScripts/scripts/ipp_serial_magic.pl
===================================================================
--- /branches/pap_magic/ippScripts/scripts/ipp_serial_magic.pl	(revision 24149)
+++ /branches/pap_magic/ippScripts/scripts/ipp_serial_magic.pl	(revision 24149)
@@ -0,0 +1,157 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config qw( caturi );
+use Data::Dumper;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my ($dbname,                    # Database name to use
+    $identifier,                # Identifier to process
+    $workdir_default,           # Default working directory
+    $verbose,                   # Verbose operations?
+    $no_op,                     # No operations?
+    $no_update,                 # No updating?
+    );
+GetOptions( 'dbname=s' => \$dbname,
+            'magic_id=s' => \$identifier,
+            'workdir=s' => \$workdir_default,
+            'verbose' => \$verbose,
+            'no-op' => \$no_op,
+            'no-update' => \$no_update,
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Required options: --dbname --workdir",
+           -exitval => 3,
+           ) unless
+    defined $dbname;
+
+$workdir_default = `pwd` unless defined $workdir_default;
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser
+my $ipprc = PS::IPP::Config->new; # IPP Configuration
+
+# Look for programs we need
+my $missing_tools;
+my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $magic_tree = can_run('magic_tree.pl') or (warn "Can't find magic_tree.pl" and $missing_tools = 1);
+my $magic_process = can_run('magic_process.pl') or (warn "Can't find magic_process.pl" and $missing_tools = 1);
+die "Can't find required tools.\n" if $missing_tools;
+
+# Tree
+{
+    my @whole;                      # The whole list for processing
+    {
+        my $command = "$magictool -totree -dbname $dbname"; # Command to run
+        $command .= " -magic_id $identifier" if defined $identifier;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run( command => $command, verbose => 1 );
+        die "Unable to get magic tree list: $error_code\n" if not $success;
+        @whole = split /\n/, join( '', @$stdout_buf );
+    }
+
+    my @single = ();
+
+    while ( scalar @whole > 0 ) {
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+                die "Unable to parse output from magictool.\n";
+
+            foreach my $item (@$list) {
+                my $magic_id = $item->{magic_id};
+                my $exp_id = $item->{exp_id};
+                my $camera = $item->{camera};
+                my $tess_id = $item->{tess_id};
+                my $ra = $item->{ra};
+                my $dec = $item->{decl};
+                my $workdir = $item->{workdir};
+                $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL");
+
+                my $outroot = caturi( $workdir, $exp_id, "$exp_id.mgc.$magic_id" );
+                $ipprc->outroot_prepare( $outroot );
+
+                my $command = "$magic_tree --magic_id $magic_id --camera $camera --tess_id $tess_id --outroot $outroot --ra $ra --dec $dec --dbname $dbname";
+                $command .= " --verbose" if defined $verbose;
+                $command .= " --no-op" if defined $no_op;
+                $command .= " --no-update" if defined $no_update;
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run( command => $command, verbose => 1 );
+                die "Unable to do magic tree on $magic_id: $error_code\n" if not $success;
+            }
+
+            @single = ();
+        }
+    }
+}
+
+# Process leaf
+{
+    my @whole;                      # The whole list for processing
+    {
+        my $command = "$magictool -toprocess -dbname $dbname"; # Command to run
+        $command .= " -magic_id $identifier" if defined $identifier;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run( command => $command, verbose => 1 );
+        die "Unable to get magic tree list: $error_code\n" if not $success;
+        @whole = split /\n/, join( '', @$stdout_buf );
+    }
+
+    my @single = ();
+
+    while ( scalar @whole > 0 ) {
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+                die "Unable to parse output from magictool.\n";
+
+            foreach my $item (@$list) {
+                my $magic_id = $item->{magic_id};
+                my $exp_id = $item->{exp_id};
+                my $camera = $item->{camera};
+                my $node = $item->{node};
+                my $workdir = $item->{workdir};
+                $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL");
+
+                my $outroot = caturi( $workdir, $exp_id, "$exp_id.mgc.$magic_id.$node" );
+                $ipprc->outroot_prepare( $outroot );
+
+                my $command = "$magic_process --magic_id $magic_id --camera $camera --node $node --outroot $outroot --dbname $dbname";
+                $command .= " --verbose" if defined $verbose;
+                $command .= " --no-op" if defined $no_op;
+                $command .= " --no-update" if defined $no_update;
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run( command => $command, verbose => 1 );
+                die "Unable to do magic process on $magic_id $node: $error_code\n" if not $success;
+            }
+
+            @single = ();
+        }
+    }
+}
+
+
+
+END {
+    my $status = $?;
+    system("sync") == 0
+        or die "failed to execute sync: $!" ;
+    $? = $status;
+}
+
+
+__END__
+
+
Index: /branches/pap_magic/ippScripts/scripts/magic_definerun.pl
===================================================================
--- /branches/pap_magic/ippScripts/scripts/magic_definerun.pl	(revision 24148)
+++ /branches/pap_magic/ippScripts/scripts/magic_definerun.pl	(revision 24149)
@@ -164,4 +164,7 @@
 &my_die("failed to retrieve new magic run information", $PS_EXIT_CONFIG_ERROR) if !$magic_id;
 
+### This is left over from when diffs were composed of a single skycell
+### When we're not too busy, it should be deleted in favour of:
+### magicRun JOIN diffSkyfile USING(diff_id) WHERE diffSkyfile.fault = 0 AND diffSkyfile.quality = 0
 foreach my $diff_skyfile (@$inputs) {
     my $skycell_id = $diff_skyfile->{skycell_id};
@@ -171,6 +174,7 @@
     }
     my $diff_id = $diff_skyfile->{diff_id};
-    my $command = "$magictool -addinputskyfile -magic_id $magic_id -diff_id $diff_id";
-    $command .= "  -node $skycell_id";
+    my $command = "$magictool -addinputskyfile";
+    $command .= " -magic_id $magic_id";
+    $command .= " -node $skycell_id";
     $command .= " -dbname $dbname" if $dbname;
 
Index: anches/pap_magic/ippScripts/scripts/magic_mask.pl
===================================================================
--- /branches/pap_magic/ippScripts/scripts/magic_mask.pl	(revision 24148)
+++ 	(revision )
@@ -1,168 +1,0 @@
-#!/usr/bin/env perl
-
-use Carp;
-use warnings;
-use strict;
-
-## report the program and machine
-use Sys::Hostname;
-my $host = hostname();
-print "\n\n";
-print "Starting script $0 on $host\n\n";
-
-use vars qw( $VERSION );
-$VERSION = '0.01';
-
-use IPC::Cmd 0.36 qw( can_run run );
-use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::List qw( parse_md_list );
-
-use Astro::FITS::CFITSIO qw( :constants );
-Astro::FITS::CFITSIO::PerlyUnpacking(1);
-
-use PS::IPP::Config 1.01 qw( :standard );
-
-use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
-use Pod::Usage qw( pod2usage );
-
-# Look for programs we need
-my $missing_tools;
-my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
-my $streaksremove = can_run('streaksremove') or (warn "Can't find streaksremove" and $missing_tools = 1);
-if ($missing_tools) {
-    warn("Can't find required tools.");
-    exit($PS_EXIT_CONFIG_ERROR);
-}
-
-# Parse the command-line arguments
-my ($magic_id, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
-GetOptions(
-           'magic_id=s'      => \$magic_id,   # Magic identifier
-           'camera=s'        => \$camera,     # Camera name
-           'dbname=s'        => \$dbname,     # Database name
-           'outroot=s'       => \$outroot,    # Output root name
-           'save-temps'      => \$save_temps, # Save temporary files?
-           'verbose'         => \$verbose,    # Print stuff?
-           'no-update'       => \$no_update,  # Don't update the database?
-           'no-op'           => \$no_op,      # Don't do any operations?
-           'logfile=s'       => \$logfile,   # Redirect output?
-           ) or pod2usage( 2 );
-
-pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --magic_id --camera --outroot",
-           -exitval => 3) unless
-    defined $magic_id and
-    defined $camera and
-    defined $outroot;
-
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $PS_EXIT_SYS_ERROR ) if $logfile;
-
-my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-### Get a list of inputs
-my $inputs;                     # List of inputs
-{
-    my $command = "$magictool -inputs -magic_id $magic_id -node root"; # Command to run
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform magictool -inputs: $error_code", $magic_id, $error_code);
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR);
-
-    $inputs = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR);
-}
-
-### Do the heavy lifting
-{
-    my $command = "$streaksremove --streaks"; # Command to execute
-    my @basenames;              # List of base names
-
-    # Concatenate the names
-    foreach my $node (@$inputs) {
-        push @basenames, $ipprc->file_resolve( $node->{path_base} );
-    }
-    $command .= ' ' . join(' ', @basenames);
-
-    unless ($no_op) {
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform RemoveStreaks: $error_code", $magic_id, $error_code);
-        }
-
-        foreach my $basename (@basenames) {
-            file_check( $basename . '.mask' );
-        }
-    } else {
-        print "Skipping command: $command\n";
-    }
-}
-
-
-
-### Input mask into database
-{
-    my $command = "$magictool -addmask";
-    $command   .= " -magic_id $magic_id";
-    $command   .= " -uri $outroot";
-    $command   .= " -dbname $dbname" if defined $dbname;
-
-    # Add the processed file to the database
-    unless ($no_update) {
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $error_code);
-        }
-    } else {
-        print "Skipping command: $command\n";
-    }
-}
-
-
-
-### Pau.
-
-sub file_check
-{
-    my $file = shift;           # Name of file
-    &my_die("Unable to find output file: $file", $magic_id, $PS_EXIT_SYS_ERROR) unless
-        $ipprc->file_exists($file);
-}
-
-sub my_die
-{
-    my $msg = shift;            # Warning message on die
-    my $magic_id = shift;       # Magic identifier
-    my $exit_code = shift;      # Exit code to add
-
-    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
-
-    carp($msg);
-    if (defined $magic_id and not $no_update) {
-        my $command = "$magictool -addmask";
-        $command .= " -magic_id $magic_id";
-        $command .= " -fault $exit_code";
-        $command .= " -dbname $dbname" if defined $dbname;
-        system($command);
-    }
-    exit $exit_code;
-}
-
-END {
-    my $status = $?;
-    system("sync") == 0
-        or die "failed to execute sync: $!" ;
-    $? = $status;
-}
-
-__END__
Index: /branches/pap_magic/ippScripts/scripts/magic_process.pl
===================================================================
--- /branches/pap_magic/ippScripts/scripts/magic_process.pl	(revision 24148)
+++ /branches/pap_magic/ippScripts/scripts/magic_process.pl	(revision 24149)
@@ -37,5 +37,5 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $outroot, $template_uri, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
 GetOptions(
@@ -45,5 +45,4 @@
            'dbname=s'        => \$dbname,     # Database name
            'outroot=s'       => \$outroot,    # Output root name
-           'template_uri=s'  => \$template_uri,# uri of diff template
            'save-temps'      => \$save_temps, # Save temporary files?
            'verbose'         => \$verbose,    # Print stuff?
@@ -59,6 +58,5 @@
     defined $node and
     defined $camera and
-    defined $outroot and
-    defined $template_uri;
+    defined $outroot;
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $node, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
@@ -138,7 +136,7 @@
         }
 
-        my $template = $ipprc->file_resolve($template_uri);
-        &my_die("failed to resolve template_uri: $template_uri", $magic_id, $node, $PS_EXIT_DATA_ERROR)
-            if !($template);
+        my $template = resolve_template($innode);
+        &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR)
+            unless defined $template;
 
         $command .= " --detect --image $image --mask $mask --weight $weight -k $template";
@@ -189,13 +187,13 @@
             foreach my $innode (@$inputs) {
                 # root for inputs from previous stage
-                my $in_uri = $innode->{uri};
-                print $sfh "$in_uri\n";
-
-                cat_list_to_list($in_fh, $in_uri, "input.list");
+                my $in_path_base = $innode->{magic_path_base};
+                print $sfh "$in_path_base\n";
+
+                cat_list_to_list($in_fh, $in_path_base, "input.list");
                 # build input lists by combining the lists from
                 # previous stages
-                cat_list_to_list($ifh, $in_uri, "image.list");
-                cat_list_to_list($mfh, $in_uri, "mask.list");
-                cat_list_to_list($wfh, $in_uri, "weight.list");
+                cat_list_to_list($ifh, $in_path_base, "image.list");
+                cat_list_to_list($mfh, $in_path_base, "mask.list");
+                cat_list_to_list($wfh, $in_path_base, "weight.list");
             }
             close $in_fh;
@@ -241,5 +239,5 @@
     $command   .= " -magic_id $magic_id";
     $command   .= " -node $node";
-    $command   .= " -uri $outroot";
+    $command   .= " -path_base $outroot";
     $command   .= " -dbname $dbname" if defined $dbname;
 
@@ -307,8 +305,8 @@
 sub cat_list_to_list   {
     my $out = shift;        # output file handle
-    my $uri = shift;        # uri to append ...
+    my $path_base = shift;  # path_base to append ...
     my $extension = shift;  # ... the extension to
 
-    my $filename = "$uri.$extension";
+    my $filename = "$path_base.$extension";
 
     my $in;
@@ -319,14 +317,46 @@
 }
 
-sub resolve_inputs {
-
+sub resolve_inputs
+{
     my $node = shift;
-    my $input_base = $node->{path_base};
-
-    my $image = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT", $input_base));
-    my $mask = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
-    my $weight= $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.VARIANCE", $input_base));
-
-    return ($image, $mask, $weight);
+    my $input_base = $node->{diff_path_base};
+
+    my ($image, $mask, $variance); # Names to return
+    if ($node->{inverse}) {
+        $image = "PPSUB.INVERSE";
+        $mask = "PPSUB.INVERSE.MASK";
+        $variance = "PPSUB.INVERSE.VARIANCE";
+    } else {
+        $image = "PPSUB.OUTPUT";
+        $mask = "PPSUB.OUTPUT.MASK";
+        $variance = "PPSUB.OUTPUT.VARIANCE";
+    }
+
+    $image = $ipprc->file_resolve($ipprc->filename($image, $input_base));
+    $mask = $ipprc->file_resolve($ipprc->filename($mask, $input_base));
+    $variance= $ipprc->file_resolve($ipprc->filename($variance, $input_base));
+
+    return ($image, $mask, $variance);
+}
+
+sub resolve_template
+{
+    my $node = shift;
+
+    my $image;                  # Name of template to return
+    my $path_base;              # Base name for name
+    if (defined $node->{warp_path_base} and $node->{warp_path_base} ne "NULL") {
+        $path_base = $node->{warp_path_base};
+        $image = "PSWARP.OUTPUT";
+    } elsif (defined $node->{stack_path_base} and $node->{stack_path_base} ne "NULL") {
+        $path_base = $node->{stack_path_base};
+        $image = "PPSTACK.OUTPUT";
+    } else {
+        return undef;
+    }
+
+    $image = $ipprc->file_resolve($ipprc->filename($image, $path_base));
+
+    return $image
 }
 
Index: /branches/pap_magic/ippTasks/magic.pro
===================================================================
--- /branches/pap_magic/ippTasks/magic.pro	(revision 24148)
+++ /branches/pap_magic/ippTasks/magic.pro	(revision 24149)
@@ -280,5 +280,4 @@
     book getword magicToProcess $pageName camera -var CAMERA
     book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE
-    book getword magicToProcess $pageName template_uri -var TEMPLATE_URI
     book getword magicToProcess $pageName dbname -var DBNAME
 
@@ -302,5 +301,5 @@
     mkdir $outpath
 
-    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --template_uri $TEMPLATE_URI --logfile $logfile
+    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --logfile $logfile
     add_standard_args run
 
Index: /branches/pap_magic/ippTools/share/magictool_inputs.sql
===================================================================
--- /branches/pap_magic/ippTools/share/magictool_inputs.sql	(revision 24148)
+++ /branches/pap_magic/ippTools/share/magictool_inputs.sql	(revision 24149)
@@ -1,39 +1,74 @@
-SELECT *
-FROM (
--- Single skycells
+-- Get input details for a magic node
 SELECT
     magicRun.magic_id,
-    magicRun.state,
-    magicInputSkyfile.node,
-    diffSkyfile.diff_id,
-    diffSkyfile.uri,
-    diffSkyfile.path_base,
-    diffSkyfile.fault,
-    diffSkyfile.quality
+    magicRun.inverse, -- Using the inverse subtraction?
+    magicRun.diff_id,
+    magicInputs.node,
+    -- Only one of diff_path_base and magic_path_base should be non-NULL
+    -- If diff_path_base is non-NULL, then only one of warp_path_base and stack_path_base should be non-NULL
+    magicInputs.diff_path_base, -- path_base for the diff (if any)
+    magicInputs.warp_path_base, -- path_base for the template warp (if any)
+    magicInputs.stack_path_base, -- path_base for the template stack (if any)
+    magicInputs.magic_path_base -- path_base for child nodes (if any)
 FROM magicRun
-JOIN magicInputSkyfile
-USING(magic_id)
-JOIN diffSkyfile
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-UNION
--- Merged skycells
-SELECT
-    magicRun.magic_id,
-    magicRun.state,
-    magicTree.node,
-    0,   -- no diff_id
-    magicNodeResult.uri,
-    NULL, -- magicNodeResult doesn't have a path_base
-    magicNodeResult.fault,
-    0 as quality
-FROM magicTree
-JOIN magicRun
-    USING(magic_id)
-JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.dep = magicNodeResult.node
-) as Foo
-WHERE
-    fault = 0
-    AND quality = 0
+JOIN (
+    -- Single skycells: have uri=NULL
+    SELECT
+        magic_id,
+        magicRun.diff_id,
+        skycell_id AS node,
+        diffSkyfile.path_base AS diff_path_base,
+        warpSkyfile.path_base AS warp_path_base,
+        stackSumSkyfile.path_base AS stack_path_base,
+        NULL AS magic_path_base
+    FROM magicRun
+    JOIN magicTree USING(magic_id)
+    JOIN magicInputSkyfile USING(magic_id, node)
+    JOIN diffRun USING(diff_id)
+    JOIN diffSkyfile
+        ON diffSkyfile.diff_id = magicRun.diff_id
+        AND diffSkyfile.skycell_id = magicInputSkyfile.node
+    JOIN magicNodeResult USING(magic_id, node)
+    JOIN (
+        -- Template for non-inverse
+        SELECT
+            magic_id,
+            skycell_id,
+            warp1 AS warp_id,
+            stack1 AS stack_id
+        FROM magicRun
+        JOIN diffInputSkyfile USING(diff_id)
+        WHERE magicRun.inverse = 0
+            -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s
+        UNION
+        -- Template for inverse
+        SELECT
+            magic_id,
+            skycell_id,
+            warp2 AS warp_id,
+            stack2 AS stack_id
+        FROM magicRun
+        JOIN diffInputSkyfile USING(diff_id)
+        WHERE magicRun.inverse = 1
+            -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s
+        ) AS diffTemplates USING(magic_id, skycell_id)
+    LEFT JOIN warpSkyfile USING(warp_id, skycell_id)
+    LEFT JOIN stackSumSkyfile USING(stack_id)
+    -- WHERE hook (magicRun.magic_id, magicTree.node) %s
+    UNION
+    -- Merged skycells: have diff_id=0, various_path_base=NULL
+    SELECT
+        magicTree.magic_id,
+        0 AS diff_id,
+        magicTree.dep,
+        NULL AS diff_path_base,
+        NULL AS warp_path_base,
+        NULL AS stack_path_base,
+        magicNodeResult.path_base
+    FROM magicTree
+    JOIN magicRun USING(magic_id)
+    JOIN magicNodeResult
+        ON magicTree.magic_id = magicNodeResult.magic_id
+        AND magicTree.dep = magicNodeResult.node
+    -- WHERE hook (magicRun.magic_id, magicTree.node) %s
+    ) AS magicInputs USING(magic_id)
Index: /branches/pap_magic/ippTools/share/magictool_inputskyfile.sql
===================================================================
--- /branches/pap_magic/ippTools/share/magictool_inputskyfile.sql	(revision 24148)
+++ /branches/pap_magic/ippTools/share/magictool_inputskyfile.sql	(revision 24149)
@@ -1,8 +1,8 @@
 SELECT
     magicInputSkyfile.*,
-    diffSkyfile.uri
+    magicRun.inverse,
+    diffSkyfile.path_base
 FROM magicRun
-JOIN magicInputSkyfile
-    USING(magic_id)
+JOIN magicInputSkyfile USING(magic_id)
 JOIN diffSkyfile
     ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
Index: /branches/pap_magic/ippTools/share/magictool_toprocess_inputs.sql
===================================================================
--- /branches/pap_magic/ippTools/share/magictool_toprocess_inputs.sql	(revision 24148)
+++ /branches/pap_magic/ippTools/share/magictool_toprocess_inputs.sql	(revision 24149)
@@ -1,26 +1,13 @@
-SELECT * FROM
-(SELECT
+SELECT
     magicTree.*,
+    magicRun.workdir,
     rawExp.exp_id,
     rawExp.camera,
---    diffSkyfile.path_base,
-    magicRun.workdir,
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad,
-    warpSkyfile.uri as template_uri
+    magicNodeResult.magic_id IS TRUE AS done,
+    magicNodeResult.fault IS TRUE AS bad
 FROM magicTree
 JOIN magicRun USING(magic_id)
-JOIN magicInputSkyfile USING(magic_id, node)
-JOIN diffSkyfile 
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-JOIN diffInputSkyfile 
-    ON diffSkyfile.diff_id = diffInputSkyfile.diff_id
-    AND diffSkyfile.skycell_id = diffInputSkyfile.skycell_id
-JOIN warpSkyfile
-    ON diffInputSkyfile.warp2 = warpSkyfile.warp_id
-    AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
 JOIN rawExp USING(exp_id)
 LEFT JOIN magicNodeResult
@@ -31,36 +18,3 @@
     AND magicNodeResult.magic_id IS NULL
     AND magicNodeResult.node IS NULL
-    -- where hook 1 %s
-UNION
-SELECT
-    magicTree.*,
-    rawExp.exp_id,
-    rawExp.camera,
---    diffSkyfile.path_base,
-    magicRun.workdir,
-    -- convert magic_id into a boolean value (1 or 0)
-    -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad,
-    stackSumSkyfile.uri as template_uri
-FROM magicTree
-JOIN magicRun USING(magic_id)
-JOIN magicInputSkyfile USING(magic_id, node)
-JOIN diffSkyfile 
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-JOIN diffInputSkyfile 
-    ON diffSkyfile.diff_id = diffInputSkyfile.diff_id
-    AND diffSkyfile.skycell_id = diffInputSkyfile.skycell_id
-JOIN stackSumSkyfile
-    ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id
-JOIN rawExp USING(exp_id)
-LEFT JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.node = magicNodeResult.node
-WHERE
-    magicRun.state = 'new'
-    AND magicNodeResult.magic_id IS NULL
-    AND magicNodeResult.node IS NULL
-    -- where hook 2 %s
-) as foo
+-- WHERE hook %s
Index: /branches/pap_magic/ippTools/share/magictool_toprocess_tree.sql
===================================================================
--- /branches/pap_magic/ippTools/share/magictool_toprocess_tree.sql	(revision 24148)
+++ /branches/pap_magic/ippTools/share/magictool_toprocess_tree.sql	(revision 24149)
@@ -1,12 +1,11 @@
 SELECT
     magicTree.*,
-    exp_id,
-    camera,
     magicRun.workdir,
+    rawExp.exp_id,
+    rawExp.camera,
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad,
-    'notused' as template_uri
+    magicNodeResult.magic_id IS TRUE AS done,
+    magicNodeResult.fault IS TRUE AS bad,
 FROM magicTree
 JOIN magicRun USING(magic_id)
@@ -17,5 +16,5 @@
 WHERE
     magicRun.state = 'new'
-    -- where hook %s
+-- WHERE hook %s
 ORDER BY
     magicRun.magic_id
Index: /branches/pap_magic/ippTools/share/pxadmin_create_tables.sql
===================================================================
--- /branches/pap_magic/ippTools/share/pxadmin_create_tables.sql	(revision 24148)
+++ /branches/pap_magic/ippTools/share/pxadmin_create_tables.sql	(revision 24149)
@@ -972,5 +972,4 @@
         diff_id BIGINT,
         skycell_id VARCHAR(64),
-        uri VARCHAR(255),
         path_base VARCHAR(255),
         data_state VARCHAR(64),
@@ -1050,5 +1049,5 @@
         magic_id BIGINT,
         node VARCHAR(64),
-        uri VARCHAR(255),
+        path_base VARCHAR(255),
         fault SMALLINT,
         PRIMARY KEY(magic_id, node),
Index: /branches/pap_magic/ippTools/src/difftool.c
===================================================================
--- /branches/pap_magic/ippTools/src/difftool.c	(revision 24148)
+++ /branches/pap_magic/ippTools/src/difftool.c	(revision 24149)
@@ -489,5 +489,4 @@
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false);
     PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (fault == 0), false);
 
@@ -523,5 +522,4 @@
                            diff_id,
                            skycell_id,
-                           uri,
                            path_base,
                            "full",
Index: /branches/pap_magic/ippTools/src/difftoolConfig.c
===================================================================
--- /branches/pap_magic/ippTools/src/difftoolConfig.c	(revision 24148)
+++ /branches/pap_magic/ippTools/src/difftoolConfig.c	(revision 24149)
@@ -92,5 +92,4 @@
     psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
-    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0,            "define URI of file", 0);
     psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", 0);
     psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg",  0,            "define exposure background", NAN);
Index: /branches/pap_magic/ippTools/src/magictool.c
===================================================================
--- /branches/pap_magic/ippTools/src/magictool.c	(revision 24148)
+++ /branches/pap_magic/ippTools/src/magictool.c	(revision 24149)
@@ -131,4 +131,5 @@
     psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns
     PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "==");
+    PXOPT_COPY_S64(config->args, diffWhere, "-diff_id", "diff_id", "==");
 
     psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else
@@ -391,5 +392,4 @@
     // required
     PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
     PXOPT_LOOKUP_STR(node, config->args, "-node", true, false);
 
@@ -397,5 +397,4 @@
             config->dbh,
             magic_id,
-            diff_id,
             node
     );
@@ -612,7 +611,17 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-node", "node", "==");
+    // Regrettably, there are multiple WHERE hooks which call the same things different names
+    psMetadata *templatesWhere = psMetadataAlloc(); // WHERE for selecting template
+    psMetadata *magicWhere = psMetadataAlloc();     // WHERE for selecting magic runs
+    psMetadata *finalWhere = psMetadataAlloc();     // WHERE at end
+
+    PXOPT_COPY_S64(config->args, templatesWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, templatesWhere, "-node", "diffInputSkyfile.skycell_id", "==");
+
+    PXOPT_COPY_S64(config->args, magicWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, magicWhere, "-node", "magicTree.node", "==");
+
+    PXOPT_COPY_S64(config->args, finalWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, finalWhere, "-node", "magicInputs.node", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -625,10 +634,27 @@
     }
 
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+    psString templatesWhereStr = psStringCopy(""); // WHERE for selecting template
+    psString magicWhereStr = psStringCopy("");     // WHERE for selecting magic runs
+
+    if (psListLength(templatesWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(templatesWhere, NULL);
+        psStringAppend(&templatesWhereStr, "\nAND %s", whereClause);
         psFree(whereClause);
     }
-    psFree(where);
+    psFree(templatesWhere);
+
+    if (psListLength(magicWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(magicWhere, NULL);
+        psStringAppend(&magicWhereStr, "\nWHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(magicWhere);
+
+    if (psListLength(finalWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(finalWhere, NULL);
+        psStringAppend(&query, "\nWHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(finalWhere);
 
     // treat limit == 0 as "no limit"
@@ -639,9 +665,15 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+    if (!p_psDBRunQueryF(config->dbh, query,
+                         templatesWhereStr, templatesWhereStr,
+                         magicWhereStr, magicWhereStr)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(templatesWhereStr);
+        psFree(magicWhereStr);
         psFree(query);
         return false;
     }
+    psFree(templatesWhereStr);
+    psFree(magicWhereStr);
     psFree(query);
 
@@ -964,5 +996,5 @@
 
     // optional
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
 
     // default values
@@ -972,5 +1004,5 @@
                                magic_id,
                                node,
-                               uri,
+                               path_base,
                                fault
         )) {
Index: /branches/pap_magic/ippTools/src/magictoolConfig.c
===================================================================
--- /branches/pap_magic/ippTools/src/magictoolConfig.c	(revision 24148)
+++ /branches/pap_magic/ippTools/src/magictoolConfig.c	(revision 24149)
@@ -53,4 +53,5 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb",       0, "define dvodb", NULL);
     psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search diff_id", 0);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL);
@@ -80,5 +81,4 @@
     psMetadata *addinputskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
-    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", 0);
     psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL);
 
@@ -127,5 +127,5 @@
     psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
     psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL);
-    psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL);
+    psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base", NULL);
     psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);