Changeset 24174 for trunk

Timestamp:

May 13, 2009, 12:37:33 PM (17 years ago)

Author:

Paul Price

Message:

Merging branches/pap_magic@24173 into trunk. One conflict in dbconfig/changes.txt resolved easily.

Location:

trunk

Files:

• . (modified) (1 prop)
• Nebulous (modified) (1 prop)
• Nebulous-Server (modified) (1 prop)
• Nebulous/bin (modified) (1 prop)
• Nebulous/examples (modified) (1 prop)
• Nebulous/lib (modified) (1 prop)
• Nebulous/nebclient (modified) (1 prop)
• Nebulous/nebclient/tests/tap (modified) (1 prop)
• Nebulous/t (modified) (1 prop)
• Ohana (modified) (1 prop)
• dbconfig/changes.txt (modified) (1 diff)
• dbconfig/config.md (modified) (1 diff)
• dbconfig/diff.md (modified) (2 diffs)
• dbconfig/magic.md (modified) (3 diffs)
• ippScripts/Build.PL (modified) (2 diffs)
• ippScripts/scripts/diff_skycell.pl (modified) (5 diffs)
• ippScripts/scripts/ipp_serial_magic.pl (copied) (copied from branches/pap_magic/ippScripts/scripts/ipp_serial_magic.pl )
• ippScripts/scripts/magic_definerun.pl (modified) (2 diffs)
• ippScripts/scripts/magic_destreak.pl (modified) (5 diffs)
• ippScripts/scripts/magic_mask.pl (deleted)
• ippScripts/scripts/magic_process.pl (modified) (9 diffs)
• ippScripts/scripts/magic_tree.pl (modified) (4 diffs)
• ippTasks/diff.pro (modified) (2 diffs)
• ippTasks/magic.pro (modified) (5 diffs)
• ippTasks/simtest.basic.auto (modified) (2 diffs)
• ippTools/configure.ac (modified) (1 diff)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/chiptool_export_imfile.sql (modified) (1 prop)
• ippTools/share/chiptool_export_processed_imfile.sql (modified) (1 prop)
• ippTools/share/difftool_definebyquery.sql (deleted)
• ippTools/share/difftool_definebyquery_part1.sql (deleted)
• ippTools/share/difftool_definebyquery_part2.sql (deleted)
• ippTools/share/difftool_definebyquery_temp_create.sql (deleted)
• ippTools/share/difftool_definewarpstack.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpstack.sql )
• ippTools/share/difftool_definewarpstack_part1.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpstack_part1.sql )
• ippTools/share/difftool_definewarpstack_part2.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpstack_part2.sql )
• ippTools/share/difftool_definewarpstack_temp_create.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpstack_temp_create.sql )
• ippTools/share/difftool_definewarpwarp_select.sql (modified) (2 diffs)
• ippTools/share/difftool_definewarpwarp_temp_create.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpwarp_temp_create.sql )
• ippTools/share/difftool_definewarpwarp_temp_insert.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpwarp_temp_insert.sql )
• ippTools/share/difftool_definewarpwarp_test.sql (copied) (copied from branches/pap_magic/ippTools/share/difftool_definewarpwarp_test.sql )
• ippTools/share/difftool_skyfile.sql (modified) (1 diff)
• ippTools/share/difftool_todiffskyfile.sql (modified) (1 diff)
• ippTools/share/magicdstool_todestreak.sql (modified) (6 diffs)
• ippTools/share/magictool_definebyquery_insert.sql (modified) (1 diff)
• ippTools/share/magictool_definebyquery_select.sql (modified) (1 diff)
• ippTools/share/magictool_inputs.sql (modified) (1 diff)
• ippTools/share/magictool_inputskyfile.sql (modified) (1 diff)
• ippTools/share/magictool_toprocess_inputs.sql (modified) (1 diff)
• ippTools/share/magictool_toprocess_tree.sql (modified) (1 diff)
• ippTools/share/magictool_totree.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (6 diffs)
• ippTools/share/pxadmin_drop_tables.sql (modified) (1 diff)
• ippTools/src/difftool.c (modified) (21 diffs)
• ippTools/src/difftool.h (modified) (1 diff)
• ippTools/src/difftoolConfig.c (modified) (5 diffs)
• ippTools/src/magictool.c (modified) (17 diffs)
• ippTools/src/magictoolConfig.c (modified) (5 diffs)
• ppImage (modified) (1 prop)
• ppStack (modified) (1 prop)
• ppSub (modified) (1 prop)
• psModules (modified) (1 prop)
• psastro (modified) (1 prop)
• psastro/src/psastroFindChip.c (modified) (1 prop)
• psconfig (modified) (1 prop)
• psphot (modified) (1 prop)

trunk/dbconfig/changes.txt

@@ -1105 +1105 @@
 ALTER TABLE receiveFileset ADD COLUMN dirinfo VARCHAR(255) AFTER state;
 
+-- Playing with the diffs for warp-warp
+ALTER TABLE diffRun ADD COLUMN bothways TINYINT DEFAULT 0 AFTER tess_id;
+UPDATE diffRun SET bothways = 1 WHERE reduction = 'WARPWARP';
+ALTER TABLE diffRun DROP FOREIGN KEY diffRun_ibfk_1;
+ALTER TABLE diffRun DROP COLUMN exp_id;
+ALTER TABLE diffRun ADD KEY(label);
+ALTER TABLE diffRun ADD COLUMN exposure TINYINT DEFAULT 0 AFTER bothways;
+UPDATE diffRun, diffInputSkyfile SET diffRun.exposure = 1 WHERE diffRun.diff_id = diffInputSkyfile.diff_id and diffInputSkyfile.warp1 IS NOT NULL;
+ALTER TABLE stackRun ADD KEY(label);
+
+ALTER TABLE magicRun ADD COLUMN inverse TINYINT NOT NULL DEFAULT 0 AFTER diff_id;
+ALTER TABLE diffSkyfile DROP COLUMN uri;
+
+DROP TABLE IF EXISTS magicSkyfileMask;
+ALTER TABLE magicInputSkyfile DROP FOREIGN KEY magicInputSkyfile_ibfk_2;
+ALTER TABLE magicInputSkyfile DROP KEY diff_id;
+ALTER TABLE magicInputSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY(magic_id,node);
+ALTER TABLE magicInputSkyfile DROP COLUMN diff_id;
+ALTER TABLE magicNodeResult CHANGE COLUMN uri path_base VARCHAR(255);
+
+-- Version 1.1.53

trunk/dbconfig/config.md

@@ -2 +2 @@
     pkg_name        STR     ippdb
     pkg_namespace   STR     ippdb
-    pkg_version     STR     1.1.52
+    pkg_version     STR     1.1.53
 END

trunk/dbconfig/diff.md

@@ -10 +10 @@
     registered  TAI         NULL
     tess_id     STR         64      # Key
-    exp_id      S64         0       # fkey(exp_id) ref rawExp(exp_id)
+    bothways    BOOL        f
+    exposure    BOOL        f
     magicked    BOOL        f
 END
@@ -32 +33 @@
     diff_id      S64        0       # Primary Key fkey(diff_id) ref diffRun(diff_id)
     skycell_id   STR        64      # 
-    uri          STR        255
     path_base    STR        255
     data_state   STR        64

trunk/dbconfig/magic.md

@@ -6 +6 @@
     exp_id      S64         0       # Key
     diff_id     S64         0       # Key
+    inverse     BOOL        FALSE           
     state       STR         64      # Key
     workdir     STR         255
@@ -15 +16 @@
 END
 
+### This is left over from when diffs were composed of a single skycell
+### When we're not too busy, it should be deleted in favour of:
+### magicRun JOIN diffSkyfile USING(diff_id) WHERE diffSkyfile.fault = 0 AND diffSkyfile.quality = 0
 magicInputSkyfile METADATA
     magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
-    diff_id     S64         0       # Primary Key fkey(diff_id) ref diffRun(diff_id)
     node        STR         64      #
 END
@@ -30 +33 @@
     magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
     node        STR         64      # Primary Key fkey(magic_id, node) ref magicTree(magic_id, node)
-    uri         STR         255
+    path_base   STR         255
     fault       S16         0       # Key
 END

trunk/ippScripts/Build.PL

@@ -56 +56 @@
         scripts/magic_tree.pl
         scripts/magic_process.pl
-        scripts/magic_mask.pl
         scripts/magic_destreak.pl
         scripts/ippdb.pl
@@ -73 +72 @@
         scripts/ipp_serial_stack.pl
         scripts/ipp_serial_mops.pl
+        scripts/ipp_serial_magic.pl
         scripts/ipp_mops_translate.pl
         scripts/ipp_simulation_data.pl

trunk/ippScripts/scripts/diff_skycell.pl

@@ -35 +35 @@
 }
 
-my ($diff_id, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect);
+my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $verbose, $no_update, $no_op, $redirect);
 my ($skycell_id, $diff_skyfile_id);
 GetOptions(
@@ -44 +44 @@
     'threads=s'         => \$threads,   # Number of threads to use
     'outroot=s'         => \$outroot, # Output root name
+    'inverse'           => \$inverse, # Make inverse subtraction?
     'reduction=s'       => \$reduction, # Reduction class
     'verbose'           => \$verbose,   # Print to stdout
@@ -96 +97 @@
 my $camera;                     # Camera
 my ($inputMagic, $templateMagic); # Are the inputs been magicked?
-my $inverse = 1;                # Generate inverse subtraction if warp-warp
 foreach my $file (@$files) {
     if (defined $file->{template} and $file->{template}) {
@@ -107 +107 @@
             # template is a stack so it doesn't need to be magicked
             $templateMagic = 1;
-            $inverse = 0;
             ## use an explicit stack name for psphot output objects
         } else {
@@ -123 +122 @@
             $inputVariance = "PPSTACK.OUTPUT.VARIANCE";
             $inputSources = "PSPHOT.OUT.CMF.MEF";  ## this must be consistent with the value in stack_skycell.pl:161
-            $inverse = 0;
         } else {
             $inputMask = "PSWARP.OUTPUT.MASK";

trunk/ippScripts/scripts/magic_definerun.pl

@@ -164 +164 @@
 &my_die("failed to retrieve new magic run information", $PS_EXIT_CONFIG_ERROR) if !$magic_id;
 
+### This is left over from when diffs were composed of a single skycell
+### When we're not too busy, it should be deleted in favour of:
+### magicRun JOIN diffSkyfile USING(diff_id) WHERE diffSkyfile.fault = 0 AND diffSkyfile.quality = 0
 foreach my $diff_skyfile (@$inputs) {
     my $skycell_id = $diff_skyfile->{skycell_id};
@@ -171 +174 @@
     }
     my $diff_id = $diff_skyfile->{diff_id};
-    my $command = "$magictool -addinputskyfile -magic_id $magic_id -diff_id $diff_id";
-    $command .= "  -node $skycell_id";
+    my $command = "$magictool -addinputskyfile";
+    $command .= " -magic_id $magic_id";
+    $command .= " -node $skycell_id";
     $command .= " -dbname $dbname" if $dbname;
 

trunk/ippScripts/scripts/magic_destreak.pl

@@ -35 +35 @@
 
 # Parse the command-line arguments
-my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
+my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $inverse, $cam_path_base);
 my ($outroot, $recoveryroot);
 my ($replace, $remove, $release);
@@ -49 +49 @@
            'uri=s'          => \$uri,        # uri of the input image
            'path_base=s'    => \$path_base,  # path_base of the input
+           'inverse'        => \$inverse,    # Inverse subtraction?
            'cam_path_base=s'=> \$cam_path_base,  # path_base of the associated camera run
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
@@ -183 +184 @@
 my $image = $uri;
 
-my ($mask, $ch_mask, $weight, $astrom);
+my ($image, $mask, $ch_mask, $weight, $astrom);
 
 if ($stage eq "raw") {
+    $image = $uri;
     $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
     $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id) if $release ;
@@ -191 +193 @@
     # we use the mask output from the camera stage for input and replace
     # the output of the chip stage with that mask as well.
+    $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
     $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
     $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
@@ -196 +199 @@
     $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
 } elsif ($stage eq "warp") {
+    $image  = $ipprc->filename("PSWARP.OUTPUT", $path_base);
     $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
     $weight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base);
 } elsif ($stage eq "diff") {
-    $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
-    $weight = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
+    my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images
+    $image  = $ipprc->filename($name, $path_base);
+    $mask   = $ipprc->filename("$name.MASK", $path_base);
+    $weight = $ipprc->filename("$name.VARIANCE", $path_base);
 }
 

trunk/ippScripts/scripts/magic_process.pl

@@ -37 +37 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $outroot, $template_uri, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
 GetOptions(
@@ -45 +45 @@
            'dbname=s'        => \$dbname,     # Database name
            'outroot=s'       => \$outroot,    # Output root name
-           'template_uri=s'  => \$template_uri,# uri of diff template
            'save-temps'      => \$save_temps, # Save temporary files?
            'verbose'         => \$verbose,    # Print stuff?
@@ -59 +58 @@
     defined $node and
     defined $camera and
-    defined $outroot and
-    defined $template_uri;
+    defined $outroot;
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $node, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
@@ -138 +136 @@
         }
 
-        my $template = $ipprc->file_resolve($template_uri);
-        &my_die("failed to resolve template_uri: $template_uri", $magic_id, $node, $PS_EXIT_DATA_ERROR)
-            if !($template);
+        my $template = resolve_template($innode);
+        &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR)
+            unless defined $template;
 
         $command .= " --detect --image $image --mask $mask --weight $weight -k $template";
@@ -189 +187 @@
             foreach my $innode (@$inputs) {
                 # root for inputs from previous stage
-                my $in_uri = $innode->{uri};
-                print $sfh "$in_uri\n";
-
-                cat_list_to_list($in_fh, $in_uri, "input.list");
+                my $in_path_base = $innode->{magic_path_base};
+                print $sfh "$in_path_base\n";
+
+                cat_list_to_list($in_fh, $in_path_base, "input.list");
                 # build input lists by combining the lists from
                 # previous stages
-                cat_list_to_list($ifh, $in_uri, "image.list");
-                cat_list_to_list($mfh, $in_uri, "mask.list");
-                cat_list_to_list($wfh, $in_uri, "weight.list");
+                cat_list_to_list($ifh, $in_path_base, "image.list");
+                cat_list_to_list($mfh, $in_path_base, "mask.list");
+                cat_list_to_list($wfh, $in_path_base, "weight.list");
             }
             close $in_fh;
@@ -243 +241 @@
     $command   .= " -magic_id $magic_id";
     $command   .= " -node $node";
-    $command   .= " -uri $outroot";
+    $command   .= " -path_base $outroot";
     $command   .= " -dbname $dbname" if defined $dbname;
 
@@ -262 +260 @@
     my $resolved = $ipprc->file_resolve($streaks_file);
 
-    my $fh;
-    open $fh, "<$resolved" or
-        &my_die("failed to open streaks file $streaks_file", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
-    # the first line in the streaks file contains the number of streaks found
-    my $num_streaks = <$fh>;
-    chomp $num_streaks;
-    close $fh;
-    print "$num_streaks streaks found on magicRun $magic_id\n" if $verbose;
+    my $num_streaks = -1;
+    unless ($no_op) {
+        my $fh;
+        open $fh, "<$resolved" or
+            &my_die("failed to open streaks file $streaks_file", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+        # the first line in the streaks file contains the number of streaks found
+        $num_streaks = <$fh>;
+        chomp $num_streaks;
+        close $fh;
+        print "$num_streaks streaks found on magicRun $magic_id\n" if $verbose;
+    }
 
     my $command = "$magictool -addmask";
@@ -309 +310 @@
 sub cat_list_to_list   {
     my $out = shift;        # output file handle
-    my $uri = shift;        # uri to append ...
+    my $path_base = shift;  # path_base to append ...
     my $extension = shift;  # ... the extension to
 
-    my $filename = "$uri.$extension";
+    my $filename = "$path_base.$extension";
 
     my $in;
@@ -321 +322 @@
 }
 
-sub resolve_inputs {
-
+sub resolve_inputs
+{
     my $node = shift;
-    my $input_base = $node->{path_base};
-
-    my $image = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT", $input_base));
-    my $mask = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
-    my $weight= $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.VARIANCE", $input_base));
-
-    return ($image, $mask, $weight);
+    my $input_base = $node->{diff_path_base};
+
+    my ($image, $mask, $variance); # Names to return
+    if ($node->{inverse}) {
+        $image = "PPSUB.INVERSE";
+        $mask = "PPSUB.INVERSE.MASK";
+        $variance = "PPSUB.INVERSE.VARIANCE";
+    } else {
+        $image = "PPSUB.OUTPUT";
+        $mask = "PPSUB.OUTPUT.MASK";
+        $variance = "PPSUB.OUTPUT.VARIANCE";
+    }
+
+    $image = $ipprc->file_resolve($ipprc->filename($image, $input_base));
+    $mask = $ipprc->file_resolve($ipprc->filename($mask, $input_base));
+    $variance= $ipprc->file_resolve($ipprc->filename($variance, $input_base));
+
+    return ($image, $mask, $variance);
+}
+
+sub resolve_template
+{
+    my $node = shift;
+
+    my $image;                  # Name of template to return
+    my $path_base;              # Base name for name
+    if (defined $node->{warp_path_base} and $node->{warp_path_base} ne "NULL") {
+        $path_base = $node->{warp_path_base};
+        $image = "PSWARP.OUTPUT";
+    } elsif (defined $node->{stack_path_base} and $node->{stack_path_base} ne "NULL") {
+        $path_base = $node->{stack_path_base};
+        $image = "PPSTACK.OUTPUT";
+    } else {
+        return undef;
+    }
+
+    $image = $ipprc->file_resolve($ipprc->filename($image, $path_base));
+
+    return $image
 }
 

trunk/ippScripts/scripts/magic_tree.pl

@@ -73 +73 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-### Get a list of warpSkyfiles
-### This is a workaround to allow processing with older diffSkyfiles that don't have
-### a wcs. If warp_id is provided, we get the wcs from there
-my %warpSkyfiles;                   # hash of warps
-if ($warp_id) {
-    my $command = "$warptool -warped -warp_id $warp_id"; # Command to run
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform warptool -warped: $error_code", $magic_id, $error_code);
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR);
-
-    my $warps = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR);
-
-    # make a hash indexed by skycell_id
-    foreach my $warp ( @$warps ) {
-        my $skycell_id = $warp->{skycell_id};
-        die "failed to lookup skycell_id from warp" if !$skycell_id;
-        $warpSkyfiles{$skycell_id} = $warp;
-    }
-}
 ### Get a list of skycells
 my @skycells;                   # List of skycells
@@ -126 +99 @@
 my @fields;
 foreach my $input ( @skycells ) {
-    # the filename method doesn't add the $skycell_id
-
-    # my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
-    #my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
-
-#    my $skycellBase = "$outroot/$skycell_id";
-#    my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $skycellBase);
-#    $ipprc->skycell_file($tess_id, $skycell_id, $skyfile, $verbose) or &my_die("Unable to generate skycells $skycell_id", $magic_id, $PS_EXIT_PROG_ERROR);
-
-    my $skyfile;
-    my $skycell_id = $input->{node};
-    if ($warp_id) {
-        # Applying the workaround
-        my $warp = $warpSkyfiles{$skycell_id};
-        die "warpSkyfile for $skycell_id not found" if !$warp;
-        my $skyfileBase = $warp->{path_base};
-        $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $skyfileBase);
-    } else {
-        # using the diffSkyfile
-        $skyfile = $input->{uri};
-    }
-    my $skyfileResolved = $ipprc->file_resolve( $skyfile );
-
-#    my ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfileResolved );
-#    &my_die("Unable to read skycell header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status;
+    # We use the WCS in the diff image
+    my $name = "PPSUB.OUTPUT"; # Name of file
+    my $skycell_id = $input->{node}; # Name of skycell
+    my $skyfile = $ipprc->filename($name, $input->{path_base}, $skycell_id); # Filename for diff
+    my $skyfileResolved = $ipprc->file_resolve( $skyfile ); # Resolved filename
 
     my ($header, $status) = (undef, 0);
@@ -162 +115 @@
 
     # Get the useful header keywords
-#    my $naxis1 = $$header{'NAXIS1'} or &my_die("Can't find NAXIS1", $magic_id, $PS_EXIT_SYS_ERROR);
     my $naxis1 = $$header{'NAXIS1'};
     my $naxis2;
@@ -181 +133 @@
         $naxis2 = $$header{'ZNAXIS2'} or &my_die("Can't find ZNAXIS2", $magic_id, $PS_EXIT_SYS_ERROR);
     }
-    my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
     my $crota1 = $$header{'CROTA1'};
     my $crota2 = $$header{'CROTA2'};

trunk/ippTasks/diff.pro

@@ -146 +146 @@
     book getword diffSkyfile $pageName skycell_id -var SKYCELL_ID
     book getword diffSkyfile $pageName camera -var CAMERA
+    book getword diffSkyfile $pageName bothways -var BOTHWAYS
     book getword diffSkyfile $pageName workdir -var WORKDIR_TEMPLATE
     book getword diffSkyfile $pageName dbname -var DBNAME
@@ -164 +165 @@
 
     $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --skycell_id $SKYCELL_ID --diff_skyfile_id $DIFF_SKYFILE_ID --outroot $outroot --redirect-output
+    if ($BOTHWAYS > 1)
+       $run = $run --inverse
+    end
     add_standard_args run
 

trunk/ippTasks/magic.pro

@@ -280 +280 @@
     book getword magicToProcess $pageName camera -var CAMERA
     book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE
-    book getword magicToProcess $pageName template_uri -var TEMPLATE_URI
     book getword magicToProcess $pageName dbname -var DBNAME
 
@@ -302 +301 @@
     mkdir $outpath
 
-    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --template_uri $TEMPLATE_URI --logfile $logfile
+    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --outroot $outroot --logfile $logfile
     add_standard_args run
 
@@ -354 +353 @@
       if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0
     end
-#    XXX magicDSRun doesn't have label so can't use add_poll_args
-#    add_poll_args run
+    add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -410 +409 @@
     book getword magicToDS $pageName uri -var URI
     book getword magicToDS $pageName path_base -var PATH_BASE
+    book getword magicToDS $pageName inverse -var INVERSE
     book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE
     book getword magicToDS $pageName outroot -var OUTROOT
@@ -424 +424 @@
 
     $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --remove $REMOVE 
+    if ($INVERSE > 0)
+       # Inverse subtraction
+       $run = $run --inverse
+    end
 
     add_standard_args run

trunk/ippTasks/simtest.basic.auto

@@ -58 +58 @@
 END
 
-### Stack automation???
+### Stack automation
 automate METADATA
   name       STR STACK
@@ -64 +64 @@
 END
 
-### Diff automation???
+### Diff automation
 automate METADATA
   name       STR DIFF
-  regular    STR "difftool -definebyquery -label proc -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@"
+  regular    STR "difftool -definewarpstack -label proc -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@"
 END
 

trunk/ippTools/configure.ac

@@ -18 +18 @@
 PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
 PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
-PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.52]) 
+PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.53]) 
 
 PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}"

trunk/ippTools/share/Makefile.am

@@ -81 +81 @@
      dettool_tostacked.sql \
      difftool_completed_runs.sql \
-     difftool_definebyquery_part1.sql \
-     difftool_definebyquery_part2.sql \
-     difftool_definebyquery_temp_create.sql \
+     difftool_definewarpstack_part1.sql \
+     difftool_definewarpstack_part2.sql \
+     difftool_definewarpstack_temp_create.sql \
+     difftool_definewarpwarp_temp_create.sql \
+     difftool_definewarpwarp_temp_insert.sql \
      difftool_definewarpwarp_select.sql \
      difftool_definewarpwarp_insert.sql \

trunk/ippTools/share/difftool_definewarpwarp_select.sql

@@ -1 +1 @@
 SELECT
     inputWarpRun.warp_id AS input_warp_id,
+    inputWarpRun.tess_id AS tess_id,
     inputRawExp.exp_id AS input_exp_id,
-    inputWarpRun.tess_id AS tess_id,
+    -- The following trick pulls out the warp_id that has the smallest distance
     SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id
 FROM warpRun AS inputWarpRun
@@ -9 +10 @@
 JOIN chipRun AS inputChipRun USING(chip_id)
 JOIN rawExp AS inputRawExp USING(exp_id)
--- To find exposures that haven't been diffed:%s LEFT JOIN diffRun ON diffRun.exp_id = inputRawExp.exp_id
+-- To find exposures that haven't been diffed, insert newline here:%s LEFT JOIN diffs USING(exp_id)
 JOIN warpRun AS templateWarpRun
     ON templateWarpRun.warp_id != inputWarpRun.warp_id -- Don't use self as template!

trunk/ippTools/share/difftool_skyfile.sql

@@ -8 +8 @@
     stack1,
     warp2,
-    stack2,
-    exp_id
+    stack2
 FROM diffRun
-JOIN diffSkyfile
-    USING(diff_id)
-JOIN diffInputSkyfile
-    USING (diff_id, skycell_id)
+JOIN diffSkyfile USING(diff_id)
+JOIN diffInputSkyfile USING(diff_id, skycell_id)
 JOIN warpRun
     ON warpRun.warp_id = diffInputSkyfile.warp1
-JOIN rawExp
-    USING(exp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)

trunk/ippTools/share/difftool_todiffskyfile.sql

@@ -9 +9 @@
     diffRun.tess_id,
     diffRun.label,
-    diffRun.state
+    diffRun.state,
+    diffRun.bothways
 FROM diffRun
 JOIN diffInputSkyfile USING(diff_id)

trunk/ippTools/share/magicdstool_todestreak.sql

@@ -13 +13 @@
     rawImfile.uri as uri,
     NULL as path_base,
+    magicRun.inverse,
     camProcessedExp.path_base as cam_path_base,
     outroot,
@@ -44 +45 @@
     chipProcessedImfile.uri,
     chipProcessedImfile.path_base,
+    magicRun.inverse,
     camProcessedExp.path_base as cam_path_base,
     outroot,
@@ -51 +53 @@
 FROM magicDSRun
 JOIN magicMask USING (magic_id)
+JOIN magicRun USING(magic_id)
 JOIN camProcessedExp USING(cam_id)
 JOIN chipRun ON chip_id = stage_id
@@ -79 +82 @@
     warpSkyfile.uri,
     warpSkyfile.path_base,
+    magicRun.inverse,
     NULL as cam_path_base,
     outroot,
@@ -113 +117 @@
     magicRun.diff_id as stage_id,
     diffSkyfile.skycell_id as component,
-    diffSkyfile.uri,
+    NULL AS uri,
     diffSkyfile.path_base,
+    magicRun.inverse,
     NULL as cam_path_base,
     outroot,
@@ -126 +131 @@
 JOIN magicInputSkyfile USING(magic_id)
 JOIN diffSkyfile
-    ON  magicInputSkyfile.diff_id = diffSkyfile.diff_id
+    ON  magicRun.diff_id = diffSkyfile.diff_id
     AND magicInputSkyfile.node = diffSkyfile.skycell_id
 LEFT JOIN magicDSFile

trunk/ippTools/share/magictool_definebyquery_insert.sql

@@ -3 +3 @@
 SELECT
     @MAGIC_ID@, -- Update this with the appropriate magic_id
-    diff_id,
     skycell_id
 FROM diffSkyfile

trunk/ippTools/share/magictool_definebyquery_select.sql

@@ -1 +1 @@
 -- Get a list of exposures on which magic may be performed
-SELECT 
+SELECT
     exp_id,
-    MAX(diffRun.diff_id) AS diff_id
-FROM diffRun
-JOIN rawExp USING(exp_id)
+    MAX(diffWarps.diff_id) AS diff_id,
+    -- The following trick pulls out the 'inverse' value for the maximum diff_id
+    CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id), ',', 1), UNSIGNED) AS inverse
+FROM (
+    -- Forward diffs
+    SELECT
+        diffRun.diff_id,
+        warp1 AS warp_id,
+        0 AS inverse
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp1 IS NOT NULL
+        AND diffRun.exposure = 1
+    -- WHERE hook %s
+    UNION
+    -- Backward diffs
+    SELECT
+        diffRun.diff_id,
+        warp2 AS warp_id,
+        1 AS inverse
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp2 IS NOT NULL
+        AND diffRun.exposure = 1
+        AND diffRun.bothways = 1
+    -- WHERE hook %s
+    ) AS diffWarps
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
 LEFT JOIN magicRun USING(exp_id)
 -- WHERE hook %s

trunk/ippTools/share/magictool_inputs.sql

@@ -1 @@
-SELECT *
-FROM (
--- Single skycells
+-- Get input details for a magic node
 SELECT
     magicRun.magic_id,
-    magicRun.state,
-    magicInputSkyfile.node,
-    diffSkyfile.diff_id,
-    diffSkyfile.uri,
-    diffSkyfile.path_base,
-    diffSkyfile.fault,
-    diffSkyfile.quality
+    magicRun.inverse, -- Using the inverse subtraction?
+    magicRun.diff_id,
+    magicInputs.node,
+    -- Only one of diff_path_base and magic_path_base should be non-NULL
+    -- If diff_path_base is non-NULL, then only one of warp_path_base and stack_path_base should be non-NULL
+    magicInputs.diff_path_base, -- path_base for the diff (if any)
+    magicInputs.warp_path_base, -- path_base for the template warp (if any)
+    magicInputs.stack_path_base, -- path_base for the template stack (if any)
+    magicInputs.magic_path_base -- path_base for child nodes (if any)
 FROM magicRun
-JOIN magicInputSkyfile
-USING(magic_id)
-JOIN diffSkyfile
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-UNION
--- Merged skycells
-SELECT
-    magicRun.magic_id,
-    magicRun.state,
-    magicTree.node,
-    0,   -- no diff_id
-    magicNodeResult.uri,
-    NULL, -- magicNodeResult doesn't have a path_base
-    magicNodeResult.fault,
-    0 as quality
-FROM magicTree
-JOIN magicRun
-    USING(magic_id)
-JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.dep = magicNodeResult.node
-) as Foo
-WHERE
-    fault = 0
-    AND quality = 0
+JOIN (
+    -- Single skycells: have uri=NULL
+    SELECT
+        magic_id,
+        magicRun.diff_id,
+        skycell_id AS node,
+        diffSkyfile.path_base AS diff_path_base,
+        warpSkyfile.path_base AS warp_path_base,
+        stackSumSkyfile.path_base AS stack_path_base,
+        NULL AS magic_path_base
+    FROM magicRun
+    JOIN magicTree USING(magic_id)
+    JOIN magicInputSkyfile USING(magic_id, node)
+    JOIN diffRun USING(diff_id)
+    JOIN diffSkyfile
+        ON diffSkyfile.diff_id = magicRun.diff_id
+        AND diffSkyfile.skycell_id = magicInputSkyfile.node
+    JOIN (
+        -- Template for non-inverse
+        SELECT
+            magic_id,
+            skycell_id,
+            warp1 AS warp_id,
+            stack1 AS stack_id
+        FROM magicRun
+        JOIN diffInputSkyfile USING(diff_id)
+        WHERE magicRun.inverse = 0
+            -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s
+        UNION
+        -- Template for inverse
+        SELECT
+            magic_id,
+            skycell_id,
+            warp2 AS warp_id,
+            stack2 AS stack_id
+        FROM magicRun
+        JOIN diffInputSkyfile USING(diff_id)
+        WHERE magicRun.inverse = 1
+            -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s
+        ) AS diffTemplates USING(magic_id, skycell_id)
+    LEFT JOIN warpSkyfile USING(warp_id, skycell_id)
+    LEFT JOIN stackSumSkyfile USING(stack_id)
+    -- WHERE hook (magicRun.magic_id, magicTree.node) %s
+    UNION
+    -- Merged skycells: have diff_id=0, various_path_base=NULL
+    SELECT
+        magicTree.magic_id,
+        0 AS diff_id,
+        magicTree.dep,
+        NULL AS diff_path_base,
+        NULL AS warp_path_base,
+        NULL AS stack_path_base,
+        magicNodeResult.path_base AS magic_path_base
+    FROM magicTree
+    JOIN magicRun USING(magic_id)
+    JOIN magicNodeResult
+        ON magicTree.magic_id = magicNodeResult.magic_id
+        AND magicTree.dep = magicNodeResult.node
+    -- WHERE hook (magicRun.magic_id, magicTree.node) %s
+    ) AS magicInputs USING(magic_id)

trunk/ippTools/share/magictool_inputskyfile.sql

@@ -1 +1 @@
 SELECT
     magicInputSkyfile.*,
-    diffSkyfile.uri
+    magicRun.inverse,
+    diffSkyfile.path_base
 FROM magicRun
-JOIN magicInputSkyfile
-    USING(magic_id)
+JOIN magicInputSkyfile USING(magic_id)
 JOIN diffSkyfile
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
+    ON diffSkyfile.diff_id = magicRun.diff_id
+    AND diffSkyfile.skycell_id = magicInputSkyfile.node
 WHERE
     magicRun.state = 'new'

trunk/ippTools/share/magictool_toprocess_inputs.sql

@@ -1 @@
-SELECT * FROM
-(SELECT
+SELECT
     magicTree.*,
+    magicRun.workdir,
     rawExp.exp_id,
     rawExp.camera,
---    diffSkyfile.path_base,
-    magicRun.workdir,
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad,
-    warpSkyfile.uri as template_uri
-FROM magicTree
-JOIN magicRun USING(magic_id)
+    magicNodeResult.magic_id IS TRUE AS done,
+    magicNodeResult.fault IS TRUE AS bad
+FROM magicRun
+JOIN magicTree USING(magic_id)
 JOIN magicInputSkyfile USING(magic_id, node)
-JOIN diffSkyfile 
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-JOIN diffInputSkyfile 
-    ON diffSkyfile.diff_id = diffInputSkyfile.diff_id
-    AND diffSkyfile.skycell_id = diffInputSkyfile.skycell_id
-JOIN warpSkyfile
-    ON diffInputSkyfile.warp2 = warpSkyfile.warp_id
-    AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
 JOIN rawExp USING(exp_id)
-LEFT JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.node = magicNodeResult.node
+JOIN diffSkyfile -- only get nodes that match a skycell
+    ON diffSkyfile.diff_id = magicRun.diff_id
+    AND diffSkyfile.skycell_id = magicInputSkyfile.node
+LEFT JOIN magicNodeResult USING(magic_id, node)
 WHERE
     magicRun.state = 'new'
     AND magicNodeResult.magic_id IS NULL
     AND magicNodeResult.node IS NULL
-    -- where hook 1 %s
-UNION
-SELECT
-    magicTree.*,
-    rawExp.exp_id,
-    rawExp.camera,
---    diffSkyfile.path_base,
-    magicRun.workdir,
-    -- convert magic_id into a boolean value (1 or 0)
-    -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad,
-    stackSumSkyfile.uri as template_uri
-FROM magicTree
-JOIN magicRun USING(magic_id)
-JOIN magicInputSkyfile USING(magic_id, node)
-JOIN diffSkyfile 
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-JOIN diffInputSkyfile 
-    ON diffSkyfile.diff_id = diffInputSkyfile.diff_id
-    AND diffSkyfile.skycell_id = diffInputSkyfile.skycell_id
-JOIN stackSumSkyfile
-    ON diffInputSkyfile.stack2 = stackSumSkyfile.stack_id
-JOIN rawExp USING(exp_id)
-LEFT JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.node = magicNodeResult.node
-WHERE
-    magicRun.state = 'new'
-    AND magicNodeResult.magic_id IS NULL
-    AND magicNodeResult.node IS NULL
-    -- where hook 2 %s
-) as foo
+-- WHERE hook %s

trunk/ippTools/share/magictool_toprocess_tree.sql

@@ -1 +1 @@
 SELECT
     magicTree.*,
-    exp_id,
-    camera,
     magicRun.workdir,
+    rawExp.exp_id,
+    rawExp.camera,
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad,
-    'notused' as template_uri
+    magicNodeResult.magic_id IS TRUE AS done,
+    magicNodeResult.fault IS TRUE AS bad
 FROM magicTree
 JOIN magicRun USING(magic_id)
 JOIN rawExp USING(exp_id)
-LEFT JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.node = magicNodeResult.node
+LEFT JOIN magicNodeResult USING(magic_id, node)
 WHERE
     magicRun.state = 'new'
-    -- where hook %s
+-- WHERE hook %s
 ORDER BY
     magicRun.magic_id

trunk/ippTools/share/magictool_totree.sql

@@ -6 +6 @@
     ra,
     decl,
-    tess_id
+    diffRun.tess_id
 FROM magicRun
+JOIN diffRun USING(diff_id)
 JOIN rawExp USING(exp_id)
 LEFT JOIN magicTree

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -877 +877 @@
         KEY(state),
         KEY(skycell_id),
-        KEY(tess_id)
+        KEY(tess_id),
+        KEY(label)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -933 +934 @@
         registered DATETIME,
         tess_id VARCHAR(64),
-        exp_id  BIGINT,
+        bothways TINYINT DEFAULT 0,
+        exposure TINYINT DEFAULT 0,
         magicked TINYINT,
         PRIMARY KEY(diff_id),
@@ -939 +941 @@
         KEY(state),
         KEY(tess_id),
+        KEY(label),
         FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -969 +972 @@
         diff_id BIGINT,
         skycell_id VARCHAR(64),
-        uri VARCHAR(255),
         path_base VARCHAR(255),
         data_state VARCHAR(64),
@@ -1006 +1008 @@
         exp_id BIGINT,
         diff_id BIGINT,
+        inverse TINYINT NOT NULL DEFAULT 0,
         state VARCHAR(64),
         workdir VARCHAR(255),
@@ -1046 +1049 @@
         magic_id BIGINT,
         node VARCHAR(64),
-        uri VARCHAR(255),
+        path_base VARCHAR(255),
         fault SMALLINT,
         PRIMARY KEY(magic_id, node),

trunk/ippTools/share/pxadmin_drop_tables.sql

@@ -48 +48 @@
 DROP TABLE IF EXISTS magicNodeResult;
 DROP TABLE IF EXISTS magicMask;
-DROP TABLE IF EXISTS magicSkyfileMask;
 DROP TABLE IF EXISTS magicDSRun;
 DROP TABLE IF EXISTS magicDSFile;

trunk/ippTools/src/difftool.c

@@ -40 +40 @@
 static bool revertdiffskyfileMode(pxConfig *config);
 static bool definepoprunMode(pxConfig *config);
-static bool definebyqueryMode(pxConfig *config);
+static bool definewarpstackMode(pxConfig *config);
 static bool definewarpwarpMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
@@ -79 +79 @@
         MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
-        MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY,         definebyqueryMode);
+        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,         definewarpstackMode);
         MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
@@ -116 +116 @@
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
     PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
+    PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
+    PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
 
     // default
@@ -133 +134 @@
             registered,
             tess_id,
-            exp_id,
+            bothways,
+            exposure,
             false
     );
@@ -487 +489 @@
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false);
     PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (fault == 0), false);
 
@@ -521 +522 @@
                            diff_id,
                            skycell_id,
-                           uri,
                            path_base,
                            "full",
@@ -771 +771 @@
                          psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
                          psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
-                         psS64 exp_id, // exposure id for input_warp_id (if defined)
                          pxConfig *config // Configuration
                          )
@@ -802 +801 @@
             registered,
             tess_id,
-            exp_id,
+            false,
+            false,
             false       // magicked
     );
@@ -882 +882 @@
     PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
     PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
@@ -902 +901 @@
         psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                 "No input has been defined (-input_stack_id or -input_warp_id)");
-        return false;
-    }
-    if (input_warp_id && !exp_id) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                "-exp_id is required with -input_warp_id.");
         return false;
     }
@@ -917 +911 @@
                       template_warp_id ? template_warp_id : PS_MAX_S64,
                       template_stack_id ? template_stack_id : PS_MAX_S64,
-                      exp_id ? exp_id : PS_MAX_S64,
                       config)) {
         psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
@@ -935 +928 @@
 
 
-static bool definebyqueryMode(pxConfig *config)
+static bool definewarpstackMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -960 +953 @@
     PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
     PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
     // find all things to queue
-    psString query = pxDataGet("difftool_definebyquery_part1.sql");
+    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -1057 +1051 @@
     }
 
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
     // create temporary table
-    query = pxDataGet("difftool_definebyquery_temp_create.sql");
+    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1070 +1075 @@
     query = NULL;
 
-    psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql");
+    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");
 
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
@@ -1183 +1188 @@
                 registered,
                 tess_id,
-                exp_id,
+                true,
+                false,
                 false       // magicked
         );
@@ -1284 +1290 @@
     // Haversine formula for great circle distance
     PXOPT_COPY_F32(config->args, selectWhere, "-distance",
-                   "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + COS(inputRawExp.decl)*COS(templateRawExp.decl)*POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<=");
+                   "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + "
+                   "COS(inputRawExp.decl)*COS(templateRawExp.decl)*"
+                   "POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<=");
 
     PXOPT_LOOKUP_BOOL(backwards, config->args, "-backwards", false);
@@ -1308 +1316 @@
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        return false;
+    }
+
+    if (!rerun) {
+        // Need to build table of exposures with diffs
+        psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL
+        if (!tempCreate) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        if (!p_psDBRunQuery(config->dbh, tempCreate)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to create temp table: %s", tempCreate);
+            psFree(tempCreate);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(tempCreate);
+
+        psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table
+        if (!tempInsert) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psString where = psStringCopy(""); // WHERE for insertion
+        if (label) {
+            psStringAppend(&where, "\nAND diffRun.label = '%s'", label);
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, tempInsert, where, where)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to insert into temp table: %s", tempInsert);
+            psFree(tempInsert);
+            psFree(where);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(where);
+        psFree(tempInsert);
+    }
+
+    // Get list of warps to diff
     psString select = pxDataGet("difftool_definewarpwarp_select.sql");
     if (!select) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
     }
@@ -1323 +1400 @@
     psFree(selectWhere);
 
-    if (!rerun) {
-        psStringAppend(&whereClause, "\n%s (diffRun.diff_id IS NULL", whereClause ? "AND" : "WHERE");
-        if (label) {
-            psStringAppend(&whereClause, " OR diffRun.label != '%s'", label);
-        }
-        psStringAppend(&whereClause, ")");
-    }
-
     if (!available) {
         psStringAppend(&whereClause, "\n%s inputWarpRun.state = 'full'", whereClause ? "AND" : "WHERE");
     }
 
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(select);
-        psFree(whereClause);
-        psFree(insertWhere);
-        return false;
+    if (!rerun) {
+        psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE");
     }
 
     if (!p_psDBRunQueryF(config->dbh, select,
-                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffRun?
+                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffs?
                          whereClause)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause);
@@ -1414 +1479 @@
         psMetadata *row = results->data[i]; // Result row from query
 
-        psS64 exp_id = psMetadataLookupS64(NULL, row, "input_exp_id");
         psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id");
         const char *template = psMetadataLookupStr(NULL, row, "template_warp_id");
         const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id");
-        if (!exp_id || !input_id || !template || !tess_id) {
+        if (!input_id || !template || !tess_id) {
             psError(PXTOOLS_ERR_PROG, false, "Identifiers not found");
             psFree(list);
@@ -1430 +1494 @@
 
         diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered,
-                                          tess_id, exp_id, false); // Run to insert
+                                          tess_id, true, true, false); // Run to insert
         if (!diffRunInsertObject(config->dbh, run)) {
             psError(PS_ERR_UNKNOWN, false, "database error");

trunk/ippTools/src/difftool.h

@@ -34 +34 @@
     DIFFTOOL_MODE_REVERTDIFFSKYFILE,
     DIFFTOOL_MODE_DEFINEPOPRUN,
-    DIFFTOOL_MODE_DEFINEBYQUERY,
+    DIFFTOOL_MODE_DEFINEWARPSTACK,
     DIFFTOOL_MODE_DEFINEWARPWARP,
     DIFFTOOL_MODE_PENDINGCLEANUPRUN,

trunk/ippTools/src/difftoolConfig.c

@@ -49 +49 @@
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0,            "define workdir (required)", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id",  0,            "define tessellation ID (required)", NULL);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways",  0,            "do the subtraction both ways?", false);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure",  0,            "subtraction for entire exposure?", false);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label",  0,            "define label", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction",  0,            "define reduction class", NULL);
@@ -90 +92 @@
     psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
-    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0,            "define URI of file", 0);
     psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", 0);
     psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg",  0,            "define exposure background", NAN);
@@ -142 +143 @@
     psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-reduction",  0,            "define reduction class", NULL);
     psMetadataAddBool(definepoprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
-    psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-exp_id", 0,              "define exposure ID for template", 0);
     psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_warp_id", 0,            "define warp ID for template", 0);
     psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_stack_id", 0,            "define stack ID for template", 0);
@@ -148 +148 @@
     psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-input_stack_id", 0,            "define stack ID for input", 0);
 
-    // -definebyquery
-    psMetadata *definebyqueryArgs = psMetadataAlloc();
-    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
-    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
-    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",  0, "define reduction class", NULL);
-    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    // -definewarpstack
+    psMetadata *definewarpstackArgs = psMetadataAlloc();
+    psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
+    psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
+    psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-reduction",  0, "define reduction class", NULL);
+    psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false);
 
     // -definewarpwarp
@@ -239 +240 @@
     PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs);
     PXOPT_ADD_MODE("-definepoprun",     "", DIFFTOOL_MODE_DEFINEPOPRUN,      definepoprunArgs);
-    PXOPT_ADD_MODE("-definebyquery",    "", DIFFTOOL_MODE_DEFINEBYQUERY,     definebyqueryArgs);
+    PXOPT_ADD_MODE("-definewarpstack",  "", DIFFTOOL_MODE_DEFINEWARPSTACK,   definewarpstackArgs);
     PXOPT_ADD_MODE("-definewarpwarp",   "", DIFFTOOL_MODE_DEFINEWARPWARP,    definewarpwarpArgs);
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);

trunk/ippTools/src/magictool.c

@@ -127 +127 @@
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
-
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns
+    PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "==");
+    PXOPT_COPY_S64(config->args, diffWhere, "-diff_id", "diff_id", "==");
+
+    psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else
+    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "==");
 
     // Get list of exposures ready to magic
@@ -146 +149 @@
         PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
 
-        psString queryWhere = NULL;     // WHERE conditions for entire query
+        psString diffWhereStr = NULL;   // WHERE conditions for diffRuns
+        psString queryWhereStr = NULL;  // WHERE conditions for entire query
         if (!available) {
-            psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'");
-        } else {
-            // what if no skycells for the diff run completed?
-        }
+            psStringAppend(&diffWhereStr, "\nAND diffRun.state = 'full'");
+        }
+        // what if no skycells for the diff run completed?
+
         if (!rerun) {
-            const char *newWhere = " magic_id IS NULL"; // String to add
-            if (queryWhere) {
-                psStringAppend(&queryWhere, " AND %s", newWhere);
-            } else {
-                psStringAppend(&queryWhere, "\nWHERE %s", newWhere);
-            }
-        }
+            psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE");
+        }
+
         // now add the user specified qualifiers
-        if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&queryWhere, "\n%s %s", queryWhere == NULL ? "WHERE" : "AND", whereClause);
+        if (psListLength(diffWhere->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(diffWhere, NULL);
+            psStringAppend(&diffWhereStr, "\nAND %s", whereClause);
             psFree(whereClause);
         }
-        psFree(where);
-        if (!queryWhere) {
-            psStringAppend(&queryWhere, " ");
-        }
-
-        if (!p_psDBRunQueryF(config->dbh, query, queryWhere)) {
+        if (psListLength(queryWhere->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(queryWhere, NULL);
+            psStringAppend(&queryWhereStr, "\n%s %s", queryWhereStr ? "AND" : "WHERE", whereClause);
+            psFree(whereClause);
+        }
+        psFree(diffWhere);
+        psFree(queryWhere);
+
+        // Ensure the WHERE strings have something
+        if (!diffWhereStr) {
+            diffWhereStr = psStringCopy("");
+        }
+        if (!queryWhereStr) {
+            queryWhereStr = psStringCopy("");
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(queryWhere);
+            psFree(diffWhereStr);
+            psFree(queryWhereStr);
             psFree(query);
             return false;
         }
-        psFree(queryWhere);
+        psFree(diffWhereStr);
+        psFree(queryWhereStr);
         psFree(query);
     }
@@ -201 +214 @@
     }
 
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+
     // Parse the list of exposures ready to magic
-
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -215 +240 @@
         psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
         psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier
+        bool inverse = psMetadataLookupU64(NULL, row, "inverse"); // Inverse subtraction? Note types!
 
         // create a new magicRun for this group
-        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "new", workdir, "dirty", label, dvodb, registered, 0);
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label,
+                                            dvodb, registered, 0);
         if (!run) {
             psAbort("failed to alloc magicRun object");
@@ -296 +323 @@
 
     // optional
+    PXOPT_LOOKUP_BOOL(inverse, config->args, "-inverse", false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
@@ -305 +333 @@
             exp_id,
             diff_id ? diff_id : PS_MAX_S64,
+            inverse,
             "reg",      // state
             workdir,
@@ -363 +392 @@
     // required
     PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
     PXOPT_LOOKUP_STR(node, config->args, "-node", true, false);
 
@@ -369 +397 @@
             config->dbh,
             magic_id,
-            diff_id,
             node
     );
@@ -563 +590 @@
     PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
 
-    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'run' WHERE fault != 0");
+    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0");
 
     if (psListLength(where->list)) {
@@ -584 +611 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-node", "node", "==");
+    // Regrettably, there are multiple WHERE hooks which call the same things different names
+    psMetadata *templatesWhere = psMetadataAlloc(); // WHERE for selecting template
+    psMetadata *magicWhere = psMetadataAlloc();     // WHERE for selecting magic runs
+
+    PXOPT_COPY_S64(config->args, templatesWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, templatesWhere, "-node", "diffInputSkyfile.skycell_id", "==");
+
+    PXOPT_COPY_S64(config->args, magicWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, magicWhere, "-node", "magicTree.node", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -597 +630 @@
     }
 
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+    psString templatesWhereStr = psStringCopy(""); // WHERE for selecting template
+    psString magicWhereStr = psStringCopy("");     // WHERE for selecting magic runs
+
+    if (psListLength(templatesWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(templatesWhere, NULL);
+        psStringAppend(&templatesWhereStr, "\nAND %s", whereClause);
         psFree(whereClause);
     }
-    psFree(where);
+    psFree(templatesWhere);
+
+    if (psListLength(magicWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(magicWhere, NULL);
+        psStringAppend(&magicWhereStr, "\nWHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(magicWhere);
 
     // treat limit == 0 as "no limit"
@@ -611 +654 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+    if (!p_psDBRunQueryF(config->dbh, query,
+                         templatesWhereStr, templatesWhereStr,
+                         magicWhereStr, magicWhereStr)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(templatesWhereStr);
+        psFree(magicWhereStr);
         psFree(query);
         return false;
     }
+    psFree(templatesWhereStr);
+    psFree(magicWhereStr);
     psFree(query);
 
@@ -704 +753 @@
 
             bool status = false;
-            psS32 done = psMetadataLookupS32(&status, data, "done");
+            psS64 done = psMetadataLookupS64(&status, data, "done");
             if (!status) {
                 psAbort("failed to lookup value for done column");
             }
-            psS16 bad = psMetadataLookupS16(&status, data, "bad");
+            psS64 bad = psMetadataLookupS64(&status, data, "bad");
             if (!status) {
                 psAbort("failed to lookup value for bad column");
@@ -738 +787 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
     pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
 
@@ -794 +843 @@
     if (limit) {
         if (psArrayLength(output) >= limit) {
-            // we've found enough (not that limit was applied to the query so '> limit' won't happen)
+            // we've found enough (note that limit was applied to the query so '> limit' won't happen)
             // negative simple so the default is true
             if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
@@ -936 +985 @@
 
     // optional
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
 
     // default values
@@ -944 +993 @@
                                magic_id,
                                node,
-                               uri,
+                               path_base,
                                fault
         )) {

trunk/ippTools/src/magictoolConfig.c

@@ -53 +53 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb",       0, "define dvodb", NULL);
     psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search diff_id", 0);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL);
@@ -58 +59 @@
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false);
 
     // -definerun
@@ -65 +67 @@
     // following argument is for compatability
     psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff_id", 0);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-inverse", 0, "use the inverse subtraction?", false);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL);
@@ -78 +81 @@
     psMetadata *addinputskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
-    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", 0);
     psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL);
 
@@ -125 +127 @@
     psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
     psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL);
-    psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL);
+    psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base", NULL);
     psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);