Index: /branches/cnb_branches/cnb_branch_20090301/Branches.txt
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Branches.txt	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Branches.txt	(revision 24244)
@@ -1,2 +1,6 @@
+sj_branches/sj_SDSSaddstar_branch_r23928_20090420
+
+* Ohana: some proposed changes to, and some questions about, 
+  SDSS-related routines in addstar.
 
 Converted to SVN 2009-02-19
Index: /branches/cnb_branches/cnb_branch_20090301/DataStore/scripts/dsproductls
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/DataStore/scripts/dsproductls	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/DataStore/scripts/dsproductls	(revision 24244)
@@ -16,10 +16,11 @@
 use Pod::Usage qw( pod2usage );
 
-my ($uri, $last_fileset, $timeout);
+my ($uri, $last_fileset, $timeout, $extra);
 
 GetOptions(
     'uri|u=s'           => \$uri,
     'last_fileset|l=s'  => \$last_fileset,
-    'timeout|t=s'         => \$timeout,
+    'timeout|t=s'       => \$timeout,
+    'extra|e'         => \$extra,
 ) or pod2usage( 2 );
 
@@ -59,5 +60,12 @@
 print "# uri fileset datetime type\n";
 foreach my $fs (@$data) {
-    print $fs->uri, " ", $fs->fileset, " ", $fs->datetime, " ", $fs->type, "\n";
+    print $fs->uri, " ", $fs->fileset, " ", $fs->datetime, " ", $fs->type;
+    if ($extra) {
+        my $cols = $fs->extra;
+        foreach my $col (@$cols) {
+            print " ", $col;
+        }
+    }
+    print "\n";
 }
 
@@ -84,5 +92,5 @@
 The URI of the file to be retrieved.
 
-=item * --lst_fileset|-l <filesetid>
+=item * --last_fileset|-l <filesetid>
 
 The FileSet ID of the last FileSet that you've seen.
@@ -94,4 +102,10 @@
 The ammount of time (in seconds) to wait for a response from the DataStore
 after making an HTTP request.  The default is 30s.
+
+Optional.
+
+=item * --extra|-e
+
+Print out any extra columns in the listing.
 
 Optional.
Index: /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsfsindex
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsfsindex	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsfsindex	(revision 24244)
@@ -112,2 +112,6 @@
     print "$line\n";
 }
+if ($print_header) {
+    # empty file set
+    print "$header\n";
+}
Index: /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsprodindex
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsprodindex	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsprodindex	(revision 24244)
@@ -27,43 +27,30 @@
 }
 
-# build the header line
-my $header = "# filesetID|time registered     |type     |";
+# first get the column names
 
-# now add the product specific columns that are defined
-# not particularly elegant. I guess I should have fetched the row as an array
-# note: we don't do any formatting for these columns. The width of the string
-# in the database should be padded if a nice width is desired
+# we have at least 3 columns
+my @header_col = ("# filesetID", "time registered", "type");
+my $numCols = 3;
+
+# now add any product specific columns that are defined. Note if there is no header value
+# then any values in the rows will be ignored
 my $last_prod_col = -1;
-if (defined $prod->{prod_col_0}) {
-    $last_prod_col = 0;
-    $header .= "$prod->{prod_col_0}|";
-    if (defined $prod->{prod_col_1}) {
-        $last_prod_col = 1;
-        $header .= "$prod->{prod_col_1}|";
-        if (defined $prod->{prod_col_2}) {
-            $last_prod_col = 2;
-            $header .= "$prod->{prod_col_2}|";
-            if (defined $prod->{prod_col_3}) {
-                $last_prod_col = 3;
-                $header .= "$prod->{prod_col_3}|";
-                if (defined $prod->{prod_col_4}) {
-                    $last_prod_col = 4;
-                    $header .= "$prod->{prod_col_4}|";
-                    if (defined $prod->{prod_col_5}) {
-                        $last_prod_col = 5;
-                        $header .= "$prod->{prod_col_5}|";
-                        if (defined $prod->{prod_col_6}) {
-                            $last_prod_col = 6;
-                            $header .= "$prod->{prod_col_6}|";
-                            if (defined $prod->{prod_col_7}) {
-                                $last_prod_col = 7;
-                                $header .= "$prod->{prod_col_7}|";
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
+for (my $i = 0; $i < 8; $i++) {
+    my $col_key = "prod_col_$i";
+    my $col_name = $prod->{$col_key};
+
+    last if ! defined $col_name;
+
+    $header_col[$numCols++] = $col_name;
+    $last_prod_col = $i;
+}
+
+# now set up arrays for the values for each column. 
+my @column;
+for (my $i = 0; $i < $numCols; $i++) {
+
+    my @col = ($header_col[$i]);
+
+    $column[$i] = \@col;
 }
     
@@ -89,5 +76,5 @@
         # XXX: the spec doesn't say what should happen in this case.
         # Here we follow the conductor implementation and return the whole list.
-        # This may not be the right thing to do.
+        # This may not be the right thing to do. It might be better to throw an error
     }
 }
@@ -96,9 +83,9 @@
 $stmt->execute();
 
-# XXX: dsproductls expects a header even if there are no matching filesets
-print "$header\n";
-
+# we at least have the header row to output
+my $numRows = 1;
 while( my $row = $stmt->fetchrow_hashref()) {
 
+    $numRows++;
     my $daytime = $row->{reg_time};
     (my $date, my $time) = split " ", $daytime;
@@ -108,31 +95,50 @@
     my $line = sprintf "%-11s|%-20s|%-9s|", $row->{fileset_name}, $reg_time, $row->{type};
 
-    # now add the product specific columns that are defined
-    # again using an array would have made this look nicer
-    if ($last_prod_col >= 0) {
-        $line .= "$row->{prod_col_0}|";
-        if ($last_prod_col >= 1) {
-            $line .= "$row->{prod_col_1}|";
-            if ($last_prod_col >= 2) {
-                $line .= "$row->{prod_col_2}|";
-                if ($last_prod_col >= 3) {
-                    $line .= "$row->{prod_col_3}|";
-                    if ($last_prod_col >= 4) {
-                        $line .= "$row->{prod_col_4}|";
-                        if ($last_prod_col >= 5) {
-                            $line .= "$row->{prod_col_5}|";
-                            if ($last_prod_col >= 6) {
-                                $line .= "$row->{prod_col_6}|";
-                                if ($last_prod_col >= 7) {
-                                    $line .= "$row->{prod_col_7}|";
-                                }
-                            }
-                        }
-                    }
-                }
-            }
+    # this probably would have been simpler had I used fetchrow array
+    push @{$column[0]}, $row->{fileset_name};
+    push @{$column[1]}, $reg_time;
+    push @{$column[2]}, $row->{type};
+    
+    for (my $i = 0; $i <= $last_prod_col; $i++) {
+        my $col_key = "prod_col_$i";
+        my $col_val = $row->{$col_key};
+
+        if (!defined($col_val)) {
+            $col_val = "null";
+        }
+        push @{$column[$i+3]}, $col_val;
+    }
+}
+
+# find the largest value width in each column
+my @col_widths;
+for (my $c = 0; $c < $numCols; $c++) {
+    $col_widths[$c] = 0;
+    my $col = $column[$c];
+    for (my $i = 0; $i < $numRows; $i++) {
+        my $val = $col->[$i];
+        if (!defined($val)) {
+            die "value for column $c in row $i is null";
+            next;
+        }
+        my $len = length($val);
+        if ($len > $col_widths[$c]) {
+            $col_widths[$c] = $len;
         }
     }
+}
 
-    print "$line\n";
+# print out the results
+for (my $r = 0; $r < $numRows; $r++) {
+    for (my $c = 0; $c < $numCols; $c++) {
+        my $val = $column[$c]->[$r];
+        my $width = $col_widths[$c];
+        if (defined($val)) {
+            printf "%-*s|", $width, $val;
+        } else {
+            die "value for column $c in row $r is null";
+        }
+    }
+    print "\n";
 }
+    
Index: /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsprodtool
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsprodtool	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsprodtool	(revision 24244)
@@ -102,8 +102,9 @@
     if ($err);
 
-my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER');
-$dbname      = metadataLookupStr($siteConfig, 'DBNAME') unless defined $dbname;
-my $dbuser   = metadataLookupStr($siteConfig, 'DBUSER');
-my $dbpass   = metadataLookupStr($siteConfig, 'DBPASSWORD');
+my $dbserver = metadataLookupStr($siteConfig, 'DS_DBSERVER');
+$dbname      = metadataLookupStr($siteConfig, 'DS_DBNAME')
+    unless defined $dbname;
+my $dbuser   = metadataLookupStr($siteConfig, 'DS_DBUSER');
+my $dbpass   = metadataLookupStr($siteConfig, 'DS_DBPASSWORD');
 exit ($PS_EXIT_CONFIG_ERROR) unless defined $dbserver and $dbname and $dbuser and $dbpass;
 
Index: /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsreg
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsreg	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/scripts/dsreg	(revision 24244)
@@ -31,4 +31,5 @@
 my $fileset;
 my $filelist;
+my $empty;
 
 my $fstype;
@@ -64,4 +65,5 @@
         'product=s'     =>      \$product,
         'list=s'        =>      \$filelist,
+        'empty'         =>      \$empty,
         'type=s'        =>      \$fstype,
         'datapath=s'    =>      \$datapath,
@@ -95,12 +97,12 @@
     $fileset = $add;
     $err .= "--type is required\n" unless defined $fstype;
-    $err .= "--list is required\n" unless defined $filelist;
+    $err .= "--list is required\n" unless defined $filelist or $empty;
     if ($linkfiles and $copyfiles) {
         $err .= "only one of --link and --copy is allowed\n";
     }
     if (($linkfiles or $copyfiles) and !$datapath and !$abspath) {
-        $err .= "need to specify datapath with --link or --copy\n";
-    }
-    if ($linkfiles && (substr($datapath, 0, 1) ne "/")) {
+        $err .= "need to specify datapath or abspath with --link and --copy\n";
+    }
+    if (!$abspath && $linkfiles && (substr($datapath, 0, 1) ne "/")) {
         $err .= "datapath must begin with / when using --link\n";
     }
@@ -135,12 +137,13 @@
     unless defined stat("$dsroot/$product/index.txt");
 
-$dbname      = metadataLookupStr($siteConfig, 'DBNAME') unless defined $dbname;
-my $dbserver = metadataLookupStr($siteConfig, 'DBSERVER');
-my $dbuser   = metadataLookupStr($siteConfig, 'DBUSER');
-my $dbpass   = metadataLookupStr($siteConfig, 'DBPASSWORD');
+$dbname      = metadataLookupStr($siteConfig, 'DS_DBNAME') unless defined $dbname;
+my $dbserver = metadataLookupStr($siteConfig, 'DS_DBSERVER');
+my $dbuser   = metadataLookupStr($siteConfig, 'DS_DBUSER');
+my $dbpass   = metadataLookupStr($siteConfig, 'DS_DBPASSWORD');
 exit ($PS_EXIT_CONFIG_ERROR) unless defined $dbserver and $dbname and $dbuser and $dbpass;
 
 my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
-my %conn_attrs = (PrintError => 1, RaiseError => 1, AutoCommit => 0);
+# mysql cookbook says 'PrintError can interfere with failure detection in some cases'
+my %conn_attrs = (PrintError => 0, RaiseError => 1, AutoCommit => 0);
 
 my $dbh = DBI->connect_cached($dsn, $dbuser, $dbpass, \%conn_attrs)
@@ -209,4 +212,5 @@
     if ($@) { # an error occured
         print STDERR "transaction failed, rolling back error was:\n$@\n";
+        # roll back within eval to prevent rollback failure from terminating the script
         eval {$dbh->rollback();};
         cleanup();
@@ -251,107 +255,110 @@
     }
 
+
     # Note: There is a race condition here. Somebody could sneak in now and add a fileset with
     # fileset_name = $fileset. So later we'll check again that only one fileset with the name exists
 
-
-    my $in;
-    if ($filelist eq "-") {
-        $in = *STDIN;
-    } else {
-        open $in, "<$filelist" or die  "cannot open filelist file $filelist\n";
-    }
-
-    my $lineno = 0;
+    ## create the fileset directory
+    if (! -e $fileset_dir) {
+        if (!mkdir $fileset_dir) {
+            die("failed trying to create fileset directory $fileset_dir");
+        }
+    }
+
     my @files;
-    while (<$in>) {
-        chomp;
-        $lineno++;
-
-        my $filename;
-        my $bytes;
-        my $md5sum;
-        my $filetype;
-
-        my @fields = split /\|/;
-        if (@fields < 4) {
-            die "malformed input line at $filelist line $lineno: $_\n";
-        }
-        $filename = shift @fields;
-        $bytes    = shift @fields;  # if this is null it will be calculated below
-        $md5sum   = shift @fields;  # if this is null it will be calculated below
-        $filetype = shift @fields;
-
-        # make sure the length of the type specific columns is 8
-        while (@fields < 8) {
-            push @fields, undef;
-        }
-        my $ts_cols = [@fields];
-
-        my $hashref = {
-            'file'      => $filename,
-            'bytes'     => $bytes,
-            'md5sum'    => $md5sum,
-            'type'      => $filetype,
-            'ts_cols'   => $ts_cols,
-        };
-
-        push @files, $hashref;
-    }
-
-    die("empty filelist\n") if (@files == 0);
-
-    # Prepare the fileset directory
-    if ($linkfiles or $copyfiles) {
-        ## create the fileset directory
-        if (! -e $fileset_dir) {
-            if (!mkdir $fileset_dir) {
-                die("failed trying to create fileset directory $fileset_dir");
-            }
-        }
-        eval {
-            ## then copy or symlink the files into place
-            foreach my $ref (@files) {
-                my $src = (defined $abspath ? '' : "$datapath/") . "$ref->{file}";
-		my $filename = fileparse($ref->{file}, ());
-                my $dest = "$fileset_dir/$filename";
-
-                if ($linkfiles) {
-                    if (! symlink $src, $dest) {
-                        die("failed trying to link $src to $dest");
-                    }
-                } else {
-                    if (!copy($src, $dest)) {
-                        die("copy($src, $dest) failed");
+    if (!$empty) {
+        my $in;
+        if ($filelist eq "-") {
+            $in = *STDIN;
+        } else {
+            open $in, "<$filelist" or die  "cannot open filelist file $filelist\n";
+        }
+
+        my $lineno = 0;
+        while (<$in>) {
+            chomp;
+            $lineno++;
+
+            my $filename;
+            my $bytes;
+            my $md5sum;
+            my $filetype;
+
+            my @fields = split /\|/;
+            if (@fields < 4) {
+                die "malformed input line at $filelist line $lineno: $_\n";
+            }
+            $filename = shift @fields;
+            $bytes    = shift @fields;  # if this is null it will be calculated below
+            $md5sum   = shift @fields;  # if this is null it will be calculated below
+            $filetype = shift @fields;
+
+            # make sure the length of the type specific columns is 8
+            while (@fields < 8) {
+                push @fields, undef;
+            }
+            my $ts_cols = [@fields];
+
+            my $hashref = {
+                'file'      => $filename,
+                'bytes'     => $bytes,
+                'md5sum'    => $md5sum,
+                'type'      => $filetype,
+                'ts_cols'   => $ts_cols,
+            };
+
+            push @files, $hashref;
+        }
+
+        die("empty filelist\n") if (@files == 0);
+
+        # Prepare the fileset directory
+        if ($linkfiles or $copyfiles) {
+            eval {
+                ## then copy or symlink the files into place
+                foreach my $ref (@files) {
+                    my $src = (defined $abspath ? '' : "$datapath/") . "$ref->{file}";
+                    my $filename = fileparse($ref->{file}, ());
+                    my $dest = "$fileset_dir/$filename";
+
+                    if ($linkfiles) {
+                        if (! symlink $src, $dest) {
+                            die("failed trying to link $src to $dest");
+                        }
+                    } else {
+                        if (!copy($src, $dest)) {
+                            die("copy($src, $dest) failed");
+                        }
                     }
                 }
-            }
-        };
-        if ($@) { # an error occured
-            print STDERR "error preparing the fileset directory: $@\n";
-
-            if (!$no_cleanup && system "rm -r $fileset_dir") {
-                die("failed to remove $fileset_dir");
-            }
-            exit $PS_EXIT_UNKNOWN_ERROR;
-        }
-    }
-
-    # now calculate the md5sum and file size if they weren't provided
-    foreach my $file (@files) {
-	my $filename = fileparse($file->{file}, ());
-        my $path = "$fileset_dir/$filename";
-        if (! -e $path ) {
-            die "file $path not found";
-        }
-        if (!$file->{bytes}) {
-            my @finfo = stat($path);
-            unless (@finfo) {
-                die ("can't stat $path");
-            }
-            $file->{bytes}  = $finfo[7];
-        }
-        if (!$file->{md5sum}) {
-            # Get MD5 sum
-            $file->{md5sum} = file_md5_hex($path);
+            };
+            if ($@) { # an error occured
+                print STDERR "error preparing the fileset directory: $@\n";
+
+                if (!$no_cleanup && system "rm -r $fileset_dir") {
+                    die("failed to remove $fileset_dir");
+                }
+                exit $PS_EXIT_UNKNOWN_ERROR;
+            }
+        }
+
+        # now calculate the md5sum and file size if they weren't provided
+        foreach my $file (@files) {
+            my $filename = fileparse($file->{file}, ());
+            my $path = "$fileset_dir/$filename";
+            if (! -e $path ) {
+                die "file $path not found";
+            }
+            if (!$file->{bytes}) {
+                my @finfo = stat($path);
+                unless (@finfo) {
+                    die ("can't stat $path");
+                }
+                $file->{bytes}  = $finfo[7];
+            }
+            if (!$file->{md5sum}) {
+                # Get MD5 sum
+                $file->{md5sum} = file_md5_hex($path);
+            }
         }
     }
@@ -471,4 +478,5 @@
     --copy      Copy files from datapath to Data Store
     --datapath  path to source files for --copy or --link
+    --abspath   path to source files is absoloute (--datapath not needed)
     --ps0 - ps7 Optional product specific data
     --rm        with --del remove the fileset directory from the Data Store
Index: /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/web/cgi/dsgetindex
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/web/cgi/dsgetindex	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/DataStoreServer/web/cgi/dsgetindex	(revision 24244)
@@ -62,4 +62,6 @@
     # used by the pretty printer
     @path = split(/\//, $directories);
+    # shift off the empty bit
+    shift @path;
 
 } else {
@@ -167,28 +169,30 @@
 # XXX:
 # quick hacky pretty printer
-# Stolen from Eric's Data Store mock up
-#
-
+# Adapted from Eric's Data Store mock up
+#
+
+sub print_free_space {
+    use Filesys::Df;
+	# get free space
+	my $ref = df($DS_ROOT);
+	printf '<p>Free space: %.2f G (%.1f%%)</p>',
+		$ref->{bfree}/(1024*1024),
+		100.0*$ref->{bfree}/$ref->{blocks};
+        print "\n";
+}
 sub pprint_pre {
-    #use Filesys::Df;
 
     print header('text/html', '200 OK');
     print start_html('Data Store');
 
-	# get free space
-#	my $ref = df($DS_ROOT);
-#	printf '<p>Free space: %.2fG (%.1f%%)</p>',
-#		$ref->{bfree}/(1024*1024),
-#		100.0*$ref->{bfree}/$ref->{blocks};
-#        print "\n";
 
 	# return link
         print a({-href=>"./index.txt"}, "Text Version");
 
-	if ($#path > 1) {
+	if ($#path >= 1) {
             print "&nbsp&nbsp&nbsp&nbsp\n";
 
-		my $up = join('/', @path[0..$#path-1]);
-	    print a({-href=>"$up"}, "Up to $up");
+            my $up = join('/', @path[0..$#path-1]);
+	    print a({-href=>".."}, "Up to $up");
 
 	}
@@ -204,4 +208,5 @@
     print p();
 
+    print_free_space();
     print end_html();
 }
@@ -232,10 +237,10 @@
     else {
 		# assumes id is always first field
-	    my $wpath = join('/',@path)."/$toks[0]";
-
-		# if this is a file request, use a download cgi
-		if ($wpath =~ /\.fits\s*$/) {
-			$wpath = '/ds-cgi/dsfits.cgi?'.substr($wpath, 4); # strip' /ds/'
-		}
+	    my $wpath = "./$toks[0]";
+
+            # if this is a file request, use a download cgi
+            if ($wpath =~ /\.fits\s*$/) {
+                    $wpath = '/ds-cgi/dsfits.cgi?'.substr($wpath, 4); # strip' /ds/'
+            }
 
         print a({-href=>$wpath}, $toks[0]);
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/Build.PL
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/Build.PL	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/Build.PL	(revision 24244)
@@ -15,21 +15,24 @@
         'DBI'                   => '1.53',
         'Digest::SHA1'          => 0,
+        'File::Basename'        => 0,
         'File::ExtAttr'         => '1.03',
         'File::Mountpoint'      => '0.01',
         'File::Path'            => '1.08',
         'File::Spec'            => 0,
-        'Filesys::Df'           => '0.92',
         'File::Spec::Functions' => 0,
         'File::Temp'            => 0,
+        'Filesys::Df'           => '0.92',
+        'Log::Dispatch::Email::MailSend' => 0,
         'Log::Log4perl'         => '0.48',
         'Net::Server::Daemonize'=> '0.05',
         'Params::Validate'      => '0.73',
         'SOAP::Lite'            => '0.69',
+        'SQL::Interp'           => '1.06',
         'URI'                   => '1.30',
-        'SQL::Interp'           => '1.06',
     },
     build_requires      => {
         'Test::More'            => '0.49',
         'Test::URI'             => '1.06',
+        'Test::DBUnit'          => '0.20',
     },
     recommends          => {
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/Changes
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/Changes	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/Changes	(revision 24244)
@@ -3,5 +3,18 @@
 0.17
     - retry database transactions when a deadlock is detected
-
+    - add log4perl logging to nebdiskd
+    - base the on disk directory hashing of files only on the dirname()
+      component of keys
+    - add email logging of events to nebdiskd
+    - attempt to optimize _is_valid_object_key() by eliminating an unused join
+    - rename Nebulous::Keys class -> Nebulous::Key
+    - select neb db to use by hashing only the directory component of the key
+    - disallow Nebulous::Server->rename_object() when it would cause the db
+      hash of a key to change
+    - create a pseduo directory structure on key creation
+    - Nebulous::Key parsing and testing improvements
+    - change 'log_level' param to 'trace'
+    - refactor ->find_objects() functionality
+      
 0.16
     - add so_id/name idxs to storage_object_xattr table
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/MANIFEST
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/MANIFEST	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/MANIFEST	(revision 24244)
@@ -4,5 +4,4 @@
 MANIFEST
 README
-Todo
 bin/neb-admin
 bin/neb-fsck
@@ -19,5 +18,5 @@
 examples/uri_test.pl
 init.d/nebdiskd
-lib/Nebulous/Keys.pm
+lib/Nebulous/Key.pm
 lib/Nebulous/Server.pm
 lib/Nebulous/Server.pod
@@ -33,4 +32,5 @@
 t/00_distribution.t 
 t/01_load.t
+t/02_config.t
 t/02_server_setup.t
 t/03_server_create_object.t
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/bin/neb-admin
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/bin/neb-admin	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/bin/neb-admin	(revision 24244)
@@ -174,4 +174,8 @@
         # if the copies xattr is unset and the object has 2 or more instances, we
         # will assume a target of 2 copies
+	
+	# XXX change this: there should be no default value or we will
+	# have a race condition.  user.copies should never get set by
+	# any client until AFTER a file is no longer in use.
         my $copies = $obj->{copies} || 2;
 
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/bin/nebdiskd
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/bin/nebdiskd	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/bin/nebdiskd	(revision 24244)
@@ -16,4 +16,5 @@
 use File::Spec;
 use Filesys::Df;
+use Log::Log4perl;
 use Nebulous::Server::SQL;
 use Net::Server::Daemonize qw( daemonize unlink_pid_file );
@@ -85,4 +86,35 @@
     unless $db && $dbuser && $dbpass;
 
+# start up logging
+my $conf = '
+    log4perl.category.nebdiskd = WARN, Screen, SERVERLOGFILE, Mailer
+
+    log4perl.appender.Screen        = Log::Log4perl::Appender::Screen
+    log4perl.appender.Screen.stderr = 1
+    log4perl.appender.Screen.layout = Log::Log4perl::Layout::PatternLayout
+    log4perl.appender.Screen.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
+
+    log4perl.appender.SERVERLOGFILE           = Log::Log4perl::Appender::File
+    log4perl.appender.SERVERLOGFILE.filename  = /tmp/nebdiskd.log
+    log4perl.appender.SERVERLOGFILE.mode      = append
+    log4perl.appender.SERVERLOGFILE.layout    = Log::Log4perl::Layout::PatternLayout
+    log4perl.appender.SERVERLOGFILE.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
+
+    log4perl.filter.MatchWarn  = Log::Log4perl::Filter::LevelMatch
+    log4perl.filter.MatchWarn.LevelToMatch  = WARN
+    log4perl.filter.MatchWarn.AcceptOnMatch = off
+
+    log4perl.appender.Mailer         = Log::Dispatch::Email::MailSend
+    log4perl.appender.Mailer.to      = ps-ipp-ops@ifa.hawaii.edu
+    log4perl.appender.Mailer.subject = nebdiskd alert
+    log4perl.appender.AppError.Filter= MatchWarn
+    log4perl.appender.Mailer.layout = Log::Log4perl::Layout::PatternLayout
+    log4perl.appender.Mailer.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
+';
+Log::Log4perl::init(\$conf);
+my $log = Log::Log4perl::get_logger("nebdiskd");
+$log->level('WARN');
+$log->level('DEBUG') if $debug;
+
 daemonize(
     $user,     # User
@@ -104,8 +136,8 @@
                 poll_interval   => $poll_interval,
                 debug           => $debug,
-        ) or die "poll_mounts() should not have returned";
+        );
     };
-    if ($!) {
-        warn $@;
+    if ($@) {
+        $log->warn($@);
     }
 
@@ -113,5 +145,5 @@
 }
 
-die "poll loop exited -- THIS SHOULD NOT HAPPEN";
+$log->logdie("poll loop exited -- THIS SHOULD NOT HAPPEN");
 
 
@@ -141,14 +173,14 @@
         # determine valid mountpoints
         foreach my $mnt (@$mounts) {
-            print "checking $mnt\n" if $debug;
+            $log->debug("checking $mnt");
             # this /SHOULD/ fail if the mount point is handled by the
             # automounter and it fails to mount
             eval {
                 unless (is_mountpoint($mnt)) {
-                    die "$mnt is not a valid mountpoint\n";
+                    $log->warn("$mnt is not a valid mountpoint");
                 }
             };
             if ($@) {
-                print $@ if $debug;
+                $log->warn($@);
                 $d_query->execute($mnt);
                 next;
@@ -160,10 +192,10 @@
             my $dev_info = df($mnt, 1024);
             unless (defined $dev_info) {
-                print "can't find device info for $mnt\n" if $debug;
+                $log->error("can't find device info for $mnt");
                 next;
             }
 
             $r_query->execute($mnt, @$dev_info{qw( blocks used )});
-            print "adding $mnt to db\n" if $debug;
+            $log->debug("adding $mnt to db");
 
         }
@@ -172,10 +204,10 @@
 
         $dbh->commit;
-        print "commited to database\n" if $debug;
+        $log->debug("commited to database");
     };
     if ($@) {
         $dbh->rollback;
-        print "rolledback transaction\n" if $debug;
-        warn $@;
+        $log->debug("rolledback transaction");
+        $log->logdie($@);
     }
 }
@@ -188,6 +220,6 @@
 
     if (!-f $rcfile) {
-        open(my $fh, '>', $rcfile) or die "can't open file: $!";
-        close($fh) or die "can't close file:$!";
+        open(my $fh, '>', $rcfile) or $log->logdie("can't open file: $!");
+        close($fh) or $log->logdie("can't close file:$!");
     }
 
@@ -226,6 +258,6 @@
     };
     if ($@) { 
-        $db->rollback;
-        die $@;
+        $dbh->rollback;
+        $log->logdie($@);
     }
 
@@ -242,7 +274,7 @@
     foreach my $path (@$mounts) {
         if (stat File::Spec->canonpath($path)) {
-            print "stated $path\n" if $debug;
+            $log->debug("stated $path");
         } else {
-            warn "can not stat path: $path";
+            $log->warn("can not stat path: $path");
         }
     }
@@ -263,9 +295,9 @@
   ### get the currently listed pid
   if( ! open(_PID,$pid_file) ){
-    die "Couldn't open existant pid_file \"$pid_file\" [$!]\n";
+    $log->logdie("Couldn't open existant pid_file \"$pid_file\" [$!]");
   }
   my $_current_pid = <_PID>;
   close _PID;
-  my $current_pid = $_current_pid =~ /^(\d{1,10})/ ? $1 : die "Couldn't find pid in existing pid_file";
+  my $current_pid = $_current_pid =~ /^(\d{1,10})/ ? $1 : $log->logdie("Couldn't find pid in existing pid_file");
 
   my $exists = undef;
@@ -289,10 +321,10 @@
 
         if( $current_pid == $$ ){
-            warn "Pid_file created by this same process. Doing nothing.\n";
+            $log->warn("Pid_file created by this same process. Doing nothing.");
             return 1;
         }else{
             kill 'TERM', $current_pid
-                or die "Failed to signal process ($current_pid)";
-            unlink $pid_file || die "Couldn't remove pid_file \"$pid_file\" [$!]\n";
+                or $log->logdie("Failed to signal process ($current_pid)");
+            unlink $pid_file || $log->logdie("Couldn't remove pid_file \"$pid_file\" [$!]");
         }
     }
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/docs/apache2-mod_perl-startup.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/docs/apache2-mod_perl-startup.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/docs/apache2-mod_perl-startup.pl	(revision 24244)
@@ -0,0 +1,44 @@
+use lib qw(/home/httpd/perl);
+
+# enable if the mod_perl 1.0 compatibility is needed
+#use Apache2::compat ();
+
+use ModPerl::Util (); #for CORE::GLOBAL::exit
+
+use Apache2::RequestRec ();
+use Apache2::RequestIO ();
+use Apache2::RequestUtil ();
+
+use Apache2::ServerRec ();
+use Apache2::ServerUtil ();
+use Apache2::Connection ();
+use Apache2::Log ();
+
+use APR::Table ();
+
+use ModPerl::Registry ();
+
+use Apache2::Const -compile => ':common';
+use APR::Const -compile => ':common';
+
+use Apache::DBI;
+use DBI;
+use Nebulous::Server::SOAP;
+use Nebulous::Server::Apache;
+use Nebulous::Server;
+
+my $dsn         = 'DBI:mysql:database=nebulous:host=ipp019.ifa.hawaii.edu';
+my $dbuser      = 'nebulous';
+my $dbpasswd    = '@neb@';
+
+#$Apache::DBI::DEBUG = 1;
+#Apache::DBI->connect_on_init( $dsn, $dbuser, $dbpasswd );
+Apache::DBI->setPingTimeOut($dsn, 10);
+Nebulous::Server::SOAP->new_on_init(
+    dsn         => $dsn,
+    dbuser      => $dbuser,
+    dbpasswd    => $dbpasswd,
+    log_level   => 'all',
+);
+
+1;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/docs/install.txt
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/docs/install.txt	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/docs/install.txt	(revision 24244)
@@ -56,10 +56,20 @@
 
     neb-initdb
-    neb-addvol --name ipp000.0 --uri file:///data/ipp000.0/nebulous
-    neb-addvol --name ipp000.1 --uri file:///data/ipp000.1/nebulous
-    neb-addvol --name ipp002.0 --uri file:///data/ipp002.0/nebulous
-    neb-addvol --name ipp003.0 --uri file:///data/ipp003.0/nebulous
-
-#    perl -e 'for (1..24) { $i = sprintf "po%02d", $_ ; system "neb-addvol --name $i --uri file:/$i/nebulous" }'
+
+    neb-voladd --vhost ipp000 --vname ipp000.0 --uri file:///data/ipp000.0/nebulous
+    neb-voladd --vhost ipp001 --vname ipp000.1 --uri file:///data/ipp000.1/nebulous
+    neb-voladd --vhost ipp002 --vname ipp002.0 --uri file:///data/ipp002.0/nebulous
+    neb-voladd --vhost ipp003 --vname ipp003.0 --uri file:///data/ipp003.0/nebulous
+    neb-voladd --vhost ipp022 --vname ipp022.0 --uri file:///data/ipp022.0/nebulous
+
+    # or in bulk:
+    
+    for i in ipp005 ipp006 ipp007 ipp009 ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp020 ipp021 ipp022 ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029 ipp030 ipp031 ipp032 ipp033 ipp034 ipp035 
+    do 
+        /usr/bin/neb-voladd --vname ${i}.0 --vhost $i --uri file:///data/${i}.0/nebulous_beta
+    done
+
+#    perl -e 'for (1..24) { $i = sprintf "po%02d", $_ ; system "neb-voladd
+#    --vname $i --vhost $i --uri file:/$i/nebulous" }'
     .
     .
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Key.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Key.pm	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Key.pm	(revision 24244)
@@ -0,0 +1,279 @@
+# Copyright (c) 2004  Joshua Hoblitt
+#
+# $Id: Keys.pm,v 1.3 2008-09-09 02:18:47 jhoblitt Exp $
+
+package Nebulous::Key;
+
+use strict;
+use warnings FATAL => qw( all );
+
+our $VERSION = '0.02';
+
+use base qw( Exporter Class::Accessor::Fast );
+
+use File::Spec;
+use URI::file;
+use URI;
+use overload '""' => \&_stringify_key;
+
+our @EXPORT_OK = qw(
+    parse_neb_key
+    parse_neb_volume
+);
+
+__PACKAGE__->mk_ro_accessors(qw( path volume soft_volume ));
+
+sub parse_neb_key
+{
+    my ($key, $volume) = @_;
+    return unless defined $key;
+
+    # white space is not allowed
+    if ($key =~ qr/\s+/) {
+        die "keys and URIs may not contain whitespace";
+    }
+
+    my $uri = URI->new($key);
+    my $scheme = $uri->scheme;
+    my $path = $uri->opaque;
+    
+    my $volume_name;
+    my $soft_volume;
+    # if this is a valid uri 
+    if (defined $scheme) {
+        # if so, does it use the neb scheme?
+        die "URI does not use the 'neb' scheme"
+            unless $scheme eq 'neb'; 
+
+        # neb:path (not allowed)
+        # neb://<volume name>/...
+        # neb:/path... (leading '/' is stripped)
+        # neb:///path... (same as neb:/path)
+
+        # volume specifier must be at least one character
+        # strip off volume component if it exists
+        $path =~ s|^//([^/]+)||;
+
+        # a new is not allowed to be just a volume specifier, it must have a
+        # path component to it
+        unless (length $path) {
+            die "neb URI scheme requires a path component"; 
+        }
+        
+        # ignore key supplied volume name if one is passed as a parameter
+        $volume ||= $1;
+        my $volume_info = parse_neb_volume($volume);
+
+        # magically eat the "any" volume
+        if (defined $volume_info->{volume} and $volume_info->{volume} ne 'any') {
+            $volume_name = $volume_info->{volume};
+            $soft_volume = $volume_info->{soft_volume};
+        }
+
+        # require a leading slash if there is no volume name
+        if ((not defined $volume_name) and (not $path =~ m|^/|)) {
+            die "neb URI scheme requires a leading slash, eg. neb:/";
+        }
+    } else {
+        my $volume_info = parse_neb_volume($volume);
+
+        # magically eat the "any" volume
+        if (defined $volume_info->{volume} and $volume_info->{volume} ne 'any') {
+            $volume_name = $volume_info->{volume};
+            $soft_volume = $volume_info->{soft_volume};
+        }
+    }
+
+    # strip leading slashes
+    $path =~ s|^/+||;
+
+    # strip leading '.' or '..'
+    $path =~ s|^\.{1,2}||;
+
+    # remove multiple /'s and trailing slashes
+    $path = File::Spec->canonpath($path);
+
+    return __PACKAGE__->new({
+        volume      => $volume_name,
+        soft_volume => $soft_volume,
+        path        => $path,
+    });
+}
+
+
+sub parse_neb_volume
+{
+    my $volume = shift;
+    return unless defined $volume;
+
+    my $soft_volume;
+    # check to see if there is a tilde and remove it if found
+    unless (defined $volume and $volume =~ s/^~//) {
+        $soft_volume = 1;
+    }
+
+    return({ volume => $volume, soft_volume => $soft_volume });
+}
+
+
+sub _stringify_key
+{
+    my $self = shift;
+
+    my $path        = $self->path;
+    my $volume      = $self->volume || "";
+    my $soft_volume = $self->soft_volume ? '~' : "";
+
+    return "neb://${soft_volume}${volume}/$path";
+}
+
+
+1;
+
+__END__
+
+=pod
+
+=head1 NAME
+
+Nebulous::Key - Nebulous Keys Explained
+
+=head1 DESCRIPTION
+
+The Nebulous system interprets it's storage object keys in a I<slightly>
+magical manner.  It supports both plain vanilla C<key strings> and keys in the
+form of an L<URI>.  In the L<URI> form a specific storage volume can be implied
+(as a request, not a requirement).  Both key forms are subject to mangling such
+that I<IF> the key I<looks> like a C<POSIX> semantics file path, it is
+canocalized.
+
+=head1 RESERVED SYNTAX
+
+This particular syntax is disallowed and is reserved for future use.
+
+    <neb:<phrase>/<path>
+
+=head1 EXAMPLES
+
+=head2 Key Strings
+
+    key:        foo/bar/baz/quix
+    mangled to: foo/bar/baz/quix
+    volume:     any
+
+    key:        /foo/bar/baz/quix
+    mangled to: foo/bar/baz/quix
+    volume:     any
+
+    key:        //foo/bar/baz/quix
+    mangled to: foo/bar/baz/quix
+    volume:     any
+
+    key:        ///foo/bar/baz/quix
+    mangled to: foo/bar/baz/quix
+    volume:     any
+
+    key:        foo////bar/baz/quix
+    mangled to: foo/bar/baz/quix
+    volume:     any
+
+    key:        foo/bar/baz/quix/
+    mangled to: foo/bar/baz/quix
+    volume:     any
+
+=head2 URI w/ volume name
+
+neb://<volume>/<path>
+
+    key:        neb://foo/bar/baz/quix
+    mangled to: bar/baz/quix
+    volume:     foo
+
+    key:        neb://foo//bar/baz/quix
+    mangled to: bar/baz/quix
+    volume:     foo
+
+    key:        neb://foo///bar/baz/quix
+    mangled to: bar/baz/quix
+    volume:     foo
+
+    key:        neb://foo/bar///baz/quix
+    mangled to: bar/baz/quix
+    volume:     foo
+
+    key:        neb://foo/bar/baz/quix/
+    mangled to: bar/baz/quix
+    volume:     foo
+
+=head2 URI w/o volume name
+
+neb:/<path>
+
+    key:        neb:/foo/bar/baz/quix
+    mangled to: foo/bar/baz/quix
+    volume:     any
+
+    key:        neb:///foo/bar/baz/quix
+    mangled to: foo/bar/baz/quix
+    volume:     any
+
+    key:        neb://///foo/bar/baz/quix
+    mangled to: foo/bar/baz/quix
+    volume:     any
+
+=head2 Forbidden Keys
+
+=head3 Key with whitespace
+
+    "/ foo/bar/baz/quix"
+    " /foo/bar/baz/quix"
+    "/foo/bar/baz/quix "
+
+=head3 URI with whitespace
+
+    "neb ://foo/bar/baz/quix"
+    "neb:// foo/bar/baz/quix"
+    " neb://foo/bar/baz/quix"
+    "neb://foo/bar/baz/quix "
+
+=head3 URI with out a volume requires a leading slash
+
+    neb:foo/bar/baz/quix
+
+=head1 CREDITS
+
+Just me, myself, and I.
+
+=head1 SUPPORT
+
+Please contact the author directly via e-mail.
+
+=head1 AUTHOR
+
+Joshua Hoblitt <jhoblitt@cpan.org>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2008  Joshua Hoblitt.  All rights reserved.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place - Suite 330, Boston, MA  02111-1307, USA.
+
+The full text of the license can be found in the LICENSE file included with
+this module, or in the L<perlgpl> Pod as supplied with Perl 5.8.1 and later.
+
+=head1 SEE ALSO
+
+L<Nebulous::Server>, L<Nebulous::Client>, L<nebclient(3)>
+
+=cut
Index: anches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Keys.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Keys.pm	(revision 24243)
+++ 	(revision )
@@ -1,267 +1,0 @@
-# Copyright (c) 2004  Joshua Hoblitt
-#
-# $Id: Keys.pm,v 1.3 2008-09-09 02:18:47 jhoblitt Exp $
-
-package Nebulous::Keys;
-
-use strict;
-use warnings FATAL => qw( all );
-
-our $VERSION = '0.02';
-
-use base qw( Exporter Class::Accessor::Fast );
-
-use File::Spec;
-use URI::file;
-use URI;
-use overload '""' => \&_stringify_key;
-
-our @EXPORT_OK = qw(
-    parse_neb_key
-    parse_neb_volume
-);
-
-__PACKAGE__->mk_ro_accessors(qw( path volume soft_volume ));
-
-sub parse_neb_key
-{
-    my ($key, $volume) = @_;
-    return unless defined $key;
-
-    # white space is not allowed
-    if ($key =~ qr/\s+/) {
-        die "keys and URIs may not contain whitespace";
-    }
-
-    my $uri = URI->new($key);
-    my $scheme = $uri->scheme;
-    my $path = $uri->opaque;
-    
-    my $volume_name;
-    my $soft_volume;
-    # if this is a valid uri 
-    if (defined $scheme) {
-        # if so, does it use the neb scheme?
-        die "URI does not use the 'neb' scheme"
-            unless $scheme eq 'neb'; 
-
-        # neb:path (not allowed)
-        # neb://<volume name>/...
-        # neb:/path... (leading '/' is stripped)
-        # neb:///path... (same as neb:/path)
-
-        # volume specifier must be at least one character
-        # strip off volume component if it exists
-        $path =~ s|^//([^/]+)||;
-        
-        # ignore key supplied volume name if one is passed as a parameter
-        $volume ||= $1;
-        my $volume_info = parse_neb_volume($volume);
-
-        # magically eat the "any" volume
-        if (defined $volume_info->{volume} and $volume_info->{volume} ne 'any') {
-            $volume_name = $volume_info->{volume};
-            $soft_volume = $volume_info->{soft_volume};
-        }
-
-        # require a leading slash if there is no volume name
-        if ((not defined $volume_name) and (not $path =~ m|^/|)) {
-            die "neb URI scheme requires a leading slash, eg. neb:/";
-        }
-    } else {
-        my $volume_info = parse_neb_volume($volume);
-
-        # magically eat the "any" volume
-        if (defined $volume_info->{volume} and $volume_info->{volume} ne 'any') {
-            $volume_name = $volume_info->{volume};
-            $soft_volume = $volume_info->{soft_volume};
-        }
-    }
-
-    # strip leading slashes
-    $path =~ s|^/+||;
-
-    # remove multiple /'s and trailing slashes
-    $path = File::Spec->canonpath($path);
-
-    return __PACKAGE__->new({
-        volume      => $volume_name,
-        soft_volume => $soft_volume,
-        path        => $path,
-    });
-}
-
-sub parse_neb_volume
-{
-    my $volume = shift;
-    return unless defined $volume;
-
-    my $soft_volume;
-    # check to see if there is a tilde and remove it if found
-    unless (defined $volume and $volume =~ s/^~//) {
-        $soft_volume = 1;
-    }
-
-    return({ volume => $volume, soft_volume => $soft_volume });
-}
-
-sub _stringify_key
-{
-    my $self = shift;
-
-    my $path        = $self->path;
-    my $volume      = $self->volume || "";
-    my $soft_volume = $self->soft_volume ? '~' : "";
-
-    return "neb://${soft_volume}${volume}/$path";
-}
-
-1;
-
-__END__
-
-=pod
-
-=head1 NAME
-
-Nebulous::Keys - Nebulous Keys Explained
-
-=head1 DESCRIPTION
-
-The Nebulous system interprets it's storage object keys in a I<slightly>
-magical manner.  It supports both plain vanilla C<key strings> and keys in the
-form of an L<URI>.  In the L<URI> form a specific storage volume can be implied
-(as a request, not a requirement).  Both key forms are subject to mangling such
-that I<IF> the key I<looks> like a C<POSIX> semantics file path, it is
-canocalized.
-
-=head1 RESERVED SYNTAX
-
-This particular syntax is disallowed and is reserved for future use.
-
-    <neb:<phrase>/<path>
-
-=head1 EXAMPLES
-
-=head2 Key Strings
-
-    key:        foo/bar/baz/quix
-    mangled to: foo/bar/baz/quix
-    volume:     any
-
-    key:        /foo/bar/baz/quix
-    mangled to: foo/bar/baz/quix
-    volume:     any
-
-    key:        //foo/bar/baz/quix
-    mangled to: foo/bar/baz/quix
-    volume:     any
-
-    key:        ///foo/bar/baz/quix
-    mangled to: foo/bar/baz/quix
-    volume:     any
-
-    key:        foo////bar/baz/quix
-    mangled to: foo/bar/baz/quix
-    volume:     any
-
-    key:        foo/bar/baz/quix/
-    mangled to: foo/bar/baz/quix
-    volume:     any
-
-=head2 URI w/ volume name
-
-neb://<volume>/<path>
-
-    key:        neb://foo/bar/baz/quix
-    mangled to: bar/baz/quix
-    volume:     foo
-
-    key:        neb://foo//bar/baz/quix
-    mangled to: bar/baz/quix
-    volume:     foo
-
-    key:        neb://foo///bar/baz/quix
-    mangled to: bar/baz/quix
-    volume:     foo
-
-    key:        neb://foo/bar///baz/quix
-    mangled to: bar/baz/quix
-    volume:     foo
-
-    key:        neb://foo/bar/baz/quix/
-    mangled to: bar/baz/quix
-    volume:     foo
-
-=head2 URI w/o volume name
-
-neb:/<path>
-
-    key:        neb:/foo/bar/baz/quix
-    mangled to: foo/bar/baz/quix
-    volume:     any
-
-    key:        neb:///foo/bar/baz/quix
-    mangled to: foo/bar/baz/quix
-    volume:     any
-
-    key:        neb://///foo/bar/baz/quix
-    mangled to: foo/bar/baz/quix
-    volume:     any
-
-=head2 Forbidden Keys
-
-=head3 Key with whitespace
-
-    "/ foo/bar/baz/quix"
-    " /foo/bar/baz/quix"
-    "/foo/bar/baz/quix "
-
-=head3 URI with whitespace
-
-    "neb ://foo/bar/baz/quix"
-    "neb:// foo/bar/baz/quix"
-    " neb://foo/bar/baz/quix"
-    "neb://foo/bar/baz/quix "
-
-=head3 URI with out a volume requires a leading slash
-
-    neb:foo/bar/baz/quix
-
-=head1 CREDITS
-
-Just me, myself, and I.
-
-=head1 SUPPORT
-
-Please contact the author directly via e-mail.
-
-=head1 AUTHOR
-
-Joshua Hoblitt <jhoblitt@cpan.org>
-
-=head1 COPYRIGHT
-
-Copyright (C) 2008  Joshua Hoblitt.  All rights reserved.
-
-This program is free software; you can redistribute it and/or modify it under
-the terms of the GNU General Public License as published by the Free Software
-Foundation; either version 2 of the License, or (at your option) any later
-version.
-
-This program is distributed in the hope that it will be useful, but WITHOUT ANY
-WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
-PARTICULAR PURPOSE.  See the GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License along with
-this program; if not, write to the Free Software Foundation, Inc., 59 Temple
-Place - Suite 330, Boston, MA  02111-1307, USA.
-
-The full text of the license can be found in the LICENSE file included with
-this module, or in the L<perlgpl> Pod as supplied with Perl 5.8.1 and later.
-
-=head1 SEE ALSO
-
-L<Nebulous::Server>, L<Nebulous::Client>, L<nebclient(3)>
-
-=cut
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server.pm	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server.pm	(revision 24244)
@@ -9,5 +9,5 @@
 no warnings qw( uninitialized );
 
-our $VERSION = '0.15';
+our $VERSION = '0.17';
 
 use base qw( Class::Accessor::Fast );
@@ -15,13 +15,14 @@
 use DBI;
 use Digest::SHA1 qw( sha1_hex );
+use File::Basename qw( basename dirname fileparse );
 use File::ExtAttr qw( setfattr );
 use File::Path;
 use File::Spec;
-use Log::Log4perl;
+use Log::Log4perl qw( :levels );
+use Nebulous::Key qw( parse_neb_key parse_neb_volume );
 use Nebulous::Server::Config;
 use Nebulous::Server::Log;
 use Nebulous::Server::SQL;
-use Nebulous::Keys qw( parse_neb_key parse_neb_volume );
-use Params::Validate qw( validate_pos SCALAR SCALARREF UNDEF );
+use Params::Validate qw( validate validate_pos SCALAR SCALARREF UNDEF BOOLEAN );
 use URI::file;
 
@@ -36,9 +37,18 @@
 
     # let Nebulous::Server::Config validate our params
-    my $config = Nebulous::Server::Config->init( @_ );
+    my $config = Nebulous::Server::Config->new( @_ );
+
+    return $class->new_from_config($config);
+}
+
+
+sub new_from_config
+{
+    my ($class, $config) = @_;
 
     # log4perl is not avaliable until we call init()
     Nebulous::Server::Log->init($config);
     my $log = Log::Log4perl::get_logger( "Nebulous::Server" );
+    $log->level($config->trace);
 
     my $sql = Nebulous::Server::SQL->new;
@@ -51,21 +61,34 @@
     $self->config($config);
 
-    # ask for the db handle as a means of validating the database parameters
-    $self->db;
-
     $log->debug( "leaving" );
-    
+
     return $self;
 }
 
-
-sub db
-{
-    my $self = shift;
-
-    if (@_) {
-        $self->{db} = $_[0];
-        return $self;
-    }
+# EAM : In the 'distributed' version of Nebulous, there is a collection of N databases
+# the db_index uniquely defines the db used by a given key
+sub _db_index_for_key
+{
+    my ($self, $key) = @_;
+
+    my $config  = $self->config;
+
+    my $db_index = 0;
+    die "key not defined" unless defined $key;
+
+    # hash the key to select the correct database instance
+    # only use the first 8 hex chars... have to be careful to avoid an int
+    # overflow here
+
+    # hash only the directory component of the path and not the filename
+    my $path = dirname($key->path);
+    $db_index = unpack("h8", sha1_hex($path)) % $config->n_db;
+
+    return $db_index;
+}
+
+sub _db_for_index
+{
+    my ($self, $db_index) = @_;
 
     my $log     = $self->log;
@@ -73,11 +96,18 @@
     my $config  = $self->config;
 
+    # lookup to see if we have a stored dbh for this database
+    my $dbh = $self->{dbs}[$db_index];
     # if the dbh is still alive, return it
-    if (defined $self->{db} and $self->{db}->ping) {
+    if (defined $dbh and $dbh->ping) {
         $log->debug("db handle is still alive");
-        return $self->{db};
+        return $dbh;
     }
     # otherwise create a new connection
     $log->debug("db handle is dead/unopened");
+
+    # lookup database info
+    my $db_config = $config->db($db_index);
+    die "can't find database configuration info for database # $db_index"
+        unless $db_config;
 
     # if we're running under mod_perl & Apache::DBI is loaded we want to
@@ -86,10 +116,9 @@
     # processes and the database might have gone away on us.  Apache::DBI will
     # take care of getting a valid dbh back.
-    my $db;
     eval {
-        $db = DBI->connect_cached(
-            $config->dsn,
-            $config->dbuser,
-            $config->dbpasswd,
+        $dbh = DBI->connect_cached(
+            $db_config->dsn,
+            $db_config->dbuser,
+            $db_config->dbpasswd,
             {
                 RaiseError => 1,
@@ -99,20 +128,40 @@
         );
 
-        $db->do( $sql->set_transaction_model );
-        $log->debug( "connected to database: ", sub { $db->data_sources; } );
-        $db->commit;
+        $dbh->do( $sql->set_transaction_model );
+        $log->debug( "connected to database: ", sub { $dbh->data_sources; } );
+        $dbh->commit;
         $log->debug("commit");
     };
     if ( $@ ) {
-        $db->rollback if $db;
+        $dbh->rollback if $dbh;
         $log->debug("rollback");
         $log->logdie( "database error: $@" );
     }
 
-    $self->{db} = $db;
-
-    return $db;
-}
-
+    $self->{dbs}[$db_index] = $dbh;
+
+    return $dbh;
+}
+
+sub db
+{
+    my $self = shift;
+
+    my ($key) = validate_pos(@_,
+        {
+            isa => 'Nebulous::Key',
+        },
+    );
+
+    my $log     = $self->log;
+    my $sql     = $self->sql;
+    my $config  = $self->config;
+
+    die "key not defined" unless defined $key;
+    my $db_index = $self->_db_index_for_key($key);
+
+    my $dbh = $self->_db_for_index($db_index);
+    return $dbh;
+}
 
 sub create_object
@@ -137,18 +186,18 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug( "entered - @_" );
-
     # vol_name overrides the key implied volume
     $key = parse_neb_key($key, $vol_name);
     $vol_name = $key->volume;
 
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug( "entered - @_" );
+
     # the key's volume can't be validiated on input for this method so we have
     # to check it after parsing the key
     if (defined $vol_name
-        and not $self->_is_valid_volume_name($key->volume)) {
+        and not $self->_is_valid_volume_name($key, $key->volume)) {
         if ($key->soft_volume) {
             $log->warn( "$vol_name is not a known volume name" );
@@ -160,5 +209,7 @@
         
     my ($vol_id, $vol_host, $vol_path, $vol_xattr)
-        = $self->_get_storage_volume($vol_name, $key->soft_volume);
+        = $self->_get_storage_volume($key, $vol_name, $key->soft_volume);
+
+    my $parent_id = $self->_resolve_dir_parent_id(key => $key, create => 1);
 
     my $uri;
@@ -168,5 +219,5 @@
                 # create storage_object
                 my $query = $db->prepare_cached( $sql->new_object ); 
-                $query->execute('NULL', $key->path);
+                $query->execute('NULL', $key->path, basename($key->path), $parent_id);
             }
 
@@ -214,5 +265,5 @@
 
             # TODO add some stuff here to retry if unsucessful
-            $uri = $self->_create_empty_instance_file($key->path, $so_id, $ins_id, $vol_path, $vol_xattr);
+            $uri = $self->_create_empty_instance_file($key, $so_id, $ins_id, $vol_path, $vol_xattr);
             $log->debug("created $uri on volume ID: $vol_id");
 
@@ -246,4 +297,125 @@
 
 
+sub _resolve_dir_parent_id
+{
+    my $self = shift;
+
+    my %p = validate(@_,
+        {
+            key     => {
+                isa         => 'Nebulous::Key',
+            },
+            create  => {
+                type        => BOOLEAN,
+                optional    => 1,
+                default     => undef,
+            },
+            # return dir_id instead of parent_id
+            dir_id  => {
+                type        => BOOLEAN,
+                optional    => 1,
+                default     => undef,
+            },
+        }
+    );
+
+    my $key = $p{key};
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug( "entered - @_" );
+
+    # no path means '/', which has a dir_id & parent_id of 1
+    return 1 if $key->path eq '';
+
+    # resolve parent directory
+    my @dirs;
+
+    # File::Spec->splitpath was causing ->splitdir to always an extra dir
+    # named "" because of a trailing /
+    unless ($p{dir_id}) {
+        @dirs = File::Spec->splitdir(dirname($key->path));
+    } else {
+        @dirs = File::Spec->splitdir($key->path);
+    }
+    # dirname returns "." if there is no dir component to the path, we have
+    # to filter this out
+    @dirs = grep(!/^\.$/, @dirs);
+
+    $log->debug("looking for dirs - ", join(" : ", @dirs), "\n");
+
+    # start at the root dir; '/' == 1
+    my $parent_id = 1;
+    my $dir_id;
+TRANS: while (1) {
+        eval {
+            foreach my $dir (@dirs) {
+                $dir_id = undef;
+                {
+                    my $query = $db->prepare_cached($sql->get_directory); 
+                    $query->execute($parent_id, $dir);
+                    if ($query->rows) {
+                        $dir_id = $query->fetchrow_hashref->{'dir_id'};
+                        $log->debug("resolved $dir to dir_id: $dir_id");
+                    }
+                    $query->finish;
+                }
+
+                # if we found a dir_id, a row for this directory already exists
+                if (defined $dir_id) {
+                    $parent_id = $dir_id;
+                    # note that you can't exit an eval {} with next
+                    next;
+                }
+
+                # else dir doesn't exist
+                unless ($p{create}) {
+                    # resolution failed
+                    $parent_id = undef;
+                    last;
+                }
+
+                {
+                    # dir doesn't exist, create it
+                    my $query = $db->prepare_cached($sql->new_directory);
+                    $query->execute($dir, $parent_id);
+                }
+
+                # get the dir_id of the new directory entry 
+                {
+                    my $query = $db->prepare_cached($sql->last_insert_id);
+                    $query->execute();
+
+                    # the new dir_id will be the parent_id of the next
+                    # descendent directory
+                    ($parent_id) = $query->fetchrow_array;
+                    $query->finish;
+                }
+                $log->logdie("failed to get LAST_INSERT_ID()")
+                    unless $parent_id;
+
+                $db->commit;
+            }
+        };
+        if ($@) {
+            $db->rollback;
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie("error: $@");
+        }
+        last;
+    }
+
+    $log->debug("leaving");
+
+    return $p{dir_id} ? $dir_id : $parent_id;
+}
+
+
 sub rename_object
 {
@@ -265,14 +437,20 @@
         },
     );
-
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug("entered - @_");
 
     # ignore volumes
     $key    = parse_neb_key($key);
     $newkey = parse_neb_key($newkey);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug("entered - @_");
+
+    # XXX this may require database migration in the future
+    unless ($self->_db_index_for_key($key)
+         == $self->_db_index_for_key($newkey)) {
+        $log->logdie("can not rename objects across distributed database boundaries");
+    }
 
 TRANS: while (1) {
@@ -281,5 +459,5 @@
             my $query = $db->prepare_cached($sql->rename_object); 
             # this SQL statment takes the new key name as the first param
-            my $rows = $query->execute($newkey->path, $key->path);
+            my $rows = $query->execute($newkey->path, basename($newkey->path), $self->_resolve_dir_parent_id(key => $newkey, create => 1), $key->path);
 
             # if we affected more then one row something very bad has happened.
@@ -309,5 +487,4 @@
 }
 
-
 sub swap_objects
 {
@@ -328,10 +505,4 @@
         },
     );
-
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug("entered - @_");
 
     # ignore volumes
@@ -339,49 +510,66 @@
     $key2 = parse_neb_key($key2);
 
-    # order of operations for the swap:
+    my $log = $self->log;
+    my $sql = $self->sql;
+
+    my $dbidx1 = $self->_db_index_for_key($key1);
+    my $dbidx2 = $self->_db_index_for_key($key2);
+    die "cannot swap keys not stored on the same database" unless ($dbidx1 == $dbidx2);
+
+    my $dbh1 = $self->_db_for_index($dbidx1);
+    my $dbh2 = $self->_db_for_index($dbidx2);
+    die "different db handles for the same db?" unless ($dbh1 == $dbh2);
+
+    $log->debug("entered - @_");
+
+    # order of operations for the swap with a single db is:
     # key1 -> key1.swap
     # key2 -> key1
     # key1.swap -> key2
 
-TRANS: while (1) {
+    my $db = $dbh1;
+  TRANS: while (1) {
         eval {
             {
-                # key1 -> key1.swap
-                my $query = $db->prepare_cached($sql->rename_object); 
-                # this SQL statment takes the new key name as the first param
-                my $rows = $query->execute($key1->path . ".swap", $key1->path);
-
-                # if we affected more then one row something very bad has happened.
-                unless ($rows == 1) {
-                    $query->finish;
-                    $log->logdie("affected row count is $rows instead of 1");
-                }
-            }
-
-            {
-                # key2 -> key1
-                my $query = $db->prepare_cached($sql->rename_object); 
-                # this SQL statment takes the new key name as the first param
-                my $rows = $query->execute($key1->path, $key2->path);
-
-                # if we affected more then one row something very bad has happened.
-                unless ($rows == 1) {
-                    $query->finish;
-                    $log->logdie("affected row count is $rows instead of 1");
-                }
-            }
-
-            {
-                # key1.swap -> key2
-                my $query = $db->prepare_cached($sql->rename_object); 
-                # this SQL statment takes the new key name as the first param
-                my $rows = $query->execute($key2->path, $key1->path . ".swap");
-
-                # if we affected more then one row something very bad has happened.
-                unless ($rows == 1) {
-                    $query->finish;
-                    $log->logdie("affected row count is $rows instead of 1");
-                }
-            }
+              # key1 -> key1.swap
+              my $query = $db->prepare_cached($sql->rename_object); 
+              # this SQL statment takes the new key name as the first param
+              # XXX currently using a bogus dir_id -- this may cause a problem
+              # someday but it's unlikley as it's contained entirely in the
+              # transaction
+              my $rows = $query->execute($key1->path . ".swap", basename($key1->path) . ".swap", 1, $key1->path);
+
+              # if we affected more then one row something very bad has happened.
+              unless ($rows == 1) {
+                  $query->finish;
+                  $log->logdie("affected row count is $rows instead of 1");
+              }
+          }
+
+          {
+              # key2 -> key1
+              my $query = $db->prepare_cached($sql->rename_object); 
+              # this SQL statment takes the new key name as the first param
+              my $rows = $query->execute($key1->path, basename($key1->path), $self->_resolve_dir_parent_id(key => $key1, create => 1), $key2->path);
+
+              # if we affected more then one row something very bad has happened.
+              unless ($rows == 1) {
+                  $query->finish;
+                  $log->logdie("affected row count is $rows instead of 1");
+              }
+          }
+
+          {
+              # key1.swap -> key2
+              my $query = $db->prepare_cached($sql->rename_object); 
+              # this SQL statment takes the new key name as the first param
+              my $rows = $query->execute($key2->path, basename($key2->path), $self->_resolve_dir_parent_id(key => $key2, create => 1), $key1->path . ".swap");
+
+              # if we affected more then one row something very bad has happened.
+              unless ($rows == 1) {
+                  $query->finish;
+                  $log->logdie("affected row count is $rows instead of 1");
+              }
+          }
 
             $db->commit;
@@ -397,6 +585,6 @@
             $log->logdie("database error: $@");
         }
-        last;
-    }
+      last;
+  }
 
     $log->debug("leaving");
@@ -405,4 +593,18 @@
 }
 
+# EAM : from JH, below is a possible option to swap instances across
+# dbs.  I recommend that this action be disallowed, and instead such
+# operations be implemented only as a combination of copy and delete
+
+# order of operations for the swap between two dbs is:
+# key1 start transaction
+# key1 -> read all instances
+# key1 -> remove all instances
+# key2 start transaction
+# key2 -> read all instances
+# key2 -> remove all instances
+# key1 -> insert key 2 instances
+# key2 -> insert key 1 instances
+# key1,2 commit
 
 sub replicate_object
@@ -439,16 +641,16 @@
     # then we should throw an error 
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug("entered - @_");
-
     # vol_name overrides the key implied volume
     $key = parse_neb_key($key, $vol_name);
     $vol_name = $key->volume;
 
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug("entered - @_");
+
     if (defined $vol_name
-        and not $self->_is_valid_volume_name($key->volume)) {
+        and not $self->_is_valid_volume_name($key, $key->volume)) {
         if ($key->soft_volume) {
             $log->warn( "$vol_name is not a known volume name" );
@@ -462,5 +664,5 @@
     if (defined $vol_name) {
         ($vol_id, $vol_host, $vol_path, $vol_xattr)
-            = $self->_get_storage_volume($vol_name);
+            = $self->_get_storage_volume($key, $vol_name);
     } else {
         ($vol_id, $vol_host, $vol_path, $vol_xattr)
@@ -507,5 +709,7 @@
 
             # TODO add some stuff here to retry if unsucessful
-            $uri = $self->_create_empty_instance_file($key->path, $so_id, $ins_id, $vol_path, $vol_xattr);
+            # XXX if this fails, it should try to generate the
+            # instance on another volume (unless !$soft_volume) 
+            $uri = $self->_create_empty_instance_file($key, $so_id, $ins_id, $vol_path, $vol_xattr);
 
             {
@@ -560,12 +764,12 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug( "entered - @_" );
-
     # ignore volume
     $key = parse_neb_key($key);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug( "entered - @_" );
 
     my $so_id;
@@ -670,12 +874,12 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug( "entered - @_" );
-
     # ignore volume
     $key = parse_neb_key($key);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug( "entered - @_" );
 
     my $so_id;
@@ -789,12 +993,12 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug("entered - @_");
-
     # ignore volume
     $key = parse_neb_key($key);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug("entered - @_");
 
 TRANS: while (1) {
@@ -863,12 +1067,12 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug("entered - @_");
-
     # ignore volume
     $key = parse_neb_key($key);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug("entered - @_");
 
     my $value;
@@ -916,12 +1120,12 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug("entered - @_");
-
     # ignore volume
     $key = parse_neb_key($key);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug("entered - @_");
 
     my @xattrs;
@@ -959,12 +1163,12 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug("entered - @_");
-
     # ignore volume
     $key = parse_neb_key($key);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug("entered - @_");
 
 TRANS: while (1) {
@@ -1000,10 +1204,10 @@
 }
 
-
+# loop over all db_index values, passing db_index to each call
 sub find_objects
 {
     my $self = shift;
 
-    my ( $pattern ) = validate_pos( @_,
+    my ($pattern) = validate_pos( @_,
         {
             type        => SCALAR,
@@ -1012,31 +1216,87 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
+    $pattern = parse_neb_key($pattern);
+
+    my $log = $self->log;
 
     $log->debug( "entered - @_" );
 
-    unless ($pattern) {
+    unless (defined $pattern) {
         $log->debug( "leaving" );
         $log->logdie("no keys found");
     }
 
-    # attempt to strip off neb:// if it exists
-    $pattern =~ s|^neb:||;
-
+    my @keys = ();
+    my $n_dbs = $self->config->n_db();
+    for (my $index = 0; $index < $n_dbs; $index ++) {
+        my $newkeys = $self->_find_objects_for_index($index, $pattern);
+        push @keys, @$newkeys;
+    }
+    $log->logdie("no keys found") unless ( scalar @keys );
+
+    $log->debug( "leaving" );
+
+    return \@keys;
+}
+
+# find matching objects from the given server
+sub _find_objects_for_index
+{
+
+    my $self    = shift;
+    my $index   = shift;
+    my $key     = shift;
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->_db_for_index($index);
+
+    $log->debug( "entered - @_" );
+
+    # first check to see if the key is an exact match
     my @keys;
     eval {
-        my $query = $db->prepare_cached( $sql->find_objects );
-        $query->execute( $pattern );
+        $log->debug("trying for an exact key match with $key");
+        my $query = $db->prepare_cached( $sql->find_object_by_ext_id );
+        $query->execute( $key->path );
+        if ($query->rows) {
+            my $ext_id = $query->fetchrow_hashref->{'ext_id'};
+            $log->debug( "pattern has an exact match" );
+            push @keys, $ext_id;
+        } else {
+            $log->debug("no exact match for key");
+        }
+        $query->finish;
+    };
+    if ($@) {
+        $db->rollback;
+        $log->logdie("database error: $@");
+    }
+
+    if (scalar @keys) {
+        # it was an exact match, so stop here
+        $log->debug("leaving");
+
+        return \@keys;
+    }
+
+    # else, assume it's a directory
+    my $dir_id = $self->_resolve_dir_parent_id(key => $key, dir_id => 1);
+    unless (defined $dir_id) {
+        $log->logdie("pattern $key does not match any key or directory");
+    }
+
+    eval {
+        $log->debug("trying for a directory match under dir: $dir_id");
+        my $query = $db->prepare_cached( $sql->find_object_by_dir_id );
+        $query->execute( $dir_id );
 
         while ( my $row = $query->fetchrow_hashref ) {
             my $key = $row->{ 'ext_id' };
             push @keys, $key if $key;
+            $log->debug( "matched $key" ) if $key;
         }
     };
     $log->logdie("database error: $@") if $@;
-
-    $log->logdie("no keys found") unless ( scalar @keys );
 
     $log->debug( "leaving" );
@@ -1070,18 +1330,18 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug("entered - @_");
-
     # vol_name overrides the key implied volume
     $key = parse_neb_key($key, $vol_name);
     $vol_name = $key->volume;
 
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug("entered - @_");
+
     # the key's volume can't be validiated on input for this method so we have
     # to check it after parsing the key
     if (defined $vol_name
-        and not $self->_is_valid_volume_name($key->volume)) {
+        and not $self->_is_valid_volume_name($key, $key->volume)) {
         if ($key->soft_volume) {
             $log->warn( "$vol_name is not a known volume name" );
@@ -1143,5 +1403,11 @@
     my $self = shift;
 
-    my ( $uri ) = validate_pos( @_,
+    my ($key, $uri) = validate_pos( @_,
+        {
+            type        => SCALAR,
+            callbacks   => {
+                'is valid object key' => sub { $self->_is_valid_object_key($_[0]) },
+            },
+        },
         {
             type => SCALAR|SCALARREF,
@@ -1149,7 +1415,10 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
+    # ignore volume
+    $key = parse_neb_key($key);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
 
     $log->debug( "entered - @_" );
@@ -1243,12 +1512,12 @@
     );
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug("entered - @_");
-
     # ignore volume
     $key = parse_neb_key($key);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
+
+    $log->debug("entered - @_");
 
     my $stat;
@@ -1271,7 +1540,10 @@
 }
 
-
+# this should have a 'db_index' as an argument
 sub mounts
 {
+    # XXX: this will only pull the mounts from one db
+    # XXX: loop over db_index and generate a single unique list
+    # XXX: or report mount info for each db server
     my $self = shift;
 
@@ -1280,5 +1552,5 @@
     my $log = $self->log;
     my $sql = $self->sql;
-    my $db  =$self->db;
+    my $db  = $self->_db_for_index(0); # XXX fix as above
 
     $log->debug("entered - @_");
@@ -1309,13 +1581,13 @@
     my $self = shift;
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  = $self->db;
+    my ($key, $name, $soft_volume) = @_;
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
 
     no warnings qw( uninitialized );
     $log->debug( "entered - @_" );
     use warnings;
-
-    my ($name, $soft_volume) = @_;
 
     my ($vol_id, $vol_host, $vol_path, $xattr);
@@ -1334,5 +1606,5 @@
                 # find it, fall back to any volume
                 if ($soft_volume) {
-                    ($vol_id, $vol_host, $vol_path, $xattr) = $self->_get_storage_volume;
+                    ($vol_id, $vol_host, $vol_path, $xattr) = $self->_get_storage_volume($key);
                     return; # this just returns out of the eval not from the subroutine
                 }
@@ -1374,15 +1646,13 @@
     my $self = shift;
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
+    my $key = shift;
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
 
     no warnings qw( uninitialized );
     $log->debug( "entered - @_" );
     use warnings;
-
-    my $key = shift;
-
-    $key = parse_neb_key($key);
 
     my ($vol_id, $vol_host, $vol_path, $xattr);
@@ -1422,13 +1692,13 @@
     my ($self, $key) = @_;
 
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
     $key = parse_neb_key($key);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
 
     my $ext_id;
     eval {
-        my $query = $db->prepare_cached( $sql->get_object ); 
+        my $query = $db->prepare_cached( $sql->check_object_name ); 
         $query->execute($key->path);
         ($ext_id) = $query->fetchrow_array;
@@ -1451,11 +1721,12 @@
 sub _is_valid_volume_name
 {
-    my ($self, $vol_name) = @_;
-
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
+    my ($self, $key, $vol_name) = @_;
+
+    $key = parse_neb_key($key);
     my $volume_info = parse_neb_volume($vol_name);
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  = $self->db($key);
 
     $vol_name = $volume_info->{volume};
@@ -1492,10 +1763,10 @@
     my $log = $self->log;
     my $sql = $self->sql;
-    my $db  = $self->db;
+    my $db  = $self->db($key);
 
     my $uri;
     eval {
-        my $storage_path = $self->_generate_storage_path($key, $vol_path);
-        my $storage_filename = $self->_generate_storage_filename($key, $ins_id);
+        my $storage_path = $self->_generate_storage_path($key->path, $vol_path);
+        my $storage_filename = $self->_generate_storage_filename($key->path, $ins_id);
         unless (-d $storage_path) {
             _retry(sub { mkpath(@_) }, $storage_path, 0, 0775)
@@ -1506,5 +1777,5 @@
         my $mode = [_retry(sub { stat($storage_path) } )]->[2] & 07777;
         unless ($mode == 0775) {
-            $log->error("$storage_path has the wrong permissions of: %04o", $mode);
+            $log->error("$storage_path has the wrong permissions of: %04x", $mode);
             _retry(sub { chmod(0775, $storage_path) }) or die "can't chmod $storage_path: $!";
         }
@@ -1522,5 +1793,5 @@
         my $path = $uri->file;
         die "can not set xattr on $path: $!"
-            unless (setfattr($path, 'user.nebulous_key', $key));
+            unless (setfattr($path, 'user.nebulous_key', $key->path));
     }
 
@@ -1621,11 +1892,12 @@
     my $log = $self->log;
     my $sql = $self->sql;
-    my $db  =$self->db;
+#    my $db  = $self->db;
 
     $log->debug( "entered" );
 
-    $self->db->disconnect;        
-
-    $log->debug( "disconnected from database: ", sub { $db->data_sources; } );
+# XXX do we need to loop over db_index?
+#    $self->db->disconnect;        
+
+#    $log->debug( "disconnected from database: ", sub { $db->data_sources; } );
 
     $log->debug( "leaving" );
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server.pod
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server.pod	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server.pod	(revision 24244)
@@ -26,5 +26,5 @@
     Nebulous::Server->find_objects( $pattern );
     Nebulous::Server->find_instances( $key, $volume );
-    Nebulous::Server->delete_instance( $uri );
+    Nebulous::Server->delete_instance( $key, $uri );
     Nebulous::Server->stat_object( $key );
     Nebulous::Server->mounts();
@@ -160,8 +160,8 @@
 Throws an exception on error.
 
-=item * delete_instance( $uri );
-
-Accepts 1 parameters, it is mandatory.  Returns Boolean true.  Throws an
-exception on error.
+=item * delete_instance( $key, $uri );
+
+Accepts 2 parameters, both mandatory.  Returns Boolean true.  C<<$uri>> must be
+an instance of C<<$key>>.  Throws an exception on error.
 
 =item * swap_objects( $key1, $key2 );
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/Config.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/Config.pm	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/Config.pm	(revision 24244)
@@ -8,10 +8,10 @@
 use warnings FATAL => qw( all );
 
-our $VERSION = 0.02;
+our $VERSION = 0.03;
 
 use base qw( Class::Accessor::Fast );
 
 use Log::Log4perl qw( :levels );
-use Params::Validate qw( validate SCALAR );
+use Params::Validate qw( validate validate_pos SCALAR );
 
 our %LEVELS = (
@@ -25,12 +25,19 @@
 );
 
-my $new_validate = {
+my $db_validate = {
+    dbindex       => {
+        type => SCALAR,
+        regex => qr/^\d+$/,
+    },
     dsn         => { type => SCALAR },
     dbuser      => { type => SCALAR },
     dbpasswd    => { type => SCALAR },
-    log_level   => {
+};
+
+my $new_validate = {
+    trace       => {
         type        => SCALAR,
         optional    => 1,
-        default     => 'all',
+        default     => 'fatal',
         callbacks   => {
             'is valid level' => sub {
@@ -39,9 +46,14 @@
         },
     },
+    dsn         => { type => SCALAR, optional => 1 },
+    dbuser      => { type => SCALAR, optional => 1 },
+    dbpasswd    => { type => SCALAR, optional => 1 },
 };
 
 __PACKAGE__->mk_ro_accessors( keys %$new_validate );
 
-sub init {
+
+sub new
+{
     my $class = shift;
 
@@ -49,13 +61,74 @@
 
     # normalize log levels to lower-case
-    $p{ log_level } = lc $p{ log_level };
-
-    my $self = \%p;
+    my $self = { trace => $LEVELS{lc($p{trace})} };
 
     bless $self, $class || ref $class;
+
+    my @dbs;
+    $self->{dbs} = \@dbs;
+
+    if (defined $p{dsn} or defined $p{dbuser} or defined $p{dbpasswd}) {
+        $self->add_db(
+            dbindex     => 0,
+            dsn         => $p{dsn},
+            dbuser      => $p{dbuser},
+            dbpasswd    => $p{dbpasswd},
+        );
+    }
 
     return $self;
 }
 
+
+sub add_db
+{
+    my $self = shift;
+    
+    my %p = validate( @_, $db_validate );
+
+    my $config_db = Nebulous::Server::Config::DB->new({
+        dsn         => $p{dsn},
+        dbuser      => $p{dbuser},
+        dbpasswd    => $p{dbpasswd},
+    });
+
+    $self->{dbs}->[$p{dbindex}] = $config_db;
+
+    return $self;
+}
+
+
+sub db 
+{
+    my $self = shift;
+
+    my ($db_index) = validate_pos( @_, { type => SCALAR, optional => 1, });
+
+    # default to 0
+    $db_index ||= 0;
+
+    return $self->{dbs}->[$db_index];
+}
+
+
+sub n_db 
+{
+    my $self = shift;
+
+    return scalar @{ $self->{dbs} };
+}
+
+
+package Nebulous::Server::Config::DB;
+
+use strict;
+use warnings FATAL => qw( all );
+
+our $VERSION = 0.01;
+
+use base qw( Class::Accessor::Fast );
+
+__PACKAGE__->mk_ro_accessors(qw( dsn dbuser dbpasswd )); 
+
 1;
 
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/Log.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/Log.pm	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/Log.pm	(revision 24244)
@@ -16,7 +16,7 @@
     my ($self, $config) = @_;
 
-    my $dsn         = $config->dsn;
-    my $dbuser      = $config->dbuser;
-    my $dbpasswd    = $config->dbpasswd;
+#    my $dsn         = $config->db->dsn;
+#    my $dbuser      = $config->db->dbuser;
+#    my $dbpasswd    = $config->db->dbpasswd;
 
     my $conf = <<END;
@@ -30,9 +30,9 @@
 #   date | hostname | priority | method/sub - message\n
 
-    log4perl.appender.SQLLOG            = Log::Log4perl::Appender::DBI
-    log4perl.appender.SQLLOG.datasource = $dsn
-    log4perl.appender.SQLLOG.username   = $dbuser
-    log4perl.appender.SQLLOG.password   = $dbpasswd
-    log4perl.appender.SQLLOG.sql        = \
+#    log4perl.appender.SQLLOG            = Log::Log4perl::Appender::DBI
+#    log4perl.appender.SQLLOG.sql        = \
+#    log4perl.appender.SQLLOG.datasource = 
+#    log4perl.appender.SQLLOG.username   = 
+#    log4perl.appender.SQLLOG.password   = 
     INSERT INTO log (timestamp, hostname, level, sub, message) \
     VALUES          (%d,        %H,       %p,    %M,  %m)
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/SOAP.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/SOAP.pm	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/SOAP.pm	(revision 24244)
@@ -1,5 +1,5 @@
 # Copyright (c) 2004  Joshua Hoblitt
 #
-# $Id: SOAP.pm,v 1.6 2008-12-14 22:54:25 eugene Exp $
+# $Id: SOAP.pm,v 1.4.32.1 2008-12-14 22:52:37 eugene Exp $
 
 package Nebulous::Server::SOAP;
@@ -17,8 +17,10 @@
 our $AUTOLOAD;
 
-our @args;
+our $config;
 our $neb;
 
-sub new_on_init {
+
+sub new_on_init
+{
     my $self = shift;
 
@@ -27,5 +29,5 @@
     require Apache2::ServerUtil;
 
-    @args = @_;
+    $config = shift;
 
     my $s = Apache2::ServerUtil->server;
@@ -35,13 +37,17 @@
 }
 
-sub init {
+
+sub init
+{
     my $self = shift;
 
-    $neb = Nebulous::Server->new(@args);        
+    $neb = Nebulous::Server->new_from_config($config);        
 
     return Apache2::Const::OK;
 }
 
-sub AUTOLOAD {
+
+sub AUTOLOAD
+{
     my $self = shift;
 
@@ -76,3 +82,4 @@
 }
 
+
 1;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/SQL.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 24244)
@@ -8,5 +8,5 @@
 use warnings FATAL => qw( all );
 
-our $VERSION = '0.02';
+our $VERSION = '0.04';
 
 use base qw( Class::Accessor::Fast );
@@ -26,6 +26,6 @@
     new_object          => qq{
         INSERT INTO storage_object
-        (so_id, ext_id, type)
-        VALUES (?, ?, 'REG_FILE')
+        (so_id, ext_id, ext_id_basename, type, dir_id)
+        VALUES (?, ?, ?, 'REG_FILE', ?)
     },
     new_object_attr  => qq{
@@ -59,4 +59,23 @@
         JOIN storage_object_attr
         USING (so_id)
+        WHERE ext_id = ?
+    },
+    get_directory       => qq{
+        SELECT
+            dir_id
+        FROM directory
+        WHERE parent_id = ?
+            AND dirname = ?
+    },
+    new_directory       => qq{
+        INSERT INTO directory
+        (dirname, parent_id)
+        VALUES (?, ?)
+    },
+    check_object_name => qq{
+        SELECT
+            so_id,
+            ext_id
+        FROM storage_object
         WHERE ext_id = ?
     },
@@ -290,12 +309,17 @@
             USING(vol_id)
     },
-    find_objects => qq{
-        SELECT *
-        FROM storage_object
-        WHERE ext_id REGEXP ?
+    find_object_by_ext_id => qq{
+        SELECT ext_id, ext_id_basename
+        FROM storage_object
+        WHERE ext_id = ?
+    },
+    find_object_by_dir_id => qq{
+        SELECT ext_id, ext_id_basename
+        FROM storage_object
+        WHERE dir_id = ?
     },
     rename_object => qq{
         UPDATE storage_object
-        SET ext_id = ?
+        SET ext_id = ?, ext_id_basename = ?, dir_id = ?
         WHERE ext_id = ?
     },
@@ -323,4 +347,27 @@
         GROUP BY so_id
         HAVING available_instances < instances OR instances < copies
+    },
+    find_objects_with_extra_instances_by_xattr => qq{
+        SELECT
+            so.so_id,
+            so.ext_id,
+            count(ins_id) as instances,
+            mv.name as volume_name,
+            mv.host as volume_host,
+            count(mv.vol_id) as available_instances,
+            xattr.value as copies
+        FROM storage_object AS so
+        JOIN storage_object_xattr as xattr
+            ON so.so_id = xattr.so_id
+            AND xattr.name = 'user.copies'
+        JOIN instance AS i
+            ON so.so_id = i.so_id
+        JOIN mountedvol AS mv
+            USING(vol_id)
+        WHERE
+            mv.available = 1
+        GROUP BY so_id
+        HAVING available_instances > copies
+        limit 5;
     },
     find_objects_with_extra_instances => qq{
@@ -380,4 +427,5 @@
 DROP TABLE IF EXISTS log;
 DROP TABLE IF EXISTS mountedvol;
+DROP TABLE IF EXISTS directory;
 DROP PROCEDURE IF EXISTS getmountedvol;
 SET FOREIGN_KEY_CHECKS=1
@@ -401,10 +449,28 @@
 
 __DATA__
+CREATE TABLE directory (
+    dir_id BIGINT NOT NULL AUTO_INCREMENT,
+    dirname CHAR(255) NOT NULL,
+    parent_id BIGINT NOT NULL,
+    FOREIGN KEY(parent_id) REFERENCES directory(dir_id),
+    PRIMARY KEY(dir_id),
+    KEY(parent_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+###
+
+INSERT INTO directory (dir_id, dirname, parent_id) VALUES (1, '/', 1);
+
+###
+
 CREATE TABLE storage_object (
     so_id BIGINT NOT NULL AUTO_INCREMENT,
     ext_id VARCHAR(255) NOT NULL UNIQUE,
+    ext_id_basename VARCHAR(255) NOT NULL,
+    dir_id BIGINT NOT NULL,
+    FOREIGN KEY(dir_id) REFERENCES directory(dir_id),
     type enum('REG_FILE'),
     PRIMARY KEY(so_id),
-    KEY(ext_id(64)),
+    KEY(dir_id),
     KEY(type)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -441,5 +507,5 @@
     type ENUM( 'read', 'write' ) NOT NULL,
     epoch TIMESTAMP,
-    KEY(so_ID)
+    KEY(so_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -455,5 +521,4 @@
     xattr BOOLEAN DEFAULT FALSE,
     PRIMARY KEY(vol_id),
-    KEY(name(16)),
     KEY(host(16)),
     KEY(path(255)),
@@ -470,5 +535,5 @@
     vol_id INT NOT NULL,
     FOREIGN KEY(vol_id) REFERENCES volume(vol_id),
-    uri VARCHAR(255) NOT NULL UNIQUE,
+    uri VARCHAR(255) NOT NULL,
     epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     mtime TIMESTAMP,
@@ -476,5 +541,5 @@
     KEY(so_id),
     KEY(vol_id),
-    KEY(uri(64))
+    KEY(uri(40))
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -515,7 +580,12 @@
     available BOOLEAN DEFAULT FALSE,
     xattr BOOLEAN DEFAULT FALSE,
+    PRIMARY KEY(mountpoint),
     KEY(vol_id),
+    KEY(name),
+    KEY(host),
+    KEY(path),
     KEY(allocate),
-    KEY(available)
+    KEY(available),
+    KEY(xattr)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/scripts/dirize.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/scripts/dirize.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/scripts/dirize.pl	(revision 24244)
@@ -0,0 +1,50 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+
+use Nebulous::Key qw( parse_neb_key );
+use Nebulous::Server;
+use Nebulous::Server::Config;
+use File::Basename qw( basename );
+
+my $config = Nebulous::Server::Config->new(
+    trace       => 'warn',
+);
+$config->add_db(
+    dbindex     => 0,
+    dsn         => 'DBI:mysql:database=nebulous:host=localhost',
+    dbuser      => 'nebulous',
+    dbpasswd    => '@neb@',
+);
+
+my $neb = Nebulous::Server->new_from_config($config);
+
+my $db = $neb->_db_for_index(0);
+
+my $n;
+{
+    my $query = $db->prepare("SELECT COUNT(*) as n FROM storage_object");
+    $query->execute;
+    $n = $query->fetchrow_hashref->{'n'};
+}
+
+my $query = $db->prepare_cached("SELECT so_id, ext_id, dir_id FROM storage_object AS so WHERE so.dir_id = 0 LIMIT 1000");
+
+my $i = 0;
+while ($query->execute and $query->rows) {
+    print "foo\n";
+    while (my $row = $query->fetchrow_hashref) {
+        $i++;
+        my $key = parse_neb_key($row->{'ext_id'});
+        my $parent_id = $neb->_resolve_dir_parent_id(key => $key, create => 1);
+
+#printf("dirizing %20s basename: %20s parent_id %10d\n", $key, basename($row->{'ext_id'}), $parent_id);
+        printf("$i/$n dirizing %s\n", $key->path);
+
+        my $q = $db->prepare_cached("UPDATE storage_object SET ext_id_basename = ?, dir_id = ? WHERE so_id = ?");
+        $q->execute(basename($row->{'ext_id'}), $parent_id, $row->{'so_id'});
+        $db->commit;
+    }
+    $query->finish;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/01_load.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/01_load.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/01_load.t	(revision 24244)
@@ -12,5 +12,5 @@
 use Test::More tests => 5;
 
-BEGIN { use_ok( 'Nebulous::Keys' ); }
+BEGIN { use_ok( 'Nebulous::Key' ); }
 BEGIN { use_ok( 'Nebulous::Server' ); }
 BEGIN { use_ok( 'Nebulous::Server::Log' ); }
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/02_config.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/02_config.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/02_config.t	(revision 24244)
@@ -3,5 +3,5 @@
 # Copryight (C) 2004-2005  Joshua Hoblitt
 #
-# $Id: 02_config.t,v 1.1 2008-12-12 21:13:41 jhoblitt Exp $
+# $Id: 02_config.t,v 1.1.2.2 2008-12-14 22:52:37 eugene Exp $
 
 use strict;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/02_server_setup.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/02_server_setup.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/02_server_setup.t	(revision 24244)
@@ -8,5 +8,5 @@
 use warnings;
 
-use Test::More tests => 2;
+use Test::More tests => 6;
 
 use lib qw( ./t ./lib );
@@ -17,23 +17,49 @@
 Test::Nebulous->setup;
 
-isa_ok(
-    Nebulous::Server->new(
-        dsn         => $NEB_DB,
-        dbuser      => $NEB_USER,
-        dbpasswd    => $NEB_PASS,
-    ),
-    "Nebulous::Server"
-);
+isa_ok(Nebulous::Server->new(), "Nebulous::Server");
 
 Test::Nebulous->setup;
 
-eval {
-    Nebulous::Server->new(
+# ->new()
+{
+    my $neb = Nebulous::Server->new( trace => 'off' );
+
+    ok($neb, "set log level");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Server->new(
         dsn         => "DBI:mysql:database=foobar:host=localhost",
         dbuser      => "baz",
         dbpasswd    =>"boo",
     );
-};
-like( $@, qr/DBI connect.*? failed/, "bad dsn/user/pass" );
+
+    ok($neb, "set database");
+}
+
+Test::Nebulous->setup;
+
+# add dbs after ->new()
+{
+    my $neb = Nebulous::Server->new;
+
+    ok($neb->config->add_db(
+        dbindex    => 0,
+        dsn         => "DBI:mysql:database=foobar:host=localhost",
+        dbuser      => "baz",
+        dbpasswd    =>"boo",
+    ), "add db");
+    
+    ok($neb->config->add_db(
+        dbindex    => 1,
+        dsn         => "DBI:mysql:database=foobar:host=localhost",
+        dbuser      => "baz",
+        dbpasswd    =>"boo",
+    ), "add dbs");
+
+    is($neb->config->n_db, 2, "n dbs")
+}
 
 Test::Nebulous->cleanup;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/03_server_create_object.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/03_server_create_object.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/03_server_create_object.t	(revision 24244)
@@ -8,8 +8,9 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 89;
+use Test::More tests => 99;
 
 use lib qw( ./t ./lib );
 
+use File::Basename qw( basename );
 use File::ExtAttr qw( getfattr );
 use Nebulous::Server;
@@ -26,4 +27,6 @@
 );
 
+use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS;
+
 Test::Nebulous->setup;
 
@@ -334,4 +337,174 @@
 }
 
+# test for properly row creation in the directories table
+
+Test::Nebulous->setup;
+
+{
+    my $key = "foo";
+    $neb->create_object($key);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        storage_object  => [so_id => 1, ext_id => $key, dir_id => 1],
+    );
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key = "a/foo";
+    $neb->create_object($key);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        storage_object  => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 2],
+    );
+
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key = "a/b/foo";
+    $neb->create_object($key);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
+        storage_object  => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 3],
+    );
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key = "a/b/c/foo";
+    $neb->create_object($key);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
+        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
+        storage_object  => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 4],
+    );
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key1 = "a/b/c/foo";
+    my $key2 = "foo";
+    $neb->create_object($key1);
+    $neb->create_object($key2);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
+        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
+        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
+    );
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key1 = "a/b/c/foo";
+    my $key2 = "a/foo";
+    $neb->create_object($key1);
+    $neb->create_object($key2);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 2],
+        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
+        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
+        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
+    );
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key1 = "a/b/c/foo";
+    my $key2 = "d/foo";
+    $neb->create_object($key1);
+    $neb->create_object($key2);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
+        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
+        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
+        directory       => [dir_id => 5, dirname => 'd', parent_id => 1],
+        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5],
+    );
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key1 = "a/b/c/foo";
+    my $key2 = "a/d/foo";
+    $neb->create_object($key1);
+    $neb->create_object($key2);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
+        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
+        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
+        directory       => [dir_id => 5, dirname => 'd', parent_id => 2],
+        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5],
+    );
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key1 = "a/b/c/foo";
+    my $key2 = "d/a/foo";
+    $neb->create_object($key1);
+    $neb->create_object($key2);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
+        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
+        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
+        directory       => [dir_id => 5, dirname => 'd', parent_id => 1],
+        directory       => [dir_id => 6, dirname => 'a', parent_id => 5],
+        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 6],
+    );
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key1 = "a/b/c/foo";
+    my $key2 = "a/b/c/d/foo";
+    $neb->create_object($key1);
+    $neb->create_object($key2);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
+        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
+        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
+        directory       => [dir_id => 5, dirname => 'd', parent_id => 4],
+        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5],
+    );
+}
+
 Test::Nebulous->setup;
 
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/08_server_delete_instance.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/08_server_delete_instance.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/08_server_delete_instance.t	(revision 24244)
@@ -3,5 +3,5 @@
 # Copryight (C) 2004-2005  Joshua Hoblitt
 #
-# $Id: 08_server_delete_instance.t,v 1.12 2008-12-14 22:54:25 eugene Exp $
+# $Id: 08_server_delete_instance.t,v 1.10.22.1 2008-12-14 22:52:37 eugene Exp $
 
 use strict;
@@ -24,7 +24,8 @@
 
 {
-    my $uri = $neb->create_object("foo");
+    my $key = "foo";
+    my $uri = $neb->create_object($key);
 
-    ok($neb->delete_instance($uri), "delete instance");
+    ok($neb->delete_instance($key, $uri), "delete instance");
 }
 
@@ -32,17 +33,18 @@
 
 {
-    my $uri1 = $neb->create_object("foo");
-    my $uri2 = $neb->replicate_object("foo");
+    my $key = "foo";
+    my $uri1 = $neb->create_object($key);
+    my $uri2 = $neb->replicate_object($key);
 
-    ok($neb->delete_instance($uri1), "delete instance");
+    ok($neb->delete_instance($key, $uri1), "delete instance");
 
-    my $locations = $neb->find_instances("foo");
+    my $locations = $neb->find_instances($key);
 
     is($locations->[0], $uri2, "instance remains");
 
-    ok($neb->delete_instance( $uri2 ), "delete instance");
+    ok($neb->delete_instance($key, $uri2), "delete instance");
 
     eval {
-        $neb->find_instances("foo");
+        $neb->find_instances($key);
     };
     like($@, qr/is valid object key/, "storage object was deleted");
@@ -52,5 +54,8 @@
 
 eval {
-    $neb->delete_instance("file:/foo");
+    my $key = "foo";
+    my $uri1 = $neb->create_object($key);
+
+    $neb->delete_instance($key, "file:/foo");
 };
 like($@, qr/no instance is associated with uri/, "uri does not exist");
@@ -61,12 +66,15 @@
     $neb->delete_instance();
 };
-like($@, qr/1 was expected/, "no params");
+like($@, qr/2 were expected/, "no params");
 
 Test::Nebulous->setup;
 
 eval {
-    $neb->delete_instance("foo", 2);
+    my $key = "foo";
+    my $uri1 = $neb->create_object($key);
+
+    $neb->delete_instance("foo", 2, 3);
 };
-like($@, qr/1 was expected/, "too many params");
+like($@, qr/2 were expected/, "too many params");
 
 Test::Nebulous->cleanup;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/10_server_is_valid_volume_name.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/10_server_is_valid_volume_name.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/10_server_is_valid_volume_name.t	(revision 24244)
@@ -23,13 +23,13 @@
 Test::Nebulous->setup;
 
-ok($neb->_is_valid_volume_name('node01'), "valid volume name");
+ok($neb->_is_valid_volume_name('foo', 'node01'), "valid volume name");
 
 Test::Nebulous->setup;
 
-ok($neb->_is_valid_volume_name('node02'), "valid volume name");
+ok($neb->_is_valid_volume_name('foo', 'node02'), "valid volume name");
 
 Test::Nebulous->setup;
 
-is($neb->_is_valid_volume_name('node99'), undef, "invalid volume name");
+is($neb->_is_valid_volume_name('foo', 'node99'), undef, "invalid volume name");
 
 Test::Nebulous->cleanup;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/12_server_find_objects.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/12_server_find_objects.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/12_server_find_objects.t	(revision 24244)
@@ -8,5 +8,5 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 6;
+use Test::More tests => 28;
 
 use lib qw( ./t ./lib );
@@ -25,6 +25,5 @@
 # search for a regex of '' should match nothing
 eval {
-    # key
-    my $uri = $neb->create_object("foo");
+    $neb->create_object("foo");
 
     my $keys = $neb->find_objects();
@@ -35,6 +34,5 @@
 
 {
-    # key
-    my $uri = $neb->create_object("foo");
+    $neb->create_object("foo");
 
     my $keys = $neb->find_objects("foo");
@@ -48,6 +46,6 @@
 {
     # key
-    my $uri1 = $neb->create_object("foo");
-    my $uri2 = $neb->replicate_object("foo");
+    $neb->create_object("foo");
+    $neb->replicate_object("foo");
 
     my $keys = $neb->find_objects("foo");
@@ -55,4 +53,120 @@
     is(scalar @$keys, 1, 'number of keys found');
     is($keys->[0], "foo", "key name");
+}
+
+Test::Nebulous->setup;
+
+{
+    # key
+    $neb->create_object("foo");
+    $neb->create_object("bar");
+
+    my $keys = $neb->find_objects("foo");
+
+    is(scalar @$keys, 1, 'number of keys found');
+    is($keys->[0], "foo", "key name");
+}
+
+# test recursive dir searching
+Test::Nebulous->setup;
+
+{
+    $neb->create_object("a/foo");
+
+    my $keys = $neb->find_objects("a");
+
+    is(scalar @$keys, 1, 'number of keys found');
+    is($keys->[0], "a/foo", "key name");
+}
+
+Test::Nebulous->setup;
+
+{
+    $neb->create_object("a/foo");
+    $neb->create_object("b/foo");
+
+    my $keys = $neb->find_objects("a");
+
+    is(scalar @$keys, 1, 'number of keys found');
+    is($keys->[0], "a/foo", "key name");
+}
+
+Test::Nebulous->setup;
+
+{
+    $neb->create_object("a/foo");
+    $neb->create_object("a/b/foo");
+
+    my $keys = $neb->find_objects("a");
+
+    is(scalar @$keys, 1, 'number of keys found');
+    is($keys->[0], "a/foo", "key name");
+}
+
+Test::Nebulous->setup;
+
+{
+    $neb->create_object("a/foo");
+    $neb->create_object("a/bar");
+
+    my $keys = $neb->find_objects("a");
+
+    is(scalar @$keys, 2, 'number of keys found');
+    is($keys->[0], "a/foo", "key name");
+    is($keys->[1], "a/bar", "key name");
+}
+
+Test::Nebulous->setup;
+
+{
+    $neb->create_object("a/foo");
+    $neb->create_object("bar");
+
+    my $keys = $neb->find_objects("a");
+
+    is(scalar @$keys, 1, 'number of keys found');
+    is($keys->[0], "a/foo", "key name");
+}
+
+Test::Nebulous->setup;
+
+{
+    $neb->create_object("a/foo");
+    $neb->create_object("foo");
+    $neb->create_object("bar");
+
+    my $keys = $neb->find_objects("/");
+
+    is(scalar @$keys, 2, 'number of keys found');
+    is($keys->[0], "foo", "key name");
+    is($keys->[1], "bar", "key name");
+}
+
+Test::Nebulous->setup;
+
+{
+    $neb->create_object("a/foo");
+    $neb->create_object("foo");
+    $neb->create_object("bar");
+
+    my $keys = $neb->find_objects(".");
+
+    is(scalar @$keys, 2, 'number of keys found');
+    is($keys->[0], "foo", "key name");
+    is($keys->[1], "bar", "key name");
+}
+
+Test::Nebulous->setup;
+
+{
+    $neb->create_object("a/foo");
+    $neb->create_object("foo");
+    $neb->create_object("bar");
+
+    my $keys = $neb->find_objects("..");
+
+    is(scalar @$keys, 2, 'number of keys found');
+    is($keys->[0], "foo", "key name");
+    is($keys->[1], "bar", "key name");
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/13_server_rename_object.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/13_server_rename_object.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/13_server_rename_object.t	(revision 24244)
@@ -8,8 +8,9 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 6;
+use Test::More tests => 8;
 
 use lib qw( ./t ./lib );
 
+use File::Basename qw( basename );
 use Nebulous::Server;
 use Test::Nebulous;
@@ -21,18 +22,48 @@
 );
 
+use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS;
+
 Test::Nebulous->setup;
 
 {
+    my $key = "bar";
     my $uri = $neb->create_object("foo");
 
-    $neb->rename_object("foo", "bar");
+    $neb->rename_object("foo", $key);
 
-    eval {
-        $neb->find_objects('^foo$');
-    };
-    like($@, qr/no keys found/, "old key name");
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        storage_object  => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 1],
+    );
+}
 
-    my $keys = $neb->find_objects('^bar$');
-    is(scalar @$keys, 1, 'number of keys found');
+Test::Nebulous->setup;
+
+{
+    my $key = "a/bar";
+    my $uri = $neb->create_object("foo");
+
+    $neb->rename_object("foo", $key);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        storage_object  => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 2],
+    );
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key = "bar";
+    my $uri = $neb->create_object("a/foo");
+
+    $neb->rename_object("a/foo", $key);
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        storage_object  => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 1],
+    );
 }
 
@@ -73,3 +104,19 @@
 like($@, qr/2 were expected/, "too many params");
 
+# test attempting to rename a key in such a way to cause the distributed
+# storage db to change
+# this must be the last test as we're messing with the $neb object
+eval {
+    $neb->config->add_db(
+        dbindex     => 1,
+        dsn         => $NEB_DB,
+        dbuser      => $NEB_USER,
+        dbpasswd    => $NEB_PASS,
+    );
+
+    $neb->create_object("a/foo");
+    $neb->rename_object("a/foo", "g/bar");
+};
+like($@, qr/rename objects across distributed database boundaries/, "rename between databases");
+
 Test::Nebulous->cleanup;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/16_server_swap_objects.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/16_server_swap_objects.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/16_server_swap_objects.t	(revision 24244)
@@ -8,8 +8,9 @@
 use warnings FATAL => qw( all );
 
-use Test::More tests => 8;
+use Test::More tests => 13;
 
 use lib qw( ./t ./lib );
 
+use File::Basename qw( basename );
 use Nebulous::Server;
 use Test::Nebulous;
@@ -21,14 +22,48 @@
 );
 
+use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS;
+
 Test::Nebulous->setup;
 
 {
-    my $uri1 = $neb->create_object("foo1");
-    my $uri2 = $neb->create_object("foo2");
+    my $key1 = "foo1";
+    my $key2 = "foo2";
+    my $uri1 = $neb->create_object($key1);
+    my $uri2 = $neb->create_object($key2);
 
-    ok($neb->swap_objects("foo1", "foo2"), "swap succeeded");
+    ok($neb->swap_objects($key1, $key2), "swap succeeded");
 
-    my $new_uri1 = ($neb->find_instances("foo1"))->[0];
-    my $new_uri2 = ($neb->find_instances("foo2"))->[0];
+    my $new_uri1 = ($neb->find_instances($key1))->[0];
+    my $new_uri2 = ($neb->find_instances($key2))->[0];
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        storage_object  => [so_id => 1, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 1],
+        storage_object  => [so_id => 2, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 1],
+    );
+
+    is($uri1, $new_uri2, "key1 -> key2");
+    is($uri2, $new_uri1, "key2 -> key1");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $key1 = "foo1";
+    my $key2 = "a/foo2";
+    my $uri1 = $neb->create_object($key1);
+    my $uri2 = $neb->create_object($key2);
+
+    ok($neb->swap_objects($key1, $key2), "swap succeeded");
+
+    my $new_uri1 = ($neb->find_instances($key1))->[0];
+    my $new_uri2 = ($neb->find_instances($key2))->[0];
+
+    expected_dataset_ok(
+        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
+        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
+        storage_object  => [so_id => 1, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 2],
+        storage_object  => [so_id => 2, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 1],
+    );
 
     is($uri1, $new_uri2, "key1 -> key2");
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/75_parse_neb_key.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/75_parse_neb_key.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/t/75_parse_neb_key.t	(revision 24244)
@@ -10,9 +10,9 @@
 use Test::More;
 
-plan tests => 80;
+plan tests => 99;
 
 use lib qw( ./t ./lib );
 
-use Nebulous::Keys qw( parse_neb_key );
+use Nebulous::Key qw( parse_neb_key );
 use Test::Nebulous;
 
@@ -224,4 +224,53 @@
 }
 
+# root volume references
+{
+    my $key = parse_neb_key('neb:///');
+
+    is($key->path, '', 'path');
+    is($key->volume, undef, 'volume name');
+    is($key->soft_volume, undef, 'soft volume name');
+}
+
+{
+    my $key = parse_neb_key('neb:///.');
+
+    is($key->path, '', 'path');
+    is($key->volume, undef, 'volume name');
+    is($key->soft_volume, undef, 'soft volume name');
+}
+
+{
+    my $key = parse_neb_key('neb:///..');
+
+    is($key->path, '', 'path');
+    is($key->volume, undef, 'volume name');
+    is($key->soft_volume, undef, 'soft volume name');
+}
+
+{
+    my $key = parse_neb_key('/');
+
+    is($key->path, '', 'path');
+    is($key->volume, undef, 'volume name');
+    is($key->soft_volume, undef, 'soft volume name');
+}
+
+{
+    my $key = parse_neb_key('.');
+
+    is($key->path, '', 'path');
+    is($key->volume, undef, 'volume name');
+    is($key->soft_volume, undef, 'soft volume name');
+}
+
+{
+    my $key = parse_neb_key('..');
+
+    is($key->path, '', 'path');
+    is($key->volume, undef, 'volume name');
+    is($key->soft_volume, undef, 'soft volume name');
+}
+
 # key w/ whitespace
 eval {
@@ -267,2 +316,8 @@
 };
 like( $@, qr/requires a leading slash/, "leading slash" );
+
+# URI w/ volume but w/o path
+eval {
+    my $key = parse_neb_key('neb://foo');
+};
+like( $@, qr/requires a path/, "no path" );
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/Build.PL
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/Build.PL	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/Build.PL	(revision 24244)
@@ -95,4 +95,5 @@
         'Nebulous::Server::SOAP'    => 0,
         'Test::Nebulous'            => 0,
+        'Test::Cmd'                 => '1.05',
         'Apache2::SOAP'             => 0,
         'Apache::DBI'               => '1.05',
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/MANIFEST
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/MANIFEST	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/MANIFEST	(revision 24244)
@@ -98,4 +98,5 @@
 t/66_client_xattr.t
 t/67_client_swap.t
+t/70_neb-ls.t
 t/90_nebclient.t
 t/TEST.PL
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/bin/neb-ls
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/bin/neb-ls	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/bin/neb-ls	(revision 24244)
@@ -1,7 +1,5 @@
 #!/usr/bin/env perl
 
-# Copyright (C) 2007-2008  Joshua Hoblitt
-#
-# $Id: neb-ls,v 1.7 2008-06-12 20:03:37 jhoblitt Exp $
+# Copyright (C) 2007-2009  Joshua Hoblitt
 
 use strict;
@@ -9,5 +7,5 @@
 
 use vars qw( $VERSION );
-$VERSION = '0.02';
+$VERSION = '0.03';
 
 use Nebulous::Client;
@@ -16,5 +14,9 @@
 use Pod::Usage qw( pod2usage );
 
-my ($server, $long, $recursive);
+my (
+    $server,
+    $long,
+#    $recursive,
+);
 
 $server = $ENV{'NEB_SERVER'} unless $server;
@@ -22,5 +24,5 @@
 GetOptions(
     'server|s=s'    => \$server,
-    'recursive|r'   => \$recursive,
+#    'recursive|r'   => \$recursive,
     'l|1'           => \$long,
 ) || pod2usage( 2 );
@@ -39,12 +41,12 @@
     unless defined $neb;
 
-# default to listing everything (bad idea?)
-$pattern ||= ".*";
+# default to listing root
+$pattern ||= "/";
 
-if ($recursive) {
-    $pattern = "^" . $pattern . ".*";
-} else {
-    $pattern = "^" . $pattern . "\$";
-}
+#if ($recursive) {
+#    $pattern = "^" . $pattern . ".*";
+#} else {
+#    $pattern = "^" . $pattern . "\$";
+#}
 
 my $keys = $neb->find_objects($pattern);
@@ -86,12 +88,13 @@
 Optional
 
-=item * --recursive|-r
-
-By default C<neb-ls> will only try to match the exact string provided to it.
-With this option set all keys which match C<<pattern>> as a REGEX or substring
-will be returned.
-
-Optional
-
+=cut
+#=item * --recursive|-r
+#
+#By default C<neb-ls> will only try to match the exact string provided to it.
+#With this option set all keys which match C<<pattern>> as a REGEX or substring
+#will be returned.
+#
+#Optional
+#
 =item * --server|-s <URL>
 
@@ -130,5 +133,5 @@
 =head1 COPYRIGHT
 
-Copyright (C) 2007-2008  Joshua Hoblitt.  All rights reserved.
+Copyright (C) 2007-2009  Joshua Hoblitt.  All rights reserved.
 
 This program is free software; you can redistribute it and/or modify it under
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/bin/neb-replicate
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/bin/neb-replicate	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/bin/neb-replicate	(revision 24244)
@@ -25,4 +25,5 @@
     'volume|v=s'    => \$volume,
     'copies|c=i'    => \$copies,
+    'set_copies=i'  => \$set_copies,
 ) || pod2usage( 2 );
 
@@ -44,14 +45,28 @@
     unless defined $neb;
 
-for (my $i = 0; $i < $copies; $i++) {
-    if (defined $volume) {
-        $neb->replicate($key, $volume)
-            or die "failed to replicate Nebulous key: $key";
-    } else {
-        $neb->replicate($key)
-            or die "failed to replicate Nebulous key: $key";
+# if replicate fails, we should still set user.copies (if requested)
+my $replication_problem;
+eval {
+    for (my $i = 0; $i < $copies; $i++) {
+        if (defined $volume) {
+            $neb->replicate($key, $volume)
+                or die "failed to replicate Nebulous key: $key";
+        } else {
+            $neb->replicate($key)
+                or die "failed to replicate Nebulous key: $key";
+        }
     }
+};
+if ($@) {
+    warn $@;
+    $replication_problem = 1;
 }
 
+if ($set_copies) {
+    $neb->setxattr($key, "user.copies", $copies, "replace")
+        or die $neb->err;
+}
+
+exit(32) if defined $replication_problem;
 
 __END__
@@ -66,5 +81,5 @@
 
     neb-replicate [--server <URL>] [--volume <volume name>]
-                  [--copies <n>] <key>
+                  [--copies <n>] [--set_copies <n>] <key>
 
 =head1 DESCRIPTION
@@ -93,5 +108,11 @@
 The number of new replications to create.
 
-Optional.  Defaults to 0.
+Optional.  Defaults to 1.
+
+=item * --set_copies|c <n>
+
+Set the C<user.copies> xattr for this storage object to C<<n>>.
+
+Optional.
 
 =back
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/lib/Nebulous/Client.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/lib/Nebulous/Client.pm	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/lib/Nebulous/Client.pm	(revision 24244)
@@ -243,5 +243,5 @@
         # if the copy failed we now have a zero length instances floating
         # around that must be removed
-        unless ($self->delete_instance("$uri")) {
+        unless ($self->delete_instance($key, "$uri")) {
             $log->logdie( "can not copy instance $uri AND FAILED TO CLEANUP EMPTY INSTANCE" );
         }
@@ -256,5 +256,5 @@
 
     unless ($src_md5 eq $dst_md5) {
-        $self->delete_instance("$uri");
+        $self->delete_instance($key, "$uri");
         $log->logdie( "md5sum mismatch" );
     }
@@ -327,5 +327,5 @@
     }
 
-    my $uri = $self->delete_instance( @$locations[0] );
+    my $uri = $self->delete_instance($key, @$locations[0]);
 
     $log->debug("leaving");
@@ -615,4 +615,8 @@
             return;
         }
+        if ($response->faultstring =~ /does not match any key or directory/) {
+            $log->debug( "leaving" );
+            return;
+        }
 
         $log->logdie("unhandled fault - ", $self->err);
@@ -813,5 +817,5 @@
     # a lock is implicitly removed when the last storage object is deleted
     foreach my $uri ( @$locations ) {
-        $self->delete_instance( $uri ) or return undef;
+        $self->delete_instance($key, $uri) or return undef;
     }
 
@@ -944,5 +948,8 @@
     my $self = shift;
 
-    my ($uri) = validate_pos(@_,
+    my ($key, $uri) = validate_pos(@_,
+        {
+            type => SCALAR,
+        },
         {
             type => SCALAR,
@@ -961,5 +968,5 @@
     $log->logdie( $@ ) if $@;
 
-    my $response = $self->{ 'server' }->delete_instance( $uri );
+    my $response = $self->{ 'server' }->delete_instance($key, $uri);
     if ( $response->fault ) {
         $self->set_err($response->faultstring);
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/lib/Nebulous/Util.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/lib/Nebulous/Util.pm	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/lib/Nebulous/Util.pm	(revision 24244)
@@ -45,4 +45,6 @@
 
 
+# XXX replace the unlink with a 'move to trash' operation
+# empty the trash with a daemon on the NSF server
 sub _nuke_file {
     my $path = shift;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/.cvsignore	(revision 24244)
@@ -0,0 +1,21 @@
+Doxyfile
+Makefile
+Makefile.in
+aclocal.m4
+autom4te.cache
+compile
+config.guess
+config.h
+config.h.in
+config.log
+config.status
+config.sub
+configure
+depcomp
+docs
+install-sh
+libtool
+ltmain.sh
+missing
+nebclient.pc
+stamp-h1
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/nebulous.wsdl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/nebulous.wsdl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/nebulous.wsdl	(revision 24244)
@@ -125,4 +125,5 @@
 
     <message name="delete_instanceRequest">
+        <part name="key" type="xsd:string" />
         <part name="uri" type="xsd:string" />
     </message>
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/.cvsignore	(revision 24244)
@@ -0,0 +1,19 @@
+.deps
+.libs
+Makefile
+Makefile.in
+SOAP.nsmap
+libnebclient.la
+nebclient.lo
+nebulous.h
+soapC.c
+soapC.lo
+soapClient.c
+soapClient.lo
+soapClientLib.c
+soapH.h
+soapServer.c
+soapServerLib.c
+soapStub.h
+stdsoap2.lo
+xmalloc.lo
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/nebclient.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/nebclient.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/nebclient.c	(revision 24244)
@@ -253,5 +253,5 @@
     }
 
-    if (!nebDeleteInstance(server, locations->URI[0])) {
+    if (!nebDeleteInstance(server, key, locations->URI[0])) {
         nebObjectInstancesFree(locations);
 
@@ -633,5 +633,5 @@
 
     for (int i = 0; i < locations->n; i++) {
-        if (!nebDeleteInstance(server, locations->URI[i])) {
+      if (!nebDeleteInstance(server, key, locations->URI[i])) {
             nebSetErr(server, "can not delete instance");
             nebObjectInstancesFree(locations);
@@ -749,5 +749,5 @@
 }
 
-bool nebDeleteInstance(nebServer *server, const char *URI)
+bool nebDeleteInstance(nebServer *server, const char *key, const char *URI)
 {
     int             response;
@@ -770,5 +770,5 @@
 
     if (soap_call_ns1__delete_USCOREinstance(server->soap, server->endpoint,
-            NULL, (char *)URI, &response) != SOAP_OK) {
+        NULL, (char *)key, (char *)URI, &response) != SOAP_OK) {
         nebFree(filename);
 
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/nebclient.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/nebclient.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/nebclient.h	(revision 24244)
@@ -282,4 +282,5 @@
 bool nebDeleteInstance(
     nebServer *server,                  ///< nebServer object
+    const char *key,                    ///< storage object key (name)
     const char *URI                     ///< URL of instance to remove
 );
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/nebulous.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/nebulous.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/nebulous.h	(revision 24244)
@@ -1,5 +1,5 @@
 /* src/nebulous.h
    Generated by wsdl2h 1.2.12 from nebulous.wsdl and typemap.dat
-   2009-03-05 21:11:05 GMT
+   2009-04-20 22:15:21 GMT
    Copyright (C) 2001-2008 Robert van Engelen, Genivia Inc. All Rights Reserved.
    This part of the software is released under one of the following licenses:
@@ -808,4 +808,5 @@
     NULL, // char *action = NULL selects default action for this operation
     // request parameters:
+    char*                               key,
     char*                               uri,
     // response parameters:
@@ -819,4 +820,5 @@
     struct soap *soap,
     // request parameters:
+    char*                               key,
     char*                               uri,
     // response parameters:
@@ -831,4 +833,5 @@
 //gsoap ns1  service method-action:	delete_USCOREinstance urn:Nebulous/Server/SOAP#delete_instance
 int ns1__delete_USCOREinstance(
+    char*                               key,	///< Request parameter
     char*                               uri,	///< Request parameter
     int                                *result	///< Response parameter
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapC.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapC.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapC.c	(revision 24244)
@@ -12,5 +12,5 @@
 #endif
 
-SOAP_SOURCE_STAMP("@(#) soapC.c ver 2.7.12 2009-03-05 21:11:05 GMT")
+SOAP_SOURCE_STAMP("@(#) soapC.c ver 2.7.12 2009-04-20 22:15:22 GMT")
 
 
@@ -1344,4 +1344,5 @@
 {
 	(void)soap; (void)a; /* appease -Wall -Werror */
+	soap_default_string(soap, &a->key);
 	soap_default_string(soap, &a->uri);
 }
@@ -1350,4 +1351,5 @@
 {
 	(void)soap; (void)a; /* appease -Wall -Werror */
+	soap_serialize_string(soap, &a->key);
 	soap_serialize_string(soap, &a->uri);
 }
@@ -1365,4 +1367,6 @@
 	if (soap_element_begin_out(soap, tag, soap_embedded_id(soap, id, a, SOAP_TYPE_ns1__delete_USCOREinstance), type))
 		return soap->error;
+	if (soap_out_string(soap, "key", -1, &a->key, ""))
+		return soap->error;
 	if (soap_out_string(soap, "uri", -1, &a->uri, ""))
 		return soap->error;
@@ -1380,4 +1384,5 @@
 SOAP_FMAC3 struct ns1__delete_USCOREinstance * SOAP_FMAC4 soap_in_ns1__delete_USCOREinstance(struct soap *soap, const char *tag, struct ns1__delete_USCOREinstance *a, const char *type)
 {
+	size_t soap_flag_key = 1;
 	size_t soap_flag_uri = 1;
 	if (soap_element_begin_in(soap, tag, 0, type))
@@ -1391,4 +1396,9 @@
 		for (;;)
 		{	soap->error = SOAP_TAG_MISMATCH;
+			if (soap_flag_key && (soap->error == SOAP_TAG_MISMATCH || soap->error == SOAP_NO_TAG))
+				if (soap_in_string(soap, "key", &a->key, "xsd:string"))
+				{	soap_flag_key--;
+					continue;
+				}
 			if (soap_flag_uri && (soap->error == SOAP_TAG_MISMATCH || soap->error == SOAP_NO_TAG))
 				if (soap_in_string(soap, "uri", &a->uri, "xsd:string"))
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapClient.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapClient.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapClient.c	(revision 24244)
@@ -10,5 +10,5 @@
 #endif
 
-SOAP_SOURCE_STAMP("@(#) soapClient.c ver 2.7.12 2009-03-05 21:11:05 GMT")
+SOAP_SOURCE_STAMP("@(#) soapClient.c ver 2.7.12 2009-04-20 22:15:21 GMT")
 
 
@@ -700,5 +700,5 @@
 }
 
-SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__delete_USCOREinstance(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *uri, int *result)
+SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__delete_USCOREinstance(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, char *uri, int *result)
 {	struct ns1__delete_USCOREinstance soap_tmp_ns1__delete_USCOREinstance;
 	struct ns1__delete_USCOREinstanceResponse *soap_tmp_ns1__delete_USCOREinstanceResponse;
@@ -708,4 +708,5 @@
 		soap_action = "urn:Nebulous/Server/SOAP#delete_instance";
 	soap->encodingStyle = "http://schemas.xmlsoap.org/soap/encoding/";
+	soap_tmp_ns1__delete_USCOREinstance.key = key;
 	soap_tmp_ns1__delete_USCOREinstance.uri = uri;
 	soap_begin(soap);
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapServer.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapServer.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapServer.c	(revision 24244)
@@ -10,5 +10,5 @@
 #endif
 
-SOAP_SOURCE_STAMP("@(#) soapServer.c ver 2.7.12 2009-03-05 21:11:05 GMT")
+SOAP_SOURCE_STAMP("@(#) soapServer.c ver 2.7.12 2009-04-20 22:15:21 GMT")
 
 
@@ -643,5 +643,5 @@
 	 || soap_end_recv(soap))
 		return soap->error;
-	soap->error = ns1__delete_USCOREinstance(soap, soap_tmp_ns1__delete_USCOREinstance.uri, &soap_tmp_int);
+	soap->error = ns1__delete_USCOREinstance(soap, soap_tmp_ns1__delete_USCOREinstance.key, soap_tmp_ns1__delete_USCOREinstance.uri, &soap_tmp_int);
 	if (soap->error)
 		return soap->error;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapStub.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapStub.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src/soapStub.h	(revision 24244)
@@ -283,4 +283,5 @@
 struct ns1__delete_USCOREinstance
 {
+	char *key;	/* optional element of type xsd:string */
 	char *uri;	/* optional element of type xsd:string */
 };
@@ -432,5 +433,5 @@
 SOAP_FMAC5 int SOAP_FMAC6 ns1__find_USCOREinstances(struct soap*, char *key, char *volume, struct ns1__find_USCOREinstancesResponse *_param_3);
 
-SOAP_FMAC5 int SOAP_FMAC6 ns1__delete_USCOREinstance(struct soap*, char *uri, int *result);
+SOAP_FMAC5 int SOAP_FMAC6 ns1__delete_USCOREinstance(struct soap*, char *key, char *uri, int *result);
 
 SOAP_FMAC5 int SOAP_FMAC6 ns1__stat_USCOREobject(struct soap*, char *key, struct ns1__stat_USCOREobjectResponse *_param_4);
@@ -467,5 +468,5 @@
 SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__find_USCOREinstances(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, char *volume, struct ns1__find_USCOREinstancesResponse *_param_3);
 
-SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__delete_USCOREinstance(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *uri, int *result);
+SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__delete_USCOREinstance(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, char *uri, int *result);
 
 SOAP_FMAC5 int SOAP_FMAC6 soap_call_ns1__stat_USCOREobject(struct soap *soap, const char *soap_endpoint, const char *soap_action, char *key, struct ns1__stat_USCOREobjectResponse *_param_4);
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/.cvsignore	(revision 24244)
@@ -0,0 +1,5 @@
+.deps
+Makefile
+Makefile.in
+.libs
+tests
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/.cvsignore	(revision 24244)
@@ -0,0 +1,13 @@
+.in
+Makefile
+Makefile.in
+aclocal.m4
+autom4te.cache
+config.log
+config.status
+configure
+libtool
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/src/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/src/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/src/.cvsignore	(revision 24244)
@@ -0,0 +1,13 @@
+.deps
+.libs
+Makefile
+Makefile.in
+libtap.la
+tap.lo
+config.h
+config.h.in
+stamp-h1
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/.cvsignore	(revision 24244)
@@ -0,0 +1,6 @@
+Makefile
+Makefile.in
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/diag/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/diag/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/diag/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/fail/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/fail/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/fail/.cvsignore	(revision 24244)
@@ -0,0 +1,15 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/.cvsignore	(revision 24244)
@@ -0,0 +1,6 @@
+Makefile
+Makefile.in
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/ok-hash/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/ok-hash/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/ok-hash/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/ok-numeric/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/ok-numeric/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/ok-numeric/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/ok/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/ok/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/ok/ok/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/pass/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/pass/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/pass/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/.cvsignore	(revision 24244)
@@ -0,0 +1,6 @@
+Makefile
+Makefile.in
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/no-tests/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/no-tests/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/no-tests/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/no_plan/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/no_plan/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/no_plan/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/not-enough-tests/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/not-enough-tests/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/not-enough-tests/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/sane/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/sane/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/sane/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/skip_all/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/skip_all/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/skip_all/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/too-many-plans/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/too-many-plans/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/too-many-plans/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/too-many-tests/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/too-many-tests/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/plan/too-many-tests/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/skip/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/skip/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/skip/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/todo/.cvsignore
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/todo/.cvsignore	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/tests/todo/.cvsignore	(revision 24244)
@@ -0,0 +1,11 @@
+.deps
+Makefile
+Makefile.in
+.libs
+test
+test.c.out
+test.pl.out
+*.bb
+*.bbg
+*.da
+gmon.out
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/ptest.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/ptest.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/ptest.pl	(revision 24244)
@@ -18,7 +18,9 @@
 $| = 1;
 
+my $prog_file = shift;
 my $key = shift || 'foobar';
 my $kids = shift || 1;
 
+require $prog_file;
 $key = "/tmp/" . $key;
 
@@ -72,41 +74,13 @@
     }
 
-#    select $sock;
-#    $| = 1;
-
-    # child
     my $fname = "${key}_$id";
 
     my $neb = Nebulous::Client::Bench->new(
 #    proxy   => 'http://localhost:80/nebulous'
-        proxy   => 'http://alala:80/nebulous',
-#        sock    => \*STDOUT,
+        proxy   => 'http://ipp008:80/nebulous'
+#        proxy   => 'http://alala:80/nebulous',
     );
 
-
-    while (1) {
-#    print $sock "$$ : i'm a little tea pot using key: $fname\n";
-        my $fh = $neb->open_create( $fname )
-            or child_die($sock, "can't create file $fname");
-        close $fh;
-
-        $fh = $neb->open( $fname, 'read' )
-            or child_die("can't open file");
-        close $fh;
-
-        $neb->lock( $fname, 'read' );
-        $neb->unlock( $fname, 'read' );
-        $neb->replicate( $fname );
-        $neb->cull( $fname );
-        $neb->find( $fname );
-        $neb->copy( $fname, $fname . "_copy" );
-        $neb->move( $fname, $fname . "_move" );
-        $neb->delete( $fname . "_copy" );
-        $neb->delete( $fname . "_move" );
-
-#    print $sock "$$ : all done!\n";
-    }
-
-    return 1;
+    test_prog($neb, $fname);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/stats.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/stats.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/stats.pl	(revision 24244)
@@ -17,4 +17,9 @@
 foreach my $line (split(/\n/, $data)) {
     my ($time, $method, $duration) = split(/\s+/, $line);
+
+    unless (defined $time and defined $method and defined $duration) {
+        warn "bad line: $line\n";
+        next;
+    }
 
     push @{$methods{$method}->{time}}, $time;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/test_prog_simple.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/test_prog_simple.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/test_prog_simple.pl	(revision 24244)
@@ -0,0 +1,31 @@
+sub test_prog
+{
+    my ($neb, $key) = @_;
+
+    while (1) {
+#    print $sock "$$ : i'm a little tea pot using key: $key\n";
+        my $fh = $neb->open_create( $key )
+            or child_die($sock, "can't create file $key");
+        close $fh;
+
+        $fh = $neb->open( $key, 'read' )
+            or child_die("can't open file");
+        close $fh;
+
+        $neb->lock( $key, 'read' );
+        $neb->unlock( $key, 'read' );
+        $neb->replicate( $key );
+        $neb->cull( $key );
+        $neb->find( $key );
+        $neb->copy( $key, $key . "_copy" );
+        $neb->move( $key, $key . "_move" );
+        $neb->delete( $key . "_copy" );
+        $neb->delete( $key . "_move" );
+
+#    print $sock "$$ : all done!\n";
+    }
+
+    return 1;
+}
+
+1;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/t/62_client_delete_instance.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/t/62_client_delete_instance.t	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/t/62_client_delete_instance.t	(revision 24244)
@@ -3,5 +3,5 @@
 # Copryight (C) 2004-2005  Joshua Hoblitt
 #
-# $Id: 62_client_delete_instance.t,v 1.3 2008-12-14 22:48:35 eugene Exp $
+# $Id: 62_client_delete_instance.t,v 1.1.38.1 2008-12-14 22:03:08 eugene Exp $
 
 use strict;
@@ -10,5 +10,5 @@
 use Apache::Test qw( -withtestmore );
 
-plan tests => 9;
+plan tests => 11;
 
 use lib qw( ./t ./lib );
@@ -26,9 +26,11 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->create( "foo" );
 
-    my $locations = $neb->find_instances( "foo" );
+    my $key = "foo";
+    $neb->create($key);
 
-    my $uri = $neb->delete_instance( @$locations[0] );
+    my $locations = $neb->find_instances($key);
+
+    my $uri = $neb->delete_instance($key, @$locations[0]);
 
     is( $uri, @$locations[0], "delete instance" );
@@ -43,10 +45,11 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->create( "foo" );
-    $neb->replicate( "foo" );
+    my $key = "foo";
+    $neb->create($key);
+    $neb->replicate($key);
 
     my $uri1 = $neb->find_instances( "foo" )->[0];
 
-    ok( $neb->delete_instance( $uri1 ), "delete instance" );
+    ok( $neb->delete_instance($key, $uri1), "delete instance" );
 
     my $uri2 = $neb->find_instances( "foo" )->[0];
@@ -54,10 +57,12 @@
     isnt( $uri1, $uri2, "other instance remains" );
 
-    ok( $neb->delete_instance( $uri2 ), "delete instance" );
+    ok( $neb->delete_instance($key, $uri2), "delete instance" );
 
     my $locations = $neb->find_instances( "foo" );
 
-    is( $locations, undef, "no remaning instances" );
+    is( $locations, undef, "no remaining instances" );
 }
+
+Test::Nebulous->setup;
 
 {
@@ -65,5 +70,7 @@
         proxy => "http://$hostport/nebulous",
     );
-    my $uri = $neb->delete_instance( "file:/foo" );
+    my $key = "foo";
+    $neb->create($key);
+    my $uri = $neb->delete_instance($key, "file:/foo" );
 
     is( $uri, undef, "uri does not exist" );
@@ -76,7 +83,7 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->delete_instance();
+    my $uri = $neb->delete_instance("foo", "file:/foo" );
 };
-like( $@, qr/1 was expected/, "no params" );
+like( $@, qr/is valid object key/, "bad object key" );
 
 Test::Nebulous->setup;
@@ -86,7 +93,27 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->delete_instance( "foo", 2 );
+    my $uri = $neb->delete_instance("foo");
 };
-like( $@, qr/1 was expected/, "too many params" );
+like( $@, qr/2 were expected/, "missing second param" );
+
+Test::Nebulous->setup;
+
+eval {
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->delete_instance();
+};
+like( $@, qr/2 were expected/, "no params" );
+
+Test::Nebulous->setup;
+
+eval {
+    my $neb = Nebulous::Client->new(
+        proxy => "http://$hostport/nebulous",
+    );
+    $neb->delete_instance("foo", 2, 3);
+};
+like( $@, qr/2 were expected/, "too many params" );
 
 Test::Nebulous->cleanup;
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/t/70_neb-ls.t
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/t/70_neb-ls.t	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/t/70_neb-ls.t	(revision 24244)
@@ -0,0 +1,206 @@
+#!/usr/bin/env perl
+
+# Copyright (C) 2009  Joshua Hoblitt
+
+=head1 NAME
+
+t/70_neb-ls.t - tests neb-ls
+
+=head1 SYNOPSIS
+    
+    prove t/70_neb-ls.t
+
+=cut
+
+use strict;
+use warnings;
+
+use Apache::Test qw( -withtestmore );
+plan tests => 31;
+
+use lib qw( ./lib ./t );
+
+use Test::Cmd;
+use Nebulous::Client;
+use Test::Nebulous;
+
+my $hostport = Apache::Test->config->{ 'hostport' };
+my $neb_url  = "http://$hostport/nebulous";
+
+
+my $cmd = 'bin/neb-ls';
+
+# last ditch effort to make sure dsget is executable
+chmod 0755, 'bin/neb-ls';
+
+my $test = Test::Cmd->new(prog => $cmd, workdir => '');
+isa_ok($test, 'Test::Cmd');
+
+# NEB_SERVER env var not set
+Test::Nebulous->setup;
+
+{
+    $test->run(args => '');
+    missing_args(2, "Required options: --server");
+}
+
+# NEB_SERVER set
+Test::Nebulous->setup;
+
+{
+    $ENV{NEB_SERVER} = $neb_url;
+
+    $test->run(args => '');
+    is($? >> 8, 0, "exit code");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('foo');
+
+    $test->run(args => '');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr/^foo$/,      "stdout");
+    like($test->stderr, qr/^$/,         "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('foo');
+    $neb->create('bar');
+
+    $test->run(args => '');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr/^foo bar$/,  "stdout");
+    like($test->stderr, qr/^$/,         "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('foo');
+    $neb->create('bar');
+
+    $test->run(args => '-l');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr/^foo\nbar\n$/,   "stdout");
+    like($test->stderr, qr/^$/,             "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('foo');
+    $neb->create('bar');
+
+    $test->run(args => '-1');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr/^foo\nbar\n$/,   "stdout");
+    like($test->stderr, qr/^$/,             "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('a/foo');
+
+    $test->run(args => '');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr||,               "stdout");
+    like($test->stderr, qr/^$/,             "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('a/foo');
+
+    $test->run(args => 'a');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr|^a/foo$|,        "stdout");
+    like($test->stderr, qr/^$/,             "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('a/foo');
+    $neb->create('a/bar');
+
+    $test->run(args => 'a');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr|^a/foo a/bar$|,  "stdout");
+    like($test->stderr, qr/^$/,             "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('a/foo');
+    $neb->create('foo');
+
+    $test->run(args => 'a');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr|^a/foo$|,        "stdout");
+    like($test->stderr, qr/^$/,             "stderr");
+}
+
+Test::Nebulous->setup;
+
+{
+    my $neb = Nebulous::Client->new(
+        proxy => $neb_url,
+    );
+    $neb->create('a/foo');
+    $neb->create('a/b/foo');
+
+    $test->run(args => 'a');
+
+    is($? >> 8, 0, "exit code");
+    like($test->stdout, qr|^a/foo$|,        "stdout");
+    like($test->stderr, qr/^$/,             "stderr");
+}
+
+Test::Nebulous->cleanup;
+
+sub missing_args
+{
+    my ($exit, $errstr) = @_;
+
+    is($? >> 8, $exit, "error code is: $exit");
+    like($test->stderr, qr/$errstr/, "error string is: $errstr");
+}
Index: /branches/cnb_branches/cnb_branch_20090301/Nebulous/t/conf/startup.pl.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Nebulous/t/conf/startup.pl.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Nebulous/t/conf/startup.pl.in	(revision 24244)
@@ -12,11 +12,25 @@
 use Test::Nebulous;
 
+#$Apache::DBI::DEBUG = 1;
 Apache::DBI->connect_on_init( $NEB_DB, $NEB_USER, $NEB_PASS );
-Nebulous::Server::SOAP->new_on_init(
+Apache::DBI->setPingTimeOut($NEB_DB, 10);
+
+my $config = Nebulous::Server::Config->new(
+    trace       => 'all',
+);
+$config->add_db(
+    dbindex     => 0,
     dsn         => $NEB_DB,
     dbuser      => $NEB_USER,
     dbpasswd    => $NEB_PASS,
-    log_level   => 'all',
 );
+#$config->add_db(
+#    dbindex     => 1,
+#    dsn         => 'DBI:mysql:database=nebulous1:host=localhost',
+#    dbuser      => $dbuser,
+#    dbpasswd    => $dbpasswd,
+#);
+
+Nebulous::Server::SOAP->new_on_init($config);
 
 1;
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/include/skycells.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/include/skycells.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/include/skycells.h	(revision 24244)
@@ -98,5 +98,5 @@
 int 	     sky_triangle_coords       	    PROTO((SkyTriangle *triangle));
 
-SkyRectangle *sky_rectangle_ring            PROTO((float dec, float dDEC, int *nring));
+SkyRectangle *sky_rectangle_ring            PROTO((float dec, float dDEC, int *nring, char *format));
 
 SkyTriangle *sky_divide_triangles      	    PROTO((SkyTriangle *in, int *ntriangles));
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/args_skycells.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/args_skycells.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/args_skycells.c	(revision 24244)
@@ -182,9 +182,10 @@
 
   fprintf (stderr, "  -mode (name)                : define the type of tessellation.  available options are:\n");
-  fprintf (stderr, "     SQUARE                   : generate rectangular skycells using icosahedron base solid (default)\n");
+  fprintf (stderr, "     SQUARE                   : generate rectangular skycells using base solid (default)\n");
   fprintf (stderr, "     TRIANGLE                 : generate triangular skycells using icosahedron base solid\n");
+  fprintf (stderr, "     RINGS                    : generate rectangular skycells using declination strips\n");
   fprintf (stderr, "     LOCAL                    : generate a local tessellation around a spot on the sky\n");
   fprintf (stderr, "                                 (Note that this tessellation does not cover the full sky)\n");
-  fprintf (stderr, "  -solid (name)               : specify the base solid\n");
+  fprintf (stderr, "  -solid (name)               : specify the base solid (default: ICOSAHEDRON)\n");
   fprintf (stderr, "                                value may be one of: TETRAHEDRON, CUBE, OCTOHEDRON, DODECAHEDRON, ICOSAHEDRON\n");
   fprintf (stderr, "                                for convenience, only the first 4 characters are required\n");
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/load_subpix.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/load_subpix.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/load_subpix.c	(revision 24244)
@@ -22,6 +22,8 @@
 
   for (i = 0; i < Nsubpix; i++) {
-    fscanf (f, "%*s %*s %lf %lf %lf %*s\n", 
-	    &Subpix[i].Amp, &Subpix[i].Phase, &Subpix[i].dM);
+      if (fscanf (f, "%*s %*s %lf %lf %lf %*s\n",
+                  &Subpix[i].Amp, &Subpix[i].Phase, &Subpix[i].dM) != 3) {
+          Shutdown("can't read subpix datafile %s", SubpixDatafile);
+      }
   }
   fclose (f);
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/sky_tessalation.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/sky_tessalation.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/sky_tessalation.c	(revision 24244)
@@ -231,8 +231,9 @@
 int sky_tessellation_rings (FITS_DB *db, int level, int Nmax) {
 
-  int j, nDEC, Nimage, Nring;
+  int j, nDEC, Nimage, Nring, Ntotal, Ndigit;
   float dec, dDEC;
   SkyRectangle *ring;
   Image *image;
+  char format[16];
 
   // The tessellation has one input parameter: the approximate cell size.  Starting with
@@ -246,11 +247,14 @@
   nDEC += 2;
 
-  // a test
-  // for (dec = 0.0 + 0.5*dDEC; dec < +90.0; dec += dDEC) {
+  // how many total projection cells for this realization?  divide sky area by cell area:
+  // this is used to set the number of digits, so it does not need to be very accurate...
+  Ntotal = 41254.2 / (dDEC*dDEC);
+  Ndigit = (int)(log10(Ntotal)) + 1 ;
+  snprintf (format, 16, "skycell.%%0%dd", Ndigit);
 
   // generate the a collection of rectangles for each ring
   for (dec = -90.0; dec < +90.0 + 0.5*dDEC; dec += dDEC) {
 
-    ring = sky_rectangle_ring (dec, dDEC, &Nring);
+    ring = sky_rectangle_ring (dec, dDEC, &Nring, format);
     if (!ring) continue;
 
@@ -534,6 +538,7 @@
 
 // define the parameters of a single sky projection center
-SkyRectangle *sky_rectangle_ring (float dec, float dDEC, int *nring) {
-
+SkyRectangle *sky_rectangle_ring (float dec, float dDEC, int *nring, char *format) {
+
+  static int Nname = 0;
   int i, NX, NY, nRA;
   SkyRectangle *ring;
@@ -631,8 +636,10 @@
     strcpy (ring[i].coords.ctype, "DEC--TAN");
 
-    ring[i].NX = NX;
-    ring[i].NY = NY;
+    ring[i].NX = NX*(1.0 + PADDING);
+    ring[i].NY = NY*(1.0 + PADDING);
     ring[i].photcode = 1; // this needs to be set more sensibly
 
+    snprintf (ring[i].name, DVO_IMAGE_NAME_LEN, format, Nname);
+    Nname++;
 
     // fprintf (stderr, "%f %f  : %f %f\n", 
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/getstar/src/Shutdown.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/getstar/src/Shutdown.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/getstar/src/Shutdown.c	(revision 24244)
@@ -21,4 +21,5 @@
   va_end (argp);
 
+  if (!db) { exit (2); }
   SetProtect (TRUE);
   gfits_db_close (db);
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/Makefile
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/Makefile	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/Makefile	(revision 24244)
@@ -60,5 +60,6 @@
 $(EXT)/fits_hdecompress.$(ARCH).o \
 $(EXT)/pliocomp.$(ARCH).o \
-$(EXT)/ricecomp.$(ARCH).o
+$(EXT)/ricecomp.$(ARCH).o \
+$(EXT)/gzip.$(ARCH).o
 
 OBJS = $(HEADER_OBJ) $(MATRIX_OBJ) $(TABLE_OBJ) $(EXTERN_OBJ)
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/extern/gzip.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/extern/gzip.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/extern/gzip.c	(revision 24244)
@@ -0,0 +1,152 @@
+# include <ohana.h>
+# include <gfitsio.h>
+# include <zlib.h>
+
+/** the two functions in this file re-implement the uncompress function of zlib.  
+    For reasons I don't understand, it seems that I need to parse the header 
+    segment of the compressed data myself rather than ship it to 'uncompress'.  
+    It could be that CFITSIO is doing something bad, or it could be my 
+    misunderstanding **/
+
+/* zlib.h -- interface of the 'zlib' general purpose compression library
+  version 1.2.3, July 18th, 2005
+
+  Copyright (C) 1995-2005 Jean-loup Gailly and Mark Adler
+
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the authors be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Jean-loup Gailly        Mark Adler
+  jloup@gzip.org          madler@alumni.caltech.edu
+
+
+  The data format used by the zlib library is described by RFCs (Request for
+  Comments) 1950 to 1952 in the files http://www.ietf.org/rfc/rfc1950.txt
+  (zlib format), rfc1951.txt (deflate format) and rfc1952.txt (gzip format).
+*/
+
+/*
+     The 'zlib' compression library provides in-memory compression and
+  decompression functions, including integrity checks of the uncompressed
+  data.  This version of the library supports only one compression method
+  (deflation) but other algorithms will be added later and will have the same
+  stream interface.
+
+     Compression can be done in a single step if the buffers are large
+  enough (for example if an input file is mmap'ed), or can be done by
+  repeated calls of the compression function.  In the latter case, the
+  application must provide more input and/or consume the output
+  (providing more output space) before each call.
+
+     The compressed data format used by default by the in-memory functions is
+  the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped
+  around a deflate stream, which is itself documented in RFC 1951.
+
+     The library also supports reading and writing files in gzip (.gz) format
+  with an interface similar to that of stdio using the functions that start
+  with "gz".  The gzip format is different from the zlib format.  gzip is a
+  gzip wrapper, documented in RFC 1952, wrapped around a deflate stream.
+
+     This library can optionally read and write gzip streams in memory as well.
+
+     The zlib format was designed to be compact and fast for use in memory
+  and on communications channels.  The gzip format was designed for single-
+  file compression on file systems, has a larger header than zlib to maintain
+  directory information, and uses a different, slower check method than zlib.
+
+     The library does not install any signal handler. The decoder checks
+  the consistency of the compressed data, so the library should never
+  crash even in case of corrupted input.
+*/
+
+// the header segments of a gzip file uses bytes and should not be swapped...
+int gfits_gz_stripheader (unsigned char *data, int *ndata) {
+  
+  int nstart, Ndata;
+
+  if (data[0] != 0x1f) return FALSE; // magic
+  if (data[1] != 0x8b) return FALSE; // magic
+  if (data[2] != 0x08) return FALSE; // DEFLATED
+  if (data[3] &  0xe0) return FALSE; // uses reserved flags?
+  
+  // data[4] - data[9] : time, xflags, OS code
+
+  nstart = 10; // data[nstart] is first byte after header
+
+  // check for an extra field
+  if (data[3] & 0x04) {
+    nstart += data[10];
+    nstart += (data[11] << 8); // data[10], data[11] are a single 2byte word that should be swapped
+  }
+  
+  // check for filename
+  if (data[3] & 0x08) {
+    while (data[nstart]) nstart ++;
+  }
+
+  // check for comment
+  if (data[3] & 0x10) {
+    while (data[nstart]) nstart ++;
+  }
+
+  // check for CRC
+  if (data[3] & 0x02) {
+    nstart ++;
+    nstart ++;
+  }
+
+  Ndata = *ndata - nstart;
+  memmove (data, &data[nstart], Ndata);
+  *ndata = Ndata;
+
+  return TRUE;
+}
+
+// XXX ??? is cfitsio writing data which is incompatible with zlib ???
+int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen)
+{
+    z_stream stream;
+    int err;
+
+    stream.next_in = (Bytef*)source;
+    stream.avail_in = (uInt)sourceLen;
+    /* Check for source > 64K on 16-bit machine: */
+    if ((uLong)stream.avail_in != sourceLen) return Z_BUF_ERROR;
+
+    stream.next_out = dest;
+    stream.avail_out = (uInt)*destLen;
+    if ((uLong)stream.avail_out != *destLen) return Z_BUF_ERROR;
+
+    stream.zalloc = (alloc_func)0;
+    stream.zfree = (free_func)0;
+
+    // MAX_WBITS = 15
+    err = inflateInit2(&stream, -15);
+    if (err != Z_OK) return err;
+
+    err = inflate(&stream, Z_FINISH);
+    if (err != Z_STREAM_END) {
+        inflateEnd(&stream);
+        if (err == Z_NEED_DICT || (err == Z_BUF_ERROR && stream.avail_in == 0))
+            return Z_DATA_ERROR;
+        return err;
+    }
+    assert (stream.total_out <= *destLen);
+    *destLen = stream.total_out;
+
+    err = inflateEnd(&stream);
+    return err;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/matrix/F_compress_M.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/matrix/F_compress_M.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/matrix/F_compress_M.c	(revision 24244)
@@ -248,5 +248,8 @@
     // XXX not certain this is the correct place to do the swap (or if zdata_pixsize is
     // the correct size to guide the swap)
-    if (!gfits_byteswap_zdata (zdata, Nzdata, zdata_pixsize)) return (FALSE);
+
+    if (strcasecmp(cmptype, "GZIP_1")) {
+      if (!gfits_byteswap_zdata (zdata, Nzdata, zdata_pixsize)) return (FALSE);
+    }
 
     // gfits_uncompress_data uncompresses from zdata to the temporary output buffer which must be allocated
@@ -496,5 +499,10 @@
 
   if (!strcasecmp(cmptype, "GZIP_1")) {
-    return (4);
+    if (out_bitpix == 8)  return (1);
+    if (out_bitpix == 16) return (2);
+    if (out_bitpix == 32) return (4);
+    if (out_bitpix == -32) return (4);
+    if (out_bitpix == -64) return (8);
+    return (1);
   }
   if (!strcasecmp(cmptype, "RICE_1")) {
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/matrix/F_uncompress_data.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/matrix/F_uncompress_data.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/matrix/F_uncompress_data.c	(revision 24244)
@@ -22,4 +22,8 @@
 int pl_l2pi (short *ll_src, int xs, int *px_dst, int npix);
 
+/* functions defined in gzip.c */
+int gfits_gz_stripheader (unsigned char *data, int *ndata);
+int gfits_uncompress (Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen);
+
 int gfits_uncompress_data (char *zdata, int Nzdata, char *cmptype, char **optname, char **optvalue, int Nopt, char *outdata, int *Nout, int out_pixsize) {
 
@@ -30,11 +34,19 @@
   if (!strcasecmp(cmptype, "GZIP_1")) {
     unsigned long tNout = *Nout * out_pixsize;
+
+    // for GZIP data, I need to check for and remove the header on the first block:
+    gfits_gz_stripheader ((unsigned char *)zdata, &Nzdata);
+
     // uncompress does not require us to know the expected number of pixel; it tells us the number
-    status = uncompress ((Bytef *) outdata, &tNout, (Bytef *) zdata, Nzdata);
+    status = gfits_uncompress ((Bytef *) outdata, &tNout, (Bytef *) zdata, Nzdata);
     if (status != Z_OK) {
       fprintf (stderr, "error in uncompress (GZIP)\n");
       return (FALSE);
     }
-    *Nout = tNout;
+    *Nout = tNout / out_pixsize;
+
+    // the resulting uncompressed data is byteswapped
+    if (!gfits_byteswap_zdata (outdata, *Nout, out_pixsize)) return (FALSE);
+
     return (TRUE);
   }
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libohana/src/gaussj.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libohana/src/gaussj.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/libohana/src/gaussj.c	(revision 24244)
@@ -1,9 +1,14 @@
 # include <ohana.h>
+# define GROWTHTEST 0
+# define MAX_RANGE 1.0e7
 
 // Gauss-Jordan elimination using full pivots based on Press et al's description.  Substantially
 // reworked for Ohana: major modifications to conform to C indexing, use a boolean to track the
 // completed pivot rows and catch the singular matrix early on.  Also, much cleaner control loops
-// than their implementation.  XXX this really needs to check on round-off errors (see version by
-// William Kahan
+// than their implementation.  (largely based on version by William Kahan)
+
+// MAX_RANGE is used to test for ill-conditioned input matrices.  For an ill-conditioned
+// matrix, one or more of the pivots trends towards zero.  Rather than allow this to go to the
+// numerical precision, I am raising an error if |growth| > 1e8
 int dgaussjordan (double **A, double **B, int N, int M) {
 
@@ -19,12 +24,13 @@
   memset (pivot, 0, N*sizeof(int));
 
+  double growth = 1.0;
+
   // determine underflow conditions
   // double underFlow = DBL_MIN;
-# if (0)
+# if (GROWTHTEST)
   double roundTest = 4.0;
   roundTest /= 3.0;
   roundTest -= 1.0;
   double epsilon = fabs(((roundTest+roundTest) - 1.0) + roundTest);
-  double growth = 1.0;
 # endif
 
@@ -57,4 +63,16 @@
     }
 
+# if (GROWTHTEST)
+    fprintf (stderr, "maxcol: %d\n", maxcol);
+    fprintf (stderr, "full A matrix:\n");
+    for (row = 0; row < N; row++) {
+	for (col = 0; col < N; col++) {
+	    fprintf (stderr, "%10.3e ", A[row][col]);
+	}
+	fprintf (stderr, "\n");
+    }
+    fprintf (stderr, "\n");
+# endif
+
     // if pivot[maxcol] is set, we have already done this row: this implies a singular matrix
     if (pivot[maxcol]) goto escape;
@@ -76,7 +94,17 @@
     for (col = 0; col < N; col++) A[maxcol][col] *= tmpval;
     for (col = 0; col < M; col++) B[maxcol][col] *= tmpval;
-    // XXX measure the pivot growth and trigger on over/under flow
-    // growth *= tmpval;
-    // fprintf (stderr, "column: %d, growth: %e, epsilon: %e\n", maxcol, growth, epsilon);
+
+    // check for ill-conditioned matrix
+    growth *= tmpval;
+
+    // report the pivot growth
+#   if (GROWTHTEST)
+    fprintf (stderr, "column: %d, maxval : %f, growth: %e, epsilon: %e\n", maxcol, tmpval, growth, epsilon);
+    fprintf (stderr, "A diagonal: ");
+    for (col = 0; col < N; col++) fprintf (stderr, "%f ", A[col][col]);
+    fprintf (stderr, "\n");
+# endif
+
+    if (fabs(growth) > MAX_RANGE) goto escape;
 
     /* adjust the elements above the pivot */
@@ -122,12 +150,13 @@
   memset (pivot, 0, N*sizeof(int));
 
+  float growth = 1.0;
+
   // determine underflow conditions
   // float underFlow = FLT_MIN;
-# if (0)
+# if (GROWTHTEST)
   float roundTest = 4.0;
   roundTest /= 3.0;
   roundTest -= 1.0;
   float epsilon = fabs(((roundTest+roundTest) - 1.0) + roundTest);
-  float growth = 1.0;
 # endif
 
@@ -179,6 +208,17 @@
     for (col = 0; col < N; col++) A[maxcol][col] *= tmpval;
     for (col = 0; col < M; col++) B[maxcol][col] *= tmpval;
-    // growth *= tmpval;
-    // fprintf (stderr, "column: %d, growth: %e, epsilon: %e\n", maxcol, growth, epsilon);
+
+    // check for ill-conditioned matrix
+    growth *= tmpval;
+
+    // report the pivot growth
+#   if (GROWTHTEST)
+    fprintf (stderr, "column: %d, maxval : %f, growth: %e, epsilon: %e\n", maxcol, tmpval, growth, epsilon);
+    fprintf (stderr, "A diagonal: ");
+    for (col = 0; col < N; col++) fprintf (stderr, "%f ", A[col][col]);
+    fprintf (stderr, "\n");
+# endif
+
+    if (fabs(growth) > MAX_RANGE) goto escape;
 
     /* adjust the elements above the pivot */
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.astro/coord_systems.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.astro/coord_systems.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.astro/coord_systems.c	(revision 24244)
@@ -123,5 +123,6 @@
   // atan2 returns -pi : +pi
   *x = DEG_RAD * atan2 (sin_x, cos_x) + transform->xo;
-  if ((*x) < 0.0) (*x) += 360;
+  if ((*x) <   0.0) (*x) += 360;
+  if ((*x) > 360.0) (*x) -= 360;
 
   // should be in range -pi/2 : +pi/2
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.data/fit1d.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.data/fit1d.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.data/fit1d.c	(revision 24244)
@@ -125,5 +125,8 @@
       }
     }
-    if (!dgaussjordan (c, b, nterm, 1)) goto escape;
+    if (!dgaussjordan (c, b, nterm, 1)) {
+	gprint (GP_ERR, "failed to fit data : ill-conditioned matrix\n");
+	goto escape;
+    }
 
     /* generate fitted values */
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.data/gaussj.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.data/gaussj.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.data/gaussj.c	(revision 24244)
@@ -47,13 +47,16 @@
   status = dgaussjordan (a, b, N, 1);
 
-  // output vector needs to be float, so re-cast it
-  ResetVector (B, OPIHI_FLT, N);
-  vf = B[0].elements.Flt;
+  // if dgaussjordan succeeds, replace the input values with the results
+  if (status) {
+    // output vector needs to be float, so re-cast it
+    ResetVector (B, OPIHI_FLT, N);
+    vf = B[0].elements.Flt;
 
-  for (i = 0; i < N; i++) {
-    for (j = 0; j < N; j++) {
-       m[i+j*N] = a[i][j];
+    for (i = 0; i < N; i++) {
+      for (j = 0; j < N; j++) {
+	m[i+j*N] = a[i][j];
+      }
+      vf[i] = b[i][0]; 
     }
-    vf[i] = b[i][0]; 
   }
 
@@ -66,5 +69,5 @@
 
   if (!status && !QUIET) {
-      gprint (GP_ERR, "failure in matrix solution\n");
+      gprint (GP_ERR, "gaussjordan: ill-conditioned matrix; input values are retained\n");
   }
   return (status);
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.data/init.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.data/init.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.data/init.c	(revision 24244)
@@ -193,4 +193,5 @@
   {1, "iminterp",     minterp,          "interpolate image pixels"},
   {1, "mkrgb",        mkrgb,            "convert 3 images to rgb jpeg (use Kapa for better control)"},
+  {1, "mset",         mset,             "insert a vector in an image"},
   {1, "imset",        mset,             "insert a vector in an image"},
   {1, "parity",       parity,           "set image parity"},
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/avextract.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/avextract.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/avextract.c	(revision 24244)
@@ -205,5 +205,5 @@
     gprint (GP_ERR, "  <photcode>:cal : first calibrated magnitude for <photcode> \n");
     gprint (GP_ERR, "  <photcode>:err : magnitude error for photcode\n");
-    gprint (GP_ERR, "  <photcode>:chipsq : chi-square of magnitude fit\n");
+    gprint (GP_ERR, "  <photcode>:chisq : raw chi-square of magnitude fit\n");
     gprint (GP_ERR, "  type : dophot type (unused)\n");
     gprint (GP_ERR, "  typefrac : dophot type fraction (unused)\n");
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/gstar.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/gstar.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/gstar.c	(revision 24244)
@@ -14,5 +14,5 @@
   int Nstars, found, GetMeasures, Nlo, Nhi;
   int SaveVectors;
-  Vector *vec1, *vec2, *vec3, *vec4;
+  Vector *vec1, *vec2, *vec3, *vec4, *vec5, *vec6;
   SkyTable *sky;
   SkyList *skylist;
@@ -37,5 +37,5 @@
 
   NPTS = 0;
-  vec1 = vec2 = vec3 = vec4 = NULL;
+  vec1 = vec2 = vec3 = vec4 = vec5 = vec6 = NULL;
   SaveVectors = FALSE;
   if ((N = get_argument (argc, argv, "-save"))) {
@@ -46,4 +46,6 @@
     if ((vec3 = SelectVector ("gs:z", ANYVECTOR, TRUE)) == NULL) return (FALSE);
     if ((vec4 = SelectVector ("gs:f", ANYVECTOR, TRUE)) == NULL) return (FALSE);
+    if ((vec5 = SelectVector ("gs:dr", ANYVECTOR, TRUE)) == NULL) return (FALSE);
+    if ((vec6 = SelectVector ("gs:dd", ANYVECTOR, TRUE)) == NULL) return (FALSE);
   }
 
@@ -87,5 +89,5 @@
   /* lock, load, unlock catalog */
   catalog.filename = skylist[0].filename[0];
-  catalog.catflags = LOAD_AVES | LOAD_MEAS | LOAD_SECF;
+  catalog.catflags = GetMeasures ? LOAD_AVES | LOAD_MEAS | LOAD_SECF : LOAD_AVES | LOAD_SECF;
   catalog.Nsecfilt = 0;
 
@@ -140,4 +142,6 @@
     ResetVector (vec3, OPIHI_FLT, NPTS);
     ResetVector (vec4, OPIHI_FLT, NPTS);
+    ResetVector (vec5, OPIHI_FLT, NPTS);
+    ResetVector (vec6, OPIHI_FLT, NPTS);
   }
 
@@ -156,19 +160,20 @@
 	gprint (GP_LOG, "%11.7f ", catalog.average[k].R);
 	gprint (GP_LOG, "%11.7f ", catalog.average[k].D);
+	gprint (GP_LOG, "%5.2f ",   3600.0*sqrt(r));
 	gprint (GP_LOG, "%3d   ",  catalog.average[k].Nmeasure);
 	gprint (GP_LOG, "%4.1f ",  0.01*catalog.average[k].Xp);
-	gprint (GP_LOG, "%5d",     catalog.average[k].flags);
-
+	gprint (GP_LOG, "%5d ",     catalog.average[k].flags);
+	
 	if (FULL_OUTPUT) {
-	    gprint (GP_LOG, "%f",     catalog.average[k].dR);
-	    gprint (GP_LOG, "%f",     catalog.average[k].dD);
-	    gprint (GP_LOG, "%f",     catalog.average[k].uR);
-	    gprint (GP_LOG, "%f",     catalog.average[k].uD);
-	    gprint (GP_LOG, "%f",     catalog.average[k].duR);
-	    gprint (GP_LOG, "%f",     catalog.average[k].duD);
-	    gprint (GP_LOG, "%f",     catalog.average[k].P);
-	    gprint (GP_LOG, "%f",     catalog.average[k].dP);
-	    gprint (GP_LOG, "%x",     catalog.average[k].objID);
-	    gprint (GP_LOG, "%x",     catalog.average[k].catID);
+	    gprint (GP_LOG, "%f ",     catalog.average[k].dR);
+	    gprint (GP_LOG, "%f ",     catalog.average[k].dD);
+	    gprint (GP_LOG, "%f ",     catalog.average[k].uR);
+	    gprint (GP_LOG, "%f ",     catalog.average[k].uD);
+	    gprint (GP_LOG, "%f ",     catalog.average[k].duR);
+	    gprint (GP_LOG, "%f ",     catalog.average[k].duD);
+	    gprint (GP_LOG, "%f ",     catalog.average[k].P);
+	    gprint (GP_LOG, "%f ",     catalog.average[k].dP);
+	    gprint (GP_LOG, "%x ",     catalog.average[k].objID);
+	    gprint (GP_LOG, "%x ",     catalog.average[k].catID);
 	}
 
@@ -210,34 +215,34 @@
 	    gprint (GP_LOG, "%20s  ",  date);
 	    gprint (GP_LOG, "%7.4f ",  catalog.measure[m].dR);
-	    gprint (GP_LOG, "%7.4f",   catalog.measure[m].dD);
+	    gprint (GP_LOG, "%7.4f ",  catalog.measure[m].dD);
 	    gprint (GP_LOG, "%4x ",    catalog.measure[m].photFlags);
 	    gprint (GP_LOG, "%3x ",    catalog.measure[m].dbFlags);
 	    gprint (GP_LOG, "%5d ",    catalog.measure[m].photcode);
-	    gprint (GP_LOG, "%-20s  ", GetPhotcodeNamebyCode (catalog.measure[m].photcode));
-	    gprint (GP_LOG, "%5.2f ",  0.01*catalog.measure[m].FWx);
+	    gprint (GP_LOG, "%-20s ",  GetPhotcodeNamebyCode (catalog.measure[m].photcode));
+ 	    gprint (GP_LOG, "%5.2f ",  0.01*catalog.measure[m].FWx);
 	    gprint (GP_LOG, "%5.2f ",  0.01*catalog.measure[m].FWy);
 
 	    if (FULL_OUTPUT) {
-		gprint (GP_LOG, "%f", catalog.measure[m].Mcal);
-		gprint (GP_LOG, "%f", catalog.measure[m].Map);
-		gprint (GP_LOG, "%f", pow(10.0, 0.4*catalog.measure[m].dt));
-		gprint (GP_LOG, "%f", 1.0 + catalog.measure[m].airmass);
-		gprint (GP_LOG, "%f", catalog.measure[m].az);
-		gprint (GP_LOG, "%f", catalog.measure[m].Xccd);
-		gprint (GP_LOG, "%f", catalog.measure[m].Yccd);
-		gprint (GP_LOG, "%d", catalog.measure[m].dXccd);
-		gprint (GP_LOG, "%d", catalog.measure[m].dYccd);
-		gprint (GP_LOG, "%f", catalog.measure[m].Sky);
-		gprint (GP_LOG, "%f", catalog.measure[m].dSky);
-		gprint (GP_LOG, "%d", catalog.measure[m].averef);
-		gprint (GP_LOG, "%d", catalog.measure[m].detID);
-		gprint (GP_LOG, "%d", catalog.measure[m].imageID);
-		gprint (GP_LOG, "%f", catalog.measure[m].psfQual);
-		gprint (GP_LOG, "%f", catalog.measure[m].psfChisq);
-		gprint (GP_LOG, "%f", catalog.measure[m].crNsigma);
-		gprint (GP_LOG, "%f", catalog.measure[m].extNsigma);
-		gprint (GP_LOG, "%f", 0.01*catalog.measure[m].FWx);
-		gprint (GP_LOG, "%f", 0.01*catalog.measure[m].FWy);
-		gprint (GP_LOG, "%f", (360.0/(float)0xffff)*catalog.measure[m].theta);
+		gprint (GP_LOG, "%f ", catalog.measure[m].Mcal);
+		gprint (GP_LOG, "%f ", catalog.measure[m].Map);
+		gprint (GP_LOG, "%f ", pow(10.0, 0.4*catalog.measure[m].dt));
+		gprint (GP_LOG, "%f ", 1.0 + catalog.measure[m].airmass);
+		gprint (GP_LOG, "%f ", catalog.measure[m].az);
+		gprint (GP_LOG, "%f ", catalog.measure[m].Xccd);
+		gprint (GP_LOG, "%f ", catalog.measure[m].Yccd);
+		gprint (GP_LOG, "%d ", catalog.measure[m].dXccd);
+		gprint (GP_LOG, "%d ", catalog.measure[m].dYccd);
+		gprint (GP_LOG, "%f ", catalog.measure[m].Sky);
+		gprint (GP_LOG, "%f ", catalog.measure[m].dSky);
+		gprint (GP_LOG, "%d ", catalog.measure[m].averef);
+		gprint (GP_LOG, "%d ", catalog.measure[m].detID);
+		gprint (GP_LOG, "%d ", catalog.measure[m].imageID);
+		gprint (GP_LOG, "%f ", catalog.measure[m].psfQual);
+		gprint (GP_LOG, "%f ", catalog.measure[m].psfChisq);
+		gprint (GP_LOG, "%f ", catalog.measure[m].crNsigma);
+		gprint (GP_LOG, "%f ", catalog.measure[m].extNsigma);
+		gprint (GP_LOG, "%f ", 0.01*catalog.measure[m].FWx);
+		gprint (GP_LOG, "%f ", 0.01*catalog.measure[m].FWy);
+		gprint (GP_LOG, "%f ", (360.0/(float)0xffff)*catalog.measure[m].theta);
 	    }
 	    gprint (GP_LOG, "\n");
@@ -251,4 +256,6 @@
 	    vec3[0].elements.Flt[N] = catalog.measure[m].airmass;
 	    vec4[0].elements.Flt[N] = catalog.measure[m].photcode;
+	    vec5[0].elements.Flt[N] = catalog.measure[m].dR;
+	    vec6[0].elements.Flt[N] = catalog.measure[m].dD;
 	    N ++;
 	    if (N == NPTS - 1) {
@@ -258,4 +265,6 @@
 	      REALLOCATE (vec3[0].elements.Flt, opihi_flt, NPTS);
 	      REALLOCATE (vec4[0].elements.Flt, opihi_flt, NPTS);
+	      REALLOCATE (vec5[0].elements.Flt, opihi_flt, NPTS);
+	      REALLOCATE (vec6[0].elements.Flt, opihi_flt, NPTS);
 	    }
 	  }
@@ -270,4 +279,6 @@
     vec3[0].Nelements = N;
     vec4[0].Nelements = N;
+    vec5[0].Nelements = N;
+    vec6[0].Nelements = N;
   }
 
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/mextract.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/mextract.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/mextract.c	(revision 24244)
@@ -226,5 +226,5 @@
     gprint (GP_ERR, "  <photcode>:cal :  calibrated magnitude for photcode \n");
     gprint (GP_ERR, "  <photcode>:err : magnitude error for photcode\n");
-    gprint (GP_ERR, "  <photcode>:chisq : chi-square of magnitude fit\n");
+    gprint (GP_ERR, "  <photcode>:chisq : raw chi-square of magnitude fit\n");
     gprint (GP_ERR, "  <photcode>:ncode : number of measurements in photcode\n");
     gprint (GP_ERR, "  <photcode>:nphot : number of measurements used for average magnitude\n");
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/mmextract.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/mmextract.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dvo/mmextract.c	(revision 24244)
@@ -348,5 +348,5 @@
     gprint (GP_ERR, "  photcode:cal :  calibrated magnitude for photcode \n");
     gprint (GP_ERR, "  photcode:err : magnitude error for photcode\n");
-    gprint (GP_ERR, "  photcode:chisq : chi-square of magnitude fit\n");
+    gprint (GP_ERR, "  photcode:chisq : raw chi-square of magnitude fit\n");
     gprint (GP_ERR, "  photcode:ncode : number of measurements in photcode\n");
     gprint (GP_ERR, "  photcode:nphot : number of measurements used for average magnitude\n");
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/lib.shell/gprint.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/lib.shell/gprint.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/lib.shell/gprint.c	(revision 24244)
@@ -306,5 +306,5 @@
     status = vfprintf (stream[0].file, format, argp);
     if (status < 0) {
-      abort();
+      return (FALSE);
     }
   } else {
Index: /branches/cnb_branches/cnb_branch_20090301/Ohana/src/tools/src/fields.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/Ohana/src/tools/src/fields.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/Ohana/src/tools/src/fields.c	(revision 24244)
@@ -60,10 +60,12 @@
   while (fscanf (stdin, "%s", filename) != EOF) {
     if (!Extnum && !Extname) {
-      GotFile &= gfits_read_header (filename, &header);
+      if (!gfits_read_header (filename, &header)) continue;
+      GotFile = TRUE;
       GotField &= print_fields (filename, NULL, &header, argc, argv);
       continue;
     }
     if (Extnum) {
-      GotFile  &= gfits_read_Xheader (filename, &header, Nextend);
+      if (!gfits_read_Xheader (filename, &header, Nextend)) continue;
+      GotFile = TRUE;
       GotField &= print_fields (filename, NULL, &header, argc, argv);
       continue;
@@ -98,5 +100,5 @@
 	Nextend ++;
 
-	GotFile = gfits_read_Xheader (filename, &header, Nextend);
+	GotFile &= gfits_read_Xheader (filename, &header, Nextend);
 	continue;
       } 
Index: /branches/cnb_branches/cnb_branch_20090301/PS-IPP-Config/lib/PS/IPP/Config.pm
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/PS-IPP-Config/lib/PS/IPP/Config.pm	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/PS-IPP-Config/lib/PS/IPP/Config.pm	(revision 24244)
@@ -564,8 +564,11 @@
 
     my $scheme = file_scheme($name); # The scheme, e.g., file://, path://
+
     if (defined $scheme and lc($scheme) eq 'neb') {
+        my $save_name = $name;
         $name = eval { $self->nebulous->create( $name ) };
+
         if ($@ or not defined $name) {
-            carp "Unable to create Nebulous handle $name";
+            carp "Unable to create Nebulous handle $save_name";
             return undef;
         }
@@ -1001,10 +1004,9 @@
     $filename =~ s/\{OUTPUT\}/$output/;
     if ($filename =~ /\{CHIP\.NAME\}/) {
-        unless (defined $component) {
-            carp "Programming error";
-            return undef;
-        }
-        $filename =~ s/\{CHIP\.NAME\}/$component/;
-        $filename =~ s/\{CELL\.NAME\}/$component/;
+        my ($chip, $cell) = ($component, $component); # Chip and cell names
+        $chip = "fpa" unless defined $chip;
+        $cell = "chip" unless defined $cell;
+        $filename =~ s/\{CHIP\.NAME\}/$chip/;
+        $filename =~ s/\{CELL\.NAME\}/$cell/;
     }
 
@@ -1043,9 +1045,7 @@
 
     if ($extname =~ /\{CHIP\.NAME\}/) {
-        unless (defined $component) {
-            carp "Programming error";
-            return undef;
-        }
-        $extname =~ s/\{CHIP\.NAME\}/$component/;
+        my $chip = $component;  # Chip name
+        $chip = "fpa" unless defined $chip;
+        $extname =~ s/\{CHIP\.NAME\}/$chip/;
     }
 
@@ -1264,11 +1264,23 @@
     $tess_dir = $self->convert_filename_absolute( $tess_dir ) or return undef;
 
-    unless ($self->file_exists( $outname )) {
-        my $outnameResolved = $self->file_create( $outname ) or return undef; # Resolved filename, for Nebulous
-        my $command = "$dvoImageExtract -D CATDIR $tess_dir $skycell_id -o $outnameResolved";
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        ( carp "Unable to perform dvoImageExtract for $tess_id $skycell_id\n" and return undef ) unless ($success and $self->file_exists( $outname ));
-    }
+    my $outnameResolved;
+
+    # check if file actually exists
+    if ($self->file_exists( $outname )) {
+	# double check that the file is not zero-length (eg: dvoImageExtract crashed)
+        $outnameResolved = $self->file_resolve( $outname, 1 ) or return undef; # Resolved filename, for Nebulous
+	my @stats = stat($outnameResolved);
+	if ($stats[7]) {
+	    return 1;
+	}
+    }
+
+    unless (defined $outnameResolved) {
+	$outnameResolved = $self->file_create( $outname ) or return undef; # Resolved filename, for Nebulous
+    }
+    my $command = "$dvoImageExtract -D CATDIR $tess_dir $skycell_id -o $outnameResolved";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $command, verbose => $verbose);
+    ( carp "Unable to perform dvoImageExtract for $tess_id $skycell_id\n" and return undef ) unless ($success and $self->file_exists( $outname ));
 
     return 1;
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/cam.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/cam.md	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/cam.md	(revision 24244)
@@ -87,4 +87,5 @@
     path_base      STR      255
     fault          S16      0       # Key NOT NULL
+    quality        S16      0
 END
 
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/changes.txt
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/changes.txt	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/changes.txt	(revision 24244)
@@ -899,4 +899,8 @@
 update chipRun SET magicked = 1 where chip_id = 11955;
 
+
+-- Version 1.1.51
+
+-- Distribution updates
 CREATE TABLE distTarget (
     target_id   BIGINT AUTO_INCREMENT,
@@ -908,5 +912,4 @@
     PRIMARY KEY(target_id)
 )  ENGINE=innodb DEFAULT CHARSET=latin1;
-
 
 CREATE TABLE rcDSProduct (
@@ -931,5 +934,4 @@
 )  ENGINE=innodb DEFAULT CHARSET=latin1;
 
-
 CREATE TABLE rcInterest (
     int_id      BIGINT AUTO_INCREMENT,
@@ -967,2 +969,159 @@
 )  ENGINE=innodb DEFAULT CHARSET=latin1;
 
+ALTER TABLE distRun ADD COLUMN no_magic TINYINT AFTER clean;
+
+
+-- Adding data quality flags
+DROP TABLE guidePendingExp;
+ALTER TABLE diffSkyfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault;
+ALTER TABLE stackSumSkyfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault;
+ALTER TABLE warpSkyfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault;
+ALTER TABLE camProcessedExp ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault;
+ALTER TABLE chipProcessedImfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault;
+ALTER TABLE rawImfile ADD COLUMN quality SMALLINT NOT NULL DEFAULT 0 AFTER fault;
+
+
+ALTER TABLE diffSkyfile ADD KEY(quality);
+ALTER TABLE stackSumSkyfile ADD KEY(quality);
+ALTER TABLE warpSkyfile ADD KEY(quality);
+ALTER TABLE camProcessedExp ADD KEY(quality);
+ALTER TABLE chipProcessedImfile ADD KEY(quality);
+ALTER TABLE rawImfile ADD KEY(quality);
+
+UPDATE TABLE warpSkyfile SET quality = 8007 WHERE ignored != 0;
+ALTER TABLE warpSkyfile DROP COLUMN ignored;
+
+-- Version 1.1.52
+
+-- drop unneeded column and the associated constraint
+ALTER TABLE rcInterest DROP FOREIGN KEY rcInterest_ibfk_3;
+ALTER TABLE rcInterest DROP COLUMN prod_id;
+
+ALTER TABLE rcDSProduct DROP column prod_root;
+
+-- just re-create these two tables
+DROP TABLE rcRun;
+DROP TABLE rcDSFileset;
+
+CREATE TABLE rcDSFileset (
+    fs_id       BIGINT AUTO_INCREMENT,
+    dist_id     BIGINT,
+    prod_id     BIGINT,
+    name        VARCHAR(255),
+    state       VARCHAR(64),
+    registered  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    fault       SMALLINT DEFAULT 0,
+    PRIMARY KEY(dist_id, prod_id),
+    KEY(fs_id),
+    FOREIGN KEY(dist_id) REFERENCES distRun(dist_id),
+    FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE rcRun (
+    rc_id       BIGINT AUTO_INCREMENT,
+    fs_id       BIGINT,
+    dest_id     BIGINT,
+    state       VARCHAR(64),
+    registered  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(rc_id),
+    FOREIGN KEY(fs_id) REFERENCES rcDSFileset(fs_id),
+    FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
+ALTER TABLE diffSkyfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
+UPDATE diffSkyfile SET data_state = 'full' WHERE data_state IS NULL;
+
+
+-- Sources from which to receive files
+CREATE TABLE receiveSource (
+    source_id BIGINT AUTO_INCREMENT, -- unique identifier
+    source VARCHAR(128) NOT NULL, -- source URI
+    product VARCHAR(64) NOT NULL, -- product of interest
+    workdir VARCHAR(255) NOT NULL, -- where to extract
+    comment VARCHAR(255),       -- for human memory
+    fileset_last VARCHAR(128),  -- last fileset seen
+    PRIMARY KEY(source_id),
+    KEY(source),
+    KEY(product),
+    KEY(comment)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Filesets to receive
+CREATE TABLE receiveFileset (
+    fileset_id BIGINT AUTO_INCREMENT, -- unique identifier
+    source_id BIGINT NOT NULL,  -- link to receiveSource
+    fileset VARCHAR(128) NOT NULL, -- fileset to receive
+    PRIMARY KEY(fileset_id),
+    KEY(source_id),
+    CONSTRAINT UNIQUE(source_id, fileset),
+    FOREIGN KEY(source_id) REFERENCES receiveSource(source_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Files to receive
+CREATE TABLE receiveFile (
+    file_id BIGINT AUTO_INCREMENT, -- unique identifier
+    fileset_id BIGINT NOT NULL,  -- link to receiveFileset
+    file VARCHAR(128) NOT NULL, -- file to receive
+    PRIMARY KEY(file_id),
+    KEY(fileset_id),
+    CONSTRAINT UNIQUE(fileset_id, file),
+    FOREIGN KEY(fileset_id) REFERENCES receiveFileset(fileset_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Result of receiving files
+CREATE TABLE receiveResult (
+    file_id BIGINT AUTO_INCREMENT, -- link to receiveFile
+    dtime_copy FLOAT,           -- Time to copy
+    dtime_extract FLOAT,        -- Time to extract
+    fault SMALLINT NOT NULL DEFAULT 0, -- Fault code
+    PRIMARY KEY(file_id),
+    KEY(fault),
+    FOREIGN KEY(file_id) REFERENCES receiveFile(file_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+ALTER TABLE rcRun ADD COLUMN fault SMALLINT DEFAULT 0;
+ALTER TABLE rcRun ADD COLUMN status_fs VARCHAR(64) AFTER state;
+
+ALTER TABLE diffSkyfile ADD COLUMN deconv_max FLOAT AFTER kernel_yy;
+
+-- This key makes it faster to find the ccd_temp for XY24
+ALTER TABLE rawImfile ADD KEY(dateobs);
+
+ALTER TABLE receiveSource ADD COLUMN status_product VARCHAR(64);
+ALTER TABLE receiveSource ADD COLUMN ds_dbname VARCHAR(64);
+ALTER TABLE receiveSource ADD COLUMN ds_dbhost VARCHAR(64);
+
+ALTER TABLE receiveFileset ADD COLUMN state VARCHAR(64);
+ALTER TABLE receiveFileset ADD COLUMN dbinfo_uri VARCHAR(255);
+ALTER TABLE receiveFileset ADD COLUMN fault SMALLINT NOT NULL DEFAULT 0;
+
+ALTER TABLE receiveFile ADD COLUMN bytes BIGINT;
+ALTER TABLE receiveFile ADD COLUMN md5sum VARCHAR(255);
+ALTER TABLE receiveFile ADD COLUMN file_type VARCHAR(64);
+ALTER TABLE receiveFile ADD COLUMN component VARCHAR(64);
+
+ALTER TABLE receiveFileset CHANGE COLUMN dbinfo_uri dbinfo VARCHAR(255);
+ALTER TABLE receiveFileset ADD COLUMN dirinfo VARCHAR(255) AFTER state;
+
+-- Playing with the diffs for warp-warp
+ALTER TABLE diffRun ADD COLUMN bothways TINYINT DEFAULT 0 AFTER tess_id;
+UPDATE diffRun SET bothways = 1 WHERE reduction = 'WARPWARP';
+ALTER TABLE diffRun DROP FOREIGN KEY diffRun_ibfk_1;
+ALTER TABLE diffRun DROP COLUMN exp_id;
+ALTER TABLE diffRun ADD KEY(label);
+ALTER TABLE diffRun ADD COLUMN exposure TINYINT DEFAULT 0 AFTER bothways;
+UPDATE diffRun, diffInputSkyfile SET diffRun.exposure = 1 WHERE diffRun.diff_id = diffInputSkyfile.diff_id and diffInputSkyfile.warp1 IS NOT NULL;
+ALTER TABLE stackRun ADD KEY(label);
+
+ALTER TABLE magicRun ADD COLUMN inverse TINYINT NOT NULL DEFAULT 0 AFTER diff_id;
+ALTER TABLE diffSkyfile DROP COLUMN uri;
+
+DROP TABLE IF EXISTS magicSkyfileMask;
+ALTER TABLE magicInputSkyfile DROP FOREIGN KEY magicInputSkyfile_ibfk_2;
+ALTER TABLE magicInputSkyfile DROP KEY diff_id;
+ALTER TABLE magicInputSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY(magic_id,node);
+ALTER TABLE magicInputSkyfile DROP COLUMN diff_id;
+ALTER TABLE magicNodeResult CHANGE COLUMN uri path_base VARCHAR(255);
+
+-- Version 1.1.53
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/chip.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/chip.md	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/chip.md	(revision 24244)
@@ -17,5 +17,5 @@
     chip_id     S64         0       # Primary Key fkey (chip_id) ref chipRun (chip_id)
     class_id    STR         64      # Primary Key
-    chip_image_id   S64     0       # Key AUTO_INCREMENT
+    chip_imfile_id   S64     0       # Key AUTO_INCREMENT
 END
 
@@ -88,4 +88,5 @@
     path_base       STR     255
     fault           S16     0       # Key NOT NULL
+    quality         S16     0
     magicked        BOOL    f
 END
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/config.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/config.md	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/config.md	(revision 24244)
@@ -2,4 +2,4 @@
     pkg_name        STR     ippdb
     pkg_namespace   STR     ippdb
-    pkg_version     STR     1.1.50
+    pkg_version     STR     1.1.53
 END
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/diff.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/diff.md	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/diff.md	(revision 24244)
@@ -10,5 +10,6 @@
     registered  TAI         NULL
     tess_id     STR         64      # Key
-    exp_id      S64         0       # fkey(exp_id) ref rawExp(exp_id)
+    bothways    BOOL	    f
+    exposure	BOOL	    f
     magicked	BOOL        f
 END
@@ -32,6 +33,6 @@
     diff_id      S64        0       # Primary Key fkey(diff_id) ref diffRun(diff_id)
     skycell_id   STR        64      # 
-    uri          STR        255
     path_base    STR        255
+    data_state   STR        64
     bg           F64        0.0
     bg_stdev     F64        0.0
@@ -46,4 +47,5 @@
     kernel_xy    F32        0.0
     kernel_yy    F32        0.0
+    deconv_max   F32        0.0
     sources      S32        0
     dtime_diff   F32        0.0
@@ -54,4 +56,5 @@
     good_frac    F32        0.0     # Key
     fault        S16        0       # Key
+    quality      S16        0
     magicked     BOOL       f
 END
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/dist.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/dist.md	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/dist.md	(revision 24244)
@@ -1,4 +1,14 @@
+distTarget METADATA
+    target_id   S64        0       # Primary Key
+    obs_mode    STR        64
+    stage       STR        64
+    clean       BOOL       f
+    state       STR        64
+    comment     STR        255
+END
+
 distRun METADATA
     dist_id     S64         0       # Primary Key
+    target_id   S64         0       # fkey(target_id) ref distTarget(target_id)
     stage       STR         64
     stage_id    S64         0
@@ -6,4 +16,5 @@
     outroot     STR         255
     clean       BOOL        f
+    no_magic    BOOL        f
     state       STR         64      # Key
     time_stamp  UTC         0001-01-01T00:00:00Z
@@ -21,10 +32,2 @@
 END
 
-distTarget METADATA
-    target_id   S64        0       # Primary Key
-    obs_mode    STR        64
-    clean       BOOL       f
-    state       STR        64
-    comment     STR        255
-END
-
Index: anches/cnb_branches/cnb_branch_20090301/dbconfig/guide.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/guide.md	(revision 24243)
+++ 	(revision )
@@ -1,5 +1,0 @@
-guidePendingExp METADATA
-    guide_id    S64         0       # Primary Key AUTO_INCREMENT
-    exp_id     S64         64      # Key
-    recipe      STR         64
-END
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/ipp.m4
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/ipp.m4	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/ipp.m4	(revision 24244)
@@ -9,6 +9,7 @@
 dnl include(telescope.md)
 dnl include(skycell.md)
-include(tasks.md)
-include(guide.md)
+include(summitcopy.md)
+include(new.md)
+include(raw.md)
 include(chip.md)
 include(cam.md)
@@ -23,2 +24,4 @@
 include(pstamp.md)
 include(dist.md)
+include(rc.md)
+include(receive.md)
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/magic.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/magic.md	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/magic.md	(revision 24244)
@@ -6,4 +6,5 @@
     exp_id      S64         0       # Key
     diff_id     S64         0       # Key
+    inverse     BOOL	    FALSE	    
     state       STR         64      # Key
     workdir     STR         255
@@ -15,7 +16,9 @@
 END
 
+### This is left over from when diffs were composed of a single skycell
+### When we're not too busy, it should be deleted in favour of:
+### magicRun JOIN diffSkyfile USING(diff_id) WHERE diffSkyfile.fault = 0 AND diffSkyfile.quality = 0
 magicInputSkyfile METADATA
     magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
-    diff_id     S64         0       # Primary Key fkey(diff_id) ref diffRun(diff_id)
     node        STR         64      #
 END
@@ -30,5 +33,5 @@
     magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
     node        STR         64      # Primary Key fkey(magic_id, node) ref magicTree(magic_id, node)
-    uri         STR         255
+    path_base   STR         255
     fault       S16         0       # Key
 END
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/new.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/new.md	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/new.md	(revision 24244)
@@ -0,0 +1,25 @@
+newExp METADATA
+    exp_id      S64         0       # Primary Key AUTO_INCREMENT
+    tmp_exp_name STR        64      # Key
+    tmp_camera    STR       64      # Key
+    tmp_telescope STR       64      # Key
+    state       STR         64      # Key
+    workdir     STR         255     # destination for output files
+    workdir_state STR       64      # key
+    reduction   STR         64      # Reduction class
+    dvodb       STR         255
+    tess_id     STR         64
+    end_stage   STR         64      # Key
+    label       STR         64      # Key
+    epoch       UTC         0001-01-01T00:00:00Z
+END
+
+# class needs to be carried here so it can go into rawImfile and be normalized
+# from there
+newImfile METADATA
+    exp_id      S64         64      # Primary Key fkey(exp_id) ref newExp(exp_id)
+    tmp_class_id STR        64      # Primary Key
+    uri         STR         255
+    epoch       UTC         0001-01-01T00:00:00Z
+END
+
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/raw.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/raw.md	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/raw.md	(revision 24244)
@@ -0,0 +1,138 @@
+# paired with rawImfile
+rawExp METADATA
+    exp_id      S64         64      # Primary Key fkey(exp_id) ref newExp(exp_id)
+    exp_name    STR         64      # Key
+    camera      STR         64
+    telescope   STR         64
+    dateobs     UTC         0001-01-01T00:00:00Z
+    exp_tag     STR         255
+    exp_type    STR         64
+    filelevel   STR         64
+    workdir     STR         255     # destination for output files
+    reduction   STR         64      # Reduction class
+    dvodb       STR         255
+    tess_id     STR         64
+    end_stage   STR         64      # Key
+    filter      STR         64
+    comment     STR         80
+    obs_mode    STR         64      # data usage goal (eg, survey name, engineering, etc)
+    obs_group   STR         64      # identifier for data block (eg, observation sequence)
+    airmass     F32         0.0
+    ra          F64         0.0
+    decl        F64         0.0
+    exp_time    F32         0.0
+    sat_pixel_frac F32      0.0
+    bg          F64         0.0
+    bg_stdev    F64         0.0
+    bg_mean_stdev   F64     0.0
+    alt         F64         0.0
+    az          F64         0.0
+    ccd_temp    F32         0.0
+    posang      F64         0.0 
+    m1_x        F32         0.0
+    m1_y        F32         0.0
+    m1_z        F32         0.0
+    m1_tip      F32         0.0
+    m1_tilt     F32         0.0
+    m2_x        F32         0.0
+    m2_y        F32         0.0
+    m2_z        F32         0.0
+    m2_tip      F32         0.0
+    m2_tilt     F32         0.0
+    env_temperature F32     0.0
+    env_humidity    F32     0.0
+    env_wind_speed  F32     0.0
+    env_wind_dir    F32     0.0
+    teltemp_m1      F32     0.0
+    teltemp_m1cell  F32     0.0
+    teltemp_m2      F32     0.0
+    teltemp_spider  F32     0.0
+    teltemp_truss   F32     0.0
+    teltemp_extra   F32     0.0
+    pon_time        F32     0.0
+    user_1      F64         0.0
+    user_2      F64         0.0
+    user_3      F64         0.0
+    user_4      F64         0.0
+    user_5      F64         0.0
+    object      STR         64
+    sun_angle   F32         0.0
+    sun_alt     F32         0.0
+    moon_angle  F32         0.0
+    moon_alt    F32         0.0
+    moon_phase  F32         0.0
+    hostname    STR         64
+    fault       S16         0       # Key NOT NULL
+    epoch       UTC         0001-01-01T00:00:00Z
+    magicked	BOOL        f
+END
+
+rawImfile METADATA
+    exp_id      S64         64      # Primary Key fkey(exp_id, tmp_class_id) ref newImfile(exp_id, tmp_class_id)
+    exp_name    STR         64      # UINDEX(exp_id, tmp_class_id)
+    camera      STR         64
+    telescope   STR         64
+    dateobs     UTC         0001-01-01T00:00:00Z
+    tmp_class_id    STR     64      # Key
+    class_id    STR         64      # Primary Key
+    uri         STR         255
+    exp_type    STR         64
+# This field is used to set the per exp filelevel. Thus the values for all of
+# the imfiles in an exposure need to be sanity checked to make sure that this
+# value is in argeement.
+    filelevel   STR         64 
+    filter      STR         64
+    comment     STR         80
+    obs_mode    STR         64      # data usage goal (eg, survey name, engineering, etc)
+    obs_group   STR         64      # identifier for data block (eg, observation sequence)
+    airmass     F32         0.0
+    ra          F64         0.0
+    decl        F64         0.0
+    exp_time    F32         0.0
+    sat_pixel_frac F32      0.0
+    bg          F64         0.0
+    bg_stdev    F64         0.0
+    bg_mean_stdev   F64     0.0
+    alt         F64         0.0
+    az          F64         0.0
+    ccd_temp    F32         0.0
+    posang      F64         0.0 
+    m1_x        F32         0.0
+    m1_y        F32         0.0
+    m1_z        F32         0.0
+    m1_tip      F32         0.0
+    m1_tilt     F32         0.0
+    m2_x        F32         0.0
+    m2_y        F32         0.0
+    m2_z        F32         0.0
+    m2_tip      F32         0.0
+    m2_tilt     F32         0.0
+    env_temperature F32     0.0
+    env_humidity    F32     0.0
+    env_wind_speed  F32     0.0
+    env_wind_dir    F32     0.0
+    teltemp_m1      F32     0.0
+    teltemp_m1cell  F32     0.0
+    teltemp_m2      F32     0.0
+    teltemp_spider  F32     0.0
+    teltemp_truss   F32     0.0
+    teltemp_extra   F32     0.0
+    pon_time        F32     0.0
+    user_1      F64         0.0
+    user_2      F64         0.0
+    user_3      F64         0.0
+    user_4      F64         0.0
+    user_5      F64         0.0
+    object      STR         64
+    sun_angle   F32         0.0
+    sun_alt     F32         0.0
+    moon_angle  F32         0.0
+    moon_alt    F32         0.0
+    moon_phase  F32         0.0
+    ignored	BOOL        f
+    hostname    STR         64
+    fault       S16         0       # Key NOT NULL
+    quality     S16         0
+    epoch       UTC         0001-01-01T00:00:00Z
+    magicked    BOOL        f
+END
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/rc.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/rc.md	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/rc.md	(revision 24244)
@@ -0,0 +1,43 @@
+rcDSProduct METADATA
+    prod_id     S64        0       # Primary Key
+    name        STR        64
+    dbname      STR        64
+    dbhost      STR        64
+END
+
+rcDSFileset METADATA
+    fs_id       S64        0       # Primary Key
+    dist_id     S64        0       # fkey(dist_id) ref distRun(dist_id)
+    prod_id     S64        0       # fkey(prod_id) ref rcDSProduct(prod_id)
+    name        STR        255
+    state       STR        64
+    fault       S16        0
+END
+
+rcDestination METADATA
+    dest_id     S64         0       # Primary Key
+    prod_id     S64         0       # fkey(prod_id) ref rcDSProduct(prod_id)
+    name        STR         64
+    status_uri  STR         255
+    comment     STR         255
+    last_fileset STR        255
+    state       STR         64      # Key
+END
+
+rcInterest METADATA
+    int_id      S64         0       # Primary Key
+    dest_id     S64         0       # fkey(dest_id) ref rcDestination(dest_id)
+    target_id   S64         0       # fkey(target_id ref distTarget(target_id)
+    state       STR         64
+END
+
+rcRun METADATA
+    rc_id       S64         0       # Primary Key
+    fs_id       S64         0       # fkey(fs_id) ref rcDSFileset(fs_id)
+    dest_id     S64         0       # Primary Key fkey(dest_id) ref rcDestination(dest_id)
+    state       STR         64
+    status_fs_name   STR         64
+    registered  UTC         0001-01-01T00:00:00Z
+    fault       S16        0
+END
+
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/receive.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/receive.md	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/receive.md	(revision 24244)
@@ -0,0 +1,40 @@
+# Tables for receiving files
+
+receiveSource	METADATA
+	source_id	S64	0	# Primary Key AUTO_INCREMENT
+	source		STR	127	# Key
+	product		STR	64	# Key
+	workdir		STR	255
+	comment		STR	255	# Key
+	fileset_last	STR	128
+        status_product  STR     64
+        ds_dbname       STR     64
+        ds_dbhost       STR     64
+END
+
+receiveFileset	METADATA
+	fileset_id	S64	0	# Primary Key AUTO_INCREMENT
+	source_id	S64	0	# Key fkey (source_id) ref receiveSource(source_id)
+	fileset		STR	128
+        state           STR     64
+        dirinfo         STR     255
+        dbinfo          STR     255
+	fault		S16	0	# Key
+END
+
+receiveFile	METADATA
+	file_id		S64	0	# Primary Key AUTO_INCREMENT
+	fileset_id	S64	0	# Key fkey (fileset_id) ref receiveFileset(fileset_id)
+	file		STR	128
+        bytes           S64     0
+        md5sum          STR     255
+        file_type       STR     64
+        component       STR     64
+END
+
+receiveResult	METADATA
+	file_id		S64	0	# Primary Key fkey (file_id) ref receiveFile(file_id)
+	dtime_copy	F32	0.0
+	dtime_extract	F32	0.0
+	fault		S16	0	# Key
+END
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/stack.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/stack.md	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/stack.md	(revision 24244)
@@ -46,3 +46,4 @@
     good_frac          F32    0.0     # Key
     fault              S16    0       # Key
+    quality            S16    0
 END
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/summitcopy.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/summitcopy.md	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/summitcopy.md	(revision 24244)
@@ -0,0 +1,60 @@
+pzDataStore METADATA
+    camera      STR         64      # Primary Key
+    telescope   STR         64      # Primary Key
+    uri         STR         255
+    epoch       UTC         0001-01-01T00:00:00Z
+END
+
+# list of source exposures -- updated as exposures are seen
+# summitExp.imfiles is updated as filesets are queried default value should be
+# -1 as NULL or 0 might be a valid value (empty fileset)
+summitExp METADATA
+    exp_name    STR         64      # Primary Key
+    camera      STR         64      # Primary Key
+    telescope   STR         64      # Primary Key
+    dateobs     UTC         NULL
+    exp_type    STR         64
+    uri         STR         255
+    imfiles     S32         0
+    fault       S16         0       # Key NOT NULL
+    epoch       UTC         0001-01-01T00:00:00Z
+END
+
+# class == type of file
+# class_id == type set id
+# list of source images -- updated as exposures/filesets are queried
+summitImfile METADATA
+    exp_name    STR         64      # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope)
+    camera      STR         64      # Primary Key
+    telescope   STR         64      # Primary Key
+    file_id     STR         64      # Key
+    bytes       S32         0
+    md5sum      STR         32
+    class       STR         64      # Primary Key
+    class_id    STR         64      # Primary Key
+    uri         STR         255
+    epoch       UTC         0001-01-01T00:00:00Z
+END
+
+# list of exposures that have had their imfiles/files registered (but not
+# downloaded) 
+pzDownloadExp METADATA
+    exp_name    STR         64      # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope)
+    camera      STR         64      # Primary Key
+    telescope   STR         64      # Primary Key
+    state       STR         64      # Key 
+    epoch       UTC         0001-01-01T00:00:00Z
+END
+
+pzDownloadImfile METADATA
+    exp_name    STR         64      # Primary Key fkey(exp_name, camera, telescope) ref pzDownloadExp(exp_name, camera, telescope)
+    camera      STR         64      # Primary Key fkey(exp_name, camera, telescope, class, class_id) ref summitImfile(exp_name, camera, telescope, class, class_id)
+    telescope   STR         64      # Primary Key
+    class       STR         64      # Primary Key
+    class_id    STR         64      # Primary Key
+    uri         STR         255
+    fault       S16         0       # Key NOT NULL
+    epoch       UTC         0001-01-01T00:00:00Z
+    hostname    STR         64
+END
+
Index: anches/cnb_branches/cnb_branch_20090301/dbconfig/tasks.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/tasks.md	(revision 24243)
+++ 	(revision )
@@ -1,243 +1,0 @@
-# $Id: tasks.md,v 1.159 2008-12-13 20:17:34 bills Exp $
-
-# this table records all exposure ID ever seen from the summit
-# exp_name == fileset
-
-# note that dec is a MySQL reserved word
-# note that use is a MySQL reserved word
-# note that exp is a MySQL reserved word
-
-# for use with this stored procedure to generate exp_ids under Mysql 5+
-#
-#CREATE FUNCTION genTag (exp_name varchar(64)) RETURNS VARCHAR(64)
-#BEGIN
-#    UPDATE expTagCounter SET counter = LAST_INSERT_ID(counter + 1);
-#    RETURN CONCAT(exp_name, '.', LAST_INSERT_ID());
-#END
-#
-#expTagCounter METADATA
-#   counter     U64         0 
-#END
-
-pzDataStore METADATA
-    camera      STR         64      # Primary Key
-    telescope   STR         64      # Primary Key
-    uri         STR         255
-    epoch       UTC         0001-01-01T00:00:00Z
-END
-
-# list of source exposures -- updated as exposures are seen
-# summitExp.imfiles is updated as filesets are queried default value should be
-# -1 as NULL or 0 might be a valid value (empty fileset)
-summitExp METADATA
-    exp_name    STR         64      # Primary Key
-    camera      STR         64      # Primary Key
-    telescope   STR         64      # Primary Key
-    dateobs     UTC         NULL
-    exp_type    STR         64
-    uri         STR         255
-    imfiles     S32         0
-    fault       S16         0       # Key NOT NULL
-    epoch       UTC         0001-01-01T00:00:00Z
-END
-
-# class == type of file
-# class_id == type set id
-# list of source images -- updated as exposures/filesets are queried
-summitImfile METADATA
-    exp_name    STR         64      # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope)
-    camera      STR         64      # Primary Key
-    telescope   STR         64      # Primary Key
-    file_id     STR         64      # Key
-    bytes       S32         0
-    md5sum      STR         32
-    class       STR         64      # Primary Key
-    class_id    STR         64      # Primary Key
-    uri         STR         255
-    epoch       UTC         0001-01-01T00:00:00Z
-END
-
-# list of exposures that have had their imfiles/files registered (but not
-# downloaded) 
-pzDownloadExp METADATA
-    exp_name    STR         64      # Primary Key fkey(exp_name, camera, telescope) ref summitExp(exp_name, camera, telescope)
-    camera      STR         64      # Primary Key
-    telescope   STR         64      # Primary Key
-    state       STR         64      # Key 
-    epoch       UTC         0001-01-01T00:00:00Z
-END
-
-pzDownloadImfile METADATA
-    exp_name    STR         64      # Primary Key fkey(exp_name, camera, telescope) ref pzDownloadExp(exp_name, camera, telescope)
-    camera      STR         64      # Primary Key fkey(exp_name, camera, telescope, class, class_id) ref summitImfile(exp_name, camera, telescope, class, class_id)
-    telescope   STR         64      # Primary Key
-    class       STR         64      # Primary Key
-    class_id    STR         64      # Primary Key
-    uri         STR         255
-    fault       S16         0       # Key NOT NULL
-    epoch       UTC         0001-01-01T00:00:00Z
-    hostname    STR         64
-END
-
-newExp METADATA
-    exp_id      S64         0       # Primary Key AUTO_INCREMENT
-    tmp_exp_name STR        64      # Key
-    tmp_camera    STR       64      # Key
-    tmp_telescope STR       64      # Key
-    state       STR         64      # Key
-    workdir     STR         255     # destination for output files
-    workdir_state STR       64      # key
-    reduction   STR         64      # Reduction class
-    dvodb       STR         255
-    tess_id     STR         64
-    end_stage   STR         64      # Key
-    label       STR         64      # Key
-    epoch       UTC         0001-01-01T00:00:00Z
-END
-
-# class needs to be carried here so it can go into rawImfile and be normalized
-# from there
-newImfile METADATA
-    exp_id      S64         64      # Primary Key fkey(exp_id) ref newExp(exp_id)
-    tmp_class_id STR        64      # Primary Key
-    uri         STR         255
-    epoch       UTC         0001-01-01T00:00:00Z
-END
-
-# paired with rawImfile
-rawExp METADATA
-    exp_id      S64         64      # Primary Key fkey(exp_id) ref newExp(exp_id)
-    exp_name    STR         64      # Key
-    camera      STR         64
-    telescope   STR         64
-    dateobs     UTC         0001-01-01T00:00:00Z
-    exp_tag     STR         255
-    exp_type    STR         64
-    filelevel   STR         64
-    workdir     STR         255     # destination for output files
-    reduction   STR         64      # Reduction class
-    dvodb       STR         255
-    tess_id     STR         64
-    end_stage   STR         64      # Key
-    filter      STR         64
-    comment     STR         80
-    obs_mode    STR         64      # data usage goal (eg, survey name, engineering, etc)
-    obs_group   STR         64      # identifier for data block (eg, observation sequence)
-    airmass     F32         0.0
-    ra          F64         0.0
-    decl        F64         0.0
-    exp_time    F32         0.0
-    sat_pixel_frac F32      0.0
-    bg          F64         0.0
-    bg_stdev    F64         0.0
-    bg_mean_stdev   F64     0.0
-    alt         F64         0.0
-    az          F64         0.0
-    ccd_temp    F32         0.0
-    posang      F64         0.0 
-    m1_x        F32         0.0
-    m1_y        F32         0.0
-    m1_z        F32         0.0
-    m1_tip      F32         0.0
-    m1_tilt     F32         0.0
-    m2_x        F32         0.0
-    m2_y        F32         0.0
-    m2_z        F32         0.0
-    m2_tip      F32         0.0
-    m2_tilt     F32         0.0
-    env_temperature F32     0.0
-    env_humidity    F32     0.0
-    env_wind_speed  F32     0.0
-    env_wind_dir    F32     0.0
-    teltemp_m1      F32     0.0
-    teltemp_m1cell  F32     0.0
-    teltemp_m2      F32     0.0
-    teltemp_spider  F32     0.0
-    teltemp_truss   F32     0.0
-    teltemp_extra   F32     0.0
-    pon_time        F32     0.0
-    user_1      F64         0.0
-    user_2      F64         0.0
-    user_3      F64         0.0
-    user_4      F64         0.0
-    user_5      F64         0.0
-    object      STR         64
-    sun_angle   F32         0.0
-    sun_alt     F32         0.0
-    moon_angle  F32         0.0
-    moon_alt    F32         0.0
-    moon_phase  F32         0.0
-    hostname    STR         64
-    fault       S16         0       # Key NOT NULL
-    epoch       UTC         0001-01-01T00:00:00Z
-    magicked	BOOL        f
-END
-
-rawImfile METADATA
-    exp_id      S64         64      # Primary Key fkey(exp_id, tmp_class_id) ref newImfile(exp_id, tmp_class_id)
-    exp_name    STR         64      # UINDEX(exp_id, tmp_class_id)
-    camera      STR         64
-    telescope   STR         64
-    dateobs     UTC         0001-01-01T00:00:00Z
-    tmp_class_id    STR     64      # Key
-    class_id    STR         64      # Primary Key
-    uri         STR         255
-    exp_type    STR         64
-# This field is used to set the per exp filelevel. Thus the values for all of
-# the imfiles in an exposure need to be sanity checked to make sure that this
-# value is in argeement.
-    filelevel   STR         64 
-    filter      STR         64
-    comment     STR         80
-    obs_mode    STR         64      # data usage goal (eg, survey name, engineering, etc)
-    obs_group   STR         64      # identifier for data block (eg, observation sequence)
-    airmass     F32         0.0
-    ra          F64         0.0
-    decl        F64         0.0
-    exp_time    F32         0.0
-    sat_pixel_frac F32      0.0
-    bg          F64         0.0
-    bg_stdev    F64         0.0
-    bg_mean_stdev   F64     0.0
-    alt         F64         0.0
-    az          F64         0.0
-    ccd_temp    F32         0.0
-    posang      F64         0.0 
-    m1_x        F32         0.0
-    m1_y        F32         0.0
-    m1_z        F32         0.0
-    m1_tip      F32         0.0
-    m1_tilt     F32         0.0
-    m2_x        F32         0.0
-    m2_y        F32         0.0
-    m2_z        F32         0.0
-    m2_tip      F32         0.0
-    m2_tilt     F32         0.0
-    env_temperature F32     0.0
-    env_humidity    F32     0.0
-    env_wind_speed  F32     0.0
-    env_wind_dir    F32     0.0
-    teltemp_m1      F32     0.0
-    teltemp_m1cell  F32     0.0
-    teltemp_m2      F32     0.0
-    teltemp_spider  F32     0.0
-    teltemp_truss   F32     0.0
-    teltemp_extra   F32     0.0
-    pon_time        F32     0.0
-    user_1      F64         0.0
-    user_2      F64         0.0
-    user_3      F64         0.0
-    user_4      F64         0.0
-    user_5      F64         0.0
-    object      STR         64
-    sun_angle   F32         0.0
-    sun_alt     F32         0.0
-    moon_angle  F32         0.0
-    moon_alt    F32         0.0
-    moon_phase  F32         0.0
-    ignored	BOOL        f
-    hostname    STR         64
-    fault       S16         0       # Key NOT NULL
-    epoch       UTC         0001-01-01T00:00:00Z
-    magicked    BOOL        f
-END
Index: /branches/cnb_branches/cnb_branch_20090301/dbconfig/warp.md
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/dbconfig/warp.md	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/dbconfig/warp.md	(revision 24244)
@@ -55,6 +55,6 @@
     ymin           S32      0
     ymax           S32      0
-    ignored        BOOL     f       # Key
     fault          S16      0       # Key
+    quality        S16      0
     magicked       BOOL     f
 END
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/Makefile.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/Makefile.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/Makefile.in	(revision 24244)
@@ -8,4 +8,7 @@
 	cp -f man/man1/* @MANDIR@/man1/
 
+clean:
+	echo "dummy clean"
+
 update:
 	mkdir -p gpcsrc
@@ -15,15 +18,26 @@
 	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/ThisIsTopLevel gpcsrc/ThisIsTopLevel
 	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/missing_protos.h gpcsrc/missing_protos.h
-# remove existing directories
-	rm -rf gpcsrc/fits
-	rm -rf gpcsrc/analysis
 # required subdirs
-	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/analysis/libpscoords gpcsrc/analysis/libpscoords
-	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/analysis/libpsf gpcsrc/analysis/libpsf
-	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/fits/libfh gpcsrc/fits/libfh
-	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/fits/libfhreg gpcsrc/fits/libfhreg
-	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/fits/burntool gpcsrc/fits/burntool
-# replace the standard gpc/Make.Common with our repaired version (g77 -> gfortran)
-	cp -f Make.Common.fixed gpcsrc/Make.Common
+	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/analysis/libpscoords gpcsrc/analysis/libpscoords.new
+	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/analysis/libpsf gpcsrc/analysis/libpsf.new
+	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/fits/libfh gpcsrc/fits/libfh.new
+	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/fits/libfhreg gpcsrc/fits/libfhreg.new
+	svn export https://svn.ifa.hawaii.edu/gpc/repos/sw/trunk/fits/burntool gpcsrc/fits/burntool.new
+# replace new files with those from the imported directories
+	cp -rf gpcsrc/analysis/libpscoords.new/* gpcsrc/analysis/libpscoords
+	cp -rf gpcsrc/analysis/libpsf.new/* gpcsrc/analysis/libpsf
+	cp -rf gpcsrc/fits/libfh.new/* gpcsrc/fits/libfh
+	cp -rf gpcsrc/fits/libfhreg.new/* gpcsrc/fits/libfhreg
+	cp -rf gpcsrc/fits/burntool.new/* gpcsrc/fits/burntool
+# remove the temp directory
+	rm -rf gpcsrc/analysis/libpscoords.new
+	rm -rf gpcsrc/analysis/libpsf.new
+	rm -rf gpcsrc/fits/libfh.new
+	rm -rf gpcsrc/fits/libfhreg.new
+	rm -rf gpcsrc/fits/burntool.new
+# modify burntool to avoid libpsf (which requires fortran)
+	cp -f burntool.nopsf/Makefile gpcsrc/fits/burntool
+	cp -f burntool.nopsf/psfstats.c gpcsrc/fits/burntool
+
 # set up links
 	ln -sf ../Make.Common gpcsrc/analysis/Make.Common
@@ -39,5 +53,3 @@
 	cd gpcsrc && make -C fits/libfhreg install
 	cd gpcsrc && make -C analysis/libpscoords install
-	cd gpcsrc && make -C analysis/libpsf install
 	cd gpcsrc && make -C fits/burntool install
-
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/burntool.nopsf/Makefile
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/burntool.nopsf/Makefile	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/burntool.nopsf/Makefile	(revision 24244)
@@ -0,0 +1,40 @@
+include ../Make.Common
+
+VERSION=0.00
+# CCWARN+=$(WERROR)
+CCDEFS+=$(VERSIONDEFS)
+CCLINK += -lm
+# $(EXECNAME): $(OBJS) libfh.a libpsf.a libpscoords.a
+$(EXECNAME): $(OBJS) libfh.a libpscoords.a
+
+include ../Make.Common
+
+# Dependencies by Make.Common $Revision: 2.17 $
+
+$(OBJ)/burnfix.o: burnfix.c burntool.h burnparams.h
+
+$(OBJ)/burntool.o: burntool.c fh/fh.h fhreg/general.h \
+  fhreg/macros.h fhreg/gpc_detector.h \
+  burntool.h burnparams.h persist_fits.h
+
+$(OBJ)/burnutils.o: burnutils.c burntool.h burnparams.h
+
+$(OBJ)/cheapfits.o: cheapfits.c
+
+$(OBJ)/persist_fits.o: persist_fits.c fh/fh.h burntool.h \
+  burnparams.h persist_fits.h
+
+$(OBJ)/persistfix.o: persistfix.c burntool.h burnparams.h
+
+$(OBJ)/persistio.o: persistio.c burntool.h burnparams.h
+
+$(OBJ)/psfstamp.o: psfstamp.c burntool.h burnparams.h \
+  pscoords/pscoords.h
+
+$(OBJ)/psfstats.o: psfstats.c burntool.h psf/psf.h
+
+$(OBJ)/sort.o: sort.c
+
+$(OBJ)/stardetect.o: stardetect.c burntool.h burnparams.h
+
+$(OBJ)/trailfit.o: trailfit.c burntool.h burnparams.h
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/burntool.nopsf/psfstats.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/burntool.nopsf/psfstats.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/burntool.nopsf/psfstats.c	(revision 24244)
@@ -0,0 +1,20 @@
+/* dummy version of psfstats.c - disable use of the psf */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "burntool.h"
+#include "math.h"
+#include "psf/psf.h"
+
+/****************************************************************/
+/* psf_stats(): Tell us about this PSF star */
+STATIC int psf_stats(int nx, int ny, IMTYPE *data, int bias, 
+		     double *fwhm, double *q)
+{
+   return(1);
+}
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/Make.Common
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/Make.Common	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/Make.Common	(revision 24244)
@@ -134,5 +134,5 @@
 TAR	 = gtar
 AR	 = ar
-FC       = gfortran
+FC       = g77
 CC	 = gcc
 LD       = gcc
@@ -673,5 +673,5 @@
 	@-rm -f $(DIR_BIN)/*.TEXT_FILE_BUSY 2> /dev/null || /bin/true
 	@ln $< $@
-	# @-setuidinst $@
+	@-setuidinst $@
 
 $(DIR_BIN)/%-$(VERSION): scripts/%.sh
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/analysis/libpscoords/pscoords.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/analysis/libpscoords/pscoords.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/analysis/libpscoords/pscoords.h	(revision 24244)
@@ -1,3 +1,3 @@
-/* Three-quarter-assed pscoords.h until jt writes me a real one */
+/* Header file for pscoords.c */
 #define NINT(x) (x<0?(int)((x)-0.5):(int)((x)+0.5))
 
@@ -128,4 +128,8 @@
 const PSC_OFFROT_T psc_otaoff[PSC_NX*PSC_NY];
 
+#define PS_scale 38.860	/* Default PS plate scale [um/arcsec] */
+#define PS_d3  1.49e-10	/* Default PS distortion [arcsec^-2] */
+#define PS_airdens 0.71	/* Default PS1 qir density (Haleakala) */
+
 /**********************/
 /* Normal Application */
@@ -147,5 +151,4 @@
  */
 
-
 /**************/
 /* Prototypes */
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/analysis/libpsf/psf.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/analysis/libpsf/psf.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/analysis/libpsf/psf.c	(revision 24244)
@@ -207,4 +207,7 @@
 /* Load up extras */
    if(psfextra != NULL) {
+/* Sanity check against no error but crazy fit! */
+      psfextra->xfw = xfwhm*binx;
+      psfextra->yfw = yfwhm*biny;
 /* Note that these major/minor things are messed up if binx!=biny */
       psfextra->majfw = wpar[9]*binx;
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/Makefile
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/Makefile	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/Makefile	(revision 24244)
@@ -5,5 +5,4 @@
 CCDEFS+=$(VERSIONDEFS)
 CCLINK += -lm
-
 # $(EXECNAME): $(OBJS) libfh.a libpsf.a libpscoords.a
 $(EXECNAME): $(OBJS) libfh.a libpscoords.a
@@ -13,15 +12,16 @@
 # Dependencies by Make.Common $Revision: 2.17 $
 
-$(OBJ)/basement.o: basement.c
-
 $(OBJ)/burnfix.o: burnfix.c burntool.h burnparams.h
 
 $(OBJ)/burntool.o: burntool.c fh/fh.h fhreg/general.h \
   fhreg/macros.h fhreg/gpc_detector.h \
-  burntool.h burnparams.h
+  burntool.h burnparams.h persist_fits.h
 
 $(OBJ)/burnutils.o: burnutils.c burntool.h burnparams.h
 
 $(OBJ)/cheapfits.o: cheapfits.c
+
+$(OBJ)/persist_fits.o: persist_fits.c fh/fh.h burntool.h \
+  burnparams.h persist_fits.h
 
 $(OBJ)/persistfix.o: persistfix.c burntool.h burnparams.h
@@ -34,4 +34,6 @@
 $(OBJ)/psfstats.o: psfstats.c burntool.h psf/psf.h
 
+$(OBJ)/sort.o: sort.c
+
 $(OBJ)/stardetect.o: stardetect.c burntool.h burnparams.h
 
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/burnparams.h	(revision 24244)
@@ -1,3 +1,5 @@
 /* A global buffer into which routines can accumulate detections */
+#ifndef _INCLUDED_burnparams_
+#define _INCLUDED_burnparams_
 
 /* '#define EXTERN' in just one file, burntool.c, to declare variables */
@@ -28,4 +30,7 @@
 EXTERN int MAX_READ_NOISE;	/* Maximum believable read noise (ADU) */
 EXTERN double MIN_EADU;		/* Minimum believable e/ADU */
+EXTERN int SAT4SURE;		/* Ignore pixels above for noise estimate */
+EXTERN double MIN_BLAST_PASS;	/* Allow blasted cells if they have BIG satfrac */
+EXTERN double MAX_BLAST_PASS;	/* But not if it's all wiped out! */
 
 EXTERN int BURN_THRESH  ;	/* Threshold for onset of burning */
@@ -48,4 +53,5 @@
 
 EXTERN double NEGLIGIBLE_TRAIL;	/* Don't sweat less than this * sigma */
+EXTERN int EXPIRE_TRAIL_TIME;	/* Expire trails after this interval */
 
-
+#endif /* _INCLUDED_burnparams_ */
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.c	(revision 24244)
@@ -20,4 +20,5 @@
 #define EXTERN  /* Define EXTERN to declare variables in params.h */
 #include "burnparams.h"
+#include "persist_fits.h"
 
 int
@@ -28,5 +29,4 @@
    HeaderUnit ehu;
    const char* ifilename = "-";
-   const char* ofilename = NULL;
    char extname[FH_MAX_STRLEN+1], otaposn[FH_MAX_STRLEN+1];
    int cellmask[MAXCELL];
@@ -34,26 +34,11 @@
    int nextend, cellxy, cell, cellcode;
    int ext, update, restore, psfsize, psfavg;
-   int outfd;
    IMTYPE *buf;
-   const char *burnfile=NULL,  *persistfile=NULL;
+   const char *burnfile=NULL,  *persistfile=NULL, *persistfitsfile=NULL;
+   const char *deltablefitsfile=NULL;
    const char *psffile=NULL, *psfstatfile=NULL;
    CELL OTA[MAXCELL];	/* Cell structure for entire OTA */
 
-   if (argc == 1)  {
-     syntax(argv[0]);
-     exit(EXIT_SUCCESS);
-   }
-
    if(argc > 1 && strncmp(argv[1], "help", 4) == 0) {
-      syntax(argv[0]);
-      exit(EXIT_SUCCESS);
-   }
-
-   if(argc > 1 && strncmp(argv[1], "--help", 6) == 0) {
-      syntax(argv[0]);
-      exit(EXIT_SUCCESS);
-   }
-
-   if(argc > 1 && strncmp(argv[1], "-h", 6) == 0) {
       syntax(argv[0]);
       exit(EXIT_SUCCESS);
@@ -71,5 +56,7 @@
       fprintf(stderr, "\rerror: `%s' is not a multi-extension FITS\n",
 	      ifilename);
+#ifndef JT2DHACK
       exit(EXIT_FAILURE);
+#endif
    }
 
@@ -91,4 +78,7 @@
    MAX_READ_NOISE = 20;		/* Maximum believable read noise (ADU) */
    MIN_EADU = 0.3;		/* Minimum believable e/ADU */
+   SAT4SURE  = 60000;		/* Ignore pixels above for noise estimate */
+   MIN_BLAST_PASS = 0.1;	/* Allow blasted cells if they have BIG satfrac */
+   MAX_BLAST_PASS = 0.9;	/* But not if it's all wiped out! */
 
    BURN_THRESH  = 30000;	/* Threshold for onset of burning */
@@ -108,5 +98,5 @@
 
    NEGLIGIBLE_TRAIL = 0.5;	/* Don't sweat less than this * sigma */
-
+   EXPIRE_TRAIL_TIME = 2000;	/* Expire a persist after this [sec] */
 
 /* Parse the args */
@@ -155,22 +145,23 @@
 	 burnfile = argv[i] + 9;
 
-/* Output file for burn streaks */
+/* Output text file for burn streaks */
       } else if(strncmp(argv[i], "out=", 4) == 0) {	/* out=fname */
 	 burnfile = argv[i] + 4;
 
-# if (0)
-/* XXX disable for now: is not yet working */
-/* Alternate Output file for deburned image */
-      } else if(strncmp(argv[i], "outimage=", 9) == 0) {	/* outimage=fname */
-	ofilename = argv[i] + 9;
-# endif
-
-/* Input file for previous burn persistence streaks */
-      } else if(strncmp(argv[i], "trailin=", 8) == 0) {	/* trailin=fname */
+/* Input text file for previous burn persistence streaks */
+      } else if(strncmp(argv[i], "infits=", 8) == 0) {	/* in=fname */
 	 persistfile = argv[i] + 8;
+
+/* Same thing, but information is stored in tables in a FITS file. */
+      } else if(strncmp(argv[i], "trailinfits=", 8) == 0) { /* infits=fname */
+	 persistfitsfile = argv[i] + 12;
 
 /* Input file for previous burn persistence streaks */
       } else if(strncmp(argv[i], "in=", 3) == 0) {	/* in=fname */
 	 persistfile = argv[i] + 3;
+
+/* Same thing, but information is stored in tables in a FITS file. */
+      } else if(strncmp(argv[i], "trailinfits=", 8) == 0) { /* trailin=fname */
+	 persistfitsfile = argv[i] + 12;
 
 /* Output file for PSF gallery */
@@ -244,4 +235,10 @@
       } else if(strncmp(argv[i], "thrpsf=", 7) == 0) {	/* thrpsf=thresh */
 	 if(sscanf(argv[i]+7, "%d", &PSF_THRESH) != 1) {
+	    fprintf(stderr, "\rerror: cannot get value from `%s'\n", argv[i]);
+	    exit(EXIT_FAILURE);
+	 }
+
+      } else if(strncmp(argv[i], "expire=", 7) == 0) {	/* expire=nsec */
+	 if(sscanf(argv[i]+7, "%d", &EXPIRE_TRAIL_TIME) != 1) {
 	    fprintf(stderr, "\rerror: cannot get value from `%s'\n", argv[i]);
 	    exit(EXIT_FAILURE);
@@ -295,4 +292,7 @@
 	 }
 
+      } else if(strncmp(argv[i], "deltables=", 8) == 0) { /* trailin=fname */
+	 deltablefitsfile = argv[i] + 10;        
+
       } else if(strncmp(argv[i], "help", 4) == 0) {	/* help output */
 	 syntax(argv[0]);
@@ -306,7 +306,22 @@
    }
 
+   /* If we're told to remove the table from a FITS file, then
+    * do nothing else. */
+   if(deltablefitsfile) {
+     fprintf(stderr, 
+             "\rRemoving burn tables from %s, all other options ignored.\n",
+             ifilename);
+     if(persist_fits_remove_tables(ihu, deltablefitsfile) != FH_SUCCESS) {
+       exit(EXIT_FAILURE);
+     }
+     else {
+       exit(EXIT_SUCCESS);
+     }
+   }
+   
+   /* If there is no other persistence info supplied, try getting
+    * it from the input FITS file. */
    if(restore && persistfile == NULL) {
-      fprintf(stderr, "\rerror: must specify an input file for restore\n");
-      exit(EXIT_FAILURE);
+     if(persistfitsfile == NULL) persistfitsfile = ifilename;
    }
 
@@ -322,4 +337,7 @@
       if(persist_read(OTA, persistfile)) exit(EXIT_FAILURE);
    }
+   else if(persistfitsfile != NULL) {
+     if(persist_fits_read(OTA, persistfitsfile) != FH_SUCCESS) exit(EXIT_FAILURE);
+   }
 
 /* Which OTA is this??? */
@@ -333,19 +351,34 @@
    }
 
-   if (ofilename) {
-     outfd = creat (ofilename, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
-     if (outfd == -1) {
-	 fprintf(stderr, "\rerror: Cannot open %s for output\n", ofilename);
-         exit(EXIT_FAILURE);
-      }
-
-     if (fh_write(ihu, outfd)) {
-	 fprintf(stderr, "\rerror: Trouble writing PHU to %s\n", ofilename);
-         exit(EXIT_FAILURE);
-      }
+   /* Warn that what you're doing might not be a good idea. */
+   {
+     fh_bool burn_applied;
+     
+     if(fh_get_bool(ihu, PHU_NAME_BURN_APPLIED, 
+                    &burn_applied) != FH_SUCCESS) {
+	if(VERBOSE > 0) {
+	   fprintf(stderr, 
+		   "warning: Unable to determine whether burn correction "
+		   "already applied - unable to find %s in primary header\n",
+		   PHU_NAME_BURN_APPLIED);
+	}
+	burn_applied = FH_FALSE;
+     }
+     if(restore && (burn_applied == FH_FALSE)) {
+       fprintf(stderr, 
+               "warning: Restoring old burns, but header indicates no burns previously corrected.\n");             
+     }
+     else if (update && (burn_applied == FH_TRUE)) {
+       fprintf(stderr, 
+               "warning: Applying burn correction, but header indicates burns previously corrected.\n");             
+       }
    }
 
 /* Look at all the MEF's extensions */
+#ifndef JT2DHACK
    for (ext = 1; ext <= nextend; ext++) {
+#else
+   for (ext = 1; ext <= MAX(1,nextend); ext++) {
+#endif
       int naxis, naxis1, naxis2, naxis3;
       int prescan1, ovrscan1, ovrscan2, pontime;
@@ -353,4 +386,10 @@
       char xtension[FH_MAX_STRLEN + 1];
 
+#ifdef JT2DHACK
+      if(nextend == 0) {
+	 ehu = ihu;
+	 sprintf(extname, "%s", "xy00");
+      } else {
+#endif
       if (!(ehu = fh_ehu(ihu, ext)) || 
 	  fh_get_str(ehu, "EXTNAME", extname, sizeof(extname)) != FH_SUCCESS) {
@@ -377,4 +416,7 @@
          fprintf(stderr,
                  "warning: Skipping non-image extension %s\n", extname);
+#ifdef JT2DHACK
+      }
+#endif
 /*
  * %%% When reading from a pipe, does the data still need
@@ -448,4 +490,9 @@
       if (fh_get_BZERO(ehu, &bzero_d) == FH_SUCCESS) BZERO = bzero_d;
 
+      if(VERBOSE & VERB_NORM) {
+	 printf("nx=%d ny=%d prex=%d postx=%d posty=%d BZERO=%.1f\n", 
+		naxis1, naxis2, prescan1, ovrscan1, ovrscan2, BZERO);
+      }
+
       buf = (IMTYPE*)malloc(naxis1*naxis2*naxis3*sizeof(short));
       if (fh_read_padded_image(ehu, fh_file_desc(ehu), buf,
@@ -471,10 +518,24 @@
 /* Get bias and sky levels */
 	 err = cell_stats(nx, ny, naxis1, naxis2, imbuf, OTA+cell);
+	 if(err) {
+	    if(VERBOSE > 0) {
+	       fprintf(stderr, "logonly: error getting bias/sky/rms for cell %d, skipping...\n", cell);
+	    }
+	    continue;
+	 }
+
+/* Does this cell get a pass because it's heavily blasted? */
+/* Turn it in to a relaxation factor for the "read noise" */
+	 i = OTA[cell].satfrac > MIN_BLAST_PASS &&
+	     OTA[cell].satfrac < MAX_BLAST_PASS &&
+	     OTA[cell].sky < TRAIL_THRESH ? 30 : 1;
 
 /* Does this cell look kosher? */
 	 if(OTA[cell].rms*OTA[cell].rms > 
-	    OTA[cell].sky/MIN_EADU + MAX_READ_NOISE*MAX_READ_NOISE) {
+	    OTA[cell].sky/MIN_EADU + i*i*MAX_READ_NOISE*MAX_READ_NOISE) {
 	    if(VERBOSE > 0) {
 	       fprintf(stderr, "logonly: cell %d is unreasonably noisy, skipping...\n", cell);
+	       fprintf(stderr, "logonly: bias = %d sky = %d rms = %d satfrac = %.2f\n", 
+		       OTA[cell].bias, OTA[cell].sky, OTA[cell].rms, OTA[cell].satfrac);
 	    }
 	    continue;
@@ -506,5 +567,4 @@
 
 	 } else {
-
 /* Restore the old burns */
 	    burn_restore(naxis1-ovrscan1, naxis2-ovrscan2, naxis1, 
@@ -514,44 +574,19 @@
 /* Write the corrected data back to the FITS. */
 	 if(update) {
-	   if (ofilename) {
-	     fh_ehu(ehu, 0);	/* Seek back to the start of data */
-
-	     if (fh_write(ehu, outfd)) {
-	       fprintf(stderr, "\rerror: Trouble writing EXT %d to %s\n", ext, ofilename);
-	       exit(EXIT_FAILURE);
-	     }
-
-	     if (fh_write_padded_image(ehu, outfd, buf,
-				       naxis1*naxis2*naxis3*sizeof(short),
-				       FH_TYPESIZE_16) != FH_SUCCESS) {
+	    fh_ehu(ehu, 0);	/* Seek back to the start of data */
+	    if (fh_write_padded_image(ehu, fh_file_desc(ehu), buf,
+				      naxis1*naxis2*naxis3*sizeof(short),
+				      FH_TYPESIZE_16) != FH_SUCCESS) {
 	       fprintf(stderr, "\rerror: failed to re-write image data for extension `%s'.\n",
 		       extname);
 	       free(buf);
 	       exit(EXIT_FAILURE);
-	     }
-	   } else {
-	     fh_ehu(ehu, 0);	/* Seek back to the start of data */
-	     if (fh_write_padded_image(ehu, fh_file_desc(ehu), buf,
-				       naxis1*naxis2*naxis3*sizeof(short),
-				       FH_TYPESIZE_16) != FH_SUCCESS) {
-	       fprintf(stderr, "\rerror: failed to re-write image data for extension `%s'.\n",
-		       extname);
-	       free(buf);
-	       exit(EXIT_FAILURE);
-	     }
-	   }
-	 }
-      }
-      
+	    }
+	 }
+      }
+
       free(buf);
    }
 
-   if (ofilename) {
-     if (close(outfd)) {
-       fprintf(stderr, "\rerror: trouble writing data to output file %s\n", ofilename);
-       exit(EXIT_FAILURE);
-     }
-   }
-      
 /* Dump out the postage stamp file */
    if(psffile != NULL) {
@@ -564,8 +599,20 @@
    }
 
+/* Write burn info to FITS file. */
+   if(update) persist_fits_write(OTA, ihu);
+
+   
+   if(restore) {    
+     /* Indicate in the header that the burns are not applied. */
+     fh_set_bool(ihu, FH_AUTO, PHU_NAME_BURN_APPLIED, 
+                 FH_FALSE, PHU_COMMENT_BURN_APPLIED);
+     fh_rewrite(ihu);
+   }
+
    fh_destroy(ihu);
 
 /* Write the persistence data for the next image */
    if(burnfile != NULL) persist_write(OTA, burnfile);
+
 
    exit(EXIT_SUCCESS);
@@ -613,5 +660,5 @@
 		      CELL *cell)
 {
-   int i, j, k, n;
+   int i, j, k, n, nsat;
 
 /* Get bias stats */
@@ -624,9 +671,20 @@
 
 /* Get sky stats */
-   for(k=n=0; k<nx*ny; k+=((617*nx)/1000)) {
+   for(k=n=nsat=0; k<nx*ny; k+=((617*nx)/1000)) {
       i = k % nx;
       j = k / nx;
-      median_buf[n++] = data[i+NX*j];
-   }
+      if(data[i+NX*j] > SAT4SURE) {
+	 nsat++;
+      } else if(data[i+NX*j] != NODATA) {
+	 median_buf[n++] = data[i+NX*j];
+      }
+   }
+
+   if(n < 20) {		/* Better have hit at least 20! */
+      cell->sky = cell->rms = 0.0;
+      return(-1);
+   }
+   cell->satfrac = ((double)nsat) / n;
+
 /* First pass at sky and quartile */
    cell->sky = int_median(n, median_buf);
@@ -656,8 +714,15 @@
    printf(" restore={t|f}  Restore the input MEF by adding input fits?\n");
    printf(" in=fname       Input file for previous burn persistence streaks\n");
+   printf(" infits=fname   Input FITS file for previous burn persistence streaks (stored\n");
+   printf("                   in table extensions).  If both this and the 'in' input\n");
+   printf("                   file option are specified, then 'in' takes precedence.\n");
    printf(" out=fname      Output file for burn streaks\n");
-   printf(" outimage=imname Output file for MEF image\n");
+
 //   printf(" trailin=fname  Input file for previous burn persistence streaks\n");
+//   printf(" trailinfits=fname  Input FITS file for previous burn persistence streaks\n");
 //   printf(" trailout=fname Output file for burn streaks\n");
+   printf(" deltables=fname Copy mef_file to new FITS file 'fname' with burn streak\n");
+   printf("                   tables removed. NOTE: if specified, all other options\n");
+   printf("                   will be ignored!\n");
    printf(" psf=fname      Output file for PSF FITS stamp gallery\n");
    printf(" psfstat=fname  Output file for PSF statistics listing\n");
@@ -676,4 +741,5 @@
    printf(" rmask=X        Diameter growth factor of burned spots\n");
    printf(" bmask=X        Box size growth of burn/star boxes\n");
+   printf(" expire=N       Retire a blasted burn after N seconds\n");
    printf(" quiet={t|f}    Quiet?\n");
    printf(" verbose=N      Set verbosity bits:\n");
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/burntool.h	(revision 24244)
@@ -1,2 +1,5 @@
+#ifndef _INCLUDED_burntool_
+#define _INCLUDED_burntool_
+
 /* Burn correction routines:
    -------------------------  
@@ -63,5 +66,5 @@
 
 #define MAXCELL	64		/* Max cells in an OTA */
-#define MAXSIZE 700		/* Maximum vertical cell size */
+#define MAXSIZE 2048		/* Maximum vertical cell size */
 
 #define STAR_RADIUS  4		/* Radius over which a star ctr must be max */
@@ -87,5 +90,6 @@
 #define BURN_PWR  1		/* Power law */
 #define BURN_EXP  2		/* Exponential */
-#define PSF_STAR  3		/* Unfitted: good psf star */
+#define BURN_BLASTED 3		/* Blasted top to bottom: flag only for IPP */
+#define PSF_STAR  9		/* Unfitted: good psf star */
 
 /* Fit error codes */
@@ -93,4 +97,5 @@
 #define FIT_TOP_ERROR  2	/* Saturation extends to top: no points */
 #define FIT_SLOPE_ERROR  3	/* Unreasonable fit */
+#define FIT_EXPIRED 9		/* Don't carry any more as persistent */
 
 /* Fit parameters */
@@ -152,4 +157,5 @@
       int rms;			/* RMS in the sky */
       int time;			/* PON time of this cell */
+      double satfrac;		/* Fraction of SAT4SURE saturated pixels */
       int nburn;		/* Number of trails left by sat stars */
       OBJBOX *burn;		/* Stars which we think left a trail */
@@ -212,2 +218,4 @@
 int write_2ddata(int nx, int ny, int *ntot, IMTYPE *data, int fd);
 int write_3dend(int *ntot, int fd);
+
+#endif /* _INCLUDED_burntool_ */
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/man/burntool.1	(revision 24244)
@@ -53,4 +53,11 @@
 	power laws; downward burns as exponentials.
 	
+	Burntool also identifies really blasted areas which are saturated from
+	top to bottom.  These cannot be fitted, but are carried along for
+	EXPIRE_TRAIL_TIME seconds (default 2000) in the persistence list.  The
+	idea is to give IPP a clue that something bad happened, and of course
+	the persistence info tells when it happened.  This expiration time
+	for carrying the region can be set with "expire=N".
+
 	Burntool finishes by rewriting the modified MEF and all of the fits in
 	an output file ("out=file_name").  If desired, burntool can be run
@@ -112,9 +119,13 @@
 	   npsf=3          Number of contributing PSF stars
 	   fwhm=5.38       PSF FWHM [pix] (0.0 if fit fails)
-	   m2=11.08        Second moment of distribution [pix^2] 
-	   qp=5.154        Plus quadrupole qxx-qyy [pix^2]	 
-	   qc=0.560        Cross quadrupole 2*qxy [pix^2]	 
-	   qt=-5.201       Tangential quadrupole [pix^2] 	  
+	   fmed=5.76       FWHM averaged over psfavg (0.0 if fit fails)
+	   m2=11.08        Second moment of distribution [pix^2] (-99.99 if fails)
+	   qp=5.154        Plus quadrupole qxx-qyy [pix^2] (-99.99 if fails)
+	   qc=0.560        Cross quadrupole 2*qxy [pix^2] (-99.99 if fails)
+	   qt=-5.201       Tangential quadrupole [pix^2] (-99.99 if fails)
 			   	q+ * cos(2*phi) + qx * sin(2*phi)
+	   qpm=5.154       Plus quadrupole averaged over psfavg
+	   qcm=0.560       Cross quadrupole averaged over psfavg
+	   qtm=-5.201      Tangential averaged over psfavg
 
 	Helpful utilities include:
@@ -148,6 +159,17 @@
 		Input file for previous burn persistence streaks
 
+	infits=fname       
+		Input FITS file for previous burn persistence streaks
+                (burn streaks stored in table extensions). If both
+                this and the 'in' input file option are specified, then
+                'in' takes precedence.
+
 	out=fname      
 		Output file for burn streaks
+
+        deltables=fname
+                Copy mef_file to new FITS file 'fname' with burn streak 
+                tables removed. NOTE: if specified, all other options 
+                will be ignored!
 
 	psf=fname      
@@ -199,4 +221,7 @@
 	bmask=X        
 		Box size growth of burn/star (BMASK_GROW)
+
+	expire=N
+		Retire a blasted burn after N seconds
 
 	quiet={t|f}    
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.c	(revision 24244)
@@ -0,0 +1,1224 @@
+/* -*- c-file-style: "Ellemtel" -*-
+ *
+ * persist_fits.c - read and write persistence info to FITS tables.
+ *
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "fh/fh.h"
+
+#include "burntool.h"
+#include "burnparams.h"
+#include "persist_fits.h"
+
+/* Default names for burn info table extensions. We'll use keyword information
+ * when reading to determine the names of the extensions, but we'll always 
+ * write them with these names. */
+#define DEFAULT_EXTNAME_AREA_TABLE "burntool_areas"
+#define DEFAULT_EXTNAME_FIT_TABLE  "burntool_fits"
+
+/* Index of all the columns of the area table. */
+typedef enum
+{
+   AREA_TABLE_COL_CELL,
+   AREA_TABLE_COL_TIME,
+   AREA_TABLE_COL_CX,
+   AREA_TABLE_COL_CY,
+   AREA_TABLE_COL_MAX,
+   AREA_TABLE_COL_Y0,
+   AREA_TABLE_COL_SX,
+   AREA_TABLE_COL_SY,
+   AREA_TABLE_COL_EX,
+   AREA_TABLE_COL_EY,
+   AREA_TABLE_COL_Y0M,
+   AREA_TABLE_COL_Y0P,
+   AREA_TABLE_COL_Y1M,
+   AREA_TABLE_COL_Y1P,
+   AREA_TABLE_COL_X0M,
+   AREA_TABLE_COL_X0P,
+   AREA_TABLE_COL_X1M,
+   AREA_TABLE_COL_X1P,
+   AREA_TABLE_COL_FUNC,
+   AREA_TABLE_COL_UP,
+   AREA_TABLE_COL_SLOPE,
+   AREA_TABLE_COL_NFIT,
+   AREA_TABLE_COL_SXFIT,
+   AREA_TABLE_COL_EXFIT,
+   
+   /* Add new columns above this line. */
+   AREA_TABLE_NUM_COLS
+} AREA_TABLE_COL_T;
+
+/* All of the columns of the area table. */
+static fhTableCol 
+area_table_cols[] = 
+{
+/*   Name     Description                              Units      Format                  Width  Dec places */
+   { "cell",  "Cell that correction area is in",       "",        FH_TABLE_FORMAT_INT,    2,     0 },
+   { "time",  "PON time when area created",            "secs",    FH_TABLE_FORMAT_INT,    10,    0 },
+   { "cx",    "Center x (position of max)",            "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "cy",    "Center y (position of max)",            "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "max",   "Max data value (above sky)",            "ADU",     FH_TABLE_FORMAT_INT,    5,     0 },
+   { "y0",    "y origin for fit (stamp sy for stars)", "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "sx",    "Left corner of area",                   "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "sy",    "Bottom corner of area",                 "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "ex",    "Right corner of area",                  "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "ey",    "Top corner of area",                    "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "y0m",   "Min y value at sx",                     "pixels",  FH_TABLE_FORMAT_INT,    5,     0 },
+   { "y0p",   "Max y value at sx",                     "pixels",  FH_TABLE_FORMAT_INT,    5,     0 },
+   { "y1m",   "Min y value at ex",                     "pixels",  FH_TABLE_FORMAT_INT,    5,     0 },
+   { "y1p",   "Max y value at ex",                     "pixels",  FH_TABLE_FORMAT_INT,    5,     0 },
+   { "x0m",   "Min x value at sy",                     "pixels",  FH_TABLE_FORMAT_INT,    5,     0 },
+   { "x0p",   "Max x value at sy",                     "pixels",  FH_TABLE_FORMAT_INT,    5,     0 },
+   { "x1m",   "Min x value at ey",                     "pixels",  FH_TABLE_FORMAT_INT,    5,     0 },
+   { "x1p",   "Max x value at ey",                     "pixels",  FH_TABLE_FORMAT_INT,    5,     0 },
+   { "func",  "What are we going to do about it?",     "",        FH_TABLE_FORMAT_INT,    3,     0 },
+   { "up",    "Trails up or down",                     "1=up 0=down", FH_TABLE_FORMAT_INT,1,     0 },
+   { "slope", "Slope of fit",                          "",        FH_TABLE_FORMAT_DOUBLE, 9,     6 },
+   { "nfit",  "Number of columns corrected",           "",        FH_TABLE_FORMAT_INT,    3,     0 },
+   { "sxfit", "Starting column for fit",              "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "exfit", "Ending column for fit",                "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+};
+
+/* Index of all of the columns in the fit table. */
+typedef enum
+{
+   FIT_TABLE_COL_CELL,
+   FIT_TABLE_COL_CX,
+   FIT_TABLE_COL_CY,
+   FIT_TABLE_COL_XFIT,
+   FIT_TABLE_COL_YFIT,
+   FIT_TABLE_COL_ZERO,
+   
+   /* Add new columns above this line. */
+   FIT_TABLE_NUM_COLS
+} FIT_TABLE_COL_T;
+
+/* All of the columns in the fit table. */
+static fhTableCol
+fit_table_cols[] = 
+{
+/*   Name     Description                              Units      Format                  Width  Dec places */
+   { "cell",  "Cell that correction area is in",       "",        FH_TABLE_FORMAT_INT,    2,     0 },
+   { "cx",    "Center x (position of max)",            "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "cy",    "Center y (position of max)",            "pixels",  FH_TABLE_FORMAT_INT,    3,     0 },
+   { "xfit",  "x of each value of the start of correction", "pixels", FH_TABLE_FORMAT_INT,3,     0 },
+   { "yfit",  "x of each value of the start of correction", "pixels", FH_TABLE_FORMAT_INT,3,     0 },
+   { "zero",  "Zero of this fit",                      "",        FH_TABLE_FORMAT_DOUBLE, 8,     4 },
+};
+
+/* Structure describing area table. Stuff like the number of
+ * rows, the overall size, etc. are calculated at runtime. */
+static fhTable
+area_table = 
+{
+   .extname = DEFAULT_EXTNAME_AREA_TABLE,
+   .num_cols = AREA_TABLE_NUM_COLS,
+   .num_rows = 0, /* Don't know yet - need to rip through the data to figure this out. */
+   .cols = area_table_cols,
+};
+
+/* This is the fit table, in all its glory. */
+static fhTable
+fit_table = 
+{
+   .extname = DEFAULT_EXTNAME_FIT_TABLE,
+   .num_cols = FIT_TABLE_NUM_COLS,
+   .num_rows = 0, /* Don't know yet - need to rip through the data to figure this out. */
+   .cols = fit_table_cols,
+};
+
+/*
+ * Finds existing fit and area tables in a FITS file.
+ *
+ * Given the primary header of a FITS file, this function
+ * tries to find the area and fit table extensions. It does this
+ * by first looking at the header to see if the extension names
+ * of the tables are listed, and if so, looks for those by name.
+ * If that information is not available for some reason, the default
+ * names are used instead.
+ *
+ * Inputs:
+ *    phu - primary header of a FITS file.
+ *
+ * Outputs:
+ *      area_hu - header of area table stored here if found, NULL otherwise.
+ *    area_name - if non-NULL, this is a buffer where the name of the 
+ *                area table extension will be copied.
+ *       fit_hu - header of fit table stored here if found, NULL otherwise.
+ *     fit_name - if non-NULL, this is a buffer where the name of the 
+ *                fit table extension will be copied.
+ *
+ * This function returns nothing.
+ */
+static void
+find_existing_tables(HeaderUnit phu, 
+                     HeaderUnit * area_hu, char * area_name, 
+                     HeaderUnit * fit_hu, char * fit_name)
+{
+   char area_extname[FH_MAX_STRLEN+1];
+   char fit_extname[FH_MAX_STRLEN+1];
+
+   /* Does the FITS file provide the names of the burn tables?
+    * If so, use those, otherwise revert to the defaults. */
+   if(fh_get_str(phu, PHU_NAME_BURN_AREA, area_extname, sizeof(area_extname)) != FH_SUCCESS)
+   {
+      snprintf(area_extname, FH_MAX_STRLEN+1, DEFAULT_EXTNAME_AREA_TABLE);
+   }
+   if(fh_get_str(phu, PHU_NAME_BURN_FIT, fit_extname, sizeof(fit_extname)) != FH_SUCCESS)
+   {
+      snprintf(fit_extname, FH_MAX_STRLEN+1, DEFAULT_EXTNAME_FIT_TABLE);
+   }
+
+   *area_hu = fh_ehu_by_extname(phu, area_extname);
+   if(area_name)
+   {
+      if(*area_hu) strcpy(area_name, area_extname);
+      else area_extname[0] = '\0';
+   }
+   *fit_hu = fh_ehu_by_extname(phu, fit_extname);
+   if(fit_name)
+   {
+      if(*fit_hu) strcpy(fit_name, fit_extname);
+      else fit_name[0] = '\0';
+   }
+}
+
+/*
+ * Determines how many rows should be in the area and fit tables.
+ *
+ * Given all of the burn information for the whole OTA, this function
+ * figures out how many rows are needed in the area and fit tables to
+ * accomodate all of that info.
+ *
+ * Inputs:
+ *    cell - array of 64 CELL structures, one for each cell in the OTA.
+ *
+ * Outputs:
+ *    num_areas_found - the number of burn areas for the OTA stored here.
+ *     num_fits_found - the number of fits across all of the burn
+ *                      areas for the OTA stored here.
+ *
+ * This function returns FH_SUCCESS if the row was successfully 
+ * written.
+ */
+static void
+calculate_rows(CELL *cell, int * num_areas_found, int * num_fits_found)
+{
+   int j, k;
+   int num_areas = 0;
+   int num_fits = 0;
+
+   for(j=0; j<MAXCELL; j++) 
+   {
+      /* First: patched up persists */
+      for(k=0; k<cell[j].npersist; k++) 
+      {
+         if(cell[j].persist[k].fiterr) continue;
+         if(cell[j].persist[k].nfit <= 0) continue;
+         num_areas++;
+         num_fits += cell[j].persist[k].nfit;
+      }
+
+      /* Second: new burns */
+      for(k=0; k<cell[j].nburn; k++) 
+      {
+	 if(!cell[j].burn[k].burned) continue;
+	 if(cell[j].burn[k].fiterr && 
+	    cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+	 if(cell[j].burn[k].nfit <= 0) continue;
+         num_areas++;
+         num_fits += cell[j].burn[k].nfit;
+      }
+   }
+
+   *num_areas_found = num_areas;
+   *num_fits_found = num_fits;
+}
+
+/*
+ * Writes one row of the area table.
+ *
+ * Given all of the data for a row of the area table, this function writes
+ * the row in the table body.
+ *
+ * Inputs:
+ *       hu - header of fit table extension.
+ *     data - points to buffer for table body where data will be written.
+ *    table - describes structure of the table.
+ *      row - row of table to be written.
+ *     area - all properties of the area.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the row was successfully 
+ * written.
+ */
+static fh_result
+write_area_row(HeaderUnit hu, void * data, fhTable * table, int row, OBJBOX * area)
+{
+   if((fh_table_write_value(table, data, row, AREA_TABLE_COL_CELL,  &(area->cell))  != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_TIME,  &(area->time))  != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_CX,    &(area->cx))    != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_CY ,   &(area->cy))    != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_MAX,   &(area->max))   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_Y0,    &(area->y0))    != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_SX,    &(area->sx))    != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_SY,    &(area->sy))    != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_EX,    &(area->ex))    != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_EY,    &(area->ey))    != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_Y0M,   &(area->y0m))   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_Y0P,   &(area->y0p))   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_Y1M,   &(area->y1m))   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_Y1P,   &(area->y1p))   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_X0M,   &(area->x0m))   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_X0P,   &(area->x0p))   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_X1M,   &(area->x1m))   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_X1P,   &(area->x1p))   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_FUNC,  &(area->func))  != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_UP,    &(area->up))    != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_SLOPE, &(area->slope)) != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_NFIT,  &(area->nfit))  != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_SXFIT, &(area->sxfit)) != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, AREA_TABLE_COL_EXFIT, &(area->exfit)) != FH_SUCCESS))
+   {
+      fprintf(stderr, "error: Error writing data to row %d of area table.\n", row);
+      return FH_BAD_VALUE;
+   }
+
+   return FH_SUCCESS;
+}
+
+/*
+ * Writes area data to a FITS file.
+ *
+ * This function iterates through all of the burn areas in all cells 
+ * and populates the body of the area table in the provided buffer. 
+ *
+ * Inputs:
+ *       hu - header of fit table extension.
+ *     data - points to buffer for table body where data will be written.
+ *    table - describes structure of the table.
+ *     cell - array of 64 CELL structures for all cells of an OTA. Burn
+ *            data from these structures will be written to the table.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the table was successfully 
+ * written.
+ */
+static fh_result
+write_area_table(HeaderUnit hu, void * data, fhTable * table, CELL * cell)
+{
+   int j, k;
+   int row=0;
+   fh_result result;
+
+   for(j=0; j<MAXCELL; j++) 
+   {
+      /* First: patched up persists */
+      for(k=0; k<cell[j].npersist; k++) 
+      {
+         if(cell[j].persist[k].fiterr) continue;
+         if(cell[j].persist[k].nfit <= 0) continue;     
+
+         result = write_area_row(hu, data, table, row++, &(cell[j].persist[k]));
+         if(result != FH_SUCCESS) return result;
+      }
+
+      /* Second: new burns */
+      for(k=0; k<cell[j].nburn; k++) 
+      {
+	 if(!cell[j].burn[k].burned) continue;
+	 if(cell[j].burn[k].fiterr && 
+	    cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+	 if(cell[j].burn[k].nfit <= 0) continue;
+
+         result = write_area_row(hu, data, table, row++, &(cell[j].burn[k]));
+         if(result != FH_SUCCESS) return result;
+      }
+   }
+
+   return FH_SUCCESS;
+}
+
+
+/*
+ * Writes one row of the fit table.
+ *
+ * Given all of the data for a row of the fit table, this function writes
+ * the row in the table body.
+ *
+ * Inputs:
+ *       hu - header of fit table extension.
+ *     data - points to buffer for table body where data will be written.
+ *    table - describes structure of the table.
+ *      row - row of table to be written.
+ *     cell - cell number for fit.
+ *       cx - center x of fit.
+ *       cy - center y of fit.
+ *     xfit - x of each value of correction start.
+ *     yfit - y of each value of correction start.
+ *     zero - zero of this fit.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the row was successfully 
+ * written.
+ */
+static fh_result
+write_fit_row(HeaderUnit hu, void * data, fhTable * table, int row, 
+              int cell, int cx, int cy, int xfit, int yfit, double zero)
+{
+   if((fh_table_write_value(table, data, row, FIT_TABLE_COL_CELL, &cell) != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, FIT_TABLE_COL_CX,   &cx)   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, FIT_TABLE_COL_CY,   &cy)   != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, FIT_TABLE_COL_XFIT, &xfit) != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, FIT_TABLE_COL_YFIT, &yfit) != FH_SUCCESS) ||
+      (fh_table_write_value(table, data, row, FIT_TABLE_COL_ZERO, &zero) != FH_SUCCESS))
+   {
+      fprintf(stderr, "error: Error writing data to row %d of fit table.\n", row);
+      return FH_BAD_VALUE;
+   }
+
+   return FH_SUCCESS;
+}
+
+/*
+ * Writes fit data to a FITS file.
+ *
+ * This function iterates through all of the burn areas in all cells 
+ * and populates the body of the fit table in the provided buffer. 
+ *
+ * Inputs:
+ *       hu - header of fit table extension.
+ *     data - points to buffer for table body where data will be written.
+ *    table - describes structure of the table.
+ *     cell - array of 64 CELL structures for all cells of an OTA. Burn
+ *            data from these structures will be written to the table.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the table was successfully 
+ * written.
+ */
+static fh_result
+write_fit_table(HeaderUnit hu, void * data, fhTable * table, CELL * cell)
+{
+   int i, j, k;
+   int row=0;
+   fh_result result;
+
+   for(j=0; j<MAXCELL; j++) 
+   {
+      /* First: patched up persists */
+      for(k=0; k<cell[j].npersist; k++) 
+      {
+         if(cell[j].persist[k].fiterr) continue;
+	 for(i=0; i<cell[j].persist[k].nfit; i++) 
+         {
+            result = write_fit_row(hu, data, table, row++,
+                                   cell[j].cell, cell[j].persist[k].cx, cell[j].persist[k].cy,
+                                   cell[j].persist[k].xfit[i], cell[j].persist[k].yfit[i], 
+                                   cell[j].persist[k].zero[i]);
+            if(result != FH_SUCCESS) return result;
+	 }
+      }
+
+      /* Second: new burns */
+      for(k=0; k<cell[j].nburn; k++) 
+      {
+	 if(!cell[j].burn[k].burned) continue;
+	 if(cell[j].burn[k].fiterr && 
+	    cell[j].burn[k].fiterr != FIT_TOP_ERROR) continue;
+
+	 for(i=0; i<cell[j].burn[k].nfit; i++) 
+         {
+            result = write_fit_row(hu, data, table, row++,
+                                   cell[j].cell, cell[j].burn[k].cx, cell[j].burn[k].cy,
+                                   cell[j].burn[k].xfit[i], cell[j].burn[k].yfit[i], 
+                                   cell[j].burn[k].zero[i]);
+            if(result != FH_SUCCESS) return result;
+	 }
+      }
+   }
+
+   return FH_SUCCESS;
+}
+
+/*
+ * Reads one row of the area table from a FITS file.
+ *
+ * Given information about an area table and a row, this function
+ * reads a single row of data from a buffer containing the table 
+ * body. The data is stored in boxbuf[row].
+ *
+ * Inputs:
+ *    table - describes structure of the table.
+ *     data - points to data from table.
+ *      row - the row to be read from the table.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the row was successfully 
+ * read.
+ */
+static fh_result
+read_area(fhTable * table, void * data, int row)
+{
+   int cell_num;
+
+   /* Get the cell number first for some sanity checking. */
+   if(fh_table_read_value(table, data, row, AREA_TABLE_COL_CELL, &cell_num) != FH_SUCCESS)
+   {
+      fprintf(stderr,
+              "error: Unable to get cell number from row %d of burn area table\n",
+              row);
+      return FH_BAD_VALUE;
+   }
+
+   /* Don't want to deal with gibberish. */
+   if((cell_num < 0) || (cell_num > MAXCELL))
+   {
+      fprintf(stderr, "error: illegal cell %d in area table row %d\n",
+              cell_num, row);
+      boxbuf[row].cell = -1;
+      boxbuf[row].nfit = 0;
+      return FH_BAD_VALUE;
+   }
+
+   boxbuf[row].cell = cell_num;
+
+   if((fh_table_read_value(table, data, row, AREA_TABLE_COL_TIME,  &(boxbuf[row].time)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_CX,    &(boxbuf[row].cx)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_CY,    &(boxbuf[row].cy)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_MAX,   &(boxbuf[row].max)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_Y0,    &(boxbuf[row].y0)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_SX,    &(boxbuf[row].sx)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_SY,    &(boxbuf[row].sy)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_EX,    &(boxbuf[row].ex)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_EY,    &(boxbuf[row].ey)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_Y0M,   &(boxbuf[row].y0m)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_Y0P,   &(boxbuf[row].y0p)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_Y1M,   &(boxbuf[row].y1m)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_Y1P,   &(boxbuf[row].y1p)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_X0M,   &(boxbuf[row].x0m)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_X0P,   &(boxbuf[row].x0p)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_X1M,   &(boxbuf[row].x1m)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_X1P,   &(boxbuf[row].x1p)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_FUNC,  &(boxbuf[row].func)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_UP,    &(boxbuf[row].up)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_SLOPE, &(boxbuf[row].slope)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_NFIT,  &(boxbuf[row].nfit)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_SXFIT, &(boxbuf[row].sxfit)) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, AREA_TABLE_COL_EXFIT, &(boxbuf[row].exfit)) != FH_SUCCESS))
+   {
+      fprintf(stderr,
+              "error: Error reading values from row %d of burn area table\n",
+              row);
+      boxbuf[row].nfit = 0;
+      return FH_BAD_VALUE;
+   }
+   
+   /* Make space for fits (to be read later). */
+   if(boxbuf[row].nfit > 0)
+   {
+      /* This shouldn't happen, but be nice and don't leak. */
+      if(boxbuf[row].zero) free(boxbuf[row].zero);
+      if(boxbuf[row].xfit) free(boxbuf[row].xfit);
+      if(boxbuf[row].yfit) free(boxbuf[row].yfit);
+
+      boxbuf[row].zero = (double *)calloc(boxbuf[row].nfit, sizeof(double));
+      boxbuf[row].xfit = (int *)calloc(boxbuf[row].nfit, sizeof(int));
+      boxbuf[row].yfit = (int *)calloc(boxbuf[row].nfit, sizeof(int));
+      boxbuf[row].fiterr = 0;
+   }
+
+   return FH_SUCCESS;
+}
+
+/*
+ * Reads the area table from a FITS file.
+ *
+ * This function fills out boxbuf[] with all of the areas in the
+ * given table. 
+ *
+ * Inputs:
+ *       hu - header of the table extension for the area table.
+ *    table - describes structure of the table.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the tables were successfully 
+ * appended to the FITS file.
+ */
+static fh_result
+read_area_table(HeaderUnit hu, fhTable * table)
+{
+   fh_result result = FH_INVALID;
+   int i;
+   int row_chars, num_cols, num_rows;
+   void * data;
+
+   if((fh_get_int(hu, "NAXIS1",  &row_chars) != FH_SUCCESS) ||
+      (fh_get_int(hu, "NAXIS2",  &num_rows) != FH_SUCCESS) ||
+      (fh_get_int(hu, "TFIELDS", &num_cols) != FH_SUCCESS))
+   {
+      fprintf(stderr, "error: Unable to find required keywords for area table dimensions\n");
+      return FH_NOT_FOUND;
+   }
+
+   /* Sanity check. */
+   if(num_cols != table->num_cols) 
+   {
+      fprintf(stderr, "error: %d-column area table found, expected %d cols.\n",
+              num_cols, table->num_cols);
+      return FH_BAD_VALUE;
+   }
+
+   if(num_rows > MAXBURN)
+   {
+      fprintf(stderr, 
+              "error: too many boxes in area table. Max is %d, got %d\n",
+              MAXBURN, num_rows);
+      return FH_BAD_VALUE;
+   }
+
+   table->num_rows = num_rows;
+   table->table_size = num_rows * row_chars;
+
+
+   if(!(data = malloc(table->table_size)))
+   {
+      fprintf(stderr, 
+              "error: Unable to allocate %d bytes for area table.\n",
+              table->table_size);
+      return FH_NO_MEMORY;
+   }
+      
+   if(fh_read_image(hu, fh_file_desc(hu), data, 
+                    table->table_size, FH_TYPESIZE_8) != FH_SUCCESS)
+   {
+      fprintf(stderr, "error: Unable to map area table body for reading.\n");
+      free(data);
+      return FH_INVALID;
+   }
+
+   for(i = 0; i < num_rows; i++)
+   {
+      if((result = read_area(table, data, i)) != FH_SUCCESS) break;
+   }
+
+   free(data);
+   return result;
+}
+
+/*
+ * Reads one row of the fit table from a FITS file.
+ *
+ * Given information about a fit table and a row, this function
+ * reads a single row of data from a buffer containing the table 
+ * body.
+ *
+ * Inputs:
+ *       hu - header of the table extension for the fit table.
+ *    table - describes structure of the table.
+ *     data - points to buffer of data from fit table.
+ *      row - the row to be read.
+ *
+ * Outputs:
+ *    cell - cell number stored here.
+ *      cx - center x from table row stored here.
+ *      cy - center y from table row stored here.
+ *    xfit - x of each value of correction start stored here.
+ *    yfit - y of each value of correction start stored here.
+ *    zero - zero of this fit stored here.
+ *
+ * This function returns FH_SUCCESS if the table row was read 
+ * successfully.
+ *
+ * Note: this function must not be called until after read_area_table()
+ *       has read out the burn areas across all cells and prepared
+ *       boxbuf[]. This function assumes that each area's fit
+ *       information has been filled out in boxbuf[], including 
+ *       allocating space for the fits. 
+ */
+static fh_result
+read_fit(HeaderUnit hu, fhTable * table, void * data, int row, 
+         int * cell, int * cx, int * cy, int * xfit, int * yfit, double * zero)
+{
+   if(fh_table_read_value(table, data, row, FIT_TABLE_COL_CELL, cell) != FH_SUCCESS)
+   {
+      fprintf(stderr,
+              "error: Unable to get cell number from row %d of fit table\n",
+              row);
+      return FH_BAD_VALUE;
+   }
+
+   if((*cell < 0) || (*cell > MAXCELL))
+   {
+      fprintf(stderr, "error: illegal cell %d in fit table row %d\n",
+              *cell, row);
+      
+      return FH_BAD_VALUE;
+   }
+
+   if((fh_table_read_value(table, data, row, FIT_TABLE_COL_CX,   cx) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, FIT_TABLE_COL_CY,   cy) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, FIT_TABLE_COL_XFIT, xfit) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, FIT_TABLE_COL_YFIT, yfit) != FH_SUCCESS) ||
+      (fh_table_read_value(table, data, row, FIT_TABLE_COL_ZERO, zero) != FH_SUCCESS))
+   {
+      fprintf(stderr,
+              "error: Error reading values from row %d of burn area table\n",
+              row);
+      return FH_BAD_VALUE;
+   }
+
+   return FH_SUCCESS;
+}
+
+/*
+ * Reads the fit table from a FITS file.
+ *
+ * This function copies the data from a given fit table to boxbuf[].
+ * Once complete, the contents of boxbuf[] should be equivalent to
+ * if persist_read() had read from a text file.
+ *
+ * Inputs:
+ *       hu - header of the table extension for the fit table.
+ *    table - describes structure of the table.
+ *    num_areas - the number of areas that were previously read from
+ *                the area table and written to boxbuf.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the table was read 
+ * successfully.
+ *
+ * Note: this function must not be called until after read_area_table()
+ *       has read out the burn areas across all cells and prepared
+ *       boxbuf[]. This function assumes that each area's fit
+ *       information has been filled out in boxbuf[], including 
+ *       allocating space for the fits. 
+ */
+static fh_result
+read_fit_table(HeaderUnit hu, fhTable * table, int num_areas)
+{
+   fh_result result;
+   int area, i;
+   int row_chars, num_cols, num_rows;
+   void * data;
+   int fit_table_row;
+
+   int cell, cx, cy, xfit, yfit;
+   double zero;
+
+   if((fh_get_int(hu, "NAXIS1",  &row_chars) != FH_SUCCESS) ||
+      (fh_get_int(hu, "NAXIS2",  &num_rows) != FH_SUCCESS) ||
+      (fh_get_int(hu, "TFIELDS", &num_cols) != FH_SUCCESS))
+   {
+      fprintf(stderr, "error: Unable to find required keywords for fit table dimensions\n");
+      return FH_NOT_FOUND;
+   }
+
+   /* Sanity check. */
+   if(num_cols != table->num_cols) 
+   {
+      fprintf(stderr, "error: %d-column fit table found, expected %d cols.\n",
+              num_cols, table->num_cols);
+      return FH_BAD_VALUE;
+   }
+
+   table->num_rows = num_rows;
+   table->table_size = num_rows * row_chars;
+   
+   /* Make space for the body of the table and copy it.
+    *
+    * %%% It would be preferable to just memory map the body of the
+    * table in the file itself, but I don't seem to be able to get 
+    * that to work for reads. */
+   if(!(data = malloc(table->table_size)))
+   {
+      fprintf(stderr, 
+              "error: Unable to allocate %d bytes for area table.\n",
+              table->table_size);
+      return FH_NO_MEMORY;
+   }
+   if(fh_read_image(hu, fh_file_desc(hu), data, table->table_size, FH_TYPESIZE_8) != FH_SUCCESS)
+   {
+      fprintf(stderr, "error: Unable to read fit table body.\n");
+      free(data);
+      return FH_INVALID;
+   }
+
+   /* For all areas, grab all of the fits for that area. This should 
+    * end up consuming the whole table. */
+   fit_table_row = 0;
+   for(area = 0; area < num_areas; area++)
+   {
+      for(i = 0; i < boxbuf[area].nfit; i++)
+      {
+         if((result = read_fit(hu, table, data, fit_table_row, 
+                               &cell, &cx, &cy, &xfit, &yfit, &zero)) != FH_SUCCESS)
+         {
+            free(data);
+            return result;
+         }
+         
+         /* This shouldn't happen, but just in case... */
+         if((cell != boxbuf[area].cell) || 
+            (cx != boxbuf[area].cx) ||
+            (cy != boxbuf[area].cy))
+         {
+            fprintf(stderr, 
+                    "error: Fit in table row %d does not match current area "
+                    "(table cell=%d cx=%d cy=%d vs. area cell=%d cx=%d cy=%d)\n",
+                    fit_table_row,
+                    cell, cx, cy, boxbuf[area].cell, boxbuf[area].cx, boxbuf[area].cy);
+            free(data);
+            return FH_BAD_VALUE;
+         }
+
+         boxbuf[area].xfit[i] = xfit;
+         boxbuf[area].yfit[i] = yfit;   
+         boxbuf[area].zero[i] = zero;      
+
+         /* Move on to the next row of the table. */
+         fit_table_row++;
+      }
+   }
+
+   free(data);
+   return FH_SUCCESS;
+}
+
+/*
+ * Retrieves burn information from a FITS file.
+ *
+ * This function reads the burn tables from an existing FITS file
+ * and builds up burntool's internal table of areas and fits from
+ * the information within. This function should be equivalent to
+ * persist_read(), except that it operates on a FITS file instead
+ * of text.
+ *
+ * Inputs:
+ *        cell - array of 64 CELL structures that describe all of the
+ *               burn info for the whole OTA. Burn info will be written
+ *               here.
+ *    filename - filename of FITS file to read from.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the tables were successfully 
+ * appended to the FITS file.
+ *
+ * Note: this function assumes that the passed file is a MEF and that
+ * any required sanity checks on the file have already been performed.
+ */
+fh_result
+persist_fits_read(CELL *cell, const char * filename)
+{
+   HeaderUnit phu;
+   HeaderUnit area_hu, fit_hu;
+   fh_result result;
+
+   if(!(phu = fh_create()))
+   {
+      fprintf(stderr, 
+              "error: unable to create header structure for FITS file \"%s\"\n",
+              filename);
+      return FH_NO_MEMORY;
+   }
+
+   /* Try to open the FITS file. */
+   if ((result = fh_file(phu, filename, FH_FILE_RDONLY)) != FH_SUCCESS)
+   {
+      fprintf(stderr, 
+              "error: unable to open FITS file \"%s\"\n",
+              filename);
+      fh_destroy(phu);
+      return result;
+   }
+
+   
+   /* Don't even try if this FITS file doesn't have any burn info in it. */
+   find_existing_tables(phu, &area_hu, NULL, &fit_hu, NULL);
+   if(!area_hu || !fit_hu) 
+   {
+      fprintf(stderr, 
+              "error: Unable to find persistence info in FITS file.\n");
+      fh_destroy(phu);
+      return FH_NOT_FOUND;
+   }
+
+   fh_table_init(&area_table);
+   fh_table_init(&fit_table);
+
+   fh_ehu_by_extname(phu, DEFAULT_EXTNAME_AREA_TABLE);
+   if((result = read_area_table(area_hu, &area_table)) != FH_SUCCESS)
+   {
+      free(area_table.strbuf);
+      free(fit_table.strbuf);
+      fh_destroy(phu);
+      return result;
+   }
+
+   fh_ehu_by_extname(phu, DEFAULT_EXTNAME_FIT_TABLE);
+   if((result = read_fit_table(fit_hu, &fit_table, area_table.num_rows)) != FH_SUCCESS)
+   {
+      free(area_table.strbuf);
+      free(fit_table.strbuf);
+      fh_destroy(phu);
+      return result;
+   }
+
+   /* Now copy over the boxes that were read to our array of cells. 
+      This section is ripped off from persist_read(). */
+   {
+      int i, k;
+      int nbox;
+
+/* Initialize the counts */
+      for(k=0; k<MAXCELL; k++) {
+         cell[k].npersist = 0;
+         cell[k].persist = NULL;
+      }
+
+/* Augment counts */
+      for(nbox = 0; nbox < area_table.num_rows; nbox++)
+      {
+         k = boxbuf[nbox].cell;
+         cell[k].npersist += 1;
+      }
+
+      
+/* Allocate some space for them */
+      for(k=0; k<MAXCELL; k++) {
+         if( (i=cell[k].npersist) > 0) {
+            cell[k].persist = (OBJBOX *)calloc(i, sizeof(OBJBOX));
+            cell[k].npersist = 0;
+         }
+      }
+/* Copy the results to the cells */
+      for(i=0; i<nbox; i++) {
+         k = boxbuf[i].cell;
+         memcpy(cell[k].persist+cell[k].npersist, boxbuf+i, sizeof(OBJBOX));
+         cell[k].npersist += 1;
+      }
+   }
+
+   fh_destroy(phu);
+   return FH_SUCCESS;
+}
+
+/*
+ * Writes burn information to a FITS file.
+ *
+ * This function writes all of the the burn information passed as 
+ * a pair of FITS table extensions - one table listing the various 
+ * areas across all of the cells in the OTA, and another indicating
+ * the various fits in those areas.
+ *
+ * Inputs:
+ *    cell - array of 64 CELL structures that describe all of the
+ *           burn info for the whole OTA.
+ *     phu - primary header for existing FITS file to be written to.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the tables were successfully 
+ * appended to the FITS file.
+ *
+ * Note: this function assumes that the passed file is a MEF and that
+ * any required sanity checks on the file have already been performed.
+ *
+ * Note: this function assumes that the burn information has been
+ * applied and sets the keyword defined by PHU_NAME_BURN_APPLIED
+ * to TRUE. 
+ */
+fh_result
+persist_fits_write(CELL *cell, HeaderUnit phu)
+{
+   int fd;
+   HeaderUnit area_hu, fit_hu;
+   void * data;
+   int nextend;
+
+   /* Don't even try if this FITS file already has burn info in it.
+    * It's a fair bit of work to try to strip an extension out of
+    * a FITS file. */
+   find_existing_tables(phu, &area_hu, NULL, &fit_hu, NULL);
+   if(area_hu || fit_hu) 
+   {
+      fprintf(stderr, 
+              "error: Unable to write correction info to FITS file. Correction FITS tables already exist.\n");
+      return FH_NO_SPACE;
+   }
+
+   /* Is there somehow no file associated with the primary header? */
+   if ((fd = fh_file_desc(phu)) == -1)
+   {
+      fprintf(stderr, "error: header passed has no associated file.\n");
+      return FH_INVALID;
+   }
+
+   /* We're going to append our tables to the end of the file, so go there now. */
+   if(lseek(fd, 0, SEEK_END) == (off_t)-1)
+   {
+      fprintf(stderr,
+              "error: Unable to seek to end of file.\n");
+      return FH_IN_ERRNO;
+   }
+
+   area_hu = fh_create();
+   fit_hu = fh_create();
+   if(!area_hu || !fit_hu) 
+   {
+      fprintf(stderr, 
+              "error: Unable to create headers for correction FITS tables.\n");
+      if(area_hu) fh_destroy(area_hu);
+      if(fit_hu) fh_destroy(fit_hu);
+      return FH_NO_MEMORY;
+   }
+
+   /* Find out how many rows are going to be in each table. */
+   calculate_rows(cell, &(area_table.num_rows), &(fit_table.num_rows));
+
+   /* Start with the area table:
+    * - Build up an extension header and write it to the file.
+    * - Make space in the FITS file for the table body.
+    * - Memory-map the area in the FITS file for writing.
+    * - Write the data to the table.
+    * - Unmap the file.
+    */
+   fh_link_ehu_to_phu(area_hu, phu);
+   if((fh_table_populate_header(area_hu, &area_table) != FH_SUCCESS) ||
+      (fh_write(area_hu, fd) != FH_SUCCESS) ||
+      (fh_reserve_padded_table(area_hu, fd) != FH_SUCCESS) ||
+      (fh_map_table(area_hu, &data, area_table.table_size) != FH_SUCCESS) ||
+      (write_area_table(area_hu, data, &area_table, cell) != FH_SUCCESS) ||
+      (fh_munmap_table(area_hu) != FH_SUCCESS))
+   {
+      fprintf(stderr, "error: Error encountered writing area table to FITS file.\n");
+      fh_destroy(fit_hu);
+      free(area_table.strbuf);
+      free(fit_table.strbuf);
+      return FH_INVALID;
+   }
+
+   /* Now the fit table:
+    * - Build up an extension header and write it to the file.
+    * - Make space in the FITS file for the table body.
+    * - Memory-map the area in the FITS file for writing.
+    * - Write the data to the table.
+    */
+   fh_link_ehu_to_phu(fit_hu, phu);
+   if((fh_table_populate_header(fit_hu, &fit_table) != FH_SUCCESS) ||
+      (fh_write(fit_hu, fd) != FH_SUCCESS) ||
+      (fh_reserve_padded_table(fit_hu, fd) != FH_SUCCESS) ||
+      (fh_map_table(fit_hu, &data, fit_table.table_size) != FH_SUCCESS) ||
+      (write_fit_table(fit_hu, data, &fit_table, cell) != FH_SUCCESS) ||
+      (fh_munmap_table(fit_hu) != FH_SUCCESS))
+   {
+      fprintf(stderr, "error: Error encountered writing area table to FITS file.\n");
+      free(area_table.strbuf);
+      free(fit_table.strbuf);
+      return FH_INVALID;
+   }
+
+   /* Finally, correct the primary header to account for the 
+    * new extensions. */
+   fh_get_int(phu, "NEXTEND", &nextend);
+   nextend += 2;
+   fh_set_int(phu, FH_AUTO, "NEXTEND", nextend, "Number of extensions");
+
+   /* Add info to the primary header indicating where to find the burn info. */
+   fh_set_str(phu, FH_AUTO, PHU_NAME_BURN_AREA, DEFAULT_EXTNAME_AREA_TABLE, PHU_COMMENT_BURN_AREA);
+   fh_set_str(phu, FH_AUTO, PHU_NAME_BURN_FIT, DEFAULT_EXTNAME_FIT_TABLE, PHU_COMMENT_BURN_FIT);
+   fh_set_bool(phu, FH_AUTO, PHU_NAME_BURN_APPLIED, FH_TRUE, PHU_COMMENT_BURN_APPLIED);  
+
+   fh_rewrite(phu);
+
+   /* Get shot of the temporary space associated with the table structures. */
+   free(area_table.strbuf);
+   free(fit_table.strbuf);
+
+   return FH_SUCCESS;
+}
+
+/*
+ * Removes the burn tables from a FITS file.
+ *
+ * This function strips out the burn tables (if present) from a FITS
+ * file, writing a new file with a couple less extensions in it.
+ *
+ * Inputs:
+ *     phu_in - primary header for existing FITS file to be copied.
+ *    fileout - filename of new FITS file.
+ *
+ * Outputs:
+ *    None.
+ *
+ * This function returns FH_SUCCESS if the new file was successfully
+ * written.
+ */
+fh_result
+persist_fits_remove_tables(HeaderUnit phu_in, const char * fileout)
+{
+   HeaderUnit phu_out;
+
+   HeaderUnit ehu_area;
+   HeaderUnit ehu_fit;
+
+   int num_extensions;
+   int num_extensions_copied = 0;
+
+   int fd_out;
+   int i;
+   fh_bool burn_applied = FH_FALSE;
+
+   char area_extname[FH_MAX_STRLEN+1];
+   char fit_extname[FH_MAX_STRLEN+1];
+
+   if(fh_get_int(phu_in, "NEXTEND", &num_extensions) != FH_SUCCESS)
+   {
+      fprintf(stderr, "error: Unable to get NEXTEND from input primary FITS header.\n");
+      return FH_INVALID;
+   }
+
+   find_existing_tables(phu_in, &ehu_area, area_extname, &ehu_fit, fit_extname);
+   if(!ehu_area || !ehu_fit) 
+   {
+      fprintf(stderr, 
+              "error: Unable to find persistence info in FITS file.\n");
+      return FH_INVALID;
+   }
+
+   /* Print a warning if the input FITS file appears to have burn 
+    * areas applied and that the output fiel is about to have that
+    * info removed. */
+   fh_get_bool(phu_in, PHU_NAME_BURN_APPLIED, &burn_applied);
+   if(burn_applied == FH_TRUE)
+   {
+      fprintf(stderr, 
+              "warning: input FITS file has burns applied to it. "
+              "%s still has those burns applied and has no built-in "
+              "knowledge of how to undo them.\n", fileout);
+   }
+
+   if((fd_out = open(fileout, O_CREAT | O_RDWR, 0644)) < 0)
+   {
+      fprintf(stderr, "error: Failed to open \"%s\" for output.\n",
+              fileout);
+      exit(EXIT_FAILURE);
+   }
+
+   /* Copy the primary header. Note that fh_merge() doesn't
+    * preserve any reserved space in the header (in our case, a
+    * big block of identical COMMENT cards). For now, make sure
+    * there's still some free space in the header by reserving 
+    * an entire block. The downside to this is obviously that the
+    * header will look a little different than the one we're copying,
+    * but I can't figure any way around this right now. */
+   if(!(phu_out = fh_create()) ||
+      (fh_merge(phu_out, phu_in) != FH_SUCCESS) ||
+      (fh_reserve(phu_out, (2880/80)) != FH_SUCCESS) ||
+      (fh_write(phu_out, fd_out) != FH_SUCCESS))
+   {
+      fprintf(stderr, "error: Unable to copy primary header to \"%s\"\n",
+              fileout);
+      fh_destroy(phu_out);
+      close(fd_out);
+      return FH_INVALID;
+   }
+
+   /* Walk through all of the extensions in the input FITS. Copy everything
+    * but the tables of burn info. */
+   for (i = 1; i <= num_extensions; i++)
+   {
+      HeaderUnit ehu_in;
+      HeaderUnit ehu_out;
+      char extname[FH_MAX_STRLEN+1];
+
+      if(!(ehu_in = fh_ehu(phu_in, i)))
+      {
+         fprintf(stderr, "error: Unable to read extension %d from input FITS file\n", i);
+         fh_destroy(phu_out);
+         close(fd_out);
+         return FH_INVALID;
+      }
+      
+      if (fh_get_str(ehu_in, "EXTNAME", extname, sizeof(extname)) != FH_SUCCESS)
+      {
+         fprintf(stderr, "error: Unable to get EXTNAME from extension %d in input FITS file.\n", i); 
+         fh_destroy(phu_out);
+         close(fd_out);
+         return FH_INVALID;
+      }
+
+      /* No match? Copy this extension in its entirety. */
+      if(strcasecmp(extname, area_extname) && strcasecmp(extname, fit_extname))
+      {
+         ehu_out = fh_create();
+
+         /* Copy the extension's header. The same thing about
+          * reserved space with fh_merge() applies here. */
+         if((fh_merge(ehu_out, ehu_in) != FH_SUCCESS) ||
+            (fh_reserve(ehu_out, (2880/80)) != FH_SUCCESS))
+         {
+            fprintf(stderr, "error: Unable to copy extension %d to %s.\n", i, fileout); 
+            fh_destroy(phu_out);
+            close(fd_out);
+            return FH_INVALID;           
+         }
+         
+         /* Apparently this can never fail. It would be handy if it just 
+          * returned FH_SUCCESS so it could be included in a big block of
+          * checks. */
+         fh_link_ehu_to_phu(ehu_out, phu_out);
+
+         /* WRite the copy of our header and all of the data that follows. */
+         if((fh_write(ehu_out, fd_out) != FH_SUCCESS) ||
+            (fh_copy_padded_image(ehu_out, fd_out, fh_file_desc(ehu_in)) != FH_SUCCESS))
+         {
+            fprintf(stderr, "error: Unable to copy extension %d to %s.\n", i, fileout); 
+            fh_destroy(phu_out);
+            close(fd_out);
+            return FH_INVALID;           
+         }
+
+         num_extensions_copied++;
+      }
+   }
+
+   /* Tweak the primary header on the new file to reflect the fact 
+    * that we just removed some extensions. */
+   fh_set_int(phu_out, FH_AUTO, "NEXTEND", 
+                 num_extensions_copied, "Number of extensions");
+   fh_remove(phu_out, PHU_NAME_BURN_AREA);
+   fh_remove(phu_out, PHU_NAME_BURN_FIT);
+   fh_rewrite(phu_out);
+
+   fh_destroy(phu_in);
+   fh_destroy(phu_out);
+   close(fd_out);
+
+   return FH_SUCCESS;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.h	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persist_fits.h	(revision 24244)
@@ -0,0 +1,34 @@
+/* -*- c-file-style: "Ellemtel" -*-
+ *
+ * persist_fits.h - definitions for reading and writing persistence info to FITS tables.
+ *
+ */
+
+#ifndef _INCLUDED_persist_fits_
+#define _INCLUDED_persist_fits_
+
+#include "fh/fh.h"
+#include "burntool.h"
+
+/* Keywords in primary header that give extension names of burn tables. */
+#define PHU_NAME_BURN_AREA       "BTOOLAR"
+#define PHU_NAME_BURN_FIT        "BTOOLFIT"
+
+/* Flag indicating whether burn correction has already been applied. */
+#define PHU_NAME_BURN_APPLIED    "BTOOLAPP"
+
+/* Comments on keywords - use these if you rewrite them for any reason. */
+#define PHU_COMMENT_BURN_AREA    "Name of extension containing burntool streak areas"
+#define PHU_COMMENT_BURN_FIT     "Name of extension containing burntool streak fits"
+#define PHU_COMMENT_BURN_APPLIED "[T=applied] Burn streaks applied to image data"
+
+fh_result
+persist_fits_read(CELL *cell, const char * filename);
+
+fh_result
+persist_fits_write(CELL *cell, HeaderUnit phu);
+
+fh_result
+persist_fits_remove_tables(HeaderUnit phu_in, const char * fileout);
+
+#endif /* _INCLUDED_persist_fits_ */
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persistfix.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persistfix.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persistfix.c	(revision 24244)
@@ -27,4 +27,12 @@
 /* Fix up all the persistence streaks */
    for(k=0; k<cell->npersist; k++) {
+
+/* Is this just a blasted area being carried for IPP? */
+      if((cell->persist)[k].func == BURN_BLASTED) {
+	 if(cell->time - (cell->persist)[k].time > EXPIRE_TRAIL_TIME) {
+	    (cell->persist)[k].fiterr = FIT_EXPIRED;
+	 }
+	 continue;
+      }
 
 /* Fit the trail */
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persistio.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persistio.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/persistio.c	(revision 24244)
@@ -233,5 +233,5 @@
    fprintf(fp,  "# Cell: %d  sky= %d   rms= %d   bias= %d\n", 
 	   cell[0].cell, cell[0].sky, cell[0].rms, cell[0].bias);
-   fprintf(fp, "#Cell time    cx  cy  max   sx  sy   ex  ey  y0m y0p y1m y1p x0m x0p x1m x1p     slope    zero\n");
+   fprintf(fp, "#Cell time    cx  cy  max   y0   sx  sy   ex  ey  y0m y0p y1m y1p x0m x0p x1m x1p F up    slope nfit sxf exf\n");
 
    for(j=0; j<MAXCELL; j++) {
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/psfstats.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/psfstats.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/psfstats.c	(revision 24244)
@@ -1,3 +1,3 @@
-/* psfstats.c - calculate some statistics about a PSF */
+/* dummy version of psfstats.c - disable use of the psf */
 #include <stdio.h>
 #include <stdlib.h>
@@ -12,11 +12,4 @@
 #include "psf/psf.h"
 
-//#define PSFSTATS		/* ignore psf stats so we can skip fortan */
-//#define TEST		/* Dump all the PSF info? */
-
-static unsigned short int *psfbuf=NULL;
-static int npsfbuf=0;
-#define PHONYSKY 1000
-
 /****************************************************************/
 /* psf_stats(): Tell us about this PSF star */
@@ -24,101 +17,4 @@
 		     double *fwhm, double *q)
 {
-
-#ifdef PSFSTATS
-
-   int i, j, r, r2, err, i0, j0;
-   double d, qxx, qxy, qyy, mx, my, sum;
-   PSF_PARAM psfout;      /* Results of fit */
-   PSF_EXTRA psfextra;    /* Extra results from fit */
-   double pi=4*atan(1.0);
-
-   if(nx*ny > npsfbuf) {
-      if(psfbuf != NULL) free(psfbuf);
-      psfbuf = (ushort *)calloc(nx*ny, sizeof(ushort));
-      npsfbuf = nx*ny;
-   }
-   for(i=0; i<nx*ny; i++) {
-      j = ((int)data[i]) - bias + PHONYSKY;
-      if(j >= 0 && j < 65535) psfbuf[i] = j;
-      else psfbuf[i] = 0;
-   }
-
-/* 2D PSF fit */
-   err = psf(nx, ny, psfbuf, PSF_2DIM, NULL, &psfout, &psfextra);
-
-#ifdef TEST
-   printf("err = %d\n", err);
-#endif
-
-   if(!err) {
-#ifdef TEST
-      printf("err = %d\n", err);
-      printf("bias = %d\n", bias);
-      printf("ix, iy = %d %d\n", psfout.ix, psfout.iy);
-      printf("x0, y0 = %.2f %.2f\n", psfout.x0, psfout.y0);
-      printf("fwhm = %.2f\n", psfout.fwhm);
-      printf("peak, bckgnd = %.1f %.1f\n", psfout.peak, psfout.bkgnd);
-      printf("flux, S/N = %.1f %.2f\n", psfout.flux, psfout.sn);
-      printf("weight = %.2f\n", psfout.weight);
-      printf("FWmaj, FWmin = %.2f %.2f\n", psfextra.majfw, psfextra.minfw);
-      printf("theta = %.1f\n", psfextra.thfw*180/pi);
-      printf("wpeak, wbkgnd = %.1f %.1f\n", psfextra.wpeak, psfextra.wbkgnd);
-      printf("dflux, dbckgnd = %.1f %.1f\n", psfextra.dflux, psfextra.dbkgnd);
-      printf("rmsbkgnd = %.1f\n", psfextra.rmsbkgnd);
-#endif
-      fwhm[0] = psfextra.majfw;
-      fwhm[1] = psfextra.minfw;
-      fwhm[2] = psfextra.thfw;
-
-/* Moments */
-      i0 = MIN(psfout.ix, nx-psfout.ix);
-      j0 = MIN(psfout.iy, ny-psfout.iy);
-   } else {
-      fwhm[0] = fwhm[1] = fwhm[2] = 0.0;
-      i0 = nx/2;
-      j0 = ny/2;
-   }
-
-   r = MIN(i0, j0) - 1;
-   sum = mx = my = qxx = qxy = qyy = 0.0;
-   for(j=j0-r; j<j0+r; j++) {
-      for(i=i0-r; i<i0+r; i++) {
-	 r2 = (i-i0)*(i-i0) + (j-j0)*(j-j0);
-	 if(r2 > r*r) continue;
-	 d = (double)data[i+j*nx] - bias;
-	 sum += d;
-	 mx += (i+0.5) * d;
-	 my += (j+0.5) * d;
-	 qxx += (i+0.5) * (i+0.5) * d;
-	 qxy += (i+0.5) * (j+0.5) * d;
-	 qyy += (j+0.5) * (j+0.5) * d;
-      }
-   }
-   if(sum > 0) {
-      mx /= sum;
-      my /= sum;
-      qxx = qxx / sum - mx*mx;
-      qxy = qxy / sum - mx*my;
-      qyy = qyy / sum - my*my;
-   } else {
-      q[0] = q[1] = q[2] = 0.0;
-   }
-
-   q[0] = (qxx + qyy) / 2;
-   q[1] = qxx - qyy;
-   q[2] = 2 * qxy;
-
-/* phi is the angular position of the OTA around the center [rad] */
-//   qtang = (qxx - qyy) * cos(2*phi) + 2 * qxy * sin(2*phi);
-
-#ifdef TEST
-   printf("Sum= %.1f mx= %.2f my= %.2f qxx= %.4f qxy= %.4f qyy= %.4f\n", 
-	  sum, mx, my, qxx, qxy, qyy);
-#endif
-
-# else
-   printf("psfstats ignored in this build\n");
-   return (1);
-#endif /* PSFSTATS */
-   return(0);
+   return(1);
 }
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/trailfit.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/trailfit.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/burntool/trailfit.c	(revision 24244)
@@ -148,8 +148,16 @@
    }
 
+/* Burn extends all the way to the top */
    if(up && y1 >= ny-1) {
       box->slope = -1.0;
-      box->nfit = xe - xs + 1;
       box->fiterr = FIT_TOP_ERROR;
+/* No fit here because the burn is to top, but the persist may be fitable */
+      if(box->sy > FIT_EDGE) {
+	 box->nfit = xe - xs + 1;
+/* However, if burn extends to bottom, persist will not be able to fit */
+      } else {
+	 box->nfit = 1;		/* Just enough to keep it alive */
+	 box->func = BURN_BLASTED;
+      }
       return(0);
    }
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfh/Makefile
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfh/Makefile	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfh/Makefile	(revision 24244)
@@ -10,8 +10,8 @@
 include ../Make.Common
 
-VERSION = 2.01
+VERSION = 2.03
 CCWARN += $(WERROR)
 CCDEFS += -DHAVE_FH_VALIDATE
-SRCS = fh.c
+SRCS = fh.c fh_table.c
 HDRS = fh.h fh_registry.h fh_registry.asm
 
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfh/fh.h	(revision 24244)
@@ -36,4 +36,7 @@
 #define FH_COUNT_CARDS "@FH_COUNT_CARDS@"  /* See also: fh_count_cards() */
 
+/* Size of buffer in fhTableCol structure for format string. */
+#define FH_TABLE_FORMAT_STR_LEN 20
+
 typedef enum /* fh_result -- Result Code for most functions in this library: */
 {  FH_SUCCESS = 0,
@@ -64,4 +67,42 @@
 } fh_bool;			/* Value for "logical" FITS cards */
 
+/* The possible field formats in an ASCII table extension. */
+typedef enum
+{
+   FH_TABLE_FORMAT_CHAR = 'A',   /* Designates a string field. */
+   FH_TABLE_FORMAT_INT = 'I',    /* Integer field. */
+   FH_TABLE_FORMAT_FLOAT = 'F',  /* Float field. */
+   FH_TABLE_FORMAT_DOUBLE = 'D'  /* Double field. */
+} fhTableFormat;
+
+typedef struct
+{
+  char * name;    /* Short string identifying this column's data. */ 
+  char * comment; /* Longer blurb about this column's data. */
+  char * units;   /* Units for column's data. */
+  fhTableFormat format; /* Type of data held in this column. */
+  int width;      /* Width of this column's data, in characters. */
+  int dec_places; /* For doubles, etc, the number of decimal places to write. */
+  /* Everything below this line is generated by fh_table_init(). These 
+   * do not need to be filled out by hand. */
+  int first_char_pos; /* Position of first character in this column. Calculated on the fly. */
+  char format_string[FH_TABLE_FORMAT_STR_LEN]; /* Format string for printf. */
+} fhTableCol;
+
+/* Describes the layout of an ASCII table. */
+typedef struct
+{
+  char * extname;    /* Name of table extension. */
+  int num_cols;      /* Number of rows in table. */
+  int num_rows;      /* Number of columns in table. */
+  fhTableCol * cols; /* Array of cols columns. */
+  /* Everything below this line is generated by fh_table_init(). These 
+   * do not need to be filled out by hand. */
+  int widest_col;    /* Width of widest column. */
+  char * strbuf;     /* Will point to buf of widest_col + 1 chars. */
+  int row_width;     /* Number of characters in a table tow. */
+  int table_size;    /* Total size of table data, in bytes. */
+} fhTable;
+
 typedef void* HeaderUnit; /* Handle to a list of FITS cards allocated by fh_create. */
 
@@ -345,3 +386,77 @@
 double fh_idx(HeaderUnit hu); /* idx of the last card returned by fh_next */
 fh_result fh_merge(HeaderUnit hu, const HeaderUnit source); /* source unchanged */
+
+/* ---------------------------------------------------------
+ * Reading and writing tables
+ * ---------------------------------------------------------
+ */
+
+fh_result
+fh_table_init(fhTable * table);
+/* Call once to initialise information about a given table. This will
+ * walk through all of the columns and fill out starting character
+ * positions and so forth, and will calculate the size of each row
+ * and the size of the table as a whole, and so forth. */
+
+fh_result
+fh_table_populate_header(HeaderUnit hu, fhTable * table);
+/* Sets up keywords for a table extension header, including all 
+ * of the column information, etc. This will initialise the table
+ * structure if necessary. This does not include any fh_write... 
+ * calls to write the header to the file. */
+
+fh_result
+fh_reserve_padded_table(HeaderUnit hu, int fd);
+/* Analogous to fh_reserve_padded_image() for tables. This reserves 
+ * space for the table based on the information in the passed extension
+ * header at the current location in the file. The header must therefore
+ * be populated prior to calling this function (use 
+ * fh_table_populate_header() to build a header from a table structure.
+ */
+
+fh_result
+fh_map_table(HeaderUnit hu, void** data, int size);
+fh_result
+fh_munmap_table(HeaderUnit hu);
+/* Analogous to fh_map_raw_image() and fh_munmap_image() for tables. 
+ * These calls give the calling program access to a memory mapped pointer
+ * to the data in the FITS file.  The data format is equivalent to what
+ * would be produced by fh_read_image() with typesize=FH_TYPESIZE_RAW.
+ * Whether or not the file can be modified through the memory-mapped
+ * pointer depends on whether fh_mode=FH_FILE_RDONLY or FH_FILE_RDWR
+ * in the call to fh_file() that opened the file.  Be sure to use
+ * fh_munmap_table() when done.
+ *
+ * `size' must match the exact number bytes of data which should be in
+ * the file, according to its NAXIS and BITPIX values, excluding padding.
+ *
+ * %%% TODO: FH_FILE_RDWR is completely unimplemented!
+ *
+ * %%% TODO: File locking is not handled properly yet.  For now, use
+ * FH_FILE_RDONLY_NOLOCK and FH_FILE_RDWR_NOLOCK, especially on MEF
+ * files where the locks are left in place until the file is closed
+ * otherwise!
+ */
+
+fh_result
+fh_table_read_value(fhTable * table, void * data,
+                    int row, int col, void * value);
+/* Reads a single value from an existing table at (row, column). 
+ * The table field is interpreted based on the format of that
+ * column in the table structure, and is passed back via the
+ * 'value' pointer. It is the caller's responsibility to ensure
+ * that 'value' points to a location with sufficient space for
+ * the data in the field, and that any casting to & from a void
+ * pointer is done correctly.
+ */
+
+fh_result
+fh_table_write_value(fhTable * table, void * data, 
+                     int row, int col, void * value);
+/* Similar to read_value() above, this function overwrites a field
+ * in a table at the given location.  The caller is repsonsible 
+ * for casting the location of the stored data to a void pointer. 
+ */
+
+
 #endif /* _INCLUDED_fh */
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfh/fh_table.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfh/fh_table.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfh/fh_table.c	(revision 24244)
@@ -0,0 +1,476 @@
+/* -*- c-file-style: "Ellemtel" -*-
+ *
+ * fh_table - manage ASCII FITS tables as part of libfh.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "fh.h"
+
+/* The FITS standard explicitly states the maximum number of 
+ * columns in a table. */
+#define MAX_COLS 999
+
+#define FH_RESERVE \
+"COMMENT  Reserved space.  This line can be used to add a new FITS card.         "
+/*
+ * Always place the END on the second line of a new 2880 block, so that
+ * there are enough blank lines after the END for expansion, but we
+ * don't require or put any blank lines before because some FITS
+ * implementations (libfh is one of them, and cfitsio too) will
+ * remove them.  Use a COMMENT instead, repeated until one occupies
+ * the first line of a new block.  Then the END will go after that.
+ * Anything which compresses and decompresses does not have to worry
+ * about keeping track of blank lines, since they are after the end.
+ * It will be forced to put back just as many as there were to begin
+ * with in order to pad the header again.
+ */
+static void
+pad_header(HeaderUnit hu)
+{
+   int blocks_orig = fh_header_blocks(hu);
+   double idx = 900000000;
+
+   do
+   {
+      fh_set_card(hu, idx++, FH_RESERVE);
+   } while (fh_header_blocks(hu) == blocks_orig);
+   fh_set_card(hu, idx++, FH_RESERVE);
+}
+
+/*
+ * Sets up table info.
+ *
+ * Inputs:
+ *    table - FITS table.
+ */
+fh_result
+fh_table_init(fhTable * table)
+{
+   int i;
+   int row_width = 0;
+
+   /* Already set up? */
+   if(table->strbuf) return FH_SUCCESS;
+
+   if((table->num_cols <= 0) || (table->num_cols > MAX_COLS))
+   {
+      fprintf(stderr, 
+              "error: %d-column table is invalid. Tables may only have 1..%d columns.\n",
+              table->num_cols, MAX_COLS);
+      return FH_INVALID;
+   }
+
+   table->widest_col = 0;
+   for(i = 0 ; i < table->num_cols; i++)
+   {
+      /* Widest column in table? */
+      if(table->cols[i].width > table->widest_col)
+      {
+         table->widest_col = table->cols[i].width;
+      }
+
+      /* Character positioning in a row in FITS starts from 1, not 0. */
+      table->cols[i].first_char_pos = row_width + 1;
+      row_width += table->cols[i].width;
+
+      /* Generate a format string that will be used later to 
+       * sprintf() values to the table. */
+      switch(table->cols[i].format)
+      {
+         case FH_TABLE_FORMAT_CHAR:
+            snprintf(table->cols[i].format_string, FH_TABLE_FORMAT_STR_LEN, 
+                     "%%%ds", table->cols[i].width);
+            break;
+            
+         case FH_TABLE_FORMAT_INT:
+            snprintf(table->cols[i].format_string, FH_TABLE_FORMAT_STR_LEN, 
+                     "%%%dd", table->cols[i].width);
+            break;
+            
+         case FH_TABLE_FORMAT_FLOAT:
+         case FH_TABLE_FORMAT_DOUBLE: /* Intentional fall-through. */
+            snprintf(table->cols[i].format_string, FH_TABLE_FORMAT_STR_LEN, 
+                     "%%%d.%df", table->cols[i].width, table->cols[i].dec_places);
+            break;
+            
+         default:
+            free(table);
+            fprintf(stderr, "error: column %d has unknown format specifier '%c'\n",
+                    i, table->cols[i].format);
+            return FH_BAD_VALUE;
+            break;
+      }
+   }
+
+   table->row_width = row_width;
+   table->table_size = row_width * table->num_rows;
+
+   /* Make a space guaranteed to be large enough to hold any
+    * of the fields, plus a null character on the end. This is 
+    * done so that we can use the string library to manipulate
+    * the contents of each field. */
+   if(!(table->strbuf = malloc(table->widest_col + 1))) 
+   {
+      fprintf(stderr, 
+              "error: Unable to malloc %d bytes for table field buffer.\n",
+              table->widest_col + 1);
+      return FH_NO_MEMORY;
+   }
+
+   return FH_SUCCESS;
+}
+
+/*
+ * Writes essential FITS header information.
+ *
+ * Given a table and a FITS header, this funciton will write the necessary 
+ * keywords to make the FITS header describe an ASCII table.
+ *
+ * Inputs:
+ *       table - FITS table.
+ *          hu - the FITS header to be written.
+ *
+ * Outputs:
+ *    none.
+ *
+ * This function returns a PASS/FAIL indication that the header was 
+ * successfully populated.
+ *
+ * NOTE: this function currently assumes that the FITS table is being
+ * written to an extension header, not a primary. 
+ *
+ * NOTE: this function does not set keywords for the extension name
+ * or other keywords that are not explicitly required for an ASCII 
+ * table extension. It is the caller's responsibility to fill out
+ * these keywords as required.
+ */
+fh_result fh_table_populate_header(HeaderUnit hu, fhTable * table)
+{
+   char keyword[FH_NAME_SIZE + 1];
+   char value[FH_MAX_STRLEN + 1];
+   int i;
+   double idx;
+
+   /* Do some sanity checks first. */
+   if(!table || !table->cols || !table->num_cols || !table->num_rows)
+   {
+      fprintf(stderr, 
+              "error: Incomplete table info passed to fh_table_populate_header()\n");
+      return FH_INVALID;
+   }
+
+   if(fh_table_init(table) != FH_SUCCESS) return FH_BAD_VALUE;
+
+   fh_set_str(hu, 0.0, "XTENSION", "TABLE", "");
+   fh_set_int(hu, 1.0, "BITPIX", 8, "8-bit ASCII");
+   fh_set_int(hu, 2.0, "NAXIS", 2, "2D table of values");
+   fh_set_int(hu, 2.1, "NAXIS1", table->row_width, 
+              "Characters in each table row");
+   fh_set_int(hu, 2.2, "NAXIS2", table->num_rows, 
+              "Number of rows in table");
+   fh_set_int(hu, 3.0, "PCOUNT", 0, 
+              "Random parameters before each array in a group");
+   fh_set_int(hu, 3.2, "GCOUNT", 1, 
+              "Number of random groups");
+   fh_set_int(hu, 4.0, "TFIELDS", table->num_cols, 
+              "Number of columns in table");
+
+   /* Write the format specifiers for each column. */
+   for(i=0; i < table->num_cols; i++)
+   {
+      idx = 5.0 + i/10.;
+      int n = i + 1; /* FITS numbering scheme starts from 1. */
+      snprintf(keyword, FH_NAME_SIZE + 1, "TFORM%d", n);
+      if((table->cols[i].format == FH_TABLE_FORMAT_FLOAT) ||
+         (table->cols[i].format == FH_TABLE_FORMAT_DOUBLE))
+      {
+         snprintf(value, FH_MAX_STRLEN + 1, "%c%d.%d", 
+                  table->cols[i].format, table->cols[i].width,
+                  table->cols[i].dec_places);
+      }
+      else
+      {
+         snprintf(value, FH_MAX_STRLEN + 1, "%c%d", 
+                  table->cols[i].format, table->cols[i].width);
+      }
+      fh_set_str(hu, idx, keyword, value, 
+                 "FITS table field format");
+      idx += 0.01;
+
+      snprintf(keyword, FH_NAME_SIZE + 1, "TBCOL%d", n);
+      fh_set_int(hu, idx, keyword, table->cols[i].first_char_pos, 
+                 "First character of column");
+   }
+
+   /* Set the name of this extension. */
+   fh_set_str(hu, 6.0, "EXTNAME", table->extname, "Extension name");
+
+   /* The FITS standard suggests that other descriptive information 
+    * that describes each column must follow after all of the format
+    * information. Presumably this is because this information, while
+    * nice to have from a self-documenting perspective, isn't strictly
+    * necessary to interpret the actual table values. */
+   for(i=0; i < table->num_cols; i++)
+   {
+      idx = 7.0 + i/10.;
+      int n = i + 1; /* FITS numbering scheme starts from 1. */
+      snprintf(keyword, FH_NAME_SIZE + 1, "TTYPE%d", n);
+      fh_set_str(hu, idx, keyword, table->cols[i].name, 
+                 "Column name");
+      
+      snprintf(keyword, FH_NAME_SIZE + 1, "TUNIT%d", n);
+      idx += 0.01;
+      fh_set_str(hu, idx, keyword, table->cols[i].units, 
+                 "Column data units");
+      
+      snprintf(keyword, FH_NAME_SIZE + 1, "TBCOM%d", n);
+      idx += 0.01;
+      fh_set_str(hu, idx, keyword, table->cols[i].comment, 
+                 "Column description");
+
+   }
+   pad_header(hu);
+   return FH_SUCCESS;
+}
+
+/*
+ * Writes a field in a table.
+ *
+ * This function is used to write a single field to a table. Given
+ * a table, information about its columns, the row and column within 
+ * the table to be written and a value, the value will be written to 
+ * the table according to the formatting instructions for that column.
+ *
+ * The value is passed as a void* so that this one function can be used
+ * for any type of data that can be put into the table. The function
+ * uses the column information to decide how to handle the pointer. It is
+ * the caller's responsibility to keep their casts and pointers straight.
+ *
+ * Inputs:
+ *    table - FITS table.
+ *     data - buffer where FITS table is to be written.
+ *      row - the row in the table where the datum should be written.
+ *      col - the column in the table where the datum should be written.
+ *    value - points to the value to be written.
+ *
+ * Outputs:
+ *    none.
+ *
+ * This function returns a FH_SUCCESS if the value was successfully written.
+ */
+fh_result
+fh_table_write_value(fhTable * table, void * data, 
+                     int row, int col, void * value)
+{
+   char * ptr;
+
+   /* Do some sanity checks first. */
+   if(!table)
+   {
+      fprintf(stderr, 
+              "error: no FITS table passed to fh_table_write_value().\n");
+      return FH_INVALID;
+   }
+   if(!table->strbuf)
+   {
+      fprintf(stderr, 
+              "error: no field buffer in this table. Was fh_table_populate_header() called already?\n");
+      return FH_INVALID;
+   }
+   if((row >= table->num_rows) || (col >= table->num_cols))
+   {
+      fprintf(stderr, 
+              "error: attempted to write value beyond edge of FITS table (row %d col %d, table is %dx%d).\n",
+              row, col, table->num_rows, table->num_cols);
+      return FH_INVALID;
+   }
+   if(!value)
+   {
+      fprintf(stderr, "error: no value given for row %d, col %d of FITS table.\n", row, col);
+      return FH_INVALID;            
+   }
+   
+   /* Figure out where this data is going to go. */
+   ptr = (char *)data +
+      (row * table->row_width) +
+      table->cols[col].first_char_pos - 1;
+
+   switch(table->cols[col].format)
+   {
+      case FH_TABLE_FORMAT_CHAR:
+         snprintf(table->strbuf, table->cols[col].width + 1, 
+                  table->cols[col].format_string, (char *)value);
+         break;
+         
+      case FH_TABLE_FORMAT_INT:
+         snprintf(table->strbuf, table->cols[col].width + 1,
+                  table->cols[col].format_string, *((int *)value));
+         break;
+         
+      case FH_TABLE_FORMAT_FLOAT:
+         snprintf(table->strbuf, table->cols[col].width + 1, 
+                  table->cols[col].format_string, *((float *)value));
+         break;
+      case FH_TABLE_FORMAT_DOUBLE:
+         snprintf(table->strbuf, table->cols[col].width + 1, 
+                  table->cols[col].format_string, *((double *)value));
+         break;
+         
+      default:
+         fprintf(stderr, 
+                 "error: column %d has unknown format specifier '%c'\n",
+                 col, table->cols[col].format);
+         return FH_BAD_VALUE;
+         break;
+   }
+
+   /* Copy the field over to the table, minus the null character. */
+   memcpy(ptr, table->strbuf, table->cols[col].width);
+
+   return FH_SUCCESS;
+}
+
+/*
+ * Reads a field in a table.
+ *
+ * This function is used to retrieve a single field in a table. Given
+ * a table, information about its columns, the row and column within 
+ * the table to be written and a value, the value will be written to 
+ * the table according to the formatting instructions for that column.
+ *
+ * The value is passed as a void* so that this one function can be used
+ * for any type of data that can be put into the table. The function
+ * uses the column information to decide how to handle the pointer. It is
+ * the caller's responsibility to ensure that there is enough space on
+ * the end of the value pointer to store the value, and that the value,
+ * once written, will be interpreted properly (e.g. don't go passing
+ * a pointer to a double for an int field).
+ *
+ * Inputs:
+ *    table - FITS table.
+ *     data - buffer where FITS table is to be written.
+ *      row - the row in the table where the datum should be written.
+ *      col - the column in the table where the datum should be written.
+ *
+ * Outputs:
+ *    value - value in field will be written here.
+ *
+ * This function returns a FH_SUCCESS if the value was successfull read.
+ */
+fh_result
+fh_table_read_value(fhTable * table, void * data, 
+                    int row, int col, void * value)
+{
+   char * ptr;
+   char * endchar;
+
+   /* Do some sanity checks first. */
+   if(!table)
+   {
+      fprintf(stderr, 
+              "error: no FITS table passed to fh_table_read_value().\n");
+      return FH_INVALID;
+   }
+   if(!table->strbuf)
+   {
+      fprintf(stderr, 
+              "error: no field buffer in this table. Was fh_table_populate_header() called already?\n");
+      return FH_INVALID;
+   }
+   if((row >= table->num_rows) || (col >= table->num_cols))
+   {
+      fprintf(stderr, 
+              "error: attempted to read value beyond edge of FITS table (row %d col %d, table is %dx%d).\n",
+              row, col, table->num_rows, table->num_cols);
+      return FH_INVALID;
+   }
+   if(!value)
+   {
+      fprintf(stderr, "error: no value given for row %d, col %d of FITS table.\n", row, col);
+      return FH_INVALID;            
+   }
+   
+   /* Figure out where this value is going to come from. */
+   ptr = (char *)data +
+      (row * table->row_width) +
+      table->cols[col].first_char_pos - 1;
+   
+   /* Grab the value, and shove a null on the end so the string
+    * library can have its way with it. */
+   memcpy(table->strbuf, ptr, table->cols[col].width);
+   table->strbuf[table->cols[col].width] = '\0';
+
+   /* Interpret the data based on the column's reported type. */
+   switch(table->cols[col].format)
+   {
+      case FH_TABLE_FORMAT_CHAR:
+         strcpy((char *)value, table->strbuf);
+         break;
+         
+      case FH_TABLE_FORMAT_INT:
+         *((int *)value) = (int)strtol(table->strbuf, &endchar, 10);
+         if(*endchar != '\0')
+         {
+            fprintf(stderr,
+                    "error: Invalid value in table at row %d, col %d. "
+                    "Expected an int, got '%s'\n",
+                    row, col, table->strbuf);
+         }
+         break;
+         
+      case FH_TABLE_FORMAT_FLOAT:
+         *((float *)value) = strtod(table->strbuf, &endchar);
+         if(*endchar != '\0')
+         {
+            fprintf(stderr,
+                    "error: Invalid value in table at row %d, col %d. "
+                    "Expected a float, got '%s'\n",
+                    row, col, table->strbuf);
+         }
+         break;
+      case FH_TABLE_FORMAT_DOUBLE:
+         *((double *)value) = strtod(table->strbuf, &endchar);
+         if(*endchar != '\0')
+         {
+            fprintf(stderr,
+                    "error: Invalid value in table at row %d, col %d. "
+                    "Expected a double, got '%s'\n",
+                    row, col, table->strbuf);
+         }
+         break;
+         
+      default:
+         fprintf(stderr, 
+                 "error: column %d has unknown format specifier '%c'\n",
+                 col, table->cols[col].format);
+         return FH_BAD_VALUE;
+         break;
+   }
+
+   return FH_SUCCESS;
+}
+
+
+/* Not sure if we need to do anything more with these right now. At 
+ * the moment, these are purely cosmetic wrappers around their 
+ * respective image functions. */
+fh_result
+fh_reserve_padded_table(HeaderUnit hu, int fd)
+{
+   return fh_reserve_padded_image(hu, fd);
+}
+
+fh_result
+fh_map_table(HeaderUnit hu, void** data, int size)
+{
+   return fh_map_raw_image(hu, data, size);
+}
+
+fh_result
+fh_munmap_table(HeaderUnit hu)
+{
+   return fh_munmap_raw_image(hu);
+}
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfhreg/gpc_telescope.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfhreg/gpc_telescope.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/gpcsrc/fits/libfhreg/gpc_telescope.h	(revision 24244)
@@ -84,5 +84,4 @@
 MK_PFL(  522.5, ROT     ,6, "Telescope rotator angle (degrees)"               )
 MK_PFL(  522.6, POSANGLE,6, "Telescope position angle (degrees)"              )
-MK_PFL(  522.7, PA_DEBUG,6, "Telescope position angle (alternate value)"      )
 MK_PFL(  523.0, COMRA   ,6, "Commanded telescope Right Ascension (degrees)"   )
 MK_PFL(  523.1, COMDEC  ,6, "Commanded telescope Declination (degrees)"       )
Index: /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/setuidinst
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/setuidinst	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/extsrc/gpcsw/setuidinst	(revision 24244)
@@ -0,0 +1,4 @@
+#!/bin/csh -f
+# the camera build tools call setuidinst, but we don't need it
+
+exit 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/Makefile.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/Makefile.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/Makefile.in	(revision 24244)
@@ -61,4 +61,5 @@
 $(DESTWWW)/detNormalizedImfile.php \
 $(DESTWWW)/detNormalizedStatImfile.php \
+$(DESTWWW)/detRegisteredImfile.php \
 $(DESTWWW)/detProcessedImfile.php \
 $(DESTWWW)/detProcessedExp.php \
@@ -93,5 +94,5 @@
 $(DESTWWW)/camPendingExp.php \
 $(DESTWWW)/camProcessedExp.php \
-$(DESTWWW)/camProcessedExp_NoImages.php \
+$(DESTWWW)/camProcessedExp_Images.php \
 $(DESTWWW)/camProcessedExp_failure.php \
 $(DESTWWW)/camIQstats.php \
@@ -126,7 +127,10 @@
 $(DESTWWW)/warpProcessedSkyfiles.php \
 $(DESTWWW)/warpFailedSkyfiles.php \
+$(DESTWWW)/diffSummary.php \
 $(DESTWWW)/diffRun.php \
 $(DESTWWW)/diffInputSkyfile.php \
-$(DESTWWW)/diffSkyfile.php \
+$(DESTWWW)/diffProcessedSkyfile.php \
+$(DESTWWW)/diffProcessedSkyfile_Images.php \
+$(DESTWWW)/diffFailedSkyfile.php \
 $(DESTWWW)/stackRun.php \
 $(DESTWWW)/stackSummary.php \
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/camProcessedExp_Images.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/camProcessedExp_Images.d	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/camProcessedExp_Images.d	(revision 24244)
@@ -0,0 +1,58 @@
+TABLE camRun, camProcessedExp, chipRun, rawExp
+TITLE Camera Processed
+FILE  camProcessedExp.php
+MENU  ipp.science.dat
+
+# the following WHERE clauses are added to all queries joined by AND
+WHERE camRun.state = 'full'
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camProcessedExp.cam_id  = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+# define image names to be used below
+# IMAGE VAR basename filerule camera class_id
+IMAGE JPEG2 $camProcessedExp.path_base PPIMAGE.JPEG2 $rawExp.camera NONE
+
+# define the arguments supplied to the links below (if any)
+ARGS  ARG1  rawImfile.exp_id=$rawExp.exp_id
+ARGS  ARG2  chipRun.chip_id=$chipRun.chip_id
+ARGS  ARG3  camRun.cam_id=$camRun.cam_id
+
+ARGS  ARG4 camRun.cam_id=$camRun.cam_id
+ARGS  ARG4 camera=$rawExp.camera
+ARGS  ARG4 basename=$camProcessedExp.path_base
+
+OP   OP1  $rawExp.ra * 57.295783
+OP   OP2  $rawExp.decl * 57.295783
+
+#     field                     	size  format  name          show   	 link to                  extras
+FIELD rawExp.exp_name,     	 	10,   %s,     Exp Name
+FIELD rawExp.exp_id,	         	 5,   %s,     Exp ID,       value,  	 rawImfile.php,           ARG1
+FIELD chipRun.chip_id,           	 5,   %s,     Chip ID,      value,  	 chipProcessedImfile.php, ARG2
+FIELD camRun.cam_id,             	 5,   %s,     Cam ID,       value,  	 camProcessedExp.php,     ARG3
+FIELD camRun.label,             	 5,   %s,     Label
+FIELD *,    	                	 8,   %s,     image,        image=JPEG2, camProcessedImfile.php,  ARG4
+FIELD rawExp.telescope,      		10,   %s,     Telescope
+FIELD rawExp.camera,         		10,   %s,     Camera
+FIELD rawExp.dateobs,        		19,   %T,     Date/Time
+FIELD rawExp.ra,           		 8,   %10.6f, RA,           op=OP1      
+FIELD rawExp.decl,         		 8,   %10.6f, DEC,          op=OP2
+FIELD rawExp.object,         		10,   %s,     Object
+FIELD rawExp.filter,         		10,   %s,     FILTER
+FIELD rawExp.exp_time,       		 5,   %.2f,     exp_time    
+FIELD rawExp.airmass,        		 5,   %.4f,     airmass     
+FIELD camProcessedExp.bg,             	 5,   %.2f,     backgnd
+FIELD camProcessedExp.bg_stdev,       	 5,   %.2f,     stdev    
+FIELD camProcessedExp.n_stars,  	 5,   %d,     Nstars
+FIELD camProcessedExp.n_astrom, 	 5,   %d,     Nastrom
+FIELD camProcessedExp.sigma_ra, 	 5,   %f,     sigma ra
+FIELD camProcessedExp.fwhm_major,     	 5,   %f,     FHWM (major)
+FIELD rawExp.comment,  	                65, %s,     Comment
+
+FIELD camProcessedExp.path_base,     	 5,   %s,     path_base,    none
+FIELD rawExp.exp_id,     		 5,   %s,     Exp ID,       none
+
+# the last two are used as arguments elsewhere, thus needed in the list, even if not displayed
+# FIELD camRun.reduction,       	         5,   %s,     reduction
+# FIELD camProcessedExp.bg_mean_stdev,  	 5,   %.2f,     &lt;backgnd&gt;
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detNormalizedImfile.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detNormalizedImfile.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detNormalizedImfile.d	(revision 24244)
@@ -1,17 +1,26 @@
-TABLE detNormalizedImfile
+TABLE detNormalizedImfile, detRun
 TITLE detNormalizedImfile
 FILE  detNormalizedImfile.php
 MENU  ipp.imfiles.dat
 
-#     field         size  format  name     show   link to     extras
-FIELD det_id,          7, %s,     det_id
-FIELD class_id,        8, %s,     class_id
-FIELD bg,              8, %s,     backgnd
-FIELD bg_stdev,        8, %s,     stdev
-FIELD bg_mean_stdev,   8, %s,     [stdev]
-FIELD iteration,       5, %s,     iteration
-#FIELD uri,           20, %s,     uri
-#FIELD path_base,     20, %s,     path_base
+WHERE detNormalizedImfile.det_id = detRun.det_id
 
-TAIL PHP insert_image ('PPIMAGE.JPEG1');
-TAIL PHP insert_log ('LOG.EXP');
+ARGS  ARG1 detNormalizedImfile.det_id=$detNormalizedImfile.det_id
+ARGS  ARG1 detNormalizedImfile.iteration=$detNormalizedImfile.iteration
+ARGS  ARG1 detNormalizedImfile.class_id=$detNormalizedImfile.class_id
+ARGS  ARG1 camera=$detRun.camera
+ARGS  ARG1 basename=$detNormalizedImfile.path_base
+ARGS  ARG1 class=$detNormalizedImfile.class_id
+
+#     field                             size  format  name      show   link to                  extras
+FIELD detNormalizedImfile.det_id,          7, %s,     det_id
+FIELD detNormalizedImfile.iteration,       5, %s,     iter
+FIELD detNormalizedImfile.class_id,        8, %s,     class_id, value, detNormalizedImfile.php, ARG1
+FIELD detNormalizedImfile.fault,           5, %d,     fault
+FIELD detNormalizedImfile.bg,              8, %f,     backgnd
+FIELD detNormalizedImfile.bg_stdev,        8, %f,     stdev
+FIELD detNormalizedImfile.bg_mean_stdev,   8, %f,     [stdev]
+FIELD detNormalizedImfile.path_base,      10, %s,     path_base
+FIELD detRun.camera,                      10, %s,     camera, none
+
+TAIL PHP insert_log ('LOG.IMFILE');
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detRegisteredImfile.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detRegisteredImfile.d	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detRegisteredImfile.d	(revision 24244)
@@ -0,0 +1,26 @@
+TABLE detRegisteredImfile, detRun
+TITLE detRegisteredImfile
+FILE  detRegisteredImfile.php
+MENU  ipp.imfiles.dat
+
+WHERE detRegisteredImfile.det_id = detRun.det_id
+
+ARGS  ARG1 detRegisteredImfile.det_id=$detRegisteredImfile.det_id
+ARGS  ARG1 detRegisteredImfile.iteration=$detRegisteredImfile.iteration
+ARGS  ARG1 detRegisteredImfile.class_id=$detRegisteredImfile.class_id
+ARGS  ARG1 camera=$detRun.camera
+ARGS  ARG1 basename=$detRegisteredImfile.path_base
+ARGS  ARG1 class=$detRegisteredImfile.class_id
+
+#     field                             size  format  name      show   link to                  extras
+FIELD detRegisteredImfile.det_id,          7, %s,     det_id
+FIELD detRegisteredImfile.iteration,       5, %s,     iter
+FIELD detRegisteredImfile.class_id,        8, %s,     class_id, value, detRegisteredImfile.php, ARG1
+FIELD detRegisteredImfile.fault,           5, %d,     fault
+FIELD detRegisteredImfile.bg,              8, %f,     backgnd
+FIELD detRegisteredImfile.bg_stdev,        8, %f,     stdev
+FIELD detRegisteredImfile.bg_mean_stdev,   8, %f,     [stdev]
+FIELD detRegisteredImfile.path_base,      10, %s,     path_base
+FIELD detRun.camera,                      10, %s,     camera, none
+
+TAIL PHP insert_log ('LOG.IMFILE');
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detResidImfile.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detResidImfile.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detResidImfile.d	(revision 24244)
@@ -20,5 +20,5 @@
 FIELD detResidImfile.fault,         5, %s,      fault
 FIELD detResidImfile.path_base,     5, %s,      path_base
-FIELD detResidImfile.uri,           5, %s,      uri
+FIELD detResidImfile.uri,           5, %s,      uri, none
 
 TAIL PHP insert_image ('PPIMAGE.JPEG1');
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detRun.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detRun.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detRun.d	(revision 24244)
@@ -14,17 +14,21 @@
 ARGS  ARG4 detResidExp.iteration=$iteration
 
-ARGS  ARG5 det_id=$det_id
+ARGS  ARG5 detStackedImfile.det_id=$det_id
+ARGS  ARG6 detNormalizedImfile.det_id=$det_id
+ARGS  ARG7 detRegisteredImfile.det_id=$det_id
 
-#     field       size  format  name            show         link to                extras
-FIELD det_id,        5, %s,     det ID,       value,       detRunSummary.php,     ARG1
+#     field       size  format  name         show       link to                   extras
+FIELD det_id,        5, %s,     det ID,      value,     detRunSummary.php,        ARG1
 FIELD iteration,     3, %s,     iter
 FIELD det_type,     12, %s,     type
 FIELD state,         5, %s,     state
-FIELD *,             5, %s,     choose,       value=input, detInputExp.php,       ARG2
-FIELD *,             5, %s,     choose,       value=proc,  detProcessedExp.php,   ARG3
-FIELD *,             5, %s,     choose,       value=stack, detStackedImfile.php,  ARG5
-FIELD *,             5, %s,     choose,       value=resid, detResidExp.php,       ARG4
+FIELD *,             5, %s,     IN,          value=IN,   detInputExp.php,          ARG2
+FIELD *,             5, %s,     P,           value=P,   detProcessedExp.php,      ARG3
+FIELD *,             5, %s,     D,           value=D,   detResidExp.php,          ARG4
+FIELD *,             5, %s,     S,           value=S,   detStackedImfile.php,     ARG5
+FIELD *,             5, %s,     N,           value=N,   detNormalizedImfile.php,  ARG6
+FIELD *,             5, %s,     R,           value=R,   detRegisteredImfile.php,  ARG7
 FIELD mode,          5, %s,     mode
-FIELD filter,       10, %s,     filter     
+FIELD filter,        5, %s,     filter     
 FIELD registered,   19, %s,     registered 
 FIELD use_begin,    19, %s,     use_begin  
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detStackedImfile.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detStackedImfile.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detStackedImfile.d	(revision 24244)
@@ -7,12 +7,14 @@
 
 # XXX change uri to path_base in db
-ARGS  ARG1 detStackedImfile.det_id=$det_id
-ARGS  ARG1 detStackedImfile.iteration=$iteration
+ARGS  ARG1 detStackedImfile.det_id=$detStackedImfile.det_id
+ARGS  ARG1 detStackedImfile.iteration=$detStackedImfile.iteration
+ARGS  ARG1 detStackedImfile.class_id=$detStackedImfile.class_id
 ARGS  ARG1 camera=$detRun.camera
 ARGS  ARG1 basename=$detStackedImfile.uri
+ARGS  ARG1 class=$detStackedImfile.class_id
 
-#     field         size  format  name      show  link to     extras
+#     field                         size  format  name        show  link to     extras
 FIELD detStackedImfile.det_id,         7,  %s,    det_id
-FIELD detStackedImfile.iteration,      5,  %s,    iteration
+FIELD detStackedImfile.iteration,      5,  %s,    iter
 FIELD detStackedImfile.class_id,       8,  %s,    class_id,   value, detStackedImfile.php, ARG1
 FIELD detStackedImfile.fault,          5,  %d,    fault
@@ -20,8 +22,8 @@
 FIELD detStackedImfile.bg_stdev,       8,  %f,    stdev
 FIELD detStackedImfile.bg_mean_stdev,  8,  %f,    [stdev]
-FIELD detStackedImfile.uri,           20,  %s,    uri
-# FIELD recipe, 20,   recipe
+FIELD detStackedImfile.uri,           10,  %s,    uri
+FIELD detRun.camera,                  10, %s,     camera, none
 
 TD_CLASS list_off $fault > 0
 
-TAIL PHP insert_log ('LOG.EXP');
+TAIL PHP insert_log ('LOG.IMFILE');
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detStackedImfile_failure.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detStackedImfile_failure.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detStackedImfile_failure.d	(revision 24244)
@@ -1,19 +1,32 @@
-TABLE detStackedImfile Failures
-TITLE detStackedImfile
+TABLE detStackedImfile, detRun
+TITLE detStackedImfile Failures
 FILE  detStackedImfile_failure.php
 MENU  ipp.detrend.dat
 
 WHERE fault > 0
+WHERE detStackedImfile.det_id = detRun.det_id
 
-#     field         size  format  name                   show  link to     extras
-FIELD det_id,         7,  %s,    det_id
-FIELD iteration,      5,  %s,    iteration
-FIELD class_id,       8,  %s,    class_id
-FIELD fault,          5,  %d,    fault
-FIELD bg,             8,  %f,    backgnd
-FIELD bg_stdev,       8,  %f,    stdev
-FIELD bg_mean_stdev,  8,  %f,    [stdev]
-FIELD uri,           20,  %s,    uri
+ARGS  ARG1 detStackedImfile.det_id    =$detStackedImfile.det_id
+ARGS  ARG1 detStackedImfile.iteration =$detStackedImfile.iteration
+ARGS  ARG1 detStackedImfile.class_id  =$detStackedImfile.class_id
+ARGS  ARG1 camera=$detRun.camera
+ARGS  ARG1 basename=$detStackedImfile.path_base
+ARGS  ARG1 class=$detStackedImfile.class_id
+
+# XXX : detStackedImfile has uri only, not path_base : fix!
+
+#     field         size  format  name       show   link to     extras
+FIELD detStackedImfile.det_id,         7,  %s,    det_id
+FIELD detStackedImfile.iteration,      5,  %s,    iteration
+FIELD detStackedImfile.class_id,       8,  %s,    class_id
+FIELD detStackedImfile.class_id,       8,  %s,    Chip ID,    value, detStackedImfile_failure.php, ARG1
+FIELD detStackedImfile.fault,          5,  %d,    fault
+FIELD detStackedImfile.bg,             8,  %f,    backgnd
+FIELD detStackedImfile.bg_stdev,       8,  %f,    stdev
+FIELD detStackedImfile.bg_mean_stdev,  8,  %f,    [stdev]
+FIELD detStackedImfile.uri,           20,  %s,    uri
 # FIELD recipe, 20,   recipe
 
 # TD_CLASS list_off $fault > 0
+
+TAIL PHP insert_log ('LOG.IMFILE');
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffFailedSkyfile.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffFailedSkyfile.d	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffFailedSkyfile.d	(revision 24244)
@@ -0,0 +1,24 @@
+TABLE diffRun, diffSkyfile
+TITLE Diff Failed Skyfiles
+FILE  diffFailedSkyfile.php
+MENU  ipp.stack.dat
+
+WHERE diffRun.diff_id             = diffSkyfile.diff_id
+WHERE diffSkyfile.fault != 0
+
+ARGS  ARG7 diffSkyfile.diff_id=$diffSkyfile.diff_id
+ARGS  ARG7 diffSkyfile.skycell_id=$diffSkyfile.skycell_id
+ARGS  ARG7 camera=GPC1
+ARGS  ARG7 basename=$diffSkyfile.path_base
+
+#     field                         size  format  name           show     link to         extras
+FIELD diffSkyfile.diff_id,             5, %d,     Diff ID
+FIELD diffSkyfile.skycell_id,          5, %s,     Skycell ID
+FIELD diffRun.label,    	       5, %s,     Label
+# FIELD diffRun.filter,    	       5, %s,     Filter
+FIELD diffRun.tess_id,    	       5, %s,     Tess ID
+FIELD diffRun.state,    	       7, %s,     State,         value,   diffProcessedSkyfile.php, ARG7
+FIELD diffSkyfile.fault,               5, %d,     fault
+FIELD diffSkyfile.path_base,           5, %s,     path_base,     none
+
+TAIL PHP insert_log ('LOG.EXP');
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffInputSkyfile.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffInputSkyfile.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffInputSkyfile.d	(revision 24244)
@@ -1,12 +1,19 @@
-TABLE diffInputSkyfile
+TABLE diffInputSkyfile, diffRun
 TITLE diffInputSkyfile
 FILE  diffInputSkyfile.php
 MENU  ipp.stack.dat
 
-#        field        size  format  name           show     link to         extras
-FIELD    diff_id,	20, %s,     diff ID
-FIELD    warp_id,	20, %s,     warp ID
-FIELD    skycell_id,	20, %s,     skycell ID
-FIELD    tess_id,	20, %s,     tessellation ID
-FIELD    kind,          20, %s,     Kind
-FIELD    template,	20, %s,     Template?
+WHERE diffInputSkyfile.diff_id = diffRun.diff_id
+
+#        field                            size format  name           show     link to         extras
+FIELD    diffInputSkyfile.diff_id,	    5, %d,     diff ID
+FIELD    diffInputSkyfile.diff_skyfile_id,  5, %d,     skyfile ID
+FIELD    diffInputSkyfile.skycell_id,	    8, %s,     skycell ID
+FIELD    diffRun.state,         	    7, %s,     state
+FIELD    diffRun.label,	    		    7, %s,     label
+FIELD    diffRun.magicked,	     	    4, %d,     Magicked
+FIELD    diffInputSkyfile.warp1,	    5, %d,     warp 1
+FIELD    diffInputSkyfile.stack1,	    5, %d,     stack 1
+FIELD    diffInputSkyfile.warp2,	    5, %d,     warp 2
+FIELD    diffInputSkyfile.stack2,	    5, %d,     stack 2
+FIELD    diffInputSkyfile.tess_id,	    8, %s,     tessellation ID
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffProcessedSkyfile.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffProcessedSkyfile.d	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffProcessedSkyfile.d	(revision 24244)
@@ -0,0 +1,34 @@
+TABLE diffRun, diffSkyfile
+TITLE Diff Processed Skyfiles
+FILE  diffProcessedSkyfile.php
+MENU  ipp.stack.dat
+
+WHERE diffRun.diff_id             = diffSkyfile.diff_id
+# WHERE diffRun.state = 'full'
+# WHERE diffSkyfile.fault = 0
+
+ARGS  ARG7 diffSkyfile.diff_id=$diffSkyfile.diff_id
+ARGS  ARG7 diffSkyfile.skycell_id=$diffSkyfile.skycell_id
+ARGS  ARG7 camera=GPC1
+ARGS  ARG7 basename=$diffSkyfile.path_base
+
+#     field                         size  format  name           show     link to         extras
+FIELD diffSkyfile.diff_id,             5, %d,     Diff ID
+FIELD diffSkyfile.skycell_id,          4, %s,     Skycell ID
+FIELD diffRun.label,    	       5, %s,     Label
+# FIELD diffRun.filter,    	       5, %s,     Filter
+FIELD diffRun.tess_id,    	       5, %s,     Tess ID
+FIELD diffRun.state,    	       7, %s,     State,         value,   diffProcessedSkyfile.php, ARG7
+FIELD diffSkyfile.bg,                  5, %.2f,   backgnd
+FIELD diffSkyfile.bg_stdev,            5, %.2f,   stdev    
+FIELD diffSkyfile.dtime_diff,          5, %.2f,   t(diff)
+FIELD diffSkyfile.dtime_match,         5, %.2f,   t(match)
+FIELD diffSkyfile.dtime_phot,          5, %.2f,   t(phot)
+FIELD diffSkyfile.dtime_script,        5, %.2f,   t(script)
+FIELD diffSkyfile.stamps_num,          5, %d,     n(stamps)
+FIELD diffSkyfile.stamps_mean,         5, %.2f,   mean(stamps)
+FIELD diffSkyfile.stamps_rms,          5, %.2f,   rms(stamps)
+FIELD diffSkyfile.sources,             5, %d,     n(sources)
+FIELD diffSkyfile.path_base,           5, %s,     path_base,     none
+
+TAIL PHP insert_log ('LOG.EXP');
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffProcessedSkyfile_Images.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffProcessedSkyfile_Images.d	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffProcessedSkyfile_Images.d	(revision 24244)
@@ -0,0 +1,41 @@
+TABLE diffRun, diffSkyfile
+TITLE Diff Processed Skyfiles
+FILE  diffProcessedSkyfile_Images.php
+MENU  ipp.stack.dat
+
+WHERE diffRun.diff_id             = diffSkyfile.diff_id
+# WHERE diffRun.state = 'full'
+# WHERE diffSkyfile.fault = 0
+
+# define image names to be used below
+# IMAGE VAR basename filerule camera class_id
+IMAGE JPEG2 $diffSkyfile.path_base PPSUB.OUTPUT.JPEG2 GPC1 NONE
+
+# XXX need to get camera from lookup
+ARGS  ARG7 diffSkyfile.diff_id=$diffSkyfile.diff_id
+ARGS  ARG7 diffSkyfile.skycell_id=$diffSkyfile.skycell_id
+ARGS  ARG7 camera=GPC1
+ARGS  ARG7 basename=$diffSkyfile.path_base
+
+#     field                         size  format  name           show     link to         extras
+FIELD diffSkyfile.diff_id,             5, %d,     Diff ID
+FIELD diffSkyfile.skycell_id,          5, %s,     Skycell ID
+FIELD diffRun.label,    	       5, %s,     Label
+FIELD *,    	                       8, %s,     image,         image=JPEG2, diffProcessedSkyfile_Images.php,  ARG7
+# FIELD diffRun.filter,    	       5, %s,     Filter
+FIELD diffRun.tess_id,    	       5, %s,     Tess ID
+FIELD diffRun.state,    	       7, %s,     State,         value,   diffProcessedSkyfile.php, ARG7
+FIELD diffSkyfile.bg,                  5, %.2f,   backgnd
+FIELD diffSkyfile.bg_stdev,            5, %.2f,   stdev    
+FIELD diffSkyfile.dtime_diff,          5, %.2f,   t(diff)
+FIELD diffSkyfile.dtime_match,         5, %.2f,   t(match)
+FIELD diffSkyfile.dtime_phot,          5, %.2f,   t(phot)
+FIELD diffSkyfile.dtime_script,        5, %.2f,   t(script)
+FIELD diffSkyfile.stamps_num,          5, %d,     n(stamps)
+FIELD diffSkyfile.stamps_mean,         5, %.2f,   mean(stamps)
+FIELD diffSkyfile.stamps_rms,          5, %.2f,   rms(stamps)
+FIELD diffSkyfile.sources,             5, %d,     n(sources)
+FIELD diffSkyfile.path_base,           5, %s,     path_base,     none
+
+TAIL PHP insert_image ('PPSUB.OUTPUT.JPEG1');
+TAIL PHP insert_log ('LOG.EXP');
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffRun.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffRun.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffRun.d	(revision 24244)
@@ -5,9 +5,12 @@
 
 #        field        size  format  name           show     link to         extras
-FIELD    diff_id,	20, %s,     diff ID
-FIELD    state,		20, %s,     state
-FIELD    workdir,	20, %s,     workdir
-FIELD    dvodb,		20, %s,     DVO database
-FIELD    registered,	20, %s,     Time registered
-FIELD    skycell_id,	20, %s,     skycell ID
-FIELD    tess_id,	20, %s,     tessellation ID
+FIELD    diff_id,	 5, %d,     diff ID
+FIELD    state,		 7, %s,     state
+FIELD    label,		 7, %s,     label
+FIELD    magicked,	 4, %d,     Magicked
+FIELD    tess_id,	 8, %s,     tess ID
+FIELD    exp_id,	 5, %d,     exp ID
+FIELD    registered,	19, %T,     Time registered
+FIELD    reduction,	10, %s,     reduction
+FIELD    workdir,	10, %s,     workdir
+FIELD    dvodb,		10, %s,     DVO database
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffSummary.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffSummary.d	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/diffSummary.d	(revision 24244)
@@ -0,0 +1,16 @@
+TABLE diffRun
+TITLE Diff Summary
+FILE  diffSummary.php
+MENU  ipp.stack.dat
+
+#     field           size  format  name         show    link to         extras
+FIELD label,    	 7, %s,     label
+FIELD workdir,    	 7, %s,     workdir
+FIELD tess_id,    	 7, %s,     tess_id
+FIELD state,    	 7, %s,     state
+FIELD count(state) as n, 7, %d,     count
+
+MODE  summary
+GROUP state
+GROUP label
+GROUP workdir
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/flatcorrRun.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/flatcorrRun.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/flatcorrRun.d	(revision 24244)
@@ -5,9 +5,10 @@
 
 #        field        size  format  name           show     link to         extras
-FIELD    corr_id,	20, %s,     correction ID
-FIELD    dvodb,		20, %s,     dvodb
-FIELD    filter,	20, %s,     filter
-FIELD    state,	        20, %s,     state
-FIELD    workdir,       20, %s,     workdir
-FIELD    label,	        20, %s,     label
-FIELD    stats,	        20, %s,     stats
+FIELD    corr_id,	 5, %d,     corr ID
+FIELD    label,	         8, %s,     label
+FIELD    filter,	 5, %s,     filter
+FIELD    state,	         5, %s,     state
+FIELD    det_type,       10, %s,    det_type
+FIELD    region,        10, %s,     region
+FIELD    dvodb,		10, %s,     dvodb
+FIELD    workdir,       10, %s,     workdir
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/warpFailedSkyfiles.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/warpFailedSkyfiles.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/warpFailedSkyfiles.d	(revision 24244)
@@ -38,5 +38,5 @@
 FIELD warpSkyfile.tess_id,    	 10, %s,     tess
 FIELD warpSkyfile.good_frac,  	  5, %s,     good_frac
-FIELD warpSkyfile.ignored,     	  5, %s,     ignored
+# FIELD warpSkyfile.ignored,     	  5, %s,     ignored
 FIELD warpSkyfile.fault,      	  5, %s,     fault
 FIELD rawExp.telescope,      	 10, %s,     Telescope
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/warpProcessedSkyfiles.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/warpProcessedSkyfiles.d	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/warpProcessedSkyfiles.d	(revision 24244)
@@ -41,5 +41,5 @@
 FIELD warpSkyfile.tess_id,    	 10, %s,     tess
 FIELD warpSkyfile.good_frac,  	  5, %s,     good_frac
-FIELD warpSkyfile.ignored,     	  5, %s,     ignored
+# FIELD warpSkyfile.ignored,     	  5, %s,     ignored
 FIELD warpSkyfile.fault,      	  5, %s,     fault
 FIELD rawExp.telescope,      	 10, %s,     Telescope
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/warpProcessedSkyfiles_Images.d
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/warpProcessedSkyfiles_Images.d	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/warpProcessedSkyfiles_Images.d	(revision 24244)
@@ -0,0 +1,64 @@
+TABLE warpSkyfile, warpRun, fakeRun, camRun, chipRun, rawExp
+TITLE Warp Processed Skyfiles
+FILE  warpProcessedSkyfiles.php
+MENU  ipp.science.dat
+
+WHERE warpSkyfile.warp_id = warpRun.warp_id
+WHERE warpRun.fake_id = fakeRun.fake_id
+WHERE fakeRun.cam_id = camRun.cam_id
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE warpSkyfile.fault = 0
+
+# define image names to be used below
+# IMAGE VAR basename filerule camera class_id
+IMAGE JPEG2 $camProcessedExp.path_base PPIMAGE.JPEG2 $rawExp.camera NONE
+
+ARGS  ARG1  rawImfile.exp_id=$rawExp.exp_id
+ARGS  ARG2  chipRun.chip_id=$chipRun.chip_id
+ARGS  ARG3  camRun.cam_id=$camRun.cam_id
+ARGS  ARG4  fakeRun.fake_id=$fakeRun.fake_id
+ARGS  ARG5  warpRun.warp_id=$warpRun.warp_id
+ARGS  ARG6  warpSkyfile.skycell_id=$warpSkyfile.skycell_id
+
+ARGS  ARG8  warpRun.warp_id=$warpRun.warp_id
+ARGS  ARG8  warpSkyCellMap.skycell_id=$warpSkyfile.skycell_id
+
+ARGS  ARG7 warpRun.warp_id=$warpRun.warp_id
+ARGS  ARG7 warpSkyfile.skycell_id=$warpSkyfile.skycell_id
+ARGS  ARG7 camera=$rawExp.camera
+ARGS  ARG7 basename=$warpSkyfile.path_base
+
+OP   OP1  $rawExp.ra * 57.295783
+OP   OP2  $rawExp.decl * 57.295783
+
+#     field                    size  format  name         show    link to         extras
+FIELD rawExp.exp_name,     	  5, %s,     Exp Name,     value,  warpStageSkyfileInputs.php, ARG8
+FIELD rawExp.exp_id,     	  5, %d,     Exp ID,       value,  rawImfile.php,           ARG1
+FIELD chipRun.chip_id,    	  7, %d,     Chip ID,      value,  chipProcessedImfile.php, ARG2
+FIELD camRun.cam_id,    	  7, %d,     Cam ID,       value,  camProcessedExp.php,     ARG3
+FIELD fakeRun.fake_id,    	  7, %d,     Fake ID,      value,  fakeProcessedImfile.php, ARG4
+FIELD warpRun.warp_id,    	  7, %d,     Warp ID,      value,  warpStageExp.php,        ARG5
+FIELD warpSkyfile.skycell_id, 	 10, %s,     skycell,      value,  warpProcessedSkyfiles.php, ARG6
+FIELD warpRun.state,    	  7, %s,     state,        value,  warpProcessedSkyfiles.php, ARG7
+FIELD warpRun.label,    	  7, %s,     label,        value
+FIELD warpSkyfile.tess_id,    	 10, %s,     tess
+FIELD warpSkyfile.good_frac,  	  5, %s,     good_frac
+# FIELD warpSkyfile.ignored,     	  5, %s,     ignored
+FIELD warpSkyfile.fault,      	  5, %s,     fault
+FIELD rawExp.telescope,      	 10, %s,     Telescope
+FIELD rawExp.camera,         	 10, %s,     Camera
+FIELD rawExp.dateobs,        	 19, %T,     Date/Time
+FIELD rawExp.ra,       	          8, %10.6f, RA,          op=OP1    
+FIELD rawExp.decl,       	  8, %10.6f, DEC,         op=OP2
+FIELD rawExp.object,         	  8, %s,     Object
+FIELD rawExp.filter,         	 10, %s,     FILTER
+FIELD rawExp.exp_time,       	  5, %.2f,   exp_time    
+FIELD rawExp.airmass,        	  5, %.4f,   airmass     
+FIELD rawExp.bg,             	  5, %.2f,   backgnd
+FIELD rawExp.bg_stdev,       	  5, %.2f,   stdev    
+FIELD rawExp.comment,  	         65, %s,     Comment
+
+FIELD warpSkyfile.path_base,      5, %s,     path_base,   none
+
+TAIL PHP insert_log ('LOG.EXP');
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/raw/ipp.science.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/raw/ipp.science.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/raw/ipp.science.dat	(revision 24244)
@@ -39,6 +39,6 @@
 menulink  | menuselect 	 | link    | Cam-Stage Exp                | camStageExp.php            
 menulink  | menuselect 	 | link    | Cam Pending Exp              | camPendingExp.php              
-menulink  | menuselect 	 | link    | Cam Processed Exp            | camProcessedExp_NoImages.php            
-menulink  | menuselect 	 | link    | Cam Proc. w/ Images          | camProcessedExp.php            
+menulink  | menuselect 	 | link    | Cam Processed Exp            | camProcessedExp.php            
+menulink  | menuselect 	 | link    | Cam Proc. w/ Images          | camProcessedExp_Images.php            
 menulink  | menuselect 	 | link    | Cam IQ Stats                 | camIQstats.php            
 menulink  | menuselect 	 | link    | Cam Failed Exp               | camProcessedExp_failure.php
@@ -60,3 +60,6 @@
 menulink  | menuselect   | link    | Warp Skyfile Inputs          | warpStageSkyfileInputs.php
 menulink  | menuselect   | link    | Warp Processed Skyfiles      | warpProcessedSkyfiles.php
+# menulink  | menuselect   | link    | Warp Processed Skyfiles      | warpProcessedSkyfiles_Images.php
 menulink  | menuselect   | link    | Warp Failed Skyfiles         | warpFailedSkyfiles.php
+
+# warp is not yet generating jpegs...
Index: /branches/cnb_branches/cnb_branch_20090301/ippMonitor/raw/ipp.stack.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippMonitor/raw/ipp.stack.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippMonitor/raw/ipp.stack.dat	(revision 24244)
@@ -34,5 +34,10 @@
 
 menutop   | menutop      | plain   | &nbsp;                       | 
+menulink  | menuselect   | link    | Diff Summary                 | diffSummary.php
 menulink  | menuselect   | link    | Diff Run                     | diffRun.php
 menulink  | menuselect   | link    | Diff Input Skyfile           | diffInputSkyfile.php
-menulink  | menuselect   | link    | Diff Skyfile                 | diffSkyfile.php
+menulink  | menuselect   | link    | Diff Processed Skyfile       | diffProcessedSkyfile.php
+menulink  | menuselect   | link    | Diff Proc. w/ Images         | diffProcessedSkyfile_Images.php
+menulink  | menuselect   | link    | Diff Failed Skyfile          | diffFailedSkyfile.php
+
+# menulink  | menuselect   | link    | Diff Skyfile                 | diffSkyfile.php
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/Build.PL
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/Build.PL	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/Build.PL	(revision 24244)
@@ -56,5 +56,4 @@
         scripts/magic_tree.pl
         scripts/magic_process.pl
-        scripts/magic_mask.pl
         scripts/magic_destreak.pl
         scripts/ippdb.pl
@@ -73,4 +72,5 @@
         scripts/ipp_serial_stack.pl
         scripts/ipp_serial_mops.pl
+        scripts/ipp_serial_magic.pl
         scripts/ipp_mops_translate.pl
         scripts/ipp_simulation_data.pl
@@ -82,4 +82,12 @@
         scripts/dist_component.pl
         scripts/dist_advancerun.pl
+        scripts/dist_make_fileset.pl
+        scripts/dist_queue_runs.pl
+        scripts/receive_source.pl
+        scripts/receive_fileset.pl
+        scripts/receive_file.pl
+        scripts/receive_advance.pl
+        scripts/receive_setstatus.pl
+        scripts/rcserver_checkstatus.pl
     )],
     dist_abstract => 'Scripts for running the Pan-STARRS IPP',
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/camera_exp.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/camera_exp.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/camera_exp.pl	(revision 24244)
@@ -171,6 +171,6 @@
     print $list1File ($ipprc->filename("PPIMAGE.BIN1", $file->{path_base}, $class_id) . "\n");
     print $list2File ($ipprc->filename("PPIMAGE.BIN2", $file->{path_base}, $class_id) . "\n");
-    print $list3File ($ipprc->file_resolve($chipObjects, 0) . "\n");
-    print $list4File ($ipprc->file_resolve($chipMask, 0) . "\n");
+    print $list3File ($chipObjects . "\n");
+    print $list4File ($chipMask . "\n");
 }
 close $list1File;
@@ -263,7 +263,6 @@
             &my_die("Unable to perform psastro: $error_code", $cam_id, $error_code);
         }
-        # XXX do we want to give an error if astrometry fails here?
-        &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects);
-
+
+        my $quality;            # Quality flag
         if ($do_stats) {
             my $fpaStatsReal = $ipprc->file_resolve($fpaStats);
@@ -282,4 +281,10 @@
             }
             chomp $cmdflags;
+
+            ($quality) = $cmdflags =~ /-quality (\d+)/;
+        }
+
+        if (!$quality) {
+            &my_die("Unable to find expected output file: $fpaObjects", $cam_id, $PS_EXIT_PROG_ERROR) unless -f $ipprc->file_resolve($fpaObjects);
         }
 
@@ -375,5 +380,5 @@
             $command .= " -addprocessedexp";
             $command .= " -uri UNKNOWN";
-            $command .= " -code $exit_code";
+            $command .= " -fault $exit_code";
             $command .= " -path_base $outroot";
             $command .= " -path_base $outroot" if defined $outroot;
@@ -381,5 +386,5 @@
         } else {
             $command .= " -updateprocessedexp";
-            $command .= " -code $exit_code";
+            $command .= " -fault $exit_code";
         }
         $command .= " -hostname $host" if defined $host;
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/chip_imfile.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/chip_imfile.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/chip_imfile.pl	(revision 24244)
@@ -52,5 +52,5 @@
     'run-state=s'       => \$run_state, # current state of the run (new, update)
     'magicked'          => \$magicked,  # magicked state of input file
-    'deburned=s'        => \$deburned,  # does deburned image exist? 
+    'deburned=s'        => \$deburned,  # does deburned image exist?
     'threads=s'         => \$threads,   # Number of threads to use for ppImage
     'verbose'           => \$verbose,   # Print to stdout
@@ -61,5 +61,5 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --exp_id --chip_id --class_id --uri --camera --outroot --run-state",
+pod2usage( -msg => "Required options: --exp_id --chip_id --chip_imfile_id --class_id --uri --camera --outroot --run-state",
            -exitval => 3) unless
     defined $exp_id and
@@ -142,6 +142,6 @@
     my $useDeburnedImage = metadataLookupBool($recipeData, 'USE.DEBURNED.IMAGE');
     if ($useDeburnedImage && $deburned) {
-	$uri =~ s/fits$/burn.fits/;
-	&my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri);
+        $uri =~ s/fits$/burn.fits/;
+        &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri);
     }
 
@@ -181,21 +181,8 @@
     ## allow the output images to be optional, depending on the recipe / reduction class
     my $outputImageExpect = metadataLookupBool($recipeData, 'CHIP.FITS');
-    if ($outputImageExpect) {
-        &my_die("Couldn't find expected output file: $outputImage\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
-    }
-
     my $outputMaskExpect = metadataLookupBool($recipeData, 'CHIP.MASK.FITS');
-    if ($outputMaskExpect) {
-        &my_die("Couldn't find expected output file: $outputMask\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
-    }
-
     my $outputWeightExpect = metadataLookupBool($recipeData, 'CHIP.VARIANCE.FITS');
-    if ($outputWeightExpect) {
-        &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
-    }
-
-    &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
-    &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
-
+
+    my $quality;                # Quality flag
     if ($do_stats) {
         my $outputStatsReal = $ipprc->file_resolve($outputStats);
@@ -214,5 +201,15 @@
         }
         chomp $cmdflags;
-    }
+        ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
+    }
+
+    if (!$quality) {
+        &my_die("Couldn't find expected output file: $outputImage\n",  $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputImageExpect or $ipprc->file_exists($outputImage);
+        &my_die("Couldn't find expected output file: $outputMask\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputMaskExpect or $ipprc->file_exists($outputMask);
+        &my_die("Couldn't find expected output file: $outputWeight\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless !$outputWeightExpect or $ipprc->file_exists($outputWeight);
+        &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
+        &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
+    }
+
 }
 
@@ -277,5 +274,5 @@
         $command .= " -chip_id $chip_id";
         $command .= " -class_id $class_id";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_correct_imfile.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_correct_imfile.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_correct_imfile.pl	(revision 24244)
@@ -132,5 +132,5 @@
         $command .= " -class_id $class_id";
         $command .= " -path_base $outroot";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_apply.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_apply.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_apply.pl	(revision 24244)
@@ -206,5 +206,5 @@
         $command .= " -class_id $class_id";
         $command .= " -path_base $outroot";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_calc.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_calc.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_calc.pl	(revision 24244)
@@ -239,5 +239,5 @@
         $command .= " -iteration $iter";
         # XXX EAM : we should add this to the db : $command .= " -path_base $outroot";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_exp.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_exp.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_exp.pl	(revision 24244)
@@ -191,5 +191,5 @@
         $command .= " -iteration $iter";
         $command .= " -path_base $outroot ";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_process_exp.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_process_exp.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_process_exp.pl	(revision 24244)
@@ -205,5 +205,5 @@
         $command .= " -det_id $det_id";
         $command .= " -exp_id $exp_id";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -path_base $outroot";
         $command .= " -dbname $dbname" if defined $dbname;
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_process_imfile.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_process_imfile.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_process_imfile.pl	(revision 24244)
@@ -195,5 +195,5 @@
         $command .= " -class_id $class_id";
         $command .= " -path_base $outroot";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_reject_exp.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_reject_exp.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_reject_exp.pl	(revision 24244)
@@ -344,5 +344,5 @@
         $command .= " -iteration $iter";
         # XXX EAM : we should add this to the db : $command .= " -path_base $outroot";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_exp.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_exp.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_exp.pl	(revision 24244)
@@ -651,5 +651,5 @@
         $command .= " -exp_id $exp_id";
         $command .= " -path_base $outroot";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_imfile.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_imfile.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_imfile.pl	(revision 24244)
@@ -265,5 +265,5 @@
         $command .= " -class_id $class_id";
         $command .= " -path_base $outroot";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_stack.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_stack.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_stack.pl	(revision 24244)
@@ -262,5 +262,5 @@
         $command .= " -class_id $class_id";
         # XXX EAM : we should add this to the db : $command .= " -path_base $outroot";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/diff_skycell.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/diff_skycell.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/diff_skycell.pl	(revision 24244)
@@ -35,5 +35,5 @@
 }
 
-my ($diff_id, $dbname, $threads, $outroot, $reduction, $verbose, $no_update, $no_op, $redirect);
+my ($diff_id, $dbname, $threads, $outroot, $reduction, $inverse, $verbose, $no_update, $no_op, $redirect);
 my ($skycell_id, $diff_skyfile_id);
 GetOptions(
@@ -44,4 +44,5 @@
     'threads=s'         => \$threads,   # Number of threads to use
     'outroot=s'         => \$outroot, # Output root name
+    'inverse'           => \$inverse, # Make inverse subtraction?
     'reduction=s'       => \$reduction, # Reduction class
     'verbose'           => \$verbose,   # Print to stdout
@@ -95,6 +96,5 @@
 my $tess_id;                    # Tesselation identifier
 my $camera;                     # Camera
-my $magicked_0;
-my $magicked_1;
+my ($inputMagic, $templateMagic); # Are the inputs been magicked?
 foreach my $file (@$files) {
     if (defined $file->{template} and $file->{template}) {
@@ -106,5 +106,5 @@
             $templateSources = "PSPHOT.OUT.CMF.MEF";  ## this must be consistent with the value in stack_skycell.pl:161
             # template is a stack so it doesn't need to be magicked
-            $magicked_1 = 1;
+            $templateMagic = 1;
             ## use an explicit stack name for psphot output objects
         } else {
@@ -112,10 +112,10 @@
             $templateVariance = "PSWARP.OUTPUT.VARIANCE";
             $templateSources = "PSWARP.OUTPUT.SOURCES";
-            $magicked_1 = $file->{magicked};
+            $templateMagic = $file->{magicked};
         }
     } else {
         $input = $file->{uri};
         $inputPath = $file->{path_base};
-        $magicked_0 = $file->{magicked};    # if input is a stack the output can't be "magicked"
+        $inputMagic = $file->{magicked};    # if input is a stack the output can't be "magicked"
         if ($file->{warp_id} == 0) {
             $inputMask = "PPSTACK.OUTPUT.MASK";
@@ -145,5 +145,4 @@
         $camera = $file->{camera};
     }
-
 }
 
@@ -158,5 +157,5 @@
 # note that difftool -inputskyfile outputs the magicked boolean as an int not T or F
 # because the output is constructed from a union of two selects
-my $magicked = $magicked_0 && $magicked_1;
+my $magicked = $inputMagic && $templateMagic;
 
 # Recipes to use based on reduction class
@@ -207,10 +206,18 @@
 my $outputMask = $ipprc->filename("PPSUB.OUTPUT.MASK", $outroot);
 my $outputVariance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $outroot);
+my $outputSources = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $outroot);
+my $jpeg1Name = $ipprc->filename("PPSUB.OUTPUT.JPEG1", $outroot);
+my $jpeg2Name = $ipprc->filename("PPSUB.OUTPUT.JPEG2", $outroot);
 my $configuration = $ipprc->filename("PPSUB.CONFIG", $outroot);
-my $outputSources = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $outroot);
-#my $bin1Name =  $ipprc->filename("PPSUB.BIN1", $outroot);
-#my $bin2Name =  $ipprc->filename("PPSUB.BIN2", $outroot);
 my $outputStats = $ipprc->filename("SKYCELL.STATS", $outroot);
 my $traceDest = $ipprc->filename("TRACE.EXP", $outroot);
+
+my ($inverseName, $inverseMask, $inverseVariance, $inverseSources);
+if ($inverse) {
+    $inverseName = $ipprc->filename("PPSUB.INVERSE", $outroot);
+    $inverseMask = $ipprc->filename("PPSUB.INVERSE.MASK", $outroot);
+    $inverseVariance = $ipprc->filename("PPSUB.INVERSE.VARIANCE", $outroot);
+    $inverseSources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $outroot);
+}
 
 my $cmdflags;
@@ -236,4 +243,5 @@
     $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF";
     $command .= " -photometry";
+    $command .= " -inverse" if $inverse;
     $command .= " -tracedest $traceDest -log $logDest";
     $command .= " -dumpconfig $configuration";
@@ -248,10 +256,4 @@
         &my_die("Unable to perform ppSub: $error_code", $diff_id, $skycell_id, $error_code);
     }
-    &my_die("Couldn't find expected output file: $outputName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
-    &my_die("Couldn't find expected output file: $outputMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
-    &my_die("Couldn't find expected output file: $outputVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance);
-#    &my_die("Couldn't find expected output file: $outputSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
-#    &my_die("Couldn't find expected output file: $bin1Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name);
-#    &my_die("Couldn't find expected output file: $bin2Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name);
 
     my $outputStatsReal = $ipprc->file_resolve($outputStats);
@@ -270,4 +272,21 @@
     }
     chomp $cmdflags;
+
+    my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
+
+    if (!$quality) {
+        &my_die("Couldn't find expected output file: $outputName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
+        &my_die("Couldn't find expected output file: $outputMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
+        &my_die("Couldn't find expected output file: $outputVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputVariance);
+        &my_die("Couldn't find expected output file: $outputSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
+        &my_die("Couldn't find expected output file: $jpeg1Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg1Name);
+        &my_die("Couldn't find expected output file: $jpeg2Name",    $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($jpeg2Name);
+        if ($inverse) {
+            &my_die("Couldn't find expected output file: $inverseName", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseName);
+            &my_die("Couldn't find expected output file: $inverseMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseMask);
+            &my_die("Couldn't find expected output file: $inverseVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseVariance);
+            &my_die("Couldn't find expected output file: $inverseSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inverseSources);
+        }
+    }
 }
 
@@ -276,5 +295,5 @@
     # Add the subtraction result
     {
-        my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -uri $outputName -path_base $outroot";
+        my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -path_base $outroot";
         $command .= " $cmdflags";
         $command .= " -magicked" if $magicked;
@@ -304,5 +323,5 @@
     warn($msg);
     if (defined $diff_id and defined $skycell_id and not $no_update) {
-        my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -code $exit_code";
+        my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -fault $exit_code";
         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
         $command .= " -hostname $host" if defined $host;
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_advancerun.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_advancerun.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_advancerun.pl	(revision 24244)
@@ -16,5 +16,5 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
-use File::Basename qw( basename );
+use File::Basename qw( dirname);
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -28,5 +28,5 @@
 
 # Parse the command-line arguments
-my ($dist_id, $stage, $stage_id, $clean, $outroot);
+my ($dist_id, $stage, $stage_id, $outdir, $clean);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
@@ -35,5 +35,6 @@
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
-           'outroot=s'      => \$outroot,    # "directory" for outputs
+           'clean'          => \$clean,      # exporting a clean run
+           'outdir=s'       => \$outdir,     # "directory" for outputs
            'save-temps'     => \$save_temps, # Save temporary files?
            'dbname=s'       => \$dbname,     # Database name
@@ -45,10 +46,10 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --dist_id --stage --stage_id --outroot",
+pod2usage( -msg => "Required options: --dist_id --stage --stage_id --outdir",
            -exitval => 3) unless
     defined $dist_id and
     defined $stage and
     defined $stage_id and
-    defined $outroot;
+    defined $outdir;
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -70,29 +71,47 @@
 my $mdcParser = PS::IPP::Metadata::Config->new;
 
-my $tool;
+my $tool_cmd;
+my $list_mode;
+my $component_key;
 if ($stage eq "raw") {
-    $tool = "regtool";
+    $tool_cmd = "$regtool -exp_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "chip") {
-    $tool = "chiptool";
+    $tool_cmd = "$chiptool -chip_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "camera") {
-    $tool = "camtool";
+    $tool_cmd = "$camtool -cam_id";
+    $list_mode = "-processedexp";
+    $component_key = "";
 } elsif ($stage eq "fake") {
-    $tool = "faketool";
+    $tool_cmd = "$faketool -fake_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "warp") {
-    $tool = "warptool";
+    $tool_cmd = "$warptool -warp_id";
+    $list_mode = "-warped";
+    $component_key = "skycell_id";
 } elsif ($stage eq "stack") {
-    $tool = "stacktool";
+    $tool_cmd = "$stacktool -stack_id";
+    $list_mode = "-sumskyfile";
+    $component_key = "skycell_id";
 } elsif ($stage eq "diff") {
-    $tool = "difftool";
+    $tool_cmd = "$difftool -diff_id";
+    $list_mode = "-diffskyfile";
+    $component_key = "skycell_id";
 } else {
     &my_die("Unexpected stage: $stage", $dist_id, $PS_EXIT_CONFIG_ERROR);
 }
 
-# XXX Should we create a file rule for this
-my $outfile = "$outroot/dbinfo.$stage.$stage_id.mdc";
-
-{
-    my $id_arg = "-$stage" . "_id";
-    my $command = "$tool -exportrun $id_arg $stage_id -outfile $outfile";
+$tool_cmd .= " $stage_id";
+
+# XXX should we create a file rule for this?
+my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc";
+
+{
+    my $command = "$tool_cmd -exportrun -outfile $dbinfo_file";
+    $command .= " -clean" if defined $clean;
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -104,8 +123,7 @@
     }
 }
-
-# set distRun.stage = 'full'
-{
-    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
+my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc";
+{
+    my $command = "$tool_cmd $list_mode";
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -116,4 +134,59 @@
         &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
     }
+    if (@$stdout_buf == 0) {
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+    my $components = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    open MANIFEST, ">$dirinfo" or
+        &my_die("Unable to open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $destdir;
+    foreach my $c (@$components) {
+        # take the workdir from the first component
+        if (!$destdir) {
+            my $workdir = $c->{workdir};
+            if ($workdir) {
+                $destdir = stripvolume($workdir, $stage);
+            } elsif ($stage eq 'raw') {
+                $destdir = 'none';
+            } else {
+                &my_die("workdir not found for open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+            }
+            print MANIFEST "destdir METADATA\n";
+            print MANIFEST "\t" , "destdir", "\tSTR\t", $destdir, "\n";
+            print MANIFEST "END\n\n";
+            print MANIFEST "components METADATA\n";
+        }
+        my $component = $c->{$component_key} ? $c->{$component_key} : "exposure";
+        my $path;
+        if ($stage eq 'raw') {
+            $path = $c->{uri};
+        } else {
+            $path = $c->{path_base};
+        }
+        &my_die("unable to find path",  $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path;
+        my $component_dir = find_componentdir($destdir, $path); 
+#        print MANIFEST "$component METADATA\n";
+        print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n";
+    }
+    print MANIFEST "END\n\n";
+    close MANIFEST or 
+        &my_die("Unable to close dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+
+{
+    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
 }
 
@@ -123,4 +196,66 @@
 ### Pau.
 
+sub stripvolume
+{
+    my $path = shift;
+    my $stage = shift;
+    my @segments;
+
+    # workdir isn't what we want for raw stage
+    return "none" if ($stage and ($stage eq 'raw'));
+
+    my $scheme = file_scheme($path);
+    my $tail;
+    if ($scheme) {
+        # strip off scheme://
+        $tail = substr($path, length($scheme) + 3);
+    } elsif (substr($path, 0, 1) eq '/') {
+        $tail = substr($path, 1);
+        $scheme = "";
+    }
+    # remove any leading / that are left
+    while ((substr($tail, 0, 1) eq '/')) {
+        $tail = substr($tail, 1);
+    }
+
+    if (($scheme eq 'neb') or ($scheme eq 'path')) {
+        my $volume;
+        ($volume, @segments) = split '/', $tail;
+
+    } elsif (!$scheme or ($scheme eq 'file')) {
+
+        # XXX Here we're assuming the /data/ipp??? structure. This won't be true when data is forwarded
+        # by remote sites. We need a way to configure this
+        my $volume;
+
+        # data/ippxxx/dirs
+        (undef, $volume, @segments) = split '/', $tail;
+    } else {
+        die( "unexpected workdir value: $path\n");
+    }
+
+    return caturi(@segments);
+}
+
+sub find_componentdir
+{
+    my $destdir = shift;
+    my $path = shift;
+
+    my $result;
+    if ($destdir eq 'none') {
+        $result = stripvolume($path);
+    } else {
+        # find location of destdir in the path
+        my $i = index($path, $destdir);
+
+        $result = substr($path, $i + length($destdir) + 1);
+
+        while (substr($result, 0, 1) eq '/') {
+            $result = substr($result, 1);
+        }
+    }
+    return dirname($result);
+}
 
 sub my_die
@@ -134,5 +269,5 @@
     my $command = "$disttool -updaterun";
     $command   .= " -dist_id $dist_id";
-    $command   .= " -code $exit_code";
+    $command   .= " -fault $exit_code";
     $command   .= " -dbname $dbname" if defined $dbname;
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_component.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_component.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_component.pl	(revision 24244)
@@ -27,4 +27,23 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
+
+# filerules that are not included in 'cleaned' distribution bundles
+# I could simplify the function below by using a two level hash but
+# that would be less clear I think
+my %chip_cleaned = ( 'PPIMAGE.CHIP' => 'image',
+                     'PPIMAGE.CHIP.MASK' => 'mask',
+                     'PPIMAGE.CHIP.VARIANCE' => 'variance' );
+my %camera_cleaned = ( 'PSASTRO.OUTPUT.MASK' => 'mask' );
+my %fake_cleaned;
+my %warp_cleaned = ( 'PSWARP.OUTPUT' => 'image',
+                     'PSWARP.OUTPUT.MASK' => 'mask',
+                     'PSWARP.OUTPUT.VARIANCE' => 'variance' );
+my %diff_cleaned = ( 'PPSUB.OUTPUT' => 'image',
+                     'PPSUB.OUTPUT.MASK' => 'mask',
+                     'PPSUB.OUTPUT.VARIANCE' => 'variance' );
+my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image',
+                      'PPSTACK.OUTPUT.MASK' => 'mask',
+                      'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
+
 
 # Look for programs we need
@@ -39,5 +58,5 @@
 # Parse the command-line arguments
 my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean);
-my ($outroot, $run_state, $data_state, $magicked);
+my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 
@@ -52,6 +71,8 @@
            'state=s'        => \$run_state,  # state of the run
            'data_state=s'   => \$data_state, # data_state for this component
+           'poor_quality'   => \$poor_quality,  # the processing for this component did not produced images
+           'no_magic'       => \$no_magic,   # magic is not required for this distribution run
            'magicked'       => \$magicked,   # magicked state for this component
-           'outroot=s'      => \$outroot,    # "directory" for outputs
+           'outdir=s'       => \$outdir,     # "directory" for outputs
            'clean'          => \$clean,      # create clean distribution
            'save-temps'     => \$save_temps, # Save temporary files?
@@ -63,5 +84,5 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --dist_id --camera --stage --stage_id --component --path_base --outroot",
+pod2usage( -msg => "Required options: --dist_id --camera --stage --stage_id --component --path_base --outdir",
            -exitval => 3) unless
     defined $dist_id and
@@ -71,9 +92,9 @@
     defined $component and
     defined $path_base and
-    defined $outroot;
+    defined $outdir;
 
 $ipprc->redirect_output($logfile) if $logfile;
 
-if (($stage eq 'raw') and !defined $chip_path_base) {
+if (($stage eq 'raw') and !$clean and !defined $chip_path_base) {
     pod2usage( -msg => "Required options: --chip_path_base for raw stage", -exitval => 3);
 }
@@ -83,31 +104,29 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $basename = basename($path_base);
-
 # making a clean bundle of raw images doesn't make sense
-if (($stage eq "raw") and $clean) {
-    # well I suppose we could ship the log file....
-    &my_die("cannot create clean run at raw stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
-}
 
 # create the output directories if it is not a nebulous path and it doesn't exist
-if (index($outroot, "neb://") != 0) {
-    if (! -e $outroot ) {
-        my $code = system "mkdir -p $outroot";
-        &my_die("cannot create output directory $outroot", $dist_id, $component,
+if (index($outdir, "neb://") != 0) {
+    if (! -e $outdir ) {
+        my $code = system "mkdir -p $outdir";
+        &my_die("cannot create output directory $outdir", $dist_id, $component,
                 $code >> 8) if $code;
     }
 }
 
-my $file_list = get_file_list($clean, $stage, $path_base);
-
-&my_die("failed to compute product list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$file_list;
-
-&my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !scalar @$file_list;
-
+# Get the list of data products for this component
+# note: We my_die in get_file_list if something goes wrong. 
+
+my $file_list = get_file_list($stage, $component, $path_base, $clean);
+
+if (($stage ne 'raw') and ($stage ne 'fake')) {
+    # If the file list is empty it is an error because we should at least get a config dump file
+    # except for fake stage which doesn't do anything yet
+    &my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list);
+}
 
 # set up directory for temporary files
 
-my $tmpdir  = "$outroot/tmpdir.$dist_id.$component.$$";
+my $tmpdir  = "$outdir/tmpdir.$dist_id.$component";
 if (-e $tmpdir) {
     if (-d $tmpdir) {
@@ -121,50 +140,95 @@
                        $PS_EXIT_UNKNOWN_ERROR);
 
-my @base_list;
+#
+# we need to run set masked pixels to NAN in the image and variance images
+# unless
+#   1. we are building a clean distribution bundle
+#   2. magic is not required for this distRun
+#   3. the processing for the component produced no images (warp or diff with bad quality for example)
+my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack')) || $no_magic || $poor_quality);
+
+my ($image, $mask, $variance);
+
+# foreach my $file_rule (keys %$file_list) {
+my $num_files = 0;
 foreach my $file (@$file_list) {
-    my $base = basename($file);
-    my $path = $ipprc->file_resolve($file);
-    # if file is not found, just skip it.
-    # This is not always the right thing to do.
-    # For example if warpSkyfile is ignored, there won't be many of the data products
-    # the new config dump will only list the files actually produced.
-    next if !$path;
-
-    push @base_list, $base;
-    symlink $path, "$tmpdir/$base";
-}
-
-if (!$clean) {
-
-    # Here is where we determine whether or not a GPC1 image can be released.
-
-    if ($camera eq "GPC1" and !$magicked) {
+    # check whether this file rule refers to an image, mask, or variance fits image
+    my $file_rule = $file->{file_rule};
+    my $image_type = get_image_type($stage, $file_rule);
+
+    # if we are building a clean bundle skip this rule
+    next if $clean && $image_type;
+
+    # if magic is required, don't ship jpegs or binned fits images
+    next if !$no_magic && (($file_rule =~ /.BIN1/) or ($file_rule =~ /.BIN2/) or ($file_rule =~ /.JPEG1/)
+            or ($file_rule =~ /.JPEG2/));
+
+    my $file_name = $file->{name};
+    my $base = basename($file_name);
+    my $path = $ipprc->file_resolve($file_name);
+
+    if (!$path) {
+        # skip this file if $poor_quality
+        # this is for compatability with older runs which don't have the files list in the
+        # config dump.
+        # Once we give up on supporting that we can remove the next line. (If the file is in the list
+        # it must exist)
+        next if $poor_quality;
+
+        &my_die("failed to resolve  $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR);
+    }
+
+    # open the file to make sure it exists (and to work around the failed mount phenomena)
+    my $fh = open_with_retries($path);
+    close $fh;
+
+    # we need to pre-process the image before adding to the bundle. Save the path names.
+    # the images will be created below
+    $num_files++;
+    if ($image_type && $nan_masked_pixels) {
+        # save the 
+        if ($image_type eq 'image') {
+            $image = $path
+        } elsif ($image_type eq 'mask') {
+            $mask = $path;
+        } elsif ($image_type eq 'variance') {
+            $variance = $path;
+        } else {
+            &my_die("invalid image type found: $image_type", $dist_id, $component,
+                       $PS_EXIT_PROG_ERROR);
+        }
+    } else {
+        # create a symbolic link from the file in the nebulous repository
+        # in the temporary directory
+        symlink $path, "$tmpdir/$base";
+    }
+}
+
+if ($nan_masked_pixels) {
+    # One last check as to whether magic has been applied to the inputs
+
+    # Note: the sql for disttool -pendingcomponent won't select a component that 
+    # requires magic and hasn't been magicked, but we check again here
+
+    if (!($magicked or $no_magic)) {
         &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR);
     }
 
+    &my_die("no image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image;
+    &my_die("no mask image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask;
+    &my_die("no variance image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
+
+    my $class_id;
     # run streaksrelease to set masked pixels to NAN
-    my ($image, $mask, $variance, $class_id);
     if ($stage eq "raw") {
         $class_id = $component;
-        $image = $path_base;
-        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $class_id);
+        # we can use the chip mask because disttool demands that magic have been run
+        # and so the camera mask and the chip mask are the same
+        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component);
+        my $fh = open_with_retries($mask);
+        close $fh;
     } elsif ($stage eq "chip") {
         $class_id = $component;
-        $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
-        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
-        $variance = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
-    } elsif ($stage eq "warp") {
-        $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
-        $variance = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base);
-    } elsif ($stage eq "diff") {
-        $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
-        $variance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
-    } else {
-        &my_die("not ready for stage: $stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
-    }
-
-    push @base_list, $image;
-    push @base_list, $mask if $mask;
-    push @base_list, $variance if $variance;
+    }
 
     my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir";
@@ -185,13 +249,14 @@
 my $bytes;
 my $md5sum;
-# create the tarfile
-{
+
+if ($num_files) {
+    # create the tarfile
     # XXX TODO: create a file rule for the tar file name
     my $tbase = basename($path_base);
     $tbase .= ".$component" if $component;
     $file_name = "$tbase.tgz";
-    my $tarfile = "$outroot/$file_name";
-
-    my $command = "tar -C $tmpdir -czhf $tarfile .";
+    my $tarfile = "$outdir/$file_name";
+
+    my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -207,9 +272,12 @@
     &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum;
 
-    my @finfo = stat($tarfile);
-    &my_die("unable to stat $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !@finfo;
-    $bytes = $finfo[7];
+    $bytes = -s $tarfile;
 
     delete_tmpdir($tmpdir);
+} else {
+    # no files for this component
+    $file_name = "none";
+    $bytes = 0;
+    $md5sum = "0";
 }
 {
@@ -234,48 +302,144 @@
 ### Pau.
 
+# return the image type (image, mask, or variance) if this file rule refers to 
+# one of the big fits files that are not included in a clean distribution
+sub get_image_type {
+    my $stage = shift;
+    my $rule = shift;
+    my $type;
+
+    if ($stage eq "raw") {
+        if ($rule eq "RAW.IMAGE") {
+            $type = 'image';
+        }
+    } elsif ($stage eq "chip") {
+        $type = $chip_cleaned{$rule};
+    } elsif ($stage eq "camera") {
+        $type = $camera_cleaned{$rule};
+    } elsif ($stage eq "fake") {
+        $type = $fake_cleaned{$rule};
+    } elsif ($stage eq "warp") {
+        $type = $warp_cleaned{$rule};
+    } elsif ($stage eq "diff") {
+        $type = $diff_cleaned{$rule};
+    } elsif ($stage eq "stack") {
+        $type = $stack_cleaned{$rule};
+    } else {
+        &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
+    }
+    return $type;
+}
+
+sub open_with_retries {
+    my $name = shift;
+    
+    my $tries = 1;
+    my $max_tries = 5;
+    my $opened = 0;
+    while (!$opened && ($tries <= $max_tries)) {
+        $opened = open(IN, "<$name");
+        if (!$opened) {
+            print STDERR "WARNING failed to open $name re-try $tries\n";
+            $tries++;
+            sleep 5;
+        }
+    }
+    
+    &my_die("failed to open $name after $max_tries tries\n", $dist_id, $component,
+                    $PS_EXIT_DATA_ERROR) if (!$opened);
+
+    return *IN;
+}
 
 sub get_file_list {
+    my $stage = shift;
+    my $component = shift;
+    my $path_base = shift;
     my $clean = shift;
-    my $stage = shift;
-    my $path_base = shift;
 
     my @file_list;
     if ($stage eq "raw") {
-        push @file_list, $path_base;
-    } else {
-        # TODO: these data will eventually come from the CONFIG dump
-
-        my $clean_mdc = get_legacy_file_mdc();
-
-        my $mdlist = $mdcParser->parse($clean_mdc) or
-                &my_die("failed to parse clean.mdc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
-
-        my $product_lists = parse_md_list($mdlist) or
-                &my_die("failed to parse metadata list", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
-
-
-        my $product_list;
-        foreach my $list (@$product_lists) {
-
-            my $list_name = $list->{STAGE};
-            if (lc ($list_name) eq lc($stage) ) {
-                $product_list = $list;
-            }
-        }
-        &my_die("failed to parse find product list for stage $stage", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$product_list;
-
-        # resolve each file rule and add it to the file_list
-        foreach my $rule (keys %$product_list) {
-            next if $rule eq "STAGE";
-            my $keep_on_clean = $product_list->{$rule};
-            my $fn = $ipprc->filename($rule, $path_base, $component) or
-                &my_die("Missing entry from camera config: $rule", $dist_id, $component,
+        # XXX: TODO for now disttool sets path_base is set to the uri of the file
+        # eventually rawImfile will have a path_base that we'll need to add '.fits'
+        # XXX do we want to distribute the registration log files?
+        if (!$clean) {
+            my %file;
+            $file{file_rule} = "RAW.IMAGE";
+            $file{name} = $path_base;
+            push @file_list, \%file;
+        }
+        return \@file_list;
+    }
+
+    # we get the list of output data products for this run from the config dump file that
+    # is created when the run is done
+    my $config_file_rule;
+    if ($stage eq "chip") {
+        $config_file_rule = "PPIMAGE.CONFIG";
+    } elsif ($stage eq "camera") {
+        $config_file_rule = "PSASTRO.CONFIG";
+    } elsif ($stage eq 'fake') {
+        # XXX: fake is a no op now return an emtpy list
+        return \@file_list;
+    } elsif ($stage eq "warp") {
+        $config_file_rule = "PSWARP.CONFIG";
+    } elsif ($stage eq "diff") {
+        $config_file_rule = "PPSUB.CONFIG";
+    } elsif ($stage eq "stack") {
+        $config_file_rule = "PPSTACK.CONFIG";
+    } else {
+        &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
+    }
+    my $config_file = $ipprc->filename($config_file_rule, $path_base, $component) or
+                &my_die("can't get filename for config dump file: $config_file_rule", $dist_id, $component,
                     $PS_EXIT_CONFIG_ERROR);
-    #        printf "%-16.16s\t%s\t%s\n",  $rule, $clean, $fn;
-            if (!$clean or $keep_on_clean) {
-                push @file_list, $fn;
-            }
-        }
-    }
+
+    my %config_file_hash;
+
+    # add the configuration file to the list
+    $config_file_hash{file_rule} = $config_file_rule;
+    $config_file_hash{name} = $config_file;
+    push @file_list, \%config_file_hash;
+
+    my $resolved = $ipprc->file_resolve($config_file);
+
+    &my_die("failed to resolve name of config dump file: $config_file_rule", $dist_id, $component,
+                    $PS_EXIT_CONFIG_ERROR) if (!$resolved);
+
+    # we don't use the mdc parser because the perl parser is way is too slow for complicated config
+    # files like this
+    my $in = open_with_retries($resolved);
+
+    my $line;
+    while ($line = <$in>) {
+        if ($line =~ /FILES.OUTPUT/) {
+            # print "found FILES.OUTPUT\n";
+            last;
+        }
+    }
+    &my_die("config dump file does not contain FILES.OUTPUT: $config_file", $dist_id, $component,
+                    $PS_EXIT_CONFIG_ERROR) if (!$line);
+
+    while ($line = <$in>) {
+        chomp $line;
+        my ($key, $type, $val) = split " ", $line;
+        # skip blank lines
+        next if !$key;
+        # we're done when we find END
+        last if $key eq "END";
+        # skip multi and other lines
+        next if ($type ne "STR");
+
+        # printf "%-32.32s   %s\n", $key, $val;
+
+        &my_die("no value found for file rule $key in $resolved", $dist_id, $component,
+                $PS_EXIT_CONFIG_ERROR) if (!$val);
+
+        my %file;
+        $file{file_rule} = $key;
+        $file{name} = $val;
+        push @file_list, \%file;
+    }
+    close $in;
 
     return \@file_list;
@@ -304,5 +468,5 @@
     $command   .= " -dist_id $dist_id";
     $command   .= " -component $component";
-    $command   .= " -code $exit_code";
+    $command   .= " -fault $exit_code";
     $command   .= " -dbname $dbname" if defined $dbname;
 
@@ -326,118 +490,8 @@
 sub delete_tmpdir
 {
-    if (!$save_temps) {
+    if (!$save_temps and $tmpdir and -e $tmpdir) {
         system "rm -r $tmpdir";
     }
 }
 
-# list of output data products for runs that were made before the configuration re-work 
-sub get_legacy_file_mdc
-{
-    my $list =
-"
-#
-# interesting things we might want to save in this file
-#   1. whether file is to be distributed in 'clean' mode
-#   2. whether file is to be cleaned (this should be the same thing)
-#   3. type of file 'no, not our business, precious'
-
-# this data should probably be computed by the relevant program and saved as part
-# of the configuration dump that is output.
-
-
-PROD_LIST MULTI
-
-PROD_LIST   METADATA
-        STAGE                    STR     RAW
-    # there is isn't really a file rule for raw images
-	DUMMY           	BOOL	F
-END
-
-# list of data products for a gpc1 chipProcessedImfile  (made by ppImage)
-PROD_LIST   METADATA
-        STAGE                    STR     CHIP
-	PPIMAGE.CONFIG  	BOOL	T
-#	PPIMAGE.CHIP    	BOOL	F
-#	PPIMAGE.CHIP.MASK	BOOL	F
-#	PPIMAGE.CHIP.VARIANCE	BOOL	F
-	PPIMAGE.BIN1    	BOOL	T
-	PPIMAGE.BIN2    	BOOL	T
-	PSPHOT.OUT.CMF.SPL	BOOL	T
-	PSPHOT.BACKMDL  	BOOL	T
-	PPIMAGE.STATS   	BOOL	T
-	LOG.IMFILE      	BOOL	T
-	TRACE.IMFILE    	BOOL	T
-    # where do we put exposure level data such as LOG.EXP ?
-END
-# exposure level output products from camera processing made by psastro and ppImage (jpegs)
-PROD_LIST   METADATA
-        STAGE                    STR     CAM
-	PSASTRO.CONFIG  	BOOL	T
-	PSASTRO.OUTPUT  	BOOL	T
-	PSASTRO.STATS   	BOOL	T
-	PPIMAGE.JPEG1   	BOOL	T
-	PPIMAGE.JPEG2   	BOOL	T
-	LOG.EXP         	BOOL	T
-	TRACE.EXP       	BOOL	T
-END
-# chip lelel output products from camera processing made by psastro
-PROD_LIST   METADATA
-        STAGE                    STR CAM_CHIP
-#	PSASTRO.OUTPUT.MASK	BOOL	F
-END
-PROD_LIST   METADATA
-        STAGE                    STR     FAKE
-#	PPSIM.OUTPUT    	BOOL	F
-END
-# list of data products for a gpc1 warpSkyfile (pswarp)
-PROD_LIST   METADATA
-        STAGE                    STR     WARP
-	PSWARP.CONFIG   	BOOL	T
-#	PSWARP.OUTPUT   	BOOL	F
-#	PSWARP.OUTPUT.MASK	BOOL	F
-#	PSWARP.OUTPUT.VARIANCE	BOOL	F
-	PSWARP.OUTPUT.SOURCES	BOOL	T
-	PSPHOT.BACKMDL.MEF  	BOOL	T
-	PSPHOT.PSF.SKY.SAVE	BOOL	T
-	SKYCELL.STATS   	BOOL	T
-	SKYCELL.TEMPLATE	BOOL	T
-	LOG.EXP         	BOOL	T
-	TRACE.EXP       	BOOL	T
-END
-# outputs from diffRun (ppSub)
-PROD_LIST   METADATA
-        STAGE                    STR     DIFF
-	PPSUB.CONFIG    	BOOL	T
-#	PPSUB.OUTPUT    	BOOL	F
-#	PPSUB.OUTPUT.MASK	BOOL	F
-#	PPSUB.OUTPUT.VARIANCE	BOOL	F
-	PPSUB.OUTPUT.KERNELS	BOOL	T
-	PPSUB.OUTPUT.JPEG1	BOOL	T
-	PPSUB.OUTPUT.JPEG2	BOOL	T
-	PSPHOT.OUT.CMF.MEF	BOOL	T
-	PSPHOT.BACKMDL.MEF	BOOL	T
-	SKYCELL.STATS   	BOOL	T
-	LOG.EXP         	BOOL	T
-	TRACE.EXP       	BOOL	T
-END
-PROD_LIST   METADATA
-        STAGE                    STR     STACK
-	PPSTACK.CONFIG  	BOOL	T
-#	PPSTACK.OUTPUT  	BOOL	F
-#	PPSTACK.OUTPUT.MASK	BOOL	F
-#	PPSTACK.OUTPUT.VARIANCE	BOOL	F
-	PPSTACK.TARGET.PSF	BOOL	T
-	PSPHOT.OUT.CMF.MEF	BOOL	T
-	PSPHOT.BACKMDL.MEF	BOOL	T
-	SKYCELL.STATS   	BOOL	T
-	PPSTACK.OUTPUT.JPEG1	BOOL	T
-	PPSTACK.OUTPUT.JPEG2	BOOL	T
-	LOG.EXP         	BOOL	T
-	TRACE.EXP       	BOOL	T
-END
-";
-    return $list;
-}
-
-
 __END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_make_fileset.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_make_fileset.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_make_fileset.pl	(revision 24244)
@@ -0,0 +1,274 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
+use Digest::MD5::File qw( file_md5_hex );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+# Look for programs we need
+my $missing_tools;
+my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
+my $dsreg   = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $prod_id, $product_name, $ds_dbhost, $ds_dbname);
+my ($dbname, $save_temps, $verbose, $no_update, $logfile);
+
+GetOptions(
+           'dist_id=s'      => \$dist_id,    # distribution run identifier
+           'dist_dir=s'     => \$dist_dir,   # directory containing dist run outputs
+           'target_id=s'    => \$target_id,  # 
+           'stage=s'        => \$stage,      # raw, chip, camera, fake, warp, stack, or diff
+           'stage_id=s'     => \$stage_id,   # exp_id, chip_id, etc.
+           'prod_id=s'      => \$prod_id,    # id for the product
+           'product_name=s' => \$product_name,  # location of the data store directory for this product
+           'ds_dbhost=s'    => \$ds_dbhost,  # database host for the datastore database
+           'ds_dbname=s'    => \$ds_dbname,  # database name for the datastore database
+           'save-temps'     => \$save_temps, # Save temporary files?
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --prod_id --ds_dbhost --ds_dbname",
+           -exitval => 3) unless
+    defined $dist_id and
+    defined $dist_dir and
+    defined $target_id and
+    defined $stage and
+    defined $stage_id and
+    defined $prod_id and
+    defined $product_name and
+    defined $ds_dbhost and
+    defined $ds_dbname;
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+my $fs_tag = get_fileset_tag($ipprc, $stage, $stage_id, $dbname);
+
+&my_die("failed to lookup fileset tag", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR) if !$fs_tag;
+
+my $fileset_name = "$fs_tag.$stage.$stage_id.$dist_id.$prod_id";
+
+print "$fileset_name\n";
+
+
+# make sure that the database info file for this run exists
+my $dbinfo_file = "dbinfo.$stage.$stage_id.mdc";
+if (! -e "$dist_dir/$dbinfo_file" ) {
+    &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+print "dbinfo file $dbinfo_file exists\n" if $verbose;
+
+# make sure that the dirinfo file for this run exists
+my $dirinfo_file = "dirinfo.$stage.$stage_id.mdc";
+if (! -e "$dist_dir/$dirinfo_file" ) {
+    &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+print "dirinfo file $dirinfo_file exists\n" if $verbose;
+
+# open the dsreg file list
+my ($listFile, $listFileName) = tempfile("/tmp/$stage.$stage_id.list.XXXX", UNLINK => !$save_temps );
+
+# add the dbinfo file to the list
+# XXX: change type from text to dbinfo and dirinfo once we define the types
+print $listFile "$dbinfo_file|||text|dbinfo|\n";
+print $listFile "$dirinfo_file|||text|dirinfo|\n";
+
+my $components;
+{
+    my $command = "$disttool -processedcomponent -dist_id $dist_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code);
+    }
+
+    my $metadata = $mdcParser->parse (join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $dist_id, $prod_id, $PS_EXIT_PROG_ERROR);
+
+    $components = parse_md_list($metadata);
+
+    my $num_components = scalar @$components;
+
+    print "distRun $dist_id has $num_components components\n";
+}
+
+foreach my $component (@$components) {
+    my $size = $component->{bytes};
+    # skip zero size components. They are placeholders to complete processing
+    next if $size == 0;
+    my $md5sum = $component->{md5sum};
+    # name of the file
+    my $name = $component->{name};
+    # component id (class_is or skycell_id)
+    my $comp_name = $component->{component};
+
+    # XXX: if tarfile is not always the right type we need to add a type to distComponent
+    print $listFile "$name|$size|$md5sum|tgz|$comp_name|\n";
+}
+
+close $listFile;
+
+{
+    # XXX: need to chose an appropriate file set type
+    my $command = "$dsreg --add $fileset_name --product $product_name --type notset --list $listFileName";
+
+    # the data store will refer to the distribution bundle via symlinks back to distRun.outdir
+    $command .= " --datapath $dist_dir --link";
+
+    # set the product specific columns in product list
+    $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $fs_tag";
+
+    $command .= " --dbname $ds_dbname";    # XXX: notyet --dbhost $ds_dbhost
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code);
+    }
+}
+
+{
+    my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -name $fileset_name";
+    $command .= " -dbname $dbname" if $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        # XXX: if we get here we the fileset has been created but the database update failed
+        # We need to have revertfileset check whether the fileset exists and do dsreg -del if it does
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code);
+    }
+}
+
+
+
+exit 0;
+
+sub getDBHandle {
+    my $ipprc = shift;
+    my $dbname = shift;
+    if (!$dbname) {
+        $dbname = metadataLookupStr($ipprc->{_siteConfig}, "DBNAME");
+    }
+    my $dbserver = metadataLookupStr($ipprc->{_siteConfig}, "DBSERVER");
+    my $dbuser = metadataLookupStr($ipprc->{_siteConfig}, "DBUSER");
+    my $dbpassword = metadataLookupStr($ipprc->{_siteConfig}, "DBPASSWORD");
+
+    die "database configuration not set up" unless defined($dbserver) and defined($dbuser)
+        and defined($dbpassword) and defined($dbname);
+
+    my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
+
+    my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) 
+        or die "Cannot connect to database.\n";
+
+    return $dbh;
+}
+
+
+sub get_fileset_tag {
+    my $ipprc = shift;
+    my $stage = shift;
+    my $stage_id = shift;
+    my $dbname = shift;
+
+    if ($stage eq 'stack') {
+        return "stack.$stage_id";
+    } elsif ($stage eq 'diff') {
+        return "diff.$stage_id";
+    }
+
+    #
+    # we are a long ways away from the rawExp in the pipeline. Rather than do some 
+    # very long joins in disttool, we look up the exp_name in the database using DBI
+    #
+    my $dbh = getDBHandle($ipprc, $dbname);
+
+    my $query; 
+
+    if ($stage eq 'raw') {
+        $query = "SELECT exp_name FROM rawExp WHERE exp_id = $stage_id";
+    } elsif ($stage eq 'chip') {
+        $query = "SELECT exp_name FROM chipRun JOIN rawExp USING(exp_id) WHERE chip_id = $stage_id";
+    } elsif ($stage eq 'camera') {
+        $query = "SELECT exp_name FROM camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"
+                    . " WHERE cam_id = $stage_id";
+    } elsif ($stage eq 'fake') {
+        $query = "SELECT exp_name FROM fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id)"
+                    . " JOIN rawExp USING(exp_id) WHERE fake_id = $stage_id";
+    } elsif ($stage eq 'warp') {
+        $query = "SELECT exp_name FROM warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id)"
+                    . " JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE warp_id = $stage_id";
+    } else {
+        &my_die("$stage is invalid value for stage\n");
+    }
+
+    my $stmt = $dbh->prepare($query);
+    $stmt->execute();
+    my $ref = $stmt->fetchrow_hashref();
+
+    my $tag = $ref->{exp_name};
+
+    return $tag;
+}
+
+sub my_die {
+    my $msg = shift;
+    my $dist_id = shift;
+    my $prod_id = shift;
+    my $fault = shift;
+
+    # TODO: disttool -adddsfileset -prod_id $prod_id -dist_id $dist_id -fault $fault
+    print STDERR "$msg\n";
+
+    my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -fault $fault";
+    $command .= " -dbname $dbname" if $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        print STDERR "Unable to perform $command error_code: $error_code\n";
+    }
+    exit $fault;
+}
+
+__END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_queue_runs.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_queue_runs.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_queue_runs.pl	(revision 24244)
@@ -0,0 +1,129 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
+use Digest::MD5::File qw( file_md5_hex );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+# Look for programs we need
+my $missing_tools;
+my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($stage, $stage_limit, $dist_root, $no_magic);
+my ($dbname, $save_temps, $verbose, $no_update, $logfile);
+
+GetOptions(
+           'dist_root=s'    => \$dist_root,  # root of distribution work area
+           'stage=s'        => \$stage,      # stage to queue
+           'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
+           'no_magic'       => \$no_magic,   # queue runs without requiring magic
+           'dbname=s'       => \$dbname,     # Database name
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database
+           'save-temps'     => \$save_temps, # Save temporary files?
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+if (0) {
+pod2usage( -msg => "Required options: --stage",
+           -exitval => 3) unless
+    defined $stage;
+}
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+if (!$dist_root) {
+    $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT");
+    &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root;
+}
+
+my ($day, $month, $year) = (localtime)[3,4,5];
+my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
+
+my $workdir = "$dist_root/$datestr";
+
+print "workdir is $workdir\n";
+
+# if stage is not supplied as an argument, loop over all stages
+my @stages;
+if ($stage) {
+    push @stages, $stage;
+} else {
+    @stages = qw( raw chip camera fake warp diff stack);
+}
+
+# XXX: how shall we deal with labels?
+my $label = 'proc';
+
+foreach my $stage (@stages) {
+    my $command = "$disttool -definebyquery -stage $stage -workdir $workdir";
+    $command .= " -no_magic" if $no_magic;
+    $command .= " -dry_run" if $no_update;
+    $command .= " -limit $stage_limit" if $stage_limit;
+    $command .= " -set_label $label";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $error_code);
+    }
+    # display the output from the command
+    print STDERR join "", @$stdout_buf if $verbose;
+}
+
+# queue rcRuns for any distRuns that have completed and have interested destinations
+{
+    my $command = "$disttool -queuercrun";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $error_code);
+    }
+    # display the output from the command
+    print STDERR join "", @$stdout_buf if $verbose;
+}
+
+exit 0;
+
+sub my_die {
+    my $msg = shift;
+    my $fault = shift;
+
+    print STDERR "$msg\n";
+
+    exit $fault;
+}
+
+__END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/fake_imfile.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/fake_imfile.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/fake_imfile.pl	(revision 24244)
@@ -216,5 +216,5 @@
         $command .= " -path_base $outroot";
         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/flatcorr_proc.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/flatcorr_proc.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/flatcorr_proc.pl	(revision 24244)
@@ -320,5 +320,5 @@
         my $command = "$flatcorr -addprocess";
 	$command .= " -corr_id $corr_id";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -hostname $host" if defined $host;
         $command .= " -dbname $dbname" if defined $dbname;
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_cleanup.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_cleanup.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_cleanup.pl	(revision 24244)
@@ -144,5 +144,5 @@
                 addFilename (\@files, "PPIMAGE.BIN2", $path_base, $class_id);
                 addFilename (\@files, "PPIMAGE.JPEG1", $path_base, $class_id);
-                addFilename (\@files, "PPIMAGE.JPEG", $path_base, $class_id);
+                addFilename (\@files, "PPIMAGE.JPEG2", $path_base, $class_id);
                 addFilename (\@files, "PPIMAGE.STATS", $path_base, $class_id);
                 addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_inject_fileset.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_inject_fileset.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_inject_fileset.pl	(revision 24244)
@@ -15,5 +15,5 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Spec;
-use PS::IPP::Config;
+use PS::IPP::Config qw( :standard );
 
 my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions
@@ -84,5 +84,6 @@
 foreach my $file ( @ARGV ) {
     # check for file existence
-    if (! -e $file) { die "file $file not found\n"; }
+    my $resolved = $ipprc->file_resolve($file);
+    if (!$resolved or ! -e $resolved) { die "file $file not found\n"; }
     if (! $exp_name) { 
 	# strip off the extension
@@ -128,6 +129,12 @@
     my $file = $ARGV[$i];
 
-    my $absfile = File::Spec->rel2abs( $file );
-    my $relfile = $ipprc->convert_filename_relative( $absfile );
+    my $relfile;
+    my $scheme = file_scheme($file);
+    if (!$scheme or $scheme ne 'neb') {
+        my $absfile = File::Spec->rel2abs( $file );
+        $relfile = $ipprc->convert_filename_relative( $absfile );
+    } else {
+        $relfile = $file;
+    }
 
     # the class_id used here is temporary : register replaces it with the true class_id
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_mops_translate.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_mops_translate.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_mops_translate.pl	(revision 24244)
@@ -15,7 +15,5 @@
 
 use constant EXTNAME => 'MOPS_TRANSIENT_DETECTIONS'; # Extension name for output table
-use constant POSITION_ERROR => 0.2 / 3600; # Error in positions, degrees
 use constant ZERO_POINT => 25;  # Magnitude zero point
-use constant MAG_ERROR => 0.1;  # Error in magnitudes
 use constant OBSERVATORY_CODE => 566; # IAU Observatory Code
 use constant FAKE_LIMITING_MAG => 23.0; # Fake limiting magnitude to report
@@ -25,5 +23,4 @@
      $extname,                  # Name of extension containing photometry
      $skycell,                  # Skycell file with WCS
-     $diff_image_id,            # difference image id
      $output,                   # Name of output file
      $save_temps,               # Save temporary files?
@@ -33,17 +30,13 @@
            'input=s'    => \$input,
            'extname=s'  => \$extname,
-           'skycell=s'  => \$skycell,
            'output=s'   => \$output,
-           'diff_image_id=s' => \$diff_image_id,
            'save-temps' => \$save_temps,
 ) or pod2usage( 2 );
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --input --extname --skycell --diff_image_id --output",
+pod2usage( -msg => "Required options: --input --extname --output",
            -exitval => 3)
     unless defined $input
     and defined $extname
-    and defined $skycell
-    and defined $diff_image_id,
     and defined $output;
 
@@ -64,7 +57,7 @@
 # Header translation table
 my $headers = {
+    # IPP name     => MOPS name, type, comment
     'FPA.RA'       => { name => 'RA',       type => TSTRING, comment => 'Right ascension of boresight'},
     'FPA.DEC'      => { name => 'DEC',      type => TSTRING, comment => 'Declination of boresight' },
-    'FPA.OBS'      => { name => 'FPA_ID',   type => TSTRING, comment => 'Exposure identifier' },
     'FPA.FILTER'   => { name => 'FILTER',   type => TSTRING, comment => 'Filter name' },
     'EXPTIME'      => { name => 'EXPTIME',  type => TDOUBLE, comment => 'Exposure time' },
@@ -72,87 +65,50 @@
     'FPA.ALT'      => { name => 'TEL_ALT',  type => TDOUBLE, comment => 'Telescope altitude' },
     'FPA.AZ'       => { name => 'TEL_AZ',   type => TDOUBLE, comment => 'Telescope azimuth' },
-            };
-
-
-# Read the skycell header for the WCS
-my $skycellResolved = $ipprc->file_resolve( $skycell );
-my $status = 0;
-my $skyfile_fits = Astro::FITS::CFITSIO::open_file( $skycellResolved, READONLY, $status ); 
-    die("failed to open skycell file: $skycellResolved: $status") if $status;
-
-my $wcsheader;
-($wcsheader, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfile_fits );
-die("Unable to read skycell header: $status") if $status;
-
-# Get the useful header keywords
-my $naxis1 = $$wcsheader{'NAXIS1'};
-my $naxis2;
-if ($naxis1) {
-    $naxis2 = $$wcsheader{'NAXIS2'} or die("Can't find NAXIS2");
-} else {
-    # if the skyfile is compressed then the WCS won't be in the primary header
-    my $hdutype;
-    $skyfile_fits->movrel_hdu(1, $hdutype, $status);
-    die("Unable to movrel_hdu: $status") if $status;
-
-    ($wcsheader, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfile_fits );
-    die("Unable to read extension header: $status") if $status;
-    my $xtension = $$wcsheader{'XTENSION'} or die("Can't find XTENSION");
-    die("XTENSION found: $xtension") if $xtension ne "'BINTABLE'";
-    $naxis1 = $$wcsheader{'ZNAXIS1'} or die("Can't find ZNAXIS1");
-    $naxis2 = $$wcsheader{'ZNAXIS2'} or die("Can't find ZNAXIS2");
-}
-
-my $ctype1 = $$wcsheader{'CTYPE1'} or die("Can't find CTYPE1");
-my $ctype2 = $$wcsheader{'CTYPE2'} or die("Can't find CTYPE2");
-my $cdelt1 = $$wcsheader{'CDELT1'} or die("Can't find CDELT1");
-my $cdelt2 = $$wcsheader{'CDELT2'} or die("Can't find CDELT2");
-my $crval1 = deg2rad($$wcsheader{'CRVAL1'}) or die("Can't find CRVAL1");
-my $crval2 = deg2rad($$wcsheader{'CRVAL2'}) or die("Can't find CRVAL2");
-my $crpix1 = $$wcsheader{'CRPIX1'} or die("Can't find CRPIX1");
-my $crpix2 = $$wcsheader{'CRPIX2'} or die("Can't find CRPIX2");
-my $pc11 = $$wcsheader{'PC001001'} or die("Can't find PC001001");
-my $pc12 = $$wcsheader{'PC001002'} or die("Can't find PC001002");
-my $pc21 = $$wcsheader{'PC002001'} or die("Can't find PC002001");
-my $pc22 = $$wcsheader{'PC002002'} or die("Can't find PC002002");
-my $crota1 = $$wcsheader{'CROTA1'};
-my $crota2 = $$wcsheader{'CROTA2'};
-
-die("Unexpected projection: $ctype1 and $ctype2.") unless $ctype1 =~ /^\'RA---TAN\s*\'$/ and
-    $ctype2 =~ /^\'DEC--TAN\s*\'$/;
-die("Can't determine size of skycell ($naxis1,$naxis2)") unless $naxis1 > 0 and $naxis2 > 0;
-die("Can't determine scale of skycell ($cdelt1,$cdelt2)") if not defined $cdelt1 or $cdelt1 == 0 or
-    not defined $cdelt2 or $cdelt2 == 0;
-die("We don't know how to handle rotations ($crota1,$crota2)") if defined $crota1 or defined $crota2;
-
-
-# Read the input table
-my $inputResolved = $ipprc->file_resolve($input);
+    'IMAGEID'      => { name => 'DIFFIMID', type => TINT,    comment => 'Difference image identifier'},
+    'FPA.OBS'      => { name => 'FPA_ID',   type => TSTRING, comment => 'Exposure identifier' },
+};
+
+
+# Read header
+my $inputResolved = $ipprc->file_resolve($input); # Resolved filename
+my $status = 0;                 # CFITSIO status
 my $inFits = Astro::FITS::CFITSIO::open_file( $inputResolved, READONLY, $status ); # FITS file handle
+die("failed to open input file: $inputResolved: $status") if $status;
 my $inHeader = $inFits->read_header(); # Header for input
 check_fitsio($status);
+
+# Read table data
 $inFits->movnam_hdu(BINARY_TBL, $extname, 0, $status) and check_fitsio($status);
 my $numRows;                    # Number of rows in table
 $inFits->get_num_rows($numRows, $status) and check_fitsio($status);
 
-my ($xCol, $yCol, $magCol, $ensCol);  # Column numbers for x and y
-$inFits->get_colnum(0, 'X_PSF', $xCol, $status) and check_fitsio($status);
-$inFits->get_colnum(0, 'Y_PSF', $yCol, $status) and check_fitsio($status);
-$inFits->get_colnum(0, 'PSF_INST_MAG', $magCol, $status) and check_fitsio($status);
-$inFits->get_colnum(0, 'EXT_NSIGMA', $ensCol, $status) and check_fitsio($status);
-
-my ($xType, $yType, $magType, $ensType);  # Column types
-$inFits->get_coltype($xCol, $xType, undef, undef, $status) and check_fitsio($status);
-$inFits->get_coltype($yCol, $yType, undef, undef, $status) and check_fitsio($status);
-$inFits->get_coltype($magCol, $magType, undef, undef, $status) and check_fitsio($status);
-$inFits->get_coltype($ensCol, $ensType, undef, undef, $status) and check_fitsio($status);
-
-my ($x, $y, $mag, $ens);              # Sources arrays
-$inFits->read_col($xType, $xCol, 1, 1, $numRows, 0, $x, undef, $status) and check_fitsio($status);
-$inFits->read_col($yType, $yCol, 1, 1, $numRows, 0, $y, undef, $status) and check_fitsio($status);
-$inFits->read_col($magType, $magCol, 1, 1, $numRows, 0, $mag, undef, $status) and check_fitsio($status);
-$inFits->read_col($ensType, $ensCol, 1, 1, $numRows, 0, $ens, undef, $status) and check_fitsio($status);
+my $ra = column($inFits, 'RA_PSF', $numRows); # Right Ascension, degrees
+my $dec = column($inFits, 'DEC_PSF', $numRows); # Declination, degrees
+my $mag = column($inFits, 'PSF_INST_MAG', $numRows); # Magnitude
+my $magErr = column($inFits, 'PSF_INST_MAG_SIG', $numRows); # Magnitude error
+my $ens = column($inFits, 'EXT_NSIGMA', $numRows); # Significance of extension
+my $xErr = column($inFits, 'X_PSF_SIG', $numRows); # Error in x position, pixels
+my $yErr = column($inFits, 'Y_PSF_SIG', $numRows); # Error in y position, pixels
+my $scale = column($inFits, 'PLTSCALE', $numRows); # Plate scale, arcsec/pixel
+my $angle = column($inFits, 'POSANGLE', $numRows); # Position angle, degrees
 
 $inFits->close_file( $status );
+
+my ($raErr, $decErr);           # Error in ra, dec
+for (my $i = 0; $i < $numRows; $i++) {
+    my $cosPA = cos($$angle[$i]);
+    my $sinPA = sin($$angle[$i]);
+
+    # XXX Not sure about the transformation here --- check
+    $$raErr[$i] = $$scale[$i] * ($cosPA * $xErr + $sinPA * $yErr) / 3600;
+    $$decErr[$i] = $$scale[$i] * ($sinPA * $xErr - $cosPA * $yErr) / 3600;
+}
+
+# Plate scales
+#my $cdelt1 = $$inHeader{'CDELT1'} or die("Can't find CDELT1");
+#my $cdelt2 = $$inHeader{'CDELT2'} or die("Can't find CDELT2");
+### XXX WCS wasn't being set in inverse diffs, but it's available elsewhere
+my $cdelt1 = $$scale[0] / 3600;
+my $cdelt2 = $$scale[1] / 3600;
 
 # Parse the list of columns
@@ -169,12 +125,10 @@
 # Convert the input data into the output formats
 for (my $i = 0; $i < $numRows; $i++) {
-    my ($ra, $dec) = pixels_to_sky($$x[$i], $$y[$i]); # Sky coordinates
-
     push @{$colData{'RA_DEG'}}, $ra;
     push @{$colData{'DEC_DEG'}}, $dec;
-    push @{$colData{'RA_SIG'}}, POSITION_ERROR;
-    push @{$colData{'DEC_SIG'}}, POSITION_ERROR;
+    push @{$colData{'RA_SIG'}}, $raErr;
+    push @{$colData{'DEC_SIG'}}, $decErr;
     push @{$colData{'FLUX'}}, $$mag[$i] + ZERO_POINT;
-    push @{$colData{'FLUX_SIG'}}, MAG_ERROR;
+    push @{$colData{'FLUX_SIG'}}, $magErr;
     push @{$colData{'ANG'}}, 0.0;
     push @{$colData{'ANG_SIG'}}, 0.0;
@@ -211,9 +165,4 @@
 }
 
-# set DIFFIMID
-# XXX TODO: we could put the diff_image_id in the .cmf file and get it from there
-$outFits->write_key( TLONGLONG, 'DIFFIMID', $diff_image_id, 'Difference image identifier', $status);
-check_fitsio( $status );
-
 # Adjust the time from start to mid-point
 {
@@ -260,4 +209,24 @@
 
 
+# Read a column specified by name
+sub column
+{
+    my $fits = shift;           # FITS file
+    my $name = shift;           # Name of column
+    my $rows = shift;           # Number of rows
+    my $status = 0;             # Status of CFITSIO
+
+    my $num;                    # Column number
+    $fits->get_colnum(0, $name, $num, $status) and check_fitsio($status);
+    my $type;                   # Type of column
+    $inFits->get_coltype($num, $type, undef, undef, $status) and check_fitsio($status);
+    my $data;                   # Array with data
+    $inFits->read_col($type, $num, 1, 1, $rows, 0, $data, undef, $status) and check_fitsio($status);
+
+    return $data;
+}
+
+
+
 
 # From Astro::FITS::CFITSIO demo
@@ -273,40 +242,4 @@
 }
 
-# Convert pixel coordinates to sky coordinates
-sub pixels_to_sky
-{
-    my ($x, $y) = @_;           # Coordinates
-
-    # Pixel coordinate relative to reference
-    $x -= $crpix1;
-    $y -= $crpix2;
-
-    # Coordinates on tangent plane
-    my $xi = $pc11 * ($x) + $pc12 * ($y);
-    my $eta = $pc21 * ($x) + $pc22 * ($y);
-    $xi *= $cdelt1;
-    $eta *= $cdelt2;
-
-    # Coordinates on rotated celestial sphere
-    my $phi = atan2($eta,$xi) + pi/2;
-    my $theta = atan(180 / pi / sqrt($xi**2 + $eta**2));
-
-    # Coordinates on celestial sphere
-    my $ra = $crval1 + atan2(cos($theta) * sin($phi),
-                             sin($theta) * cos($crval2) + cos($theta) * sin($crval2) * cos($phi));
-    my $dec = asin(sin($theta) * sin($crval2) - cos($theta) * cos($crval2) * cos($phi));
-
-    $ra = rad2deg($ra);
-    $dec = rad2deg($dec);
-
-    $ra += 360 if $ra < 0;
-    $dec += 360 if $dec < 0;
-    $ra -= 360 if $ra >= 360;
-    $dec -= 360 if $dec >= 360;
-
-    return ($ra, $dec);
-}
-
-
 __END__
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_serial_chip.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_serial_chip.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_serial_chip.pl	(revision 24244)
@@ -13,26 +13,26 @@
 use Pod::Usage qw( pod2usage );
 
-my ($dbname,			# Database name to use
-    $workdir_default,		# Default working directory
-    $verbose,			# Verbose operations?
-    $no_op,			# No operations?
-    $no_update,			# No updating?
+my ($dbname,                    # Database name to use
+    $workdir_default,           # Default working directory
+    $verbose,                   # Verbose operations?
+    $no_op,                     # No operations?
+    $no_update,                 # No updating?
     );
 GetOptions(
-	   'dbname=s' => \$dbname,
-	   'workdir=s' => \$workdir_default,
-	   'verbose' => \$verbose,
-	   'no-op' => \$no_op,
-	   'no-update' => \$no_update,
+           'dbname=s' => \$dbname,
+           'workdir=s' => \$workdir_default,
+           'verbose' => \$verbose,
+           'no-op' => \$no_op,
+           'no-update' => \$no_update,
 ) or pod2usage( 2 );
 
 pod2usage( -msg => "Required options: --dbname --workdir",
-	   -exitval => 3,
-	   ) unless
+           -exitval => 3,
+           ) unless
     defined $dbname;
 
 $workdir_default = `pwd` unless defined $workdir_default;
 
-my $mdcParser = PS::IPP::Metadata::Config->new;	# Metadata config parser
+my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser
 my $ipprc = PS::IPP::Config->new; # IPP Configuration
 
@@ -44,9 +44,9 @@
 
 # Imfile processing
-my @whole;			# The whole list for processing
+my @whole;                      # The whole list for processing
 {
     my $command = "$chiptool -pendingimfile -dbname $dbname"; # Command to run
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run( command => $command, verbose => 1 );
+        run( command => $command, verbose => 1 );
     die "Unable to get phase 2 imfile list: $error_code\n" if not $success;
     @whole = split /\n/, join( '', @$stdout_buf );
@@ -59,34 +59,36 @@
     push @single, $value;
     if ($value =~ /^\s*END\s*$/) {
-	push @single, "\n";
-	
-	my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
-	    die "Unable to parse output from chiptool.\n";
-	    
-	foreach my $item (@$list) {
-	    my $chip_id = $item->{chip_id};
-	    my $exp_id = $item->{exp_id};
-	    my $exp_tag = $item->{exp_tag};
-	    my $camera = $item->{camera};
-	    my $class_id = $item->{class_id};
-	    my $uri = $item->{uri};
-	    my $reduction = $item->{reduction};
-	    my $workdir = $item->{workdir};
-	    $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL");
+        push @single, "\n";
 
-	    my $outroot = caturi( $workdir, $exp_tag, "$exp_tag.ch.$chip_id" );
-	    $ipprc->outroot_prepare( $outroot );
+        my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+            die "Unable to parse output from chiptool.\n";
 
-	    my $command = "$chip --chip_id $chip_id --exp_id $exp_id --class_id $class_id --uri $uri --dbname $dbname --camera $camera --outroot $outroot";
-	    $command .= " --reduction $reduction" if defined $reduction;
-	    $command .= " --verbose" if defined $verbose;
-	    $command .= " --no-op" if defined $no_op;
-	    $command .= " --no-update" if defined $no_update;
-	    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-		run( command => $command, verbose => 1 );
-	    die "Unable to do phase 2 processing on $chip_id $class_id: $error_code\n" if not $success;
-	}
+        foreach my $item (@$list) {
+            my $chip_id = $item->{chip_id};
+            my $exp_id = $item->{exp_id};
+            my $exp_tag = $item->{exp_tag};
+            my $chip_imfile_id = $item->{chip_imfile_id};
+            my $camera = $item->{camera};
+            my $class_id = $item->{class_id};
+            my $uri = $item->{uri};
+            my $reduction = $item->{reduction};
+            my $state = $item->{state};
+            my $workdir = $item->{workdir};
+            $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL");
 
-	@single = ();
+            my $outroot = caturi( $workdir, $exp_tag, "$exp_tag.ch.$chip_id" );
+            $ipprc->outroot_prepare( $outroot );
+
+            my $command = "$chip --chip_id $chip_id --chip_imfile_id $chip_imfile_id --exp_id $exp_id --class_id $class_id --uri $uri --dbname $dbname --camera $camera --outroot $outroot --run-state $state";
+            $command .= " --reduction $reduction" if defined $reduction;
+            $command .= " --verbose" if defined $verbose;
+            $command .= " --no-op" if defined $no_op;
+            $command .= " --no-update" if defined $no_update;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run( command => $command, verbose => 1 );
+            die "Unable to do phase 2 processing on $chip_id $class_id: $error_code\n" if not $success;
+        }
+
+        @single = ();
 
     }
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_serial_magic.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_serial_magic.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_serial_magic.pl	(revision 24244)
@@ -0,0 +1,157 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config qw( caturi );
+use Data::Dumper;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+my ($dbname,                    # Database name to use
+    $identifier,                # Identifier to process
+    $workdir_default,           # Default working directory
+    $verbose,                   # Verbose operations?
+    $no_op,                     # No operations?
+    $no_update,                 # No updating?
+    );
+GetOptions( 'dbname=s' => \$dbname,
+            'magic_id=s' => \$identifier,
+            'workdir=s' => \$workdir_default,
+            'verbose' => \$verbose,
+            'no-op' => \$no_op,
+            'no-update' => \$no_update,
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Required options: --dbname --workdir",
+           -exitval => 3,
+           ) unless
+    defined $dbname;
+
+$workdir_default = `pwd` unless defined $workdir_default;
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Metadata config parser
+my $ipprc = PS::IPP::Config->new; # IPP Configuration
+
+# Look for programs we need
+my $missing_tools;
+my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $magic_tree = can_run('magic_tree.pl') or (warn "Can't find magic_tree.pl" and $missing_tools = 1);
+my $magic_process = can_run('magic_process.pl') or (warn "Can't find magic_process.pl" and $missing_tools = 1);
+die "Can't find required tools.\n" if $missing_tools;
+
+# Tree
+{
+    my @whole;                      # The whole list for processing
+    {
+        my $command = "$magictool -totree -dbname $dbname"; # Command to run
+        $command .= " -magic_id $identifier" if defined $identifier;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run( command => $command, verbose => $verbose );
+        die "Unable to get magic tree list: $error_code\n" if not $success;
+        @whole = split /\n/, join( '', @$stdout_buf );
+    }
+
+    my @single = ();
+
+    while ( scalar @whole > 0 ) {
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+                die "Unable to parse output from magictool.\n";
+
+            foreach my $item (@$list) {
+                my $magic_id = $item->{magic_id};
+                my $exp_id = $item->{exp_id};
+                my $camera = $item->{camera};
+                my $tess_id = $item->{tess_id};
+                my $ra = $item->{ra};
+                my $dec = $item->{decl};
+                my $workdir = $item->{workdir};
+                $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL");
+
+                my $outroot = caturi( $workdir, $exp_id, "$exp_id.mgc.$magic_id" );
+                $ipprc->outroot_prepare( $outroot );
+
+                my $command = "$magic_tree --magic_id $magic_id --camera $camera --tess_id $tess_id --outroot $outroot --ra $ra --dec $dec --dbname $dbname";
+                $command .= " --verbose" if defined $verbose;
+                $command .= " --no-op" if defined $no_op;
+                $command .= " --no-update" if defined $no_update;
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run( command => $command, verbose => $verbose );
+                die "Unable to do magic tree on $magic_id: $error_code\n" if not $success;
+            }
+
+            @single = ();
+        }
+    }
+}
+
+# Process leaf
+{
+    my @whole;                      # The whole list for processing
+    {
+        my $command = "$magictool -toprocess -dbname $dbname"; # Command to run
+        $command .= " -magic_id $identifier" if defined $identifier;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run( command => $command, verbose => $verbose );
+        die "Unable to get magic tree list: $error_code\n" if not $success;
+        @whole = split /\n/, join( '', @$stdout_buf );
+    }
+
+    my @single = ();
+
+    while ( scalar @whole > 0 ) {
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+                die "Unable to parse output from magictool.\n";
+
+            foreach my $item (@$list) {
+                my $magic_id = $item->{magic_id};
+                my $exp_id = $item->{exp_id};
+                my $camera = $item->{camera};
+                my $node = $item->{node};
+                my $workdir = $item->{workdir};
+                $workdir = $workdir_default unless (defined $workdir or $workdir ne "NULL");
+
+                my $outroot = caturi( $workdir, $exp_id, "$exp_id.mgc.$magic_id.$node" );
+                $ipprc->outroot_prepare( $outroot );
+
+                my $command = "$magic_process --magic_id $magic_id --camera $camera --node $node --outroot $outroot --dbname $dbname";
+                $command .= " --verbose" if defined $verbose;
+                $command .= " --no-op" if defined $no_op;
+                $command .= " --no-update" if defined $no_update;
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run( command => $command, verbose => $verbose );
+                die "Unable to do magic process on $magic_id $node: $error_code\n" if not $success;
+            }
+
+            @single = ();
+        }
+    }
+}
+
+
+
+END {
+    my $status = $?;
+    system("sync") == 0
+        or die "failed to execute sync: $!" ;
+    $? = $status;
+}
+
+
+__END__
+
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_serial_mops.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_serial_mops.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_serial_mops.pl	(revision 24244)
@@ -15,4 +15,5 @@
 use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Config qw( caturi );
+use Carp qw( carp );
 
 # Look for programs we need
@@ -71,12 +72,50 @@
     die "Unable to connect to database: $DBI::errstr";
 
-my $sql = "SELECT exp_id, diff_id, skycell_id, warp_id, diffSkyfile.path_base AS diff_path_base, warpSkyfile.path_base AS warp_path_base FROM diffSkyfile JOIN diffRun using(diff_id) JOIN diffInputSkyfile USING(diff_id,tess_id,skycell_id) JOIN warpRun USING(warp_id,tess_id) JOIN warpSkyfile USING(warp_id,skycell_id,tess_id) JOIN fakeRun using(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE diffSkyfile.fault = 0 AND rawExp.camera = \'$camera\'";
-$sql .= " AND diffRun.label = \'$label\'" if defined $label;
-$sql .= ';';
+my $where_label = defined $label ? "AND label = '$label'" : ""; # WHERE for label
 
-my $rows = $db->selectall_arrayref( $sql, { Slice => {} } ) or die "Unable to execute SQL: $DBI::errstr";
-$db->disconnect;
+my $sql = "
+-- Get a list of exposures on which magic may be performed
+SELECT
+    exp_id,
+    MAX(diffWarps.diff_id) AS diff_id,
+    -- The following trick pulls out the 'inverse' value for the maximum diff_id
+    CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id), ',', 1), UNSIGNED) AS inverse
+FROM (
+    -- Forward diffs
+    SELECT
+        diffRun.diff_id,
+        warp1 AS warp_id,
+        0 AS inverse
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp1 IS NOT NULL
+        AND diffRun.state = 'full'
+        AND diffRun.exposure = 1
+        $where_label
+    UNION
+    -- Backward diffs
+    SELECT
+        diffRun.diff_id,
+        warp2 AS warp_id,
+        1 AS inverse
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp2 IS NOT NULL
+        AND diffRun.state = 'full'
+        AND diffRun.exposure = 1
+        AND diffRun.bothways = 1
+        $where_label
+    ) AS diffWarps
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+WHERE rawExp.camera = '$camera'
+GROUP BY exp_id;";
 
-print "Selected " . scalar @$rows . " rows.\n";
+my $diffs = $db->selectall_arrayref( $sql, { Slice => {} } ) or die "Unable to execute SQL: $DBI::errstr";
+
+print "Selected " . scalar @$diffs . " rows.\n";
 
 $ipprc->outroot_prepare( $outroot );
@@ -84,35 +123,39 @@
 my ($dsFile, $dsName) = tempfile( "$outrootResolved.dslist.XXXX", UNLINK => !$save_temps);
 
-foreach my $row ( @$rows ) {
-    my $exp_id = $row->{exp_id};
-    my $diff_id = $row->{diff_id};
-    my $skycell_id = $row->{skycell_id};
-    my $warp_id = $row->{warp_id};
-    my $diff_base = $row->{diff_path_base};
-    my $warp_base = $row->{warp_path_base};
+foreach my $diff ( @$diffs ) {
+    my $exp_id = $diff->{exp_id};
+    my $diff_id = $diff->{diff_id};
+    my $inverse = $diff->{inverse};
 
-    my $input = $ipprc->filename("PSPHOT.OUT.CMF.MEF", $diff_base);
-    die "Can't find $input\n" unless $ipprc->file_exists($input);
-    $input = $ipprc->file_resolve($input);
+    my $sql = "SELECT * FROM diffSkyfile WHERE diff_id = $diff_id AND fault = 0 AND quality = 0;";
+    my $skycells = $db->selectall_arrayref( $sql, { Slice => {} } ) or die "Unable to execute SQL: $DBI::errstr";
 
-    my $skycell = $ipprc->filename("SKYCELL.TEMPLATE", $warp_base);
-    die "Can't find $skycell\n" unless $ipprc->file_exists($skycell);
-    $skycell = $ipprc->file_resolve($skycell);
+    foreach my $skycell ( @$skycells ) {
+        my $skycell_id = $skycell->{skycell_id};
+        my $path_base = $skycell->{path_base};
 
-    my $output = caturi( $outroot, "mops.$exp_id.$skycell_id.$diff_id.fits" );
-    $output = $ipprc->file_resolve($output);
+        my $sources = $inverse ? "PPSUB.INVERSE.SOURCES" : "PPSUB.OUTPUT.SOURCES";
 
-    unless ($no_op) {
-        $ipprc->file_prepare($output);
-        my $command = "$mops --input $input --extname SkyChip.psf --skycell $skycell --output $output";
-        my $success = run( command => $command, verbose => $verbose );
-        die "Couldn't translate $input\n" unless $success;
+        my $input = $ipprc->filename($sources, $path_base);
+        (carp "Can't find $input\n" and next) unless $ipprc->file_exists($input);
+        $input = $ipprc->file_resolve($input);
+
+        my $output = caturi( $outroot, "mops.$exp_id.$skycell_id.$diff_id.fits" );
+        $output = $ipprc->file_resolve($output);
+
+        unless ($no_op) {
+            $ipprc->file_prepare($output);
+            my $command = "$mops --input $input --extname SkyChip.psf --output $output";
+            my $success = run( command => $command, verbose => $verbose );
+            (carp "Couldn't translate $input\n" and next) unless $success;
+        }
+
+        # format: filename|filesize|md5sum|filetype|
+        # note: since we omit filesize and md5sum, dsreg will calculate them
+        print $dsFile "${output}|||ipp-mops|\n";
     }
-
-    # format: filename|filesize|md5sum|filetype|
-    # note: since we omit filesize and md5sum, dsreg will calculate them
-    print $dsFile "${output}|||ipp-mops|\n";
 }
 close $dsFile;
+$db->disconnect;
 
 # Register new files with the data store
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_definerun.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_definerun.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_definerun.pl	(revision 24244)
@@ -164,4 +164,7 @@
 &my_die("failed to retrieve new magic run information", $PS_EXIT_CONFIG_ERROR) if !$magic_id;
 
+### This is left over from when diffs were composed of a single skycell
+### When we're not too busy, it should be deleted in favour of:
+### magicRun JOIN diffSkyfile USING(diff_id) WHERE diffSkyfile.fault = 0 AND diffSkyfile.quality = 0
 foreach my $diff_skyfile (@$inputs) {
     my $skycell_id = $diff_skyfile->{skycell_id};
@@ -171,6 +174,7 @@
     }
     my $diff_id = $diff_skyfile->{diff_id};
-    my $command = "$magictool -addinputskyfile -magic_id $magic_id -diff_id $diff_id";
-    $command .= "  -node $skycell_id";
+    my $command = "$magictool -addinputskyfile";
+    $command .= " -magic_id $magic_id";
+    $command .= " -node $skycell_id";
     $command .= " -dbname $dbname" if $dbname;
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_destreak.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_destreak.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_destreak.pl	(revision 24244)
@@ -35,5 +35,5 @@
 
 # Parse the command-line arguments
-my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
+my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $inverse, $cam_path_base);
 my ($outroot, $recoveryroot);
 my ($replace, $remove, $release);
@@ -49,4 +49,5 @@
            'uri=s'          => \$uri,        # uri of the input image
            'path_base=s'    => \$path_base,  # path_base of the input
+           'inverse'        => \$inverse,    # Inverse subtraction?
            'cam_path_base=s'=> \$cam_path_base,  # path_base of the associated camera run
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
@@ -64,5 +65,5 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --outroot",
+pod2usage( -msg => "Required options: --magic_ds_id --camera --streaks --stage --stage_id --component --uri --path_base --outroot",
            -exitval => 3) unless
     defined $magic_ds_id and
@@ -72,4 +73,6 @@
     defined $stage_id and
     defined $component and
+    defined $uri and
+    defined $path_base and
     defined $outroot;
 
@@ -104,4 +107,9 @@
 }
 
+# default value is "NULL" do not use
+if (defined($recoveryroot) and ($recoveryroot = "NULL")) {
+    $recoveryroot = undef;
+}
+
 if (($stage eq "raw") and $replace and ! $recoveryroot) {
     &my_die("Can not replace raw files without defining recoveryroot.",
@@ -168,5 +176,6 @@
 
         foreach my $skycell (@$skycells) {
-            print $sfh "$skycell->{uri}\n"
+            my $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+            print $sfh "$skycell_uri\n";
         }
         close $sfh;
@@ -174,9 +183,8 @@
 }
 
-my $image = $uri;
-
-my ($mask, $ch_mask, $weight, $astrom);
+my ($image, $mask, $ch_mask, $weight, $astrom);
 
 if ($stage eq "raw") {
+    $image = $uri;
     $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
     $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id) if $release ;
@@ -184,4 +192,5 @@
     # we use the mask output from the camera stage for input and replace
     # the output of the chip stage with that mask as well.
+    $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
     $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
     $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
@@ -189,9 +198,12 @@
     $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
 } elsif ($stage eq "warp") {
+    $image  = $ipprc->filename("PSWARP.OUTPUT", $path_base);
     $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
     $weight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base);
 } elsif ($stage eq "diff") {
-    $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
-    $weight = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
+    my $name = $inverse ? "PPSUB.INVERSE" : "PPSUB.OUTPUT"; # Base name for images
+    $image  = $ipprc->filename($name, $path_base);
+    $mask   = $ipprc->filename("$name.MASK", $path_base);
+    $weight = $ipprc->filename("$name.VARIANCE", $path_base);
 }
 
@@ -272,5 +284,5 @@
     $command   .= " -magic_ds_id $magic_ds_id";
     $command   .= " -component $component";
-    $command   .= " -code $exit_code";
+    $command   .= " -fault $exit_code";
     $command   .= " -dbname $dbname" if defined $dbname;
 
Index: anches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_mask.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_mask.pl	(revision 24243)
+++ 	(revision )
@@ -1,168 +1,0 @@
-#!/usr/bin/env perl
-
-use Carp;
-use warnings;
-use strict;
-
-## report the program and machine
-use Sys::Hostname;
-my $host = hostname();
-print "\n\n";
-print "Starting script $0 on $host\n\n";
-
-use vars qw( $VERSION );
-$VERSION = '0.01';
-
-use IPC::Cmd 0.36 qw( can_run run );
-use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::List qw( parse_md_list );
-
-use Astro::FITS::CFITSIO qw( :constants );
-Astro::FITS::CFITSIO::PerlyUnpacking(1);
-
-use PS::IPP::Config 1.01 qw( :standard );
-
-use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
-use Pod::Usage qw( pod2usage );
-
-# Look for programs we need
-my $missing_tools;
-my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
-my $streaksremove = can_run('streaksremove') or (warn "Can't find streaksremove" and $missing_tools = 1);
-if ($missing_tools) {
-    warn("Can't find required tools.");
-    exit($PS_EXIT_CONFIG_ERROR);
-}
-
-# Parse the command-line arguments
-my ($magic_id, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
-GetOptions(
-           'magic_id=s'      => \$magic_id,   # Magic identifier
-           'camera=s'        => \$camera,     # Camera name
-           'dbname=s'        => \$dbname,     # Database name
-           'outroot=s'       => \$outroot,    # Output root name
-           'save-temps'      => \$save_temps, # Save temporary files?
-           'verbose'         => \$verbose,    # Print stuff?
-           'no-update'       => \$no_update,  # Don't update the database?
-           'no-op'           => \$no_op,      # Don't do any operations?
-           'logfile=s'       => \$logfile,   # Redirect output?
-           ) or pod2usage( 2 );
-
-pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --magic_id --camera --outroot",
-           -exitval => 3) unless
-    defined $magic_id and
-    defined $camera and
-    defined $outroot;
-
-my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
-$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $PS_EXIT_SYS_ERROR ) if $logfile;
-
-my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-### Get a list of inputs
-my $inputs;                     # List of inputs
-{
-    my $command = "$magictool -inputs -magic_id $magic_id -node root"; # Command to run
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform magictool -inputs: $error_code", $magic_id, $error_code);
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR);
-
-    $inputs = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR);
-}
-
-### Do the heavy lifting
-{
-    my $command = "$streaksremove --streaks"; # Command to execute
-    my @basenames;              # List of base names
-
-    # Concatenate the names
-    foreach my $node (@$inputs) {
-        push @basenames, $ipprc->file_resolve( $node->{path_base} );
-    }
-    $command .= ' ' . join(' ', @basenames);
-
-    unless ($no_op) {
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform RemoveStreaks: $error_code", $magic_id, $error_code);
-        }
-
-        foreach my $basename (@basenames) {
-            file_check( $basename . '.mask' );
-        }
-    } else {
-        print "Skipping command: $command\n";
-    }
-}
-
-
-
-### Input mask into database
-{
-    my $command = "$magictool -addmask";
-    $command   .= " -magic_id $magic_id";
-    $command   .= " -uri $outroot";
-    $command   .= " -dbname $dbname" if defined $dbname;
-
-    # Add the processed file to the database
-    unless ($no_update) {
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $error_code);
-        }
-    } else {
-        print "Skipping command: $command\n";
-    }
-}
-
-
-
-### Pau.
-
-sub file_check
-{
-    my $file = shift;           # Name of file
-    &my_die("Unable to find output file: $file", $magic_id, $PS_EXIT_SYS_ERROR) unless
-        $ipprc->file_exists($file);
-}
-
-sub my_die
-{
-    my $msg = shift;            # Warning message on die
-    my $magic_id = shift;       # Magic identifier
-    my $exit_code = shift;      # Exit code to add
-
-    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
-
-    carp($msg);
-    if (defined $magic_id and not $no_update) {
-        my $command = "$magictool -addmask";
-        $command .= " -magic_id $magic_id";
-        $command .= " -code $exit_code";
-        $command .= " -dbname $dbname" if defined $dbname;
-        system($command);
-    }
-    exit $exit_code;
-}
-
-END {
-    my $status = $?;
-    system("sync") == 0
-        or die "failed to execute sync: $!" ;
-    $? = $status;
-}
-
-__END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_process.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_process.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_process.pl	(revision 24244)
@@ -30,5 +30,5 @@
 my $missing_tools;
 my $magictool      = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
-my $removestreaks = can_run('RemoveStreaks') or (warn "Can't find RemoveStreaks" and $missing_tools = 1);
+my $detectstreaks = can_run('DetectStreaks') or (warn "Can't find DetectStreaks" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -63,10 +63,10 @@
 $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $node, $PS_EXIT_SYS_ERROR ) if $logfile;
 
-# RemoveStreaks doesn't know about nebulous. It expects to be able to append strings to outroot
+# DetectStreaks doesn't know about nebulous. It expects to be able to append strings to outroot
 # to form valid file names.
-# So forbid nebulous path in outroot. We could relax this by change RemoveStreaks to take all
-# of the file names as arguments
+# So forbid nebulous path in outroot. We could relax this by change DetectStreaks to take all
+# of the file names as arguments or by teaching it about Nebulous
 if ($outroot =~ 'neb:/') {
-    &my_die("RemoveStreaks does not support nebulous paths in outroot", $magic_id, $node, $PS_EXIT_CONFIG_ERROR);
+    &my_die("DetectStreaks does not support nebulous paths in outroot", $magic_id, $node, $PS_EXIT_CONFIG_ERROR);
 }
 
@@ -100,9 +100,10 @@
 {
     my $command;                # Command to execute
-    $command = "$removestreaks --outroot $outroot";
+    $command = "$detectstreaks --outroot $outroot";
     $command .= " --verbose" if $verbose;
 
-    # added per Paul Sydney's Jan 6, 2009
-    $command .= " -D 3";
+    # added per Paul Sydney's request Jan 6, 2009
+    # removed per Paul Sydney March 30, 2009
+    # $command .= " -D 3";
 
 
@@ -118,5 +119,5 @@
     if (scalar @$inputs == 1 and $node ne "root") {
         #
-        #  RemoveStreak --detect --image filename --mask maskname --weight weightname --outroot path_base
+        #  DetectStreaks --detect --image filename --mask maskname --weight weightname --outroot path_base
         #
         # Leaf node: 'detect' stage
@@ -131,9 +132,19 @@
         }
         my ($image, $mask, $weight) = resolve_inputs($innode);
-
-        $command .= " --detect --image $image --mask $mask --weight $weight";
+        if (!defined($image) or !defined($mask) or !defined($weight)) {
+            &my_die("failed to resolve inputs", $magic_id, $node, $PS_EXIT_DATA_ERROR);
+        }
+
+        my $template = resolve_template($innode);
+        &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR)
+            unless defined $template;
+
+        $command .= " --detect --image $image --mask $mask --weight $weight -k $template";
 
         # set threshold to 4 sigma
         $command .= ' -t 4';
+
+        # add -S
+        $command .= ' -S';
 
         # create the list of inputs used at this stage. At higher levels the
@@ -156,5 +167,5 @@
     } else {
         #
-        # RemoveStreak --merge --inputs input.list --images image0_1.list \
+        # DetectStreaks --merge --inputs input.list --images image0_1.list \
         #                      --masks mask0_1.list --weight weights0_1.list
         #                      --outroot outroot
@@ -176,13 +187,13 @@
             foreach my $innode (@$inputs) {
                 # root for inputs from previous stage
-                my $in_uri = $innode->{uri};
-                print $sfh "$in_uri\n";
-
-                cat_list_to_list($in_fh, $in_uri, "input.list");
+                my $in_path_base = $innode->{magic_path_base};
+                print $sfh "$in_path_base\n";
+
+                cat_list_to_list($in_fh, $in_path_base, "input.list");
                 # build input lists by combining the lists from
                 # previous stages
-                cat_list_to_list($ifh, $in_uri, "image.list");
-                cat_list_to_list($mfh, $in_uri, "mask.list");
-                cat_list_to_list($wfh, $in_uri, "weight.list");
+                cat_list_to_list($ifh, $in_path_base, "image.list");
+                cat_list_to_list($mfh, $in_path_base, "mask.list");
+                cat_list_to_list($wfh, $in_path_base, "weight.list");
             }
             close $in_fh;
@@ -195,4 +206,6 @@
             $command .= " --images $image_list --masks $mask_list --weights $weight_list" ;
             $command .= " --inputstreaks $streaks_list";
+            # new option to "merge duplicate streaks into unique streaks"
+            $command .= " --duplicates";
         };
         if ($@) {
@@ -202,5 +215,5 @@
 
     unless ($no_op) {
-        # RemoveStreaks fails if an output file already exists
+        # DetectStreaks fails if an output file already exists
         foreach my $output (@outputs) {
             unlink($output) if -e $output;
@@ -211,5 +224,5 @@
         unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform RemoveStreaks: $error_code", $magic_id, $node, $error_code);
+            &my_die("Unable to perform DetectStreaks: $error_code", $magic_id, $node, $error_code);
         }
 
@@ -228,5 +241,5 @@
     $command   .= " -magic_id $magic_id";
     $command   .= " -node $node";
-    $command   .= " -uri $outroot";
+    $command   .= " -path_base $outroot";
     $command   .= " -dbname $dbname" if defined $dbname;
 
@@ -247,12 +260,15 @@
     my $resolved = $ipprc->file_resolve($streaks_file);
 
-    my $fh;
-    open $fh, "<$resolved" or
-        &my_die("failed to open streaks file $streaks_file", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
-    # the first line in the streaks file contains the number of streaks found
-    my $num_streaks = <$fh>;
-    chomp $num_streaks;
-    close $fh;
-    print "$num_streaks streaks found on magicRun $magic_id\n" if $verbose;
+    my $num_streaks = -1;
+    unless ($no_op) {
+        my $fh;
+        open $fh, "<$resolved" or
+            &my_die("failed to open streaks file $streaks_file", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+        # the first line in the streaks file contains the number of streaks found
+        $num_streaks = <$fh>;
+        chomp $num_streaks;
+        close $fh;
+        print "$num_streaks streaks found on magicRun $magic_id\n" if $verbose;
+    }
 
     my $command = "$magictool -addmask";
@@ -294,8 +310,8 @@
 sub cat_list_to_list   {
     my $out = shift;        # output file handle
-    my $uri = shift;        # uri to append ...
+    my $path_base = shift;  # path_base to append ...
     my $extension = shift;  # ... the extension to
 
-    my $filename = "$uri.$extension";
+    my $filename = "$path_base.$extension";
 
     my $in;
@@ -306,14 +322,46 @@
 }
 
-sub resolve_inputs {
-
+sub resolve_inputs
+{
     my $node = shift;
-    my $input_base = $node->{path_base};
-
-    my $image = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT", $input_base));
-    my $mask = $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.MASK", $input_base));
-    my $weight= $ipprc->file_resolve($ipprc->filename("PPSUB.OUTPUT.VARIANCE", $input_base));
-
-    return ($image, $mask, $weight);
+    my $input_base = $node->{diff_path_base};
+
+    my ($image, $mask, $variance); # Names to return
+    if ($node->{inverse}) {
+        $image = "PPSUB.INVERSE";
+        $mask = "PPSUB.INVERSE.MASK";
+        $variance = "PPSUB.INVERSE.VARIANCE";
+    } else {
+        $image = "PPSUB.OUTPUT";
+        $mask = "PPSUB.OUTPUT.MASK";
+        $variance = "PPSUB.OUTPUT.VARIANCE";
+    }
+
+    $image = $ipprc->file_resolve($ipprc->filename($image, $input_base));
+    $mask = $ipprc->file_resolve($ipprc->filename($mask, $input_base));
+    $variance= $ipprc->file_resolve($ipprc->filename($variance, $input_base));
+
+    return ($image, $mask, $variance);
+}
+
+sub resolve_template
+{
+    my $node = shift;
+
+    my $image;                  # Name of template to return
+    my $path_base;              # Base name for name
+    if (defined $node->{warp_path_base} and $node->{warp_path_base} ne "NULL") {
+        $path_base = $node->{warp_path_base};
+        $image = "PSWARP.OUTPUT";
+    } elsif (defined $node->{stack_path_base} and $node->{stack_path_base} ne "NULL") {
+        $path_base = $node->{stack_path_base};
+        $image = "PPSTACK.OUTPUT";
+    } else {
+        return undef;
+    }
+
+    $image = $ipprc->file_resolve($ipprc->filename($image, $path_base));
+
+    return $image
 }
 
@@ -339,5 +387,5 @@
         $command .= " -magic_id $magic_id";
         $command .= " -node $node";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_tree.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_tree.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_tree.pl	(revision 24244)
@@ -73,31 +73,4 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-### Get a list of warpSkyfiles
-### This is a workaround to allow processing with older diffSkyfiles that don't have
-### a wcs. If warp_id is provided, we get the wcs from there
-my %warpSkyfiles;                   # hash of warps
-if ($warp_id) {
-    my $command = "$warptool -warped -warp_id $warp_id"; # Command to run
-    $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform warptool -warped: $error_code", $magic_id, $error_code);
-    }
-
-    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $magic_id, $PS_EXIT_PROG_ERROR);
-
-    my $warps = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $magic_id, $PS_EXIT_PROG_ERROR);
-
-    # make a hash indexed by skycell_id
-    foreach my $warp ( @$warps ) {
-        my $skycell_id = $warp->{skycell_id};
-        die "failed to lookup skycell_id from warp" if !$skycell_id;
-        $warpSkyfiles{$skycell_id} = $warp;
-    }
-}
 ### Get a list of skycells
 my @skycells;                   # List of skycells
@@ -126,29 +99,9 @@
 my @fields;
 foreach my $input ( @skycells ) {
-    # the filename method doesn't add the $skycell_id
-
-    # my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
-    #my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
-
-#    my $skycellBase = "$outroot/$skycell_id";
-#    my $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $skycellBase);
-#    $ipprc->skycell_file($tess_id, $skycell_id, $skyfile, $verbose) or &my_die("Unable to generate skycells $skycell_id", $magic_id, $PS_EXIT_PROG_ERROR);
-
-    my $skyfile;
-    my $skycell_id = $input->{node};
-    if ($warp_id) {
-        # Applying the workaround
-        my $warp = $warpSkyfiles{$skycell_id};
-        die "warpSkyfile for $skycell_id not found" if !$warp;
-        my $skyfileBase = $warp->{path_base};
-        $skyfile = $ipprc->filename("SKYCELL.TEMPLATE", $skyfileBase);
-    } else {
-        # using the diffSkyfile
-        $skyfile = $input->{uri};
-    }
-    my $skyfileResolved = $ipprc->file_resolve( $skyfile );
-
-#    my ($header, $status) = Astro::FITS::CFITSIO::fits_read_header( $skyfileResolved );
-#    &my_die("Unable to read skycell header: $status", $magic_id, $PS_EXIT_SYS_ERROR) if $status;
+    # We use the WCS in the diff image
+    my $name = "PPSUB.OUTPUT"; # Name of file
+    my $skycell_id = $input->{node}; # Name of skycell
+    my $skyfile = $ipprc->filename($name, $input->{path_base}, $skycell_id); # Filename for diff
+    my $skyfileResolved = $ipprc->file_resolve( $skyfile ); # Resolved filename
 
     my ($header, $status) = (undef, 0);
@@ -162,5 +115,4 @@
 
     # Get the useful header keywords
-#    my $naxis1 = $$header{'NAXIS1'} or &my_die("Can't find NAXIS1", $magic_id, $PS_EXIT_SYS_ERROR);
     my $naxis1 = $$header{'NAXIS1'};
     my $naxis2;
@@ -181,16 +133,16 @@
         $naxis2 = $$header{'ZNAXIS2'} or &my_die("Can't find ZNAXIS2", $magic_id, $PS_EXIT_SYS_ERROR);
     }
-    my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001", $magic_id, $PS_EXIT_SYS_ERROR);
-    my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $ctype1 = $$header{'CTYPE1'} or &my_die("Can't find CTYPE1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $ctype2 = $$header{'CTYPE2'} or &my_die("Can't find CTYPE2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $cdelt1 = $$header{'CDELT1'} or &my_die("Can't find CDELT1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $cdelt2 = $$header{'CDELT2'} or &my_die("Can't find CDELT2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $crval1 = $$header{'CRVAL1'} or &my_die("Can't find CRVAL1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $crval2 = $$header{'CRVAL2'} or &my_die("Can't find CRVAL2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $crpix1 = $$header{'CRPIX1'} or &my_die("Can't find CRPIX1 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $crpix2 = $$header{'CRPIX2'} or &my_die("Can't find CRPIX2 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $pc11 = $$header{'PC001001'} or &my_die("Can't find PC001001 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $pc12 = $$header{'PC001002'} or &my_die("Can't find PC001002 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $pc21 = $$header{'PC002001'} or &my_die("Can't find PC002001 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
+    my $pc22 = $$header{'PC002002'} or &my_die("Can't find PC002002 in $skyfile", $magic_id, $PS_EXIT_SYS_ERROR);
     my $crota1 = $$header{'CROTA1'};
     my $crota2 = $$header{'CROTA2'};
@@ -296,5 +248,5 @@
         my $command = "$magictool -inputtree";
         $command .= " -magic_id $magic_id";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/rcserver_checkstatus.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/rcserver_checkstatus.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/rcserver_checkstatus.pl	(revision 24244)
@@ -0,0 +1,119 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
+use Digest::MD5::File qw( file_md5_hex );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+
+use PS::IPP::Config 1.01 qw( :standard );
+
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+# Look for programs we need
+my $missing_tools;
+my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
+my $dsproductls = can_run('dsproductls') or (warn "Can't find dsproductls" and $missing_tools = 1);
+my $wget   = can_run('wget') or (warn "Can't find wget" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($dest_id, $prod_id, $status_uri, $last_fileset, $limit);
+my ($dbname, $save_temps, $verbose, $no_update, $logfile);
+
+GetOptions(
+           'dest_id=s'      => \$dest_id,    # destination identifier
+           'prod_id=s'      => \$prod_id,    # product identifier
+           'status_uri=s'   => \$status_uri, # uri for the desination's status data store product
+           'last_fileset=s' => \$last_fileset, # name of last fileset seen on this datastore
+           'limit=i'        => \$limit,      # limit on the number of filesets to process
+           'dbname=s'       => \$dbname,     # Database name
+           'save-temps'     => \$save_temps, # Save temporary files?
+           'verbose'        => \$verbose,    # Print stuff?
+           'no-update'      => \$no_update,  # Don't update the database
+           'logfile=s'      => \$logfile,
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --dest_id --status_uri",
+           -exitval => 3) unless
+    defined $dest_id and
+    defined $status_uri;
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+my $url = "$status_uri/index.txt";
+$url .= "?$last_fileset" if defined($last_fileset) and ($last_fileset ne "NULL");
+
+my @fileset_list;
+{
+    # XXX TODO: use dsproductls once it's been updated to output product type specific columns
+    my $command = "$wget --no-verbose -O - $url";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $dest_id, $error_code);
+    }
+
+    @fileset_list = split "\n", join "", @$stdout_buf;
+
+    my $numlines = scalar @fileset_list;
+}
+
+my $numFilesets = 0;
+foreach my $line (@fileset_list) {
+    # skip comment lines
+    next if $line =~ /^#/;
+    $numFilesets++;
+    last if $limit and ($numFilesets > $limit);
+
+    my ($status_fs_name, $time, $type, $fs_name, $fault) = split /\|/, $line;
+
+    my $command = "$disttool -updatercrun -dest_id $dest_id -fs_name $fs_name -status_fs_name $status_fs_name -set_last_fileset $status_fs_name -fault $fault";
+    if ($fault == 0) {
+        $command .= " -set_state full";
+    }
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $dest_id, $error_code);
+    }
+}
+
+
+
+sub my_die {
+    my $msg = shift;
+    my $dest_id = shift;
+    my $fault = shift;
+
+    die "$msg dest_id: $dest_id: $fault";
+
+    exit $fault;
+}
+
+__END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_advance.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_advance.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_advance.pl	(revision 24244)
@@ -0,0 +1,139 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "Starting script $0 on $host\n\n";
+
+use DateTime;
+my $mjd_start = DateTime->now->mjd;   # MJD of starting script
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Config 1.01 qw( :standard );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
+use Carp;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $receivetool = can_run('receivetool') or (warn "Can't find receivetool" and $missing_tools = 1);
+my $receive_setstatus = can_run('receive_setstatus.pl') or (warn "Can't find receive_setstatus.pl" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ( $fileset_id, $fileset, $dbinfo_uri, $product, $ds_dbname, $ds_dbhost, $dbname, $verbose, $no_update, $save_temps );
+
+GetOptions(
+           'fileset_id=s'      => \$fileset_id,# database id for the fileset
+           'fileset=s'         => \$fileset,   # fileset name
+           'dbinfo_uri=s'      => \$dbinfo_uri,# uri for the database info file
+           'status_product=s'  => \$product,   # Product for status update
+           'ds_dbname=s'       => \$ds_dbname, # data store host
+           'ds_dbhost=s'       => \$ds_dbhost, # data store dbname
+           'dbname=s'          => \$dbname,    # Database name
+           'verbose'           => \$verbose,   # Print to stdout
+           'no-update'         => \$no_update, # Don't update the database?
+           'save-temps'        => \$save_temps, # Save temporary files?
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --file_id --source --product --fileset --file --workdir",
+           -exitval => $PS_EXIT_CONFIG_ERROR) unless
+    defined $fileset_id;
+
+my $ipprc = PS::IPP::Config->new() or
+    &my_die( "Unable to set up", $fileset_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+
+# update the fileset entry
+
+if ($dbinfo_uri and ($dbinfo_uri ne "NULL")) {
+    my $filename = basename($dbinfo_uri);
+    my ($stage) = $filename =~ m|^dbinfo\.(\S+)\.\d+\.mdc$|; # Stage of interest
+    my $tool_name;
+    if ($stage eq "raw") {
+        $tool_name = "regtool";
+    } elsif ($stage eq "camera") {
+        $tool_name = "camtool";
+    } else {
+        $tool_name = "${stage}tool";
+    }
+    my $tool = can_run("$tool_name") or &my_die("Can't find tool to load $dbinfo_uri\n", $fileset_id, $PS_EXIT_CONFIG_ERROR);
+
+    my $file = $ipprc->file_resolve($dbinfo_uri);
+    &my_die("Unable to resolve $dbinfo_uri\n", $PS_EXIT_UNKNOWN_ERROR) unless $file;
+
+    my $command = "$tool -importrun -infile $file"; # Command to execute
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die("Unable to load $file\n", $fileset_id, $PS_EXIT_UNKNOWN_ERROR) unless $success;
+
+    # XXX: once the dbinfo file is imported we cannot revert this fileset
+}
+
+if ($product and ($product ne 'NULL')) {
+    my $command = "$receive_setstatus --dbname $ds_dbname --status_product $product";
+    $command .= " --received_fs_name $fileset --fault 0";
+    if ($ds_dbname) {
+        $command .= " --dbname $ds_dbname";
+    } else {
+        # XXX: is falling back to the other database a good idea?
+        $command .= " --dbname $dbname" if defined $dbname;
+    }
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die("Unable to set transfer status on data store for $fileset_id\n", $fileset_id, $PS_EXIT_UNKNOWN_ERROR) unless $success;
+}
+
+# All done
+{
+    my $command = "$receivetool -updatefileset -fileset_id $fileset_id -set_state full";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    unless (defined $no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        &my_die("Unable to add result for $fileset_id\n", $fileset_id, $PS_EXIT_CONFIG_ERROR) unless $success;
+    }
+}
+
+# Pau.
+
+
+sub my_die
+{
+    my $msg = shift;            # Exit message
+    my $fileset_id = shift;     # Fileset identifier
+    my $fault = shift;          # Fault code
+
+    $fault = $PS_EXIT_PROG_ERROR unless defined $fault;
+
+    carp($msg);
+    if (defined $fileset_id and not $no_update) {
+        my $command = "$receivetool -updatefileset";
+        $command .= " -fileset_id $fileset_id";
+        $command .= " -fault $fault";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+    }
+    exit $fault;
+}
+
+
+__END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_file.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_file.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_file.pl	(revision 24244)
@@ -0,0 +1,521 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "Starting script $0 on $host\n\n";
+
+use DateTime;
+my $mjd_start = DateTime->now->mjd;   # MJD of starting script
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename );
+use Carp;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $receivetool = can_run('receivetool') or (warn "Can't find receivetool" and $missing_tools = 1);
+my $dsproductls = can_run('dsfilesetls') or (warn "Can't find dsfilesetls" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my $tempdir = "/tmp";
+
+# Parse the command-line arguments
+my ( $file_id, $source, $product, $fileset, $fileset_id, $file, $component, $bytes, $md5sum, $workdir, $dirinfo_uri, $dbname, $verbose, $no_update, $save_temps );
+
+GetOptions(
+           'file_id=s'         => \$file_id, # File identifier
+           'source=s'          => \$source, # Source for data
+           'product=s'         => \$product, # Product for data
+           'fileset=s'         => \$fileset, # Fileset for data
+           'fileset_id=s'      => \$fileset_id, # database id for the fileset
+           'file=s'            => \$file, # File to retrieve
+           'component=s'       => \$component, # component for this file (class_id, skycell_id or dbinfo)
+           'bytes=i'           => \$bytes, # file size in bytes
+           'md5sum=s'          => \$md5sum, # md5sum for file from data store
+           'workdir=s'         => \$workdir, # Working directory for output
+           'dirinfo=s'    => \$dirinfo_uri, # file containing the destination directories for this component
+           'dbname=s'          => \$dbname,    # Database name
+           'verbose'           => \$verbose,   # Print to stdout
+           'no-update'         => \$no_update, # Don't update the database?
+           'save-temps'        => \$save_temps, # Save temporary files?
+           ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --file_id --source --product --fileset --file --component --workdir --bytes --md5sum --dirinfo",
+           -exitval => $PS_EXIT_CONFIG_ERROR) unless
+    defined $file_id and
+    defined $source and
+    defined $product and
+    defined $fileset and
+    defined $file and
+    defined $component and
+    defined $bytes and
+    defined $md5sum and
+    defined $dirinfo_uri and
+    defined $workdir;
+
+$tempdir .= "/$file_id";
+
+&my_die( "dirinfo is NULL for $component", $file_id, $PS_EXIT_CONFIG_ERROR )
+    if (($dirinfo_uri eq "NULL") and ($component ne "dirinfo"));
+
+my $ipprc = PS::IPP::Config->new() or
+    &my_die( "Unable to set up", $file_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+
+my $mdcParser = PS::IPP::Metadata::Config->new;
+
+
+
+# Retrieve file
+my $filename = "$tempdir/$file";  # Target filename
+{
+    my $uri = "$source/$product/$fileset/$file"; # URI for datastore file
+    my $command = "dsget --uri $uri --filename $filename"; # Command to execute
+    $command .= " --timeout 590";
+    $command .= " --bytes $bytes --md5 $md5sum";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die( "Unable to retrieve file from $uri\n", $file_id, $PS_EXIT_DATA_ERROR) unless $success;
+}
+my $mjd_copy = DateTime->now->mjd;   # MJD of finishing copy
+
+# figure out which dirinfo file to read
+my $dirinfo_file_to_read = $component eq "dirinfo" ? $filename : $dirinfo_uri;
+
+# process it
+my ($destdir, $components, $dirinfo_lines) = read_dirinfo_file($dirinfo_file_to_read, $file_id);
+
+# select a directory for the dirinfo and dbinfo files
+# XXX: perhaps this directory should be set by the script and passed in
+# rather than computed here.
+
+my ($day, $month, $year) = (localtime)[3,4,5];
+my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
+my $dir_for_info_files = caturi($workdir, $datestr, $fileset);
+
+# Deal with file
+if ($component eq 'dirinfo') {
+    # save the dirinfo file contents into the $workdir
+
+    $dirinfo_uri = caturi($dir_for_info_files, basename($filename));
+    print "dirinfo_uri: $dirinfo_uri\n" if $verbose;
+
+    my $resolved = $ipprc->file_resolve($dirinfo_uri, 'create');
+    &my_die( "failed to resolve $dirinfo_uri\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved;
+
+    print "dirinfo resolved is: $resolved\n" if $verbose;
+
+    open OUT, ">$resolved" 
+        or &my_die( "failed to open $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    print OUT @$dirinfo_lines
+        or &my_die( "failed to write $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    close OUT
+        or &my_die( "failed to close $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    
+    # update the fileset to allow processing of other files
+    my $command = "$receivetool -updatefileset -fileset_id $fileset_id";
+    $command .= " -set_state new -dirinfo $dirinfo_uri";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die( "Unable to update fileset $fileset_id to\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) unless $success;
+
+} elsif ($component eq "dbinfo") {
+
+    open INFILE, $filename or &my_die( "Can't open $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my @lines = (<INFILE>);
+
+    my $dbinfo = join "", @lines;
+
+    close INFILE;
+
+    my $dbinfo_uri = caturi($dir_for_info_files, basename($filename));
+    print "dbinfo_uri: $dbinfo_uri\n" if $verbose;
+
+    my $resolved = $ipprc->file_resolve($dbinfo_uri, 'create');
+    &my_die( "failed to resolve $dbinfo_uri\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved;
+
+    open OUT, ">$resolved" 
+        or &my_die( "failed to open $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    # We process the dbinfo file (the exported run from the distribution) line by line
+    # Rather than read it as an mdc and interptet it, we do our substitutions directly
+    # This is much faster. Parsing a mdc file for a chip run takes several seconds.
+    # we are very strict about the format of the file
+    #
+    # First comes the data for the Run
+    # Next is the data for each component
+    # The component_id (class_id, skycell_id) must come before any of the paths that we edit
+
+    # The first line tells us the run type. From this we get the stage
+    #
+    my $line = $lines[0];
+    my ($runType, $multi) = split " ", $line;
+    &my_die( "unexpected first line found in $filename: $line\n", $file_id, $PS_EXIT_UNKNOWN_ERROR)
+        if !$runType or ($multi ne 'MULTI');
+
+    my $stage;
+    my $comp_name;
+    my $current_component;
+    if ($runType eq 'rawExp') {
+        $stage = 'raw';
+        $comp_name = 'class_id';
+    } elsif ($runType eq 'chipRun') {
+        $stage = 'chip';
+        $comp_name = 'class_id';
+    } elsif ($runType eq 'camRun') {
+        $stage = 'camera';
+        $comp_name = 'exposure';
+        $current_component = $comp_name;
+    } elsif ($runType eq 'fakeRun') {
+        $stage = 'fake';
+        $comp_name = 'class_id';
+    } elsif ($runType eq 'warpRun') {
+        $stage = 'warp';
+        $comp_name = 'skycell_id';
+    } elsif ($runType eq 'diffRun') {
+        $stage = 'diff';
+        $comp_name = 'skycell_id';
+    } elsif ($runType eq 'stackRun') {
+        $stage = 'stack';
+        $comp_name = 'skycell_id';
+    } else {
+        &my_die( "unexpected run type line found in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    }
+
+    my $new_workdir_value;
+    if ($destdir eq 'none') {
+        # this only appiles to rawExp
+        $new_workdir_value = "$workdir";
+    } else {
+        $new_workdir_value = "$workdir/$destdir";
+    }
+    my $component_dir;
+    if ($current_component) {
+        $component_dir = $components->{$current_component};
+    }
+    foreach $line (@lines) {
+        my $out_line = $line;
+
+        my ($name, $type, $value) = split " ", $line;
+        # only complete lines have things that we need to examine
+        if ($name and $type and $value) {
+            my $new_value;
+            # we have a new component id, save it and look up the corresponding
+            # component_dir
+            if ($name eq $comp_name) {
+                $current_component = $value;
+                $component_dir = $components->{$current_component};
+                &my_die( "$component_dir is null for $value in $filename: $runType\n", 
+                        $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$component_dir;
+            } elsif ($name eq 'workdir') {
+                $new_value = $new_workdir_value;
+            } elsif ($name eq 'tess_id') {
+                # for tess_id strip off any directories just keep the basename.
+                # The site configuration will need to map this to a proper location
+                # XXX: Document this
+                $new_value = basename($value);
+            } elsif ((($name eq 'uri') or ($name eq 'path_base')) and ($value ne 'NULL')) {
+                &my_die( "$component_dir is null and we need it for $name", 
+                        $file_id, $PS_EXIT_PROG_ERROR) if !$component_dir;
+
+                $new_value = caturi($new_workdir_value, $component_dir, basename($value));
+            }
+
+            # if the value changed re-write the line, otherwise just print what we read
+            if ($new_value) {
+                $out_line = "   " . $name . "\t\t" . $type . "\t" . $new_value . "\n";
+            }
+        }
+
+        print OUT $out_line or &my_die( "failed to write to $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    }
+
+    close OUT
+        or &my_die( "failed to close $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    # update the fileset to allow processing of other files
+    my $command = "$receivetool -updatefileset -fileset_id $fileset_id";
+    $command .= " -dbinfo $dbinfo_uri";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die( "Unable to update fileset $fileset_id to\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) unless $success;
+
+
+} elsif ($file =~ m|.*\.tgz$|) {        # XXX: perhaps get this off of file type ?
+    # Get contents of tarball
+    my @files = ();
+    {
+        my $command = "tar -C $tempdir -tzf $filename"; # Command to execute
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        &my_die( "Unable to get listing of tar file $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) unless $success;
+
+        my @lines = split(/\n/, join "", @$stdout_buf); # Lines from output
+        foreach my $line ( @lines ) {
+            $line =~ s/\#.*$//;
+            next unless $line =~ /\S+/;
+            my @fields = split(/\s+/, $line); # Fields in line
+            my $file = $fields[0];      # Name of file
+            $file =~ s|^./||;
+            push @files, $file if $file =~ /\S+/;
+        }
+    }
+
+    # Extract files from tarball
+    {
+        my $command = "tar -C $tempdir -xzf $filename"; # Command to execute
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        &my_die( "Unable to extract tar file $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) unless $success;
+    }
+
+    my $component_dir = $components->{$component};
+    &my_die( "Unable to find component_dir for $component $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) unless $component_dir;
+
+    my $target_dir;
+    if ($destdir eq 'none') {
+        $target_dir = "$workdir";
+    } else {
+        $target_dir = "$workdir/$destdir";
+    }
+    $target_dir .= "/$component_dir";
+
+    # Move files into filesystem of choice
+    foreach my $file ( @files ) {
+        my $from = "$tempdir/$file"; # Source for file
+        my $target = "$target_dir/$file"; # Target destination for file
+
+
+        $ipprc->file_delete ($target);
+
+        my $to = $ipprc->file_create( $target ); # Target for move
+
+        if (!$to) {
+            &my_die( "failed to create: $target\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+        }
+
+        if ( $file =~ /.+\.mdc/ ) {
+            # this file is a config dump file edit the paths
+            edit_mdc_file($file_id, $from, $to, $workdir);
+        } else {
+            system("mv $from $to") == 0 or &my_die( "Unable to move $file into workdir $workdir: $!\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
+} else {
+    &my_die( "Unrecognised file: $file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+
+if (!$save_temps and -e $filename) {
+    unlink $filename or &my_die( "Unable to unlink $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+my $mjd_extract = DateTime->now->mjd;   # MJD of finishing extract
+
+
+# All done
+{
+    my $command = "$receivetool -addresult";
+    $command .= " -file_id $file_id";
+    $command .= (" -dtime_copy " . (($mjd_copy - $mjd_start) * 86400));
+    $command .= (" -dtime_extract " . (($mjd_extract - $mjd_copy) * 86400));
+    $command .= " -dbname $dbname" if defined $dbname;
+
+
+    print "$command\n";
+
+    unless (defined $no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        die "Unable to add result for $file\n" unless $success;
+    }
+}
+
+# Pau.
+
+sub read_dirinfo_file
+{
+    my $filename = shift;
+    my $file_id = shift;
+
+    my $resolved = $ipprc->file_resolve($filename);
+    &my_die("failed to resolve dirinfo file: $filename ", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$resolved;
+
+    open INFILE, $resolved or &my_die( "Can't open $resolved\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my @lines = (<INFILE>);
+
+    my $dirinfo = join "", @lines;
+
+    close INFILE;
+
+    my $metadata = $mdcParser->parse($dirinfo) or
+        &my_die("Unable to parse metadata config doc", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $array = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $dest_hash = $array->[0];
+
+    my $destdir = $dest_hash->{destdir};
+    &my_die("destdir not found in $filename", $file_id, $PS_EXIT_UNKNOWN_ERROR) if !$destdir;
+
+    my $components = $array->[1];
+
+    return ($destdir, $components, \@lines);
+}
+
+# edit a config dump file replacing the "volume" value with the new local value: $workdir
+sub edit_mdc_file
+{
+    my $file_id = shift;
+    my $src = shift;
+    my $dest = shift;
+    my $workdir = shift;
+
+    open my $IN,  "<$src" or &my_die("failed to open $src for input", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    open my $OUT, ">$dest" or &my_die("failed to open $dest for output", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    # Assumed file structure
+    # stuff
+    # FILES.INPUT metadata
+    # FILES.OUTPUT metadata
+    # more stuff
+    # only the paths in the FILES.* metadata are monkeyed with
+    my $done_editing = 0;
+    my $numFilesMD = 0;
+    foreach my $line (<$IN>) {
+        my $out_line = $line;
+        if (!$done_editing) {
+            my (@words) = split " ", $line;
+            if (scalar @words) {
+                # get rid of any leading blank words
+                while ((scalar @words) and !defined $words[0]) {
+                    shift @words;
+                }
+
+                if ($words[1] and $words[1] eq "METADATA") {
+                    if ( $words[0] =~ /^FILES\..+/ ) {
+                        $numFilesMD++;
+                    }
+                } elsif ($words[0] eq "END") {
+                    # when we get to the end of the second FILES metadata we're done editing
+                    if ($numFilesMD == 2) {
+                        $done_editing = 1;
+                    }
+                } elsif ($numFilesMD and ($words[1] eq "STR"))  {
+                    # we're processing one of the files metadata edit the path
+                    my $key = shift @words;
+                    my $type = shift @words;
+                    my $path = shift @words;
+                    my $extra = join " ", @words;
+
+                    $path = edit_path($file_id, $workdir, $path);
+
+                    $out_line = "\t" . $key ."\t" . "STR" . "\t" . $path;
+                    $out_line .= "\t" . $extra if $extra;
+                    $out_line .= "\n";
+                }
+            }
+        }
+        print $OUT $out_line;
+    }
+
+    close $IN;
+    close $OUT or &my_die("failed to close $dest", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+
+
+# XXX: this should go into a module
+# Replace 'volume portion of path with $workdir/
+# Volume is defined here by
+#   neb://volume/
+#   /xxx/xxxxx/         i.e. /data/ippxxx.y/
+#   file://xxx/xxxxx/   i.e. file://data/ippxxx.y/
+#   path://somepath/
+sub edit_path
+{
+    my $file_id = shift;
+    my $workdir = shift;
+    my $path = shift;
+
+    my $scheme = file_scheme($path);
+    my $tail;
+    if ($scheme) {
+            # strip off scheme://
+        $tail = substr($path, length($scheme) + 3);
+    } elsif (substr($path, 0, 1) eq '/') {
+        $tail = substr($path, 1);
+        $scheme = "";
+    }
+    # remove any leading / that are left
+    while ((substr($tail, 0, 1) eq '/')) {
+        $tail = substr($tail, 1);
+    }
+
+    my @segments;
+    if (($scheme eq 'neb') or ($scheme eq 'path')) {
+        my $volume;
+        ($volume, @segments) = split '/', $tail;
+
+    } elsif (!$scheme or ($scheme eq 'file')) {
+
+        # XXX Here we're assuming the /data/ipp??? structure. This won't be true when data is forwarded
+        # by remote sites. We need a way to configure this
+        my $volume;
+
+        # data/ippxxx/dirs
+        (undef, $volume, @segments) = split '/', $tail;
+    } else {
+        &my_die( "unexpected workdir value: $path\n", $file_id, $PS_EXIT_PROG_ERROR);
+    }
+
+    my $new_path = caturi($workdir, @segments);
+
+    return $new_path;
+}
+
+sub my_die
+{
+    my $msg = shift;            # Exit message
+    my $file_id = shift;        # File identifier
+    my $fault = shift;          # Fault code
+
+    $fault = $PS_EXIT_PROG_ERROR unless defined $fault;
+
+    carp($msg);
+    if (defined $file_id and not $no_update) {
+        my $command = "$receivetool -addresult";
+        $command .= " -file_id $file_id";
+        $command .= " -fault $fault";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+    }
+    exit $fault;
+}
+
+
+__END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_fileset.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_fileset.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_fileset.pl	(revision 24244)
@@ -0,0 +1,143 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "Starting script $0 on $host\n\n";
+
+use DateTime;
+my $mjd_start = DateTime->now->mjd;   # MJD of starting script
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Config 1.01 qw( :standard );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use File::Temp qw( tempfile );
+
+# Look for programs we need
+my $missing_tools;
+my $receivetool = can_run('receivetool') or (warn "Can't find receivetool" and $missing_tools = 1);
+my $dsproductls = can_run('dsfilesetls') or (warn "Can't find dsfilesetls" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ( $fileset_id, $source, $product, $fileset, $dbname, $verbose, $no_update, $save_temps );
+
+GetOptions(
+           'fileset_id=s'      => \$fileset_id, # Fileset identifier
+           'source=s'          => \$source, # Source for data
+           'product=s'         => \$product, # Product for data
+           'fileset=s'         => \$fileset, # Fileset for data
+           'dbname=s'          => \$dbname,    # Database name
+           'verbose'           => \$verbose,   # Print to stdout
+           'no-update'         => \$no_update, # Don't update the database?
+           'save-temps'        => \$save_temps, # keep temp files
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --fileset_id --source --product --fileset",
+           -exitval => $PS_EXIT_CONFIG_ERROR) unless
+    defined $fileset_id and
+    defined $source and
+    defined $product and
+    defined $fileset;
+
+# Get list of files, sanity check and then write it to a temporary file
+my $numFiles = 0;
+my ($listFile, $listName) = tempfile("/tmp/$product.$fileset.list.XXXX", UNLINK => !$save_temps);
+{
+    my $uri = "$source/$product/$fileset"; # URI for datastore fileset
+    $uri .= "/index.txt" unless $uri =~ m|/index.txt$|;
+    my $command = "dsfilesetls --uri $uri"; # Command to execute
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    &my_die( "Unable to get fileset listing from $uri\n", $fileset_id, $error_code) unless $success;
+
+    # Get files
+    my @lines = split(/\n/, join "", @$stdout_buf); # Lines from output
+    foreach my $line ( @lines ) {
+        $line =~ s/\#.*$//;
+        next unless $line =~ /\S+/;
+        my @fields = split(/\s+/, $line); # Fields in line
+        &my_die( "too few columns in fileset list entry: $line\n", $fileset_id, $PS_EXIT_DATA_ERROR) unless scalar @fields >= 6;
+
+        print $listFile "FILE$numFiles\tMETADATA\n";
+
+        print $listFile "\turi\t\tSTR\t" . $fields[0] .  "\n";
+        print $listFile "\tfile\t\tSTR\t" . $fields[1] .  "\n";
+        print $listFile "\tbytes\t\tS64\t" . $fields[2] .  "\n";
+        print $listFile "\tmd5sum\t\tSTR\t" . $fields[3] .  "\n";
+        print $listFile "\tfile_type\tSTR\t" . $fields[4] .  "\n";
+        print $listFile "\tcomponent\tSTR\t" . $fields[5] .  "\n";
+
+        print $listFile "END\n";
+        $numFiles++;
+    }
+    close $listFile;
+}
+
+# Add files
+my $new_state;
+if ($numFiles > 0) {
+    $new_state = "listed";
+    my $command = "receivetool -addfile"; # Command to execute
+    $command .= " -fileset_id $fileset_id";
+    $command .= " -file_list $listName";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    unless (defined $no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        &my_die( "Unable to add file list for fileset $fileset\n", $fileset_id, $error_code) unless $success;
+    }
+} else {
+    print STDERR "fileset $fileset has no files\n" if $verbose;
+    $new_state = "full";
+}
+
+{
+    my $command = "receivetool -updatefileset"; # Command to execute
+    $command .= " -fileset_id $fileset_id";
+    $command .= " -set_state $new_state";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    unless (defined $no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        &my_die( "Unable to set state to $new_state for fileset $fileset\n", $fileset_id, $error_code) unless $success;
+    }
+}
+
+sub my_die {
+    my $msg = shift;            # Exit message
+    my $fileset_id = shift;     # File identifier
+    my $fault = shift;          # Fault code
+
+    $fault = $PS_EXIT_PROG_ERROR unless defined $fault;
+
+    carp($msg);
+    if (not $no_update) {
+        my $command = "$receivetool -updatefileset";
+        $command .= " -fileset_id $fileset_id";
+        $command .= " -fault $fault";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        die "Unable to set fault to $fault for fileset $fileset\n" unless $success;
+    }
+    exit $fault;
+}
+__END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_setstatus.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_setstatus.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_setstatus.pl	(revision 24244)
@@ -0,0 +1,103 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "\n\n";
+print "Starting script $0 on $host\n\n";
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw( tempfile );
+use File::Basename qw( basename dirname );
+
+use PS::IPP::Config 1.01 qw( :standard );
+
+my $ipprc = PS::IPP::Config->new(); # IPP configuration
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+
+# Look for programs we need
+my $missing_tools;
+my $dsreg   = can_run('dsreg') or (warn "Can't find dsreg" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ($status_fs_name, $status_product, $received_fs_name, $fault, $status_file);
+my ($dbname, $save_temps, $verbose, $no_update, $logfile);
+
+GetOptions(
+           'status_fs_name=s'   => \$status_fs_name,
+           'status_product=s'   => \$status_product,
+           'received_fs_name=s' => \$received_fs_name,
+           'fault=i'            => \$fault,
+           'status_file=s'      => \$status_file,
+           'dbname=s'           => \$dbname,     # Database name
+           'save-temps'         => \$save_temps, # Save temporary files?
+           'verbose'            => \$verbose,    # Print stuff?
+           'logfile=s'          => \$logfile,
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --status_fs_name --status_product",
+           -exitval => 3) unless
+    defined $received_fs_name and
+    defined $status_product and
+    defined $fault;
+#    and ($fault == 0 or defined $status_file);
+
+# if a name for the status fileset is not provided use the received value
+$status_fs_name = $received_fs_name if !defined $status_fs_name;
+
+$ipprc->redirect_output($logfile) if $logfile;
+
+{
+    # if a status file is provided put it in the fileset otherwise make an empty one
+    # The status information in the produce list is enough for the server
+    my $regline = "";
+    my $fileargs = " --empty";
+
+    if ($status_file) {
+        $regline = "$status_file'|||text|'";
+
+        $fileargs = " --list - --copy --abspath";
+    }
+        
+    # XXX need to create a fileset type for this
+    my $command = "echo $regline | dsreg --add $status_fs_name --product $status_product --type notset";
+    $command .= " $fileargs --ps0 $received_fs_name --ps1 $fault";
+    $command .= " --dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_CONFIG_ERROR);
+        &my_die("Unable to perform $command error_code: $error_code", $status_fs_name, $error_code);
+    }
+}
+
+exit 0;
+
+
+sub my_die {
+    my $msg = shift;
+    my $fs_name = shift;
+    my $fault = shift;
+
+    die "$msg fs_name: $fs_name: $fault";
+
+    exit $fault;
+}
+
+__END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_source.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_source.pl	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/receive_source.pl	(revision 24244)
@@ -0,0 +1,107 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+print "Starting script $0 on $host\n\n";
+
+use DateTime;
+my $mjd_start = DateTime->now->mjd;   # MJD of starting script
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Config 1.01 qw( :standard );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $receivetool = can_run('receivetool') or (warn "Can't find receivetool" and $missing_tools = 1);
+my $dsproductls = can_run('dsproductls') or (warn "Can't find dsproductls" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+# Parse the command-line arguments
+my ( $source_id, $source, $product, $last_fileset, $dbname, $verbose, $no_update );
+
+GetOptions(
+           'source_id=s'       => \$source_id, # Source identifier
+           'source=s'          => \$source, # Source for data
+           'product=s'         => \$product, # Product for data
+           'last_fileset=s'    => \$last_fileset, # Last fileset seen
+           'dbname=s'          => \$dbname,    # Database name
+           'verbose'           => \$verbose,   # Print to stdout
+           'no-update'         => \$no_update, # Don't update the database?
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --source_id --source --product",
+           -exitval => $PS_EXIT_CONFIG_ERROR) unless
+    defined $source_id and
+    defined $source and
+    defined $product;
+
+# Get list of filesets
+my $uri = "$source/$product"; # URI for datastore product
+$uri .= "/index.txt" unless $uri =~ m|/index.txt$|;
+my $command = "dsproductls --uri $uri"; # Command to execute
+$command .= " --last_fileset $last_fileset" if defined $last_fileset;
+
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    run(command => $command, verbose => $verbose);
+die "Unable to get source listing from $source for $product\n" unless $success;
+
+# Parse list of filesets
+my @filesets = ();              # Filesets to add
+my @lines = split(/\n/, join "", @$stdout_buf); # Lines from output
+foreach my $line ( @lines ) {
+    $line =~ s/\#.*//g;
+    next unless $line =~ /\S+/;
+    $line =~ s/^\s+//;
+    my @fields = split(/\s+/, $line); # Fields in line
+    my $fileset = $fields[1];
+    push @filesets, $fileset if defined $fileset and $fileset =~ /\S+/;
+}
+
+if (scalar @filesets > 0) {
+    # Add filesets
+    {
+        my $command = "receivetool -addfileset"; # Command to execute
+        $command .= " -src_id $source_id";
+        $command .= " -fileset " . join(' -fileset ', @filesets);
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        unless (defined $no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            die "Unable to add filesets from $source for $product\n" unless $success;
+        }
+    }
+
+    # Update the last fileset seen
+    {
+        my $command = "receivetool -updatelast"; # Command to execute
+        $command .= " -src_id $source_id";
+        my $last = pop @filesets; # Last fileset
+        $command .= " -fileset $last";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        unless (defined $no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            die "Unable to update last fileset to $last\n" unless $success;
+        }
+    }
+}
+
+
+__END__
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/register_exp.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/register_exp.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/register_exp.pl	(revision 24244)
@@ -191,5 +191,5 @@
     carp($msg);
     if (defined $exp_id and not $no_update) {
-        my $command = "$regtool -addprocessedexp -exp_id $exp_id -code $exit_code";
+        my $command = "$regtool -addprocessedexp -exp_id $exp_id -fault $exit_code";
         $command .= " -hostname $host" if defined $host;
         $command .= " -dbname $dbname" if defined $dbname;
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/register_imfile.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/register_imfile.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/register_imfile.pl	(revision 24244)
@@ -226,5 +226,5 @@
         $command .= " -inst UNKNOWN";
         $command .= " -class_id $tmp_class_id";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -hostname $host" if defined $host;
         $command .= " -dbname $dbname" if defined $dbname;
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/stack_skycell.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/stack_skycell.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/stack_skycell.pl	(revision 24244)
@@ -212,4 +212,7 @@
     $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";
     $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF";
+    $command .= " -F SOURCE.PLOT.MOMENTS SOURCE.PLOT.SKY.MOMENTS";
+    $command .= " -F SOURCE.PLOT.PSFMODEL SOURCE.PLOT.SKY.PSFMODEL";
+    $command .= " -F SOURCE.PLOT.APRESID SOURCE.PLOT.SKY.APRESID";
     $command .= " -photometry";
     $command .= " -threads $threads" if defined $threads;
@@ -232,11 +235,6 @@
         &my_die("Unable to perform ppStack: $error_code", $stack_id, $error_code);
     }
-    &my_die("Couldn't find expected output file: $outputName", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
-    &my_die("Couldn't find expected output file: $outputMask", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
-    &my_die("Couldn't find expected output file: $outputWeight", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
-    &my_die("Couldn't find expected output file: $outputSources", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
-#   &my_die("Couldn't find expected output file: $bin1Name",    $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name);
-#   &my_die("Couldn't find expected output file: $bin2Name",    $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name);
-
+
+    my $quality;                # Quality flag
     if ($do_stats) {
         my $outputStatsReal = $ipprc->file_resolve($outputStats);
@@ -255,5 +253,17 @@
         }
         chomp $cmdflags;
-    }
+
+        ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
+    }
+
+    if (!$quality) {
+        &my_die("Couldn't find expected output file: $outputName", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputName);
+        &my_die("Couldn't find expected output file: $outputMask", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
+        &my_die("Couldn't find expected output file: $outputWeight", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputWeight);
+        &my_die("Couldn't find expected output file: $outputSources", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputSources);
+#       &my_die("Couldn't find expected output file: $bin1Name",    $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin1Name);
+#       &my_die("Couldn't find expected output file: $bin2Name",    $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($bin2Name);
+    }
+
 }
 
@@ -306,5 +316,5 @@
 
     if (defined $stack_id and not $no_update) {
-        my $command = "$stacktool -stack_id $stack_id -code $exit_code";
+        my $command = "$stacktool -stack_id $stack_id -fault $exit_code";
         if ($run_state eq 'new') {
             $command .= " -addsumskyfile";
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/summit_copy.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/summit_copy.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/summit_copy.pl	(revision 24244)
@@ -148,5 +148,5 @@
         $command .= " -class_id $class_id";
         $command .= " -uri $uri";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/warp_overlap.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/warp_overlap.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/warp_overlap.pl	(revision 24244)
@@ -193,5 +193,5 @@
         my $command = "$warptool -addoverlap";
         $command .= " -warp_id $warp_id";
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);
Index: /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/warp_skycell.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/warp_skycell.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/warp_skycell.pl	(revision 24244)
@@ -57,5 +57,5 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
-    -msg => "Required options: --warp_id --skycell_id --tess_dir --camera --outroot --run-state",
+    -msg => "Required options: --warp_id --warp_skyfile_id --skycell_id --tess_dir --camera --outroot --run-state",
     -exitval => 3,
 ) unless defined $warp_id
@@ -81,5 +81,8 @@
 {
     # XXX change -tess_id to -tess_dir when db schema is updated
-    my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id -tess_id $tess_dir";
+    my $command = "$warptool -scmap";
+    $command .= " -warp_id $warp_id";
+    $command .= " -skycell_id $skycell_id";
+    # $command .= " -tess_id $tess_dir";  XXX I don't think this is necessary or useful
     $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -175,5 +178,4 @@
 # Run pswarp
 my $cmdflags;
-my $accept = 1;                 # Accept the skycell?
 my $do_stats;
 unless ($no_op) {
@@ -200,5 +202,7 @@
         $do_stats = 1;
     } else {
-        $command .= " -ipprc $configuration";
+        #$command .= " -ipprc $configuration";
+        my $resolved = $ipprc->file_resolve($configuration);
+        $command .= " -ipprc $resolved";
     }
     if ($do_stats) {
@@ -215,7 +219,8 @@
 
     if ($do_stats) {
-        # Check first for the stats file, and if the ACCEPT flag is set.
+        # Check first for the stats file
         my $outputStatsReal = $ipprc->file_resolve($outputStats);
         &my_die("Couldn't find expected output file: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal;
+        &my_die("Stats file has zero size: $outputStats", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless -s $outputStatsReal;
 
         # measure chip stats
@@ -232,5 +237,7 @@
         chomp $cmdflags;
 
-        if ($cmdflags =~ /-accept/) {
+        my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
+
+        if (!$quality) {
             &my_die("Couldn't find expected output file: $outputImage", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputImage);
             &my_die("Couldn't find expected output file: $outputMask", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputMask);
@@ -249,7 +256,7 @@
             $command .= " -magicked" if $magicked;
 
-            $command .= " -uri $outputImage" if $accept;
+            $command .= " -uri $outputImage" if !$quality;
             $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
-            $command .= " $cmdflags" if $accept;
+            $command .= " $cmdflags";
             $command .= " -hostname $host"   if defined $host;
             $command .= " -dbname $dbname"   if defined $dbname;
@@ -301,4 +308,5 @@
             $command .= " -path_base $outroot";
             $command .= " -hostname $host" if defined $host;
+            $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
         } else {
             $command .= " -updateskyfile";
@@ -306,6 +314,5 @@
         $command .= " -warp_id $warp_id";
         $command .= " -skycell_id $skycell_id";
-        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
-        $command .= " -code $exit_code";
+        $command .= " -fault $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
         run(command => $command, verbose => $verbose);
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/Makefile.am	(revision 24244)
@@ -21,5 +21,7 @@
 	replicate.pro \
 	dist.pro \
-	pstamp.pro
+	rcserver.pro \
+	pstamp.pro \
+	receive.pro
 
 other_files = \
@@ -32,4 +34,5 @@
 	simtest.basic.config \
 	simtest.basic.auto \
+	simtest.nebulous.basic.auto \
 	simtest.detverify.config \
 	simtest.detverify.auto \
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/automate.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/automate.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/automate.pro	(revision 24244)
@@ -35,4 +35,7 @@
   queuesubstr tmp @DBNAME@ $3
   queuesubstr tmp @CWD@ $cwd
+
+  $username = `whoami`
+  queuesubstr tmp @USER@ $username
 
   ipptool2book tmp automate -key name -uniq -setword pantaskState INIT.BLOCK
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/camera.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/camera.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/camera.pro	(revision 24244)
@@ -73,4 +73,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = camtool -pendingexp
     if ($DB:n == 0)
@@ -84,4 +85,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -216,4 +218,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = camtool -pendingcleanuprun
     if ($DB:n == 0)
@@ -227,4 +230,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/chip.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/chip.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/chip.pro	(revision 24244)
@@ -82,4 +82,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = chiptool -pendingimfile
     if ($DB:n == 0)
@@ -93,4 +94,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -202,7 +204,11 @@
   # locked list
   task.exit    crash
-    showcommand crash
-    echo "hostname: $JOB_HOSTNAME"
-    book setword chipPendingImfile $options:0 pantaskState CRASH
+    ### Getting a lot of chip crashes (no idea why), so remove verbosity for now
+    #showcommand crash
+    #echo "hostname: $JOB_HOSTNAME"
+
+    # Set a fault code in the database
+    exec chiptool -addprocessedimfile -dbname $DBNAME -chip_id $CHIP_ID -class_id $CLASS_ID -fault $EXIT_CRASH_ERR
+    process_exit chipPendingImfile $options:0 $EXIT_CRASH_ERR
   end
 
@@ -223,6 +229,7 @@
 
   periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
+#  periods      -exec $LOADEXEC
+  periods      -exec 30
+  periods      -timeout 60
   npending     1
 
@@ -231,4 +238,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = chiptool -advanceexp -limit 10
     if ($DB:n == 0)
@@ -242,4 +250,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -283,4 +292,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = chiptool -pendingcleanuprun
     if ($DB:n == 0)
@@ -294,4 +304,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/diff.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/diff.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/diff.pro	(revision 24244)
@@ -81,4 +81,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = difftool -todiffskyfile
     if ($DB:n == 0)
@@ -92,4 +93,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -144,4 +146,5 @@
     book getword diffSkyfile $pageName skycell_id -var SKYCELL_ID
     book getword diffSkyfile $pageName camera -var CAMERA
+    book getword diffSkyfile $pageName bothways -var BOTHWAYS
     book getword diffSkyfile $pageName workdir -var WORKDIR_TEMPLATE
     book getword diffSkyfile $pageName dbname -var DBNAME
@@ -162,4 +165,7 @@
 
     $run = diff_skycell.pl --threads @MAX_THREADS@ --diff_id $DIFF_ID --skycell_id $SKYCELL_ID --diff_skyfile_id $DIFF_SKYFILE_ID --outroot $outroot --redirect-output
+    if ("$BOTHWAYS" == "T")
+       $run = $run --inverse
+    end
     add_standard_args run
 
@@ -209,4 +215,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = difftool -pendingcleanuprun
     if ($DB:n == 0)
@@ -220,4 +227,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -226,5 +234,5 @@
   task.exit    0
     # convert 'stdout' to book format
-    ipptool2book stdout diffCleanup -key diff_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    ipptool2book stdout diffCleanup -key diff_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     if ($VERBOSE > 2)
       book listbook diffCleanup
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/dist.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/dist.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/dist.pro	(revision 24244)
@@ -14,4 +14,5 @@
 $distToProcess_DB = 0
 $distToAdvance_DB = 0
+$distQueue_DB = 0
 
 ### Check status of tasks
@@ -41,4 +42,9 @@
     active true
   end
+  task dist.queueruns
+#   We aren't ready to run this task yet. It will queue much too much 
+#    active true
+    active false
+  end
 end
 macro dist.off
@@ -53,4 +59,7 @@
   end
   task dist.advance.run
+    active false
+  end
+  task dist.queueruns
     active false
   end
@@ -80,4 +89,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -131,17 +141,24 @@
     book getword distToProcess $pageName state -var STATE
     book getword distToProcess $pageName data_state -var DATA_STATE
+    book getword distToProcess $pageName quality -var QUALITY
+    book getword distToProcess $pageName no_magic -var NO_MAGIC
     book getword distToProcess $pageName magicked -var MAGICKED
-    book getword distToProcess $pageName outroot -var OUTROOT
+    book getword distToProcess $pageName outdir -var OUTDIR
     book getword distToProcess $pageName dbname -var DBNAME
-    if ("$CLEAN" = "T")
-        $CLEAN_ARG = "--clean"
-    else
-        $CLEAN_ARG = ""
+
+    $EXTRA_ARGS = ""
+    if ("$CLEAN" == "T")
+        $EXTRA_ARGS = --clean
     end
     # magicked is output as integer due to the union in the sql
     if ($MAGICKED)
-        $MAGICKED_ARG = "--magicked"
-    else
-        $MAGICKED_ARG = ""
+        $EXTRA_ARGS = $EXTRA_ARGS --magicked
+    end
+    # no_magic is output as integer due to the union in the sql
+    if ($NO_MAGIC)
+        $EXTRA_ARGS = $EXTRA_ARGS --no_magic
+    end
+    if ($QUALITY)
+        $EXTRA_ARGS = $EXTRA_ARGS --poor_quality
     end
 
@@ -150,7 +167,9 @@
     host anyhost
 
-    sprintf logfile "%s/dist.%s.%s.log" $OUTROOT $DIST_ID $COMPONENT
-
-    $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $MAGICKED_ARG $CLEAN_ARG --outroot $OUTROOT --logfile $logfile
+    sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT
+    stdout $logfile
+    stderr $logfile
+
+    $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile
 
     add_standard_args run
@@ -202,4 +221,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -246,11 +266,19 @@
     book getword distToAdvance $pageName stage   -var STAGE
     book getword distToAdvance $pageName stage_id -var STAGE_ID
-    book getword distToAdvance $pageName outroot -var OUTROOT
+    book getword distToAdvance $pageName outdir -var OUTDIR
+    book getword distToAdvance $pageName clean -var CLEAN
+    book getword distToAdvance $pageName dbname -var DBNAME
+    $EXTRA_ARGS = ""
+    if ("$CLEAN" == "T")
+        $EXTRA_ARGS = --clean
+    end
 
     host anyhost
 
-    sprintf logfile "%s/dist.advance.%s.log" $OUTROOT $DIST_ID
-
-    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outroot $OUTROOT --logfile $logfile
+    sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
+    stdout $logfile
+    stderr $logfile
+
+    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile
     add_standard_args run
 
@@ -277,2 +305,63 @@
 end
 
+task	       dist.queueruns
+#  host         local
+
+  periods      -poll $RUNPOLL
+  periods      -exec 30
+  periods      -timeout 45
+  npending     1
+
+  active false
+
+#  stdout $LOGDIR/dist.queuruns
+#  stderr $LOGDIR/dist.queueruns
+
+  task.exec
+    $MYARGS = ""
+    # assume that we need magic unless we are running simtest
+    if ($?SIMTEST_CAMERA != 0)
+        $MYARGS = --no_magic
+    end
+
+    $run = dist_queue_runs.pl $MYARGS --stage_limit 16 --logfile $LOGDIR/dist.queueruns
+
+    if ($DB:n == 0)
+       $DBNAME = DEFAULT
+    else
+      # save the DB name for add_standard_args
+      $DBNAME =  $DB:$distQueue_DB
+      $distQueue_DB ++
+      if ($distQueue_DB >= $DB:n) set distQueue_DB = 0
+    end
+
+    host anyhost
+
+    add_standard_args run
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  task.exit     $EXIT_SUCCESS
+    # nothing to do
+  end
+
+  # default exit status
+  task.exit    default
+    showcommand failure
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+  end
+
+  # operation timed out?
+  task.exit    crash
+    showcommand crash
+  end
+end
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/fake.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/fake.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/fake.pro	(revision 24244)
@@ -79,4 +79,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = faketool -pendingimfile
     if ($DB:n == 0)
@@ -90,4 +91,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -123,6 +125,12 @@
 
   task.exec
+    # fake is fast, so generate a bunch of commands in a row
     book npages fakePendingImfile -var N
-    if ($N == 0) break
+    if ($N == 0)
+       periods -exec $RUNEXEC
+       break
+    end
+    periods -exec 0.05
+
     if ($NETWORK == 0) break
     
@@ -198,5 +206,6 @@
   periods      -poll $LOADPOLL
   periods      -exec $LOADEXEC
-  periods      -timeout 30
+  periods      -exec 30
+  periods      -timeout 60
   npending     1
 
@@ -205,4 +214,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = faketool -advanceexp -limit 10
     if ($DB:n == 0)
@@ -216,4 +226,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -257,4 +268,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = faketool -pendingcleanuprun
     if ($DB:n == 0)
@@ -269,4 +281,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/flatcorr.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/flatcorr.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/flatcorr.pro	(revision 24244)
@@ -72,6 +72,6 @@
 
   # check the list of pending flatcorr runs
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
+  periods      -poll 5
+  periods      -exec 30
   periods      -timeout 60
   npending 1
@@ -137,5 +137,6 @@
       if ($flatcorr_pendingprocess_DB >= $DB:n) set flatcorr_pendingprocess_DB = 0
     end
-    # add_poll_args run
+    add_poll_args run
+    add_poll_labels run
     command $run
   end
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/magic.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/magic.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/magic.pro	(revision 24244)
@@ -98,4 +98,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -157,8 +158,4 @@
     $WORKDIR = $WORKDIR_TEMPLATE
 
-    basename $TESS_DIR -var TESS_ID
-
-    $TESS_ID = "FIXNS"
-
     sprintf outroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
 
@@ -228,4 +225,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -333,5 +331,5 @@
 
   periods      -poll $LOADPOLL
-  periods      -exec 30
+  periods      -exec $LOADEXEC
   periods      -timeout 20
   npending     1
@@ -351,6 +349,6 @@
       if ($magicToDS_DB >= $DB:n) set magicToDS_DB = 0
     end
-#    XXX magicDSRun doesn't have label so can't use add_poll_args
-#    add_poll_args run
+    add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -407,4 +405,5 @@
     book getword magicToDS $pageName uri -var URI
     book getword magicToDS $pageName path_base -var PATH_BASE
+    book getword magicToDS $pageName inverse -var INVERSE
     book getword magicToDS $pageName cam_path_base -var CAM_PATH_BASE
     book getword magicToDS $pageName outroot -var OUTROOT
@@ -421,4 +420,8 @@
 
     $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --streaks $STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --remove $REMOVE 
+    if ("$INVERSE" == "T")
+       # Inverse subtraction
+       $run = $run --inverse
+    end
 
     add_standard_args run
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/pantasks.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/pantasks.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/pantasks.pro	(revision 24244)
@@ -6,5 +6,5 @@
 $PARALLEL = 1
 $VERBOSE = 1
-$LABEL = "NONE"
+$LABEL:n = 0
 $POLLLIMIT = 32
 $LOGDIR = `pwd`
@@ -14,7 +14,8 @@
 
 $LOADPOLL = 1.0
-$LOADEXEC = 5
-$RUNPOLL = 0.5
-$RUNEXEC = 1
+$LOADEXEC = 5.0
+
+$RUNPOLL  = 0.5
+$RUNEXEC  = 1.0
 
 $EXIT_SUCCESS     = 0
@@ -24,4 +25,5 @@
 $EXIT_PROG_ERR    = 4
 $EXIT_DATA_ERR    = 5
+$EXIT_CRASH_ERR   = 256
 
 # DB lists the database names in use; the default ipprc database
@@ -33,5 +35,5 @@
 $workdir_template = `pwd`
 
-# user function to add databases by name (avoids duplicates)
+# user functions to manage databases
 macro add.database
   if ($0 != 2)
@@ -44,4 +46,5 @@
   end
 
+  local found
   $found = 0
   for i 0 $DB:n
@@ -68,4 +71,78 @@
 
   list DB -del $1
+end
+
+macro show.databases
+  if ($0 != 1)
+    echo "USAGE: show.databases"
+    break
+  end
+  if ($?DB:n == 0)
+    echo "no databases defined"
+  end
+  if ($DB:n == 0)
+    echo "no databases defined"
+  end
+
+  local i
+  for i 0 $DB:n
+    echo $DB:$i
+  end
+end
+
+# user functions to manipulate labels
+macro add.label
+  if ($0 != 2)
+    echo "USAGE: add.label (label)"
+    break
+  end
+  if ($?LABEL:n == 0)
+    list LABEL -add $1
+    return
+  end
+
+  local found
+  $found = 0
+  for i 0 $LABEL:n
+    if ($LABEL:$i == $1) 
+      $found = 1
+      echo "$LABEL:$i set"
+      last
+    end
+  end
+  
+  if ($found == 0)
+    list LABEL -add $1
+  end
+end
+
+macro del.label
+  if ($0 != 2)
+    echo "USAGE: del.label (label)"
+    break
+  end
+  if ($?LABEL:n == 0)
+    return
+  end
+
+  list LABEL -del $1
+end
+
+macro show.labels
+  if ($0 != 1)
+    echo "USAGE: show.labels"
+    break
+  end
+  if ($?LABEL:n == 0)
+    echo "no labels defined"
+  end
+  if ($LABEL:n == 0)
+    echo "no labels defined"
+  end
+
+  local i
+  for i 0 $LABEL:n
+    echo $LABEL:$i
+  end
 end
 
@@ -198,10 +275,23 @@
     end
 
-    local command
+    local command i
     $command = $$1 -limit $POLLLIMIT
 
+    $$1 = $command
+end
+
+macro add_poll_labels
+    if ($0 != 2)
+	echo "Must pass in the command of interest, and this function will supplement"
+	stop
+    end
+
+    local command i
+
+    $command = $$1
+
     # Only process the data with the specified label.
-    if ($?LABEL && $LABEL != "NONE")
-	$command = $command -label $LABEL
+    for i 0 $LABEL:n
+      $command = $command -label $LABEL:$i
     end
 
@@ -302,4 +392,12 @@
   end
 
+  # failure related to pantasks
+  if ($exitCode == $EXIT_CRASH_ERR)
+    # stop
+    showcommand "crash error"
+    book setword $bookName $pageName pantaskState CRASH_ERR
+    return
+  end
+
   # any other exit status
   showcommand failure
@@ -322,5 +420,19 @@
     book delpage $1 -key pantaskState CONFIG_ERR
     book delpage $1 -key pantaskState UNKNOWN_ERR
-  end
+    book delpage $1 -key pantaskState CRASH_ERR
+  end
+end
+
+macro set.poll
+  if ($0 != 2)
+    echo "USAGE:set.poll (value)"
+    break
+  end
+ 
+  $POLLLIMIT = $1
+end
+
+macro get.poll
+  echo "poll limit: $POLLLIMIT"
 end
 
@@ -331,4 +443,8 @@
   end
   $VERBOSE = $1
+end
+
+macro get.verbosity
+  echo "verbosity: $VERBOSE"
 end
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/rcserver.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/rcserver.pro	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/rcserver.pro	(revision 24244)
@@ -0,0 +1,273 @@
+## rcserver.pro : tasks for managing the server side of the ipp data distribution system : -*- sh -*-
+
+# test for required global variables
+check.globals
+
+#$LOGSUBDIR = $LOGDIR/rcserver
+#exec mkdir -p $LOGSUBDIR
+
+### Initialise the books containing the tasks to do
+book init rcPendingFS
+book init rcPendingDest
+
+### Database lists
+$rcPendingFS_DB = 0
+$rcPendingDest_DB = 0
+
+### Check status of tasks
+macro rcserver.status
+  book listbook rcPendingFS
+  book listbook rcPendingDest
+end
+
+### Reset tasks
+macro rcserver.reset
+  book init rcPendingFS
+  book init rcPendingDest
+end
+
+### Turn tasks on
+macro rcserver.on
+  task rcserver.makefileset.load
+    active true
+  end
+  task rcserver.makefileset.run
+    active true
+  end
+  task rcserver.checkstatus.load
+    active true
+  end
+  task rcserver.checkstatus.run
+    active true
+  end
+end
+macro rcserver.off
+  task rcserver.makefileset.load
+    active false
+  end
+  task rcserver.makefileset.run
+    active false
+  end
+  task rcserver.checkstatus.load
+    active false
+  end
+  task rcserver.checkstatus.run
+    active false
+  end
+end
+
+task	       rcserver.makefileset.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/rcserver.makefileset.load.log
+
+  task.exec
+    $run = disttool -pendingfileset
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$rcPendingFS_DB
+      $run = $run -dbname $DB:$rcPendingFS_DB
+      $rcPendingFS_DB ++
+      if ($rcPendingFS_DB >= $DB:n) set rcPendingFS_DB = 0
+    end
+    add_poll_args run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout rcPendingFS -key dist_id:prod_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook rcPendingFS
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup rcPendingFS
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       rcserver.makefileset.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    book npages rcPendingFS -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new components to process (pantaskState == INIT)
+    book getpage rcPendingFS 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book getword rcPendingFS $pageName dist_id -var DIST_ID
+    book getword rcPendingFS $pageName target_id -var TARGET_ID
+    book getword rcPendingFS $pageName stage -var STAGE
+    book getword rcPendingFS $pageName stage_id -var STAGE_ID
+    book getword rcPendingFS $pageName dist_dir -var DIST_DIR
+#    book getword rcPendingFS $pageName clean -var CLEAN
+    book getword rcPendingFS $pageName prod_id -var PROD_ID
+    book getword rcPendingFS $pageName product_name -var PRODUCT_NAME
+    book getword rcPendingFS $pageName ds_dbhost -var DS_DBHOST
+    book getword rcPendingFS $pageName ds_dbname -var DS_DBNAME
+    book getword rcPendingFS $pageName dbname -var DBNAME
+
+#    set.host.for.camera $CAMERA $MAGIC_ID
+#    set.workdir.by.camera $CAMERA $MAGIC_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+    host anyhost
+
+    sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $PROD_ID
+    stdout $logfile
+    stderr $logfile
+
+    book setword rcPendingFS $pageName pantaskState RUN
+
+    $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --prod_id $PROD_ID --product_name $PRODUCT_NAME  --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR
+
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit rcPendingFS $options:0 $JOB_STATUS
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword rcPendingFS $options:0 pantaskState TIMEOUT
+  end
+end
+
+
+task	       rcserver.checkstatus.load
+  host         local
+
+  periods      -poll $LOADPOLL
+#  periods      -exec $LOADEXEC
+  periods      -exec 20
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/rcserver.checkstatus.load.log
+
+  task.exec
+    $run = disttool -pendingdest
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$rcPendingDest_DB
+      $run = $run -dbname $DB:$rcPendingDest_DB
+      $rcPendingDest_DB ++
+      if ($rcPendingDest_DB >= $DB:n) set rcPendingDest_DB = 0
+    end
+    add_poll_args run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout rcPendingDest -key dest_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook rcPendingDest
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup rcPendingDest
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       rcserver.checkstatus.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  periods      -exec 20
+
+  task.exec
+    book npages rcPendingDest -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    # look for new components to process (pantaskState == INIT)
+    book getpage rcPendingDest 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    stdout NULL
+    stderr $LOGDIR/rcserver.checkstatus.run.log
+
+    book setword rcPendingDest $pageName pantaskState RUN
+    book getword rcPendingDest $pageName dest_id -var DEST_ID
+    book getword rcPendingDest $pageName prod_id -var PROD_ID
+    book getword rcPendingDest $pageName status_uri -var STATUS_URI
+    book getword rcPendingDest $pageName last_fileset -var LAST_FILESET
+    book getword rcPendingDest $pageName dbname -var DBNAME
+
+    host anyhost
+
+    $run = rcserver_checkstatus.pl --dest_id $DEST_ID --prod_id $PROD_ID --status_uri $STATUS_URI --last_fileset $LAST_FILESET
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit rcPendingDest $options:0 $JOB_STATUS
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword rcPendingDest $options:0 pantaskState TIMEOUT
+  end
+end
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/receive.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/receive.pro	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/receive.pro	(revision 24244)
@@ -0,0 +1,558 @@
+## receive.pro: -*- sh -*-
+
+### receivetool -definesource: add new receive source (run by user for setup)
+### receivetool -list: get list of sources (for pantasks setup)
+### 
+### receive_source.pl: launched regularly by pantasks; check source for new filesets
+### dsproductls: get available filesets from datastore
+### receivetool -addfileset: add new fileset
+### receivetool -updatelast: update last fileset for source
+### 
+### receivetool -pendingfileset: get pending fileset (for pantasks)
+### receive_fileset.pl: launched by pantasks; get list of files
+### dsfilesetls: get files for fileset
+### receivetool -addfiles: add file to receive
+### 
+### receivetool -pendingfile: get pending file (for pantasks)
+### receive_file.pl: launched by pantasks; download file, extract fileset, import
+### dsget: get file from datastore
+### receivetool -addresult: add result
+### 
+### receivetool -revert: remove result after something goes wrong
+###
+### receivetool -toadvance: filesets for which all files have been downloaded and are ready
+### to complete processing
+
+# test for required global variables
+check.globals
+
+book init receiveSource
+book init receiveFileset
+book init receiveFile
+book init receiveAdvance
+
+macro receive.status
+  book listbook receiveSource
+  book listbook receiveFileset
+  book listbook receiveFile
+  book listbook receiveAdvance
+end
+
+macro receive.reset
+  book init receiveSource
+  book init receiveFileset
+  book init receiveFile
+  book init receiveAdvance
+end
+
+macro receive.on
+  task receive.source.load
+    active true
+  end
+  task receive.source.run
+    active true
+  end
+  task receive.fileset.load
+    active true
+  end
+  task receive.fileset.run
+    active true
+  end
+  task receive.file.load
+    active true
+  end
+  task receive.file.run
+    active true
+  end
+  task receive.advance.load
+    active true
+  end
+  task receive.advance.run
+    active true
+  end
+end
+
+macro receive.off
+  task receive.source.load
+    active false
+  end
+  task receive.source.run
+    active false
+  end
+  task receive.fileset.load
+    active false
+  end
+  task receive.fileset.run
+    active false
+  end
+  task receive.file.load
+    active false
+  end
+  task receive.file.run
+    active false
+  end
+  task receive.advance.load
+    active false
+  end
+  task receive.advance.run
+    active false
+  end
+end
+
+
+# this variable will cycle through the known database names
+$receive_DB = 0
+$receive_Advance_DB = 0
+
+task	       receive.source.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/receive.source.log
+
+  task.exec
+    $run = receivetool -list
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$receive_DB
+      $run = $run -dbname $DB:$receive_DB
+      $receive_DB ++
+      if ($receive_DB >= $DB:n) set receive_DB = 0
+    end
+    add_poll_args run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout receiveSource -key source_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook receiveSource
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup receiveSource
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       receive.source.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    book npages receiveSource -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    book getpage receiveSource 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword receiveSource $pageName pantaskState RUN
+    book getword receiveSource $pageName source_id -var SOURCE_ID
+    book getword receiveSource $pageName source -var SOURCE
+    book getword receiveSource $pageName product -var PRODUCT
+    book getword receiveSource $pageName last_fileset -var FILESET
+    book getword receiveSource $pageName dbname -var DBNAME
+    book getword receiveSource $pageName state -var RUN_STATE
+
+    stdout $LOGDIR/receive.source.log
+    stderr $LOGDIR/receive.source.log
+
+    $run = receive_source.pl --source_id $SOURCE_ID --source $SOURCE --product $PRODUCT
+    if ("$FILESET" != "NULL")
+      $run = $run --last_fileset $FILESET
+    end
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit receiveSource $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword receiveSource $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword receiveSource $options:0 pantaskState TIMEOUT
+  end
+end
+
+
+task	       receive.fileset.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/receive.fileset.log
+
+  task.exec
+    $run = receivetool -pendingfileset
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$receive_DB
+      $run = $run -dbname $DB:$receive_DB
+      $receive_DB ++
+      if ($receive_DB >= $DB:n) set receive_DB = 0
+    end
+    add_poll_args run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout receiveFileset -key fileset_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook receiveFileset
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup receiveFileset
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       receive.fileset.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    book npages receiveFileset -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    book getpage receiveFileset 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword receiveFileset $pageName pantaskState RUN
+    book getword receiveFileset $pageName fileset_id -var FILESET_ID
+    book getword receiveFileset $pageName source -var SOURCE
+    book getword receiveFileset $pageName product -var PRODUCT
+    book getword receiveFileset $pageName fileset -var FILESET
+    book getword receiveFileset $pageName dbname -var DBNAME
+    book getword receiveFileset $pageName state -var RUN_STATE
+
+    stdout $LOGDIR/receive.fileset.log
+    stderr $LOGDIR/receive.fileset.log
+
+    host anyhost
+
+    $run = receive_fileset.pl --fileset_id $FILESET_ID --source $SOURCE --product $PRODUCT --fileset $FILESET
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit receiveFileset $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword receiveFileset $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword receiveFileset $options:0 pantaskState TIMEOUT
+  end
+end
+
+
+task	       receive.file.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/receive.file.log
+
+  task.exec
+    $run = receivetool -pendingfile
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$receive_DB
+      $run = $run -dbname $DB:$receive_DB
+      $receive_DB ++
+      if ($receive_DB >= $DB:n) set receive_DB = 0
+    end
+    add_poll_args run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout receiveFile -key file_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook receiveFile
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup receiveFile
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       receive.file.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 600
+
+  task.exec
+    book npages receiveFile -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    book getpage receiveFile 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword receiveFile $pageName pantaskState RUN
+    book getword receiveFile $pageName file_id -var FILE_ID
+    book getword receiveFile $pageName source -var SOURCE
+    book getword receiveFile $pageName product -var PRODUCT
+    book getword receiveFile $pageName fileset -var FILESET
+    book getword receiveFile $pageName fileset_id -var FILESET_ID
+    book getword receiveFile $pageName file -var FILE
+    book getword receiveFile $pageName bytes -var BYTES
+    book getword receiveFile $pageName md5sum -var MD5SUM
+    book getword receiveFile $pageName component -var COMPONENT
+    book getword receiveFile $pageName workdir -var WORKDIR
+    book getword receiveFile $pageName dirinfo -var DIRINFO
+    book getword receiveFile $pageName dbname -var DBNAME
+    book getword receiveFile $pageName state -var RUN_STATE
+
+    stdout $LOGDIR/receive.file.log
+    stderr $LOGDIR/receive.file.log
+
+    host anyhost
+
+    $run = receive_file.pl --file_id $FILE_ID --source $SOURCE --product $PRODUCT --fileset $FILESET --fileset_id $FILESET_ID --file $FILE --component $COMPONENT --bytes $BYTES --md5 $MD5SUM --workdir $WORKDIR --dirinfo $DIRINFO
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit receiveFile $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword receiveFile $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword receiveFile $options:0 pantaskState TIMEOUT
+  end
+end
+
+task	       receive.advance.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/receive.advance.log
+
+  task.exec
+    $run = receivetool -toadvance
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$receive_Advance_DB
+      $run = $run -dbname $DB:$receive_Advance_DB
+      $receive_Advance_DB ++
+      if ($receive_Advance_DB >= $DB:n) set receive_Advance_DB = 0
+    end
+    add_poll_args run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout receiveAdvance -key fileset_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook receiveAdvance
+    end
+
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup receiveAdvance
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       receive.advance.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    book npages receiveAdvance -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    
+    book getpage receiveAdvance 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword receiveAdvance $pageName pantaskState RUN
+    book getword receiveAdvance $pageName fileset_id -var FILESET_ID
+    book getword receiveAdvance $pageName fileset -var FILESET
+    book getword receiveAdvance $pageName dbinfo -var DBINFO
+    book getword receiveAdvance $pageName status_product -var STATUS_PRODUCT
+    book getword receiveAdvance $pageName ds_dbname -var DS_DBNAME
+    book getword receiveAdvance $pageName ds_dbhost -var DS_DBHOST
+    book getword receiveAdvance $pageName dbname -var DBNAME
+
+    stdout $LOGDIR/receive.advance.log
+    stderr $LOGDIR/receive.advance.log
+
+    $run = receive_advance.pl --fileset_id $FILESET_ID --fileset $FILESET --dbinfo $DBINFO --status_product $STATUS_PRODUCT --ds_dbname $DS_DBNAME --ds_dbhost $DS_DBHOST
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit receiveAdvance $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword receiveAdvance $options:0 pantaskState CRASH
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword receiveAdvance $options:0 pantaskState TIMEOUT
+  end
+end
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/register.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/register.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/register.pro	(revision 24244)
@@ -15,4 +15,5 @@
 
 macro register.status
+  book list
   book listbook regPendingImfile
   book listbook regPendingExp
Index: anches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.auto
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.auto	(revision 24243)
+++ 	(revision )
@@ -1,118 +1,0 @@
-
-automate MULTI
-
-# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK
-# success on CHECK -> LAUNCH
-# failure on BLOCK -> CHECK
-
-# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis
-
-automate METADATA
-  name       STR BIAS
-  check      STR "regtool -processedexp -exp_type BIAS -inst SIMTEST -dbname @DBNAME@"
-  ncheck     S32 20
-  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@"
-  block      STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@"
-END
-
-automate METADATA
-  name       STR DARK
-  check      STR "detselect -search -inst SIMTEST -det_type BIAS -dbname @DBNAME@"
-  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type DARK -select_exp_type DARK -dbname @DBNAME@"
-  block      STR "dettool -runs -active -det_type DARK -dbname @DBNAME@"
-END
- 
-automate METADATA
-  name       STR SHUTTER
-  check      STR "detselect -search -inst SIMTEST -det_type DARK -dbname @DBNAME@"
-  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@"
-  block      STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@"
-END
-
-automate METADATA
-  name       STR FLAT-r
-  check      STR "detselect -search -inst SIMTEST -det_type SHUTTER -dbname @DBNAME@"
-  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type FLAT -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@"
-  block      STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@"
-END
-
-automate METADATA
-  name       STR FLAT-i
-  check      STR "detselect -search -inst SIMTEST -det_type SHUTTER -dbname @DBNAME@"
-  launch     STR "dettool -definebyquery -workdir file://@CWD@/detwork -inst SIMTEST -det_type FLAT -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@"
-  block      STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@"
-END
-
-automate METADATA
-  name       STR OBJECT-i
-  check      STR "detselect -dbname @DBNAME@ -search    -inst SIMTEST -filter i -det_type FLAT"
-  launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst SIMTEST -filter i -label wait -set_label proc"
-  block      STR NONE
-END
-
-automate METADATA
-  name       STR OBJECT-r
-  check      STR "detselect -dbname @DBNAME@ -search    -inst SIMTEST -filter r -det_type FLAT"
-  launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst SIMTEST -filter r -label wait -set_label proc"
-  block      STR NONE
-END
-
-### Stack automation???
-automate METADATA
-  name       STR STACK
-  regular    STR "stacktool -definebyquery -all -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@"
-END
-
-### Diff automation???
-automate METADATA
-  name       STR DIFF
-  regular    STR "difftool -definebyquery -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@"
-END
-
-### Magic automation???
-#automate METADATA
-#  name       STR MAGIC
-#  regular    STR "magictool -definebyquery -workdir file://@CWD@/magic -dbname @DBNAME@"
-#END
-
-
-
-### automate METADATA
-###   name       STR OBJECT-r
-###   check      STR "detselect -search -inst SIMTEST -det_type FLAT -dbname @DBNAME@ -filter r"
-###   launch     STR "chiptool -unblock -label simtest-r -dbname @DBNAME@"
-###   block      STR "chiptool -unmasked -label simtest-r -dbname @DBNAME@"
-### END
-
-### automate METADATA
-###   name       STR OBJECT-i
-###   check      STR "detselect -search -inst SIMTEST -det_type FLAT -dbname @DBNAME@ -filter i"
-###   launch     STR "chiptool -unblock -label simtest-i -dbname @DBNAME@"
-###   block      STR "chiptool -unmasked -label simtest-i -dbname @DBNAME@"
-### END
-
-### there is a weakness in the label / block business: the labels are
-### generic, and the blocks are against those fairly generic words.
-### that makes it difficult to block and unblock science exposures
-### based on different detrend types that are available.  For example,
-### to block the r-band against the absence of the desired r-band
-### flat, we would need to specify a label specific to the r-band
-### images and remove that block when ready.
-
-### a better approach might be to modify the labels rather than the
-### blocks.  we can define, eg, chiptool -set_label -definebyquery to
-### turn assign the label names based on various properties of the
-### interesting images.
-
-### here is what the automate element might look like for such a circumstance:
-
-### automate METADATA
-###   name       STR OBJECT-i
-###   block      STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2"
-###   check      STR "detselect -dbname @DBNAME@ -search -inst SIMTEST -det_type FLAT -dbname @DBNAME@ -filter i -time_begin 2008/1/1 -time_end 2008/1/2" 
-###   launch     STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -dbname @DBNAME@ -time_begin 2008/1/1 -time_end 2008/1/2"
-### END
-
-### XXX still not quite there....
-### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ?
-### 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.basic.auto
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.basic.auto	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.basic.auto	(revision 24244)
@@ -58,14 +58,14 @@
 END
 
-### Stack automation???
+### Stack automation
 automate METADATA
   name       STR STACK
-  regular    STR "stacktool -definebyquery -all -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@"
+  regular    STR "stacktool -definebyquery -all -label proc -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@"
 END
 
-### Diff automation???
+### Diff automation
 automate METADATA
   name       STR DIFF
-  regular    STR "difftool -definebyquery -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@"
+  regular    STR "difftool -definewarpstack -label proc -workdir file://@CWD@/diff -good_frac 0.2 -dbname @DBNAME@"
 END
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.nebulous.basic.auto
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.nebulous.basic.auto	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.nebulous.basic.auto	(revision 24244)
@@ -0,0 +1,116 @@
+
+automate MULTI
+
+# we run in the sequence BLOCK -> CHECK -> LAUNCH -> BLOCK -> CHECK
+# success on CHECK -> LAUNCH
+# failure on BLOCK -> CHECK
+
+# XXX these steps all refer to "@DBNAME@"; they need to be more generic, even for the basic simtest analysis
+
+automate METADATA
+  name       STR BIAS
+  check      STR "regtool -processedexp -exp_type BIAS -inst @CAMERA@ -dbname @DBNAME@"
+  ncheck     S32 20
+  launch     STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type BIAS -select_exp_type BIAS -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type BIAS -dbname @DBNAME@"
+END
+
+automate METADATA
+  name       STR DARK
+  check      STR "detselect -search -inst @CAMERA@ -det_type BIAS -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type DARK -select_exp_type DARK -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type DARK -dbname @DBNAME@"
+END
+ 
+automate METADATA
+  name       STR SHUTTER
+  check      STR "detselect -search -inst @CAMERA@ -det_type DARK -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type SHUTTER -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type SHUTTER -dbname @DBNAME@"
+END
+
+automate METADATA
+  name       STR FLAT-r
+  check      STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type FLAT -filter r -select_exp_type FLAT -select_filter r -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type FLAT -filter r -dbname @DBNAME@"
+END
+
+automate METADATA
+  name       STR FLAT-i
+  check      STR "detselect -search -inst @CAMERA@ -det_type SHUTTER -dbname @DBNAME@"
+  launch     STR "dettool -definebyquery -workdir neb://anyhost/simtest.@USER@/detwork -inst @CAMERA@ -det_type FLAT -filter i -select_exp_type FLAT -select_filter i -dbname @DBNAME@"
+  block      STR "dettool -runs -active -det_type FLAT -filter i -dbname @DBNAME@"
+END
+
+automate METADATA
+  name       STR OBJECT-i
+  check      STR "detselect -dbname @DBNAME@ -search    -inst @CAMERA@ -filter i -det_type FLAT"
+  launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter i -label wait -set_label proc"
+  block      STR NONE
+END
+
+automate METADATA
+  name       STR OBJECT-r
+  check      STR "detselect -dbname @DBNAME@ -search    -inst @CAMERA@ -filter r -det_type FLAT"
+  launch     STR "chiptool  -dbname @DBNAME@ -updaterun -inst @CAMERA@ -filter r -label wait -set_label proc"
+  block      STR NONE
+END
+
+### Stack automation???
+automate METADATA
+  name       STR STACK
+  regular    STR "stacktool -definebyquery -all -label proc -workdir neb://anyhost/simtest.@USER@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@"
+END
+
+### Diff automation???
+automate METADATA
+  name       STR DIFF
+  regular    STR "difftool -definewarpstack -label proc -workdir neb://anyhost/simtest.@USER@/diff -good_frac 0.2 -dbname @DBNAME@"
+END
+
+#### Magic automation???
+#automate METADATA
+#  name       STR MAGIC
+#  regular    STR "magictool -definebyquery -workdir neb://anyhost/simtest.@USER@/magic -good_frac 0.2 -dbname @DBNAME@"
+#END
+
+### automate METADATA
+###   name       STR OBJECT-r
+###   check      STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter r"
+###   launch     STR "chiptool -unblock -label simtest-r -dbname @DBNAME@"
+###   block      STR "chiptool -unmasked -label simtest-r -dbname @DBNAME@"
+### END
+
+### automate METADATA
+###   name       STR OBJECT-i
+###   check      STR "detselect -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i"
+###   launch     STR "chiptool -unblock -label simtest-i -dbname @DBNAME@"
+###   block      STR "chiptool -unmasked -label simtest-i -dbname @DBNAME@"
+### END
+
+### there is a weakness in the label / block business: the labels are
+### generic, and the blocks are against those fairly generic words.
+### that makes it difficult to block and unblock science exposures
+### based on different detrend types that are available.  For example,
+### to block the r-band against the absence of the desired r-band
+### flat, we would need to specify a label specific to the r-band
+### images and remove that block when ready.
+
+### a better approach might be to modify the labels rather than the
+### blocks.  we can define, eg, chiptool -set_label -definebyquery to
+### turn assign the label names based on various properties of the
+### interesting images.
+
+### here is what the automate element might look like for such a circumstance:
+
+### automate METADATA
+###   name       STR OBJECT-i
+###   block      STR "chiptool -dbname @DBNAME@ -unmasked -label wait -filter i -time_begin 2008/1/1 -time_end 2008/1/2"
+###   check      STR "detselect -dbname @DBNAME@ -search -inst @CAMERA@ -det_type FLAT -dbname @DBNAME@ -filter i -time_begin 2008/1/1 -time_end 2008/1/2" 
+###   launch     STR "chiptool -dbname @DBNAME@ -set_label proc -label wait -dbname @DBNAME@ -time_begin 2008/1/1 -time_end 2008/1/2"
+### END
+
+### XXX still not quite there....
+### the 'checks' or maybe the 'launch' needs to include a '-n_exp_min' ?
+### 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.pro	(revision 24244)
@@ -27,4 +27,8 @@
 end
 
+$SIMTEST_RAWDIR = raw
+$SIMTEST_WORKDIR = work
+  
+
 macro simtest
   if ($0 != 4)
@@ -38,5 +42,5 @@
     echo "  SIMTEST_CAMERA   : define an alternate camera (otherwise uses entry in sequence file)"
     echo "  SIMTEST_SEQUENCE : define the set of observations generated (simtest.basic.config)"
-    echo "  SIMTEST_AUTO     : define the analysis steps to perform (simtest.auto)"
+    echo "  SIMTEST_AUTO     : define the analysis steps to perform (simtest.basic.auto)"
     echo "  SIMTEST_THREADS  : set the number of threads for the processing node (0)"
     echo ""
@@ -70,5 +74,5 @@
       # the labels "wait" and "proc" are special names used in automate.pro
     
-      $ppsim = "ppSimSequence $MODULES:0/$SIMTEST_SEQUENCE simtest.mkimages simtest.inject -path raw -workdir work -dbname $dbname -label wait -dvodb DVO -tess_id TESS"
+      $ppsim = "ppSimSequence $MODULES:0/$SIMTEST_SEQUENCE simtest.mkimages simtest.inject -path $SIMTEST_RAWDIR -workdir $SIMTEST_WORKDIR -dbname $dbname -label wait -dvodb DVO -tess_id TESS"
       if ("$PPSIM_RECIPE" != "default") 
          $ppsim = $ppsim -ppsim_recipe $PPSIM_RECIPE
@@ -95,4 +99,6 @@
 
   module.tasks
+
+  add.label proc
 
   if ($SIMTEST_THREADS == 0) 
@@ -121,4 +127,13 @@
 end
 
+macro simtest.setup.nebulous.basic
+  $PPSIM_RECIPE = default
+  $SIMTEST_SEQUENCE = simtest.basic.config
+  $SIMTEST_AUTO = simtest.nebulous.basic.auto
+  $SIMTEST_USER = `whoami`
+  $SIMTEST_RAWDIR = neb://anyhost/simtest.$SIMTEST_USER/raw
+  $SIMTEST_WORKDIR = neb://anyhost/simtest.$SIMTEST_USER/work
+end
+
 macro simtest.setup.detverify
   $PPSIM_RECIPE = default
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.stack.auto
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.stack.auto	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.stack.auto	(revision 24244)
@@ -61,5 +61,5 @@
 automate METADATA
   name       STR STACK
-  regular    STR "stacktool -definebyquery -all -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@"
+  regular    STR "stacktool -definebyquery -all -label proc -workdir file://@CWD@/stack -min_new 4 -min_frac 2 -select_good_frac_min 0.2 -dbname @DBNAME@"
 END
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/stack.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/stack.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/stack.pro	(revision 24244)
@@ -81,4 +81,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = stacktool -tosum
     if ($DB:n == 0)
@@ -92,4 +93,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -209,4 +211,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = stacktool -pendingcleanuprun
     if ($DB:n == 0)
@@ -220,4 +223,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/summit.copy.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/summit.copy.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/summit.copy.pro	(revision 24244)
@@ -501,5 +501,5 @@
         if ($pztoolClearFault_DB >= $DB:n) set pztoolClearFault_DB = 0
       end
-      periods -exec 1800
+      periods -exec 600
     end
 
@@ -595,4 +595,5 @@
         book getword pzPendingAdvance $pageName telescope -var TELESCOPE
         book getword pzPendingAdvance $pageName dbname    -var DBNAME
+        book getword pzPendingAdvance $pageName dateobs    -var DATEOBS
 
         # 2007-08-30T05:09:59Z
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/survey.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/survey.pro	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/survey.pro	(revision 24244)
@@ -0,0 +1,62 @@
+## survey.pro : tasks related to automation of the PS1 survey : -*- sh -*-
+
+# test for required global variables
+check.globals
+
+list DIFF_WW
+  ThreePi.Run2.g.v0
+  ThreePi.Run2.r.v0
+  ThreePi.Run2.i.v0
+end
+
+$DIFF_WW_CNT = 0
+
+task diff.check
+  host local
+ 
+  periods      -poll 10
+  periods      -exec 120
+  periods      -timeout 600
+  npending     1
+
+  stdout $LOGDIR/diffcheck.log
+  stderr $LOGDIR/diffcheck.log
+
+  # generate diff warp-warp runs
+  task.exec
+    if ($DIFF_WW:n == 0) break
+    $run = difftool -dbname gpc1 -definewarpwarp -distance 0.1 -good_frac 0.1
+    $run = $run -input_label $DIFF_WW:$DIFF_WW_CNT
+    $run = $run -template_label $DIFF_WW:$DIFF_WW_CNT
+    $run = $run -label $DIFF_WW:$DIFF_WW_CNT
+    $run = $run -workdir neb://@HOST@.0/gpc1/$DIFF_WW:$DIFF_WW_CNT
+    $run = $run -reduction WARPWARP
+
+    $DIFF_WW_CNT ++
+    if ($DIFF_WW_CNT >= $DIFF_WW:n) 
+      echo "reset DIFF_WW_CNT (was $DIFF_WW_CNT)"
+      set DIFF_WW_CNT = 0
+    end
+
+    command $run
+  end
+
+  # success
+  task.exit    0
+    echo "checked for new diff runs"
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
Index: /branches/cnb_branches/cnb_branch_20090301/ippTasks/warp.pro
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTasks/warp.pro	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTasks/warp.pro	(revision 24244)
@@ -47,4 +47,7 @@
     active true
   end
+  task warp.advancerun
+    active true
+  end
 end
 
@@ -61,4 +64,7 @@
   end
   task warp.skycell.run
+    active false
+  end
+  task warp.advancerun
     active false
   end
@@ -96,4 +102,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = warptool -tooverlap
     if ($DB:n == 0)
@@ -107,4 +114,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -220,4 +228,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = warptool -towarped
     if ($DB:n == 0)
@@ -231,4 +240,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
@@ -339,4 +349,56 @@
 end
 
+# this variable will cycle through the known database names
+$warp_advance_DB = 0
+
+# advance exposures for which all imfiles have completed processing
+# sets the exposure state to full and queues warp processing if requested
+task	       warp.advancerun
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec 30
+  periods      -timeout 60
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/warp.advancerun.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = warptool -advancerun -limit 10
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$warp_advance_DB
+      $run = $run -dbname $DB:$warp_advance_DB
+      $warp_advance_DB ++
+      if ($warp_advance_DB >= $DB:n) set warp_advance_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
 # select images ready for warp analysis
 # new entries are added to warpPendingImfile
@@ -355,4 +417,5 @@
 
   task.exec
+    if ($LABEL:n == 0) break
     $run = warptool -pendingcleanuprun
     if ($DB:n == 0)
@@ -366,4 +429,5 @@
     end
     add_poll_args run
+    add_poll_labels run
     command $run
   end
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/configure.ac	(revision 24244)
@@ -18,5 +18,5 @@
 PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
 PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
-PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.50]) 
+PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.53]) 
 
 PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}"
@@ -53,4 +53,6 @@
 fi
 
+AC_PROG_SED
+
 IPP_STDOPTS
 CFLAGS="${CFLAGS=} -Wall -Werror"
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/Makefile.am	(revision 24244)
@@ -1,2 +1,7 @@
+pxadmin_create_mirror_tables.sql: pxadmin_create_tables.sql
+	-$(RM) pxadmin_create_mirror_tables.sql
+	$(SED) -n -e '/^  *$$/ d' -e 's/--.*//' -e '/./ H' -e '/;/ !b' -e 'x ; s/\n//g ; /^CREATE TABLE receive/ { b print }' -e 's/AUTO_INCREMENT//g ; s/,\{0,1\} *FOREIGN *KEY *([a-zA-Z][a-zA-Z0-9\-\_, ]*) *REFERENCES *[a-zA-Z]\{1,\}([a-zA-Z][a-zA-Z0-9\-\_, ]*)//g' -e ': print' -e 's/( */(/g ; s/ *)/)/g ; s/  */ /g ; p ; s/.*//g ; x'  pxadmin_create_tables.sql > pxadmin_create_mirror_tables.sql
+	echo '-- This comment line is here to avoid empty query error.' >> pxadmin_create_mirror_tables.sql
+
 dist_pkgdata_DATA = \
      camtool_donecleanup.sql \
@@ -10,4 +15,6 @@
      camtool_reset_faulted_runs.sql \
      camtool_revertprocessedexp.sql \
+     camtool_export_run.sql \
+     camtool_export_processed_exp.sql \
      chiptool_change_exp_state.sql \
      chiptool_change_imfile_data_state.sql \
@@ -21,6 +28,7 @@
      chiptool_revertprocessedimfile.sql \
      chiptool_run.sql \
+     chiptool_export_imfile.sql \
+     chiptool_export_processed_imfile.sql \
      chiptool_export_run.sql \
-     chiptool_export_imfiles.sql \
      chiptool_unmasked.sql \
      detselect_search.sql \
@@ -73,10 +81,15 @@
      dettool_tostacked.sql \
      difftool_completed_runs.sql \
-     difftool_definebyquery_part1.sql \
-     difftool_definebyquery_part2.sql \
-     difftool_definebyquery_temp_create.sql \
+     difftool_definewarpstack_part1.sql \
+     difftool_definewarpstack_part2.sql \
+     difftool_definewarpstack_temp_create.sql \
+     difftool_definewarpwarp_temp_create.sql \
+     difftool_definewarpwarp_temp_insert.sql \
      difftool_definewarpwarp_select.sql \
      difftool_definewarpwarp_insert.sql \
      difftool_donecleanup.sql \
+     difftool_export_input_skyfile.sql \
+     difftool_export_run.sql \
+     difftool_export_skyfile.sql \
      difftool_inputskyfile.sql \
      difftool_pendingcleanuprun.sql \
@@ -86,13 +99,28 @@
      difftool_skyfile.sql \
      difftool_todiffskyfile.sql \
+     disttool_definebyquery_camera.sql \
+     disttool_definebyquery_chip.sql \
+     disttool_definebyquery_diff.sql \
+     disttool_definebyquery_fake.sql \
+     disttool_definebyquery_raw.sql \
+     disttool_definebyquery_stack.sql \
+     disttool_definebyquery_warp.sql \
      disttool_pendingcomponent.sql \
+     disttool_pendingfileset.sql \
+     disttool_pendingdest.sql \
      disttool_processedcomponent.sql \
+     disttool_queuercrun.sql \
+     disttool_revertrcrun.sql \
      disttool_revertrun_update.sql \
      disttool_revertrun_delete.sql \
+     disttool_revertfileset.sql \
      disttool_toadvance.sql \
+     disttool_updatercrun.sql \
      faketool_change_exp_state.sql \
      faketool_change_imfile_data_state.sql \
      faketool_completely_processed_exp.sql \
      faketool_donecleanup.sql \
+     faketool_export_processed_imfile.sql \
+     faketool_export_run.sql \
      faketool_find_camrun.sql \
      faketool_find_pendingexp.sql \
@@ -111,10 +139,6 @@
      magictool_addmask.sql \
      magictool_create_tmp_warpcomplete.sql \
-     magictool_definebyquery.sql \
      magictool_definebyquery_insert.sql \
      magictool_definebyquery_select.sql \
-     magictool_definebyquery_select_test.sql \
-     magictool_definebyquery_temp_create.sql \
-     magictool_definebyquery_temp_insert.sql \
      magictool_inputs.sql \
      magictool_inputskyfile.sql \
@@ -129,4 +153,5 @@
      magictool_chipprocessedimfile.sql \
      magictool_rawimfile.sql \
+     magictool_revertnode.sql \
      magicdstool_todestreak.sql \
      magicdstool_completed_runs.sql \
@@ -140,4 +165,5 @@
      pstamptool_project.sql \
      pxadmin_create_tables.sql \
+     pxadmin_create_mirror_tables.sql \
      pxadmin_drop_tables.sql \
      pztool_find_completed_exp.sql \
@@ -146,5 +172,13 @@
      pztool_revert_fileset_faults.sql \
      pztool_revertcopied.sql \
+     receivetool_list.sql \
+     receivetool_addfileset.sql \
+     receivetool_pendingfileset.sql \
+     receivetool_pendingfile.sql \
+     receivetool_revert.sql \
+     receivetool_toadvance.sql \
      regtool_create_dup_table.sql \
+     regtool_export_exp.sql \
+     regtool_export_imfile.sql \
      regtool_pendingexp.sql \
      regtool_pendingimfile.sql \
@@ -162,4 +196,7 @@
      stacktool_definebyquery_test.sql \
      stacktool_donecleanup.sql \
+     stacktool_export_input_skyfile.sql \
+     stacktool_export_run.sql \
+     stacktool_export_sum_skyfile.sql \
      stacktool_find_complete_warps.sql \
      stacktool_inputskyfile.sql \
@@ -175,4 +212,8 @@
      warptool_donecleanup.sql \
      warptool_exp.sql \
+     warptool_export_imfile.sql \
+     warptool_export_run.sql \
+     warptool_export_skyfile.sql \
+     warptool_export_skycell_map.sql \
      warptool_finished_run_select.sql \
      warptool_finish_run.sql \
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_export_processed_exp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_export_processed_exp.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_export_processed_exp.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT * from camProcessedExp
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_export_run.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_export_run.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_export_run.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT camRun.* from camRun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_chip_id.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_chip_id.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_chip_id.sql	(revision 24244)
@@ -1,31 +1,28 @@
 SELECT
-    *
-FROM
-    (SELECT DISTINCT
-        chipRun.*,
-        rawExp.camera,
-        rawExp.telescope,
-        rawExp.dateobs,
-        rawExp.exp_tag,
-        rawExp.exp_type,
-        rawExp.filelevel,
-        rawExp.filter,
-        rawExp.airmass,
-        rawExp.ra,
-        rawExp.decl,
-        rawExp.exp_time,
-        rawExp.sat_pixel_frac,
-        rawExp.bg,
-        rawExp.bg_stdev,
-        rawExp.bg_mean_stdev,
-        rawExp.alt,
-        rawExp.az,
-        rawExp.ccd_temp,
-        rawExp.posang,
-        rawExp.object,
-        rawExp.sun_angle
-    FROM chipRun
-    JOIN rawExp
-        using(exp_id)
-    WHERE
-        chipRun.state = 'full') as Foo
+    chipRun.*,
+    rawExp.camera,
+    rawExp.telescope,
+    rawExp.dateobs,
+    rawExp.exp_tag,
+    rawExp.exp_type,
+    rawExp.filelevel,
+    rawExp.filter,
+    rawExp.airmass,
+    rawExp.ra,
+    rawExp.decl,
+    rawExp.exp_time,
+    rawExp.sat_pixel_frac,
+    rawExp.bg,
+    rawExp.bg_stdev,
+    rawExp.bg_mean_stdev,
+    rawExp.alt,
+    rawExp.az,
+    rawExp.ccd_temp,
+    rawExp.posang,
+    rawExp.object,
+    rawExp.sun_angle
+FROM chipRun
+JOIN rawExp
+    using(exp_id)
+WHERE
+    chipRun.state = 'full'
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_pendingexp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_pendingexp.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_pendingexp.sql	(revision 24244)
@@ -2,29 +2,23 @@
 -- does a little more work then is necessary for -addprocessed but it seems
 -- "cleaner" to use the same query for both cases 
-SELECT * FROM
-    (SELECT
-        camRun.*,
-        rawExp.exp_tag,
-        rawExp.exp_id,
-        rawExp.exp_name,
-        rawExp.camera,
-        rawExp.telescope,
-        rawExp.filelevel
-    FROM camRun
-    JOIN chipRun
-        USING(chip_id)
---  JOIN chipProcessedImfile
---      USING(chip_id)
-    JOIN rawExp
-        USING(exp_id)
---      ON chipProcessedImfile.exp_id = rawExp.exp_id
-    LEFT JOIN camProcessedExp
-        USING(cam_id)
-    LEFT JOIN camMask
-        ON camRun.label = camMask.label
-    WHERE
-        chipRun.state = 'full'
-        AND ((camRun.state = 'new' AND camProcessedExp.cam_id IS NULL)
-            OR camRun.state = 'update')
-        AND camMask.label IS NULL
-    ) as Foo
+SELECT
+    camRun.*,
+    rawExp.exp_tag,
+    rawExp.exp_id,
+    rawExp.exp_name,
+    rawExp.camera,
+    rawExp.telescope,
+    rawExp.filelevel
+FROM camRun
+JOIN chipRun
+    USING(chip_id)
+JOIN rawExp
+    USING(exp_id)
+LEFT JOIN camProcessedExp
+    USING(cam_id)
+LEFT JOIN camMask
+    ON camRun.label = camMask.label
+WHERE
+    chipRun.state = 'full'
+    AND ((camRun.state = 'new' AND camProcessedExp.cam_id IS NULL) OR camRun.state = 'update')
+    AND camMask.label IS NULL
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_pendingimfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_pendingimfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_pendingimfile.sql	(revision 24244)
@@ -1,25 +1,20 @@
-SELECT DISTINCT * FROM
-     -- the subselect is so where criteria can be specified without knowing
-     -- which table the field came from
-    (SELECT
-        camRun.cam_id,
-        chipProcessedImfile.*,
-        rawExp.exp_name,
-        rawExp.camera,
-        rawExp.telescope,
-        rawExp.filelevel
-    FROM camRun
-    JOIN chipRun
---      ON camRun.chip_id = chipRun.chip_id
-        USING (chip_id)
-    JOIN chipProcessedImfile
---      ON camRun.chip_id = chipProcessedImfile.chip_id
-        USING (chip_id)
-    JOIN rawExp
-      ON chipProcessedImfile.exp_id = rawExp.exp_id
-    LEFT JOIN camProcessedExp
-        USING(cam_id)
-    LEFT JOIN camMask
-        ON camRun.label = camMask.label
-    WHERE
-        camMask.label IS NULL) as foo
+SELECT
+    camRun.cam_id,
+    chipProcessedImfile.*,
+    rawExp.exp_name,
+    rawExp.camera,
+    rawExp.telescope,
+    rawExp.filelevel
+FROM camRun
+JOIN chipRun
+    USING (chip_id)
+JOIN chipProcessedImfile
+    USING (chip_id)
+JOIN rawExp
+  ON chipProcessedImfile.exp_id = rawExp.exp_id
+LEFT JOIN camProcessedExp
+    USING(cam_id)
+LEFT JOIN camMask
+    ON camRun.label = camMask.label
+WHERE
+    camMask.label IS NULL
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_processedexp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_processedexp.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_processedexp.sql	(revision 24244)
@@ -1,4 +1,5 @@
-SELECT DISTINCT
+SELECT
     camProcessedExp.*,
+    camRun.workdir,
     chipRun.chip_id,
     rawExp.exp_tag,
@@ -12,6 +13,4 @@
 JOIN chipRun
     USING(chip_id)
-JOIN chipProcessedImfile
-    USING(chip_id)
 JOIN rawExp
-    ON chipProcessedImfile.exp_id = rawExp.exp_id
+    ON chipRun.exp_id = rawExp.exp_id
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_completely_processed_exp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_completely_processed_exp.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_completely_processed_exp.sql	(revision 24244)
@@ -35,3 +35,4 @@
         COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id)
         AND SUM(chipProcessedImfile.fault) = 0
+        AND SUM(chipProcessedImfile.quality > 0) != COUNT(*)
     ) as Foo
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_export_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_export_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_export_imfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    chipImfile.*
+FROM chipImfile
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_export_imfiles.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_export_imfiles.sql	(revision 24243)
+++ 	(revision )
@@ -1,3 +1,0 @@
-SELECT
-    chipProcessedImfile.*
-FROM chipProcessedImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_export_processed_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_export_processed_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_export_processed_imfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    chipProcessedImfile.*
+FROM chipProcessedImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_find_rawexp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_find_rawexp.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_find_rawexp.sql	(revision 24244)
@@ -2,6 +2,8 @@
 -- processeing
 SELECT
-    *
+    rawExp.*,
+    newExp.label
 FROM rawExp
+JOIN newExp using (exp_id)
 WHERE
     rawExp.fault = 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_processedimfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_processedimfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_processedimfile.sql	(revision 24244)
@@ -9,4 +9,5 @@
     chipProcessedImfile.path_base,
     chipRun.state,
+    chipRun.workdir,
     rawExp.exp_id,
     rawExp.exp_tag,
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_input_exp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_input_exp.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_input_exp.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detInputExp.*
+FROM detInputExp
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_normalized_exp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_normalized_exp.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_normalized_exp.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detNormalizedExp.*
+FROM detNormalizedExp
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_normalized_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_normalized_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_normalized_imfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detNormalizedImfile.*
+FROM detNormalizedImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_normalized_stat_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_normalized_stat_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_normalized_stat_imfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detNormalizedStatImfile.*
+FROM detNormalizedStatImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_processed_exp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_processed_exp.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_processed_exp.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detProcessedExp.*
+FROM detProcessedExp
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_processed_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_processed_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_processed_imfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detProcessedImfile.*
+FROM detProcessedImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_registered_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_registered_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_registered_imfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detRegisteredImfile.*
+FROM detRegisteredImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_resid_exp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_resid_exp.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_resid_exp.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detResidExp.*
+FROM detResidExp
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_resid_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_resid_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_resid_imfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detResidImfile.*
+FROM detResidImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_run.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_run.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_run.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detRun.*
+FROM detRun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_run_summary.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_run_summary.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_run_summary.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detRunSummary.*
+FROM detRunSummary
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_stacked_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_stacked_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_export_stacked_imfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    detStackedImfile.*
+FROM detStackedImfile
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definebyquery.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definebyquery.sql	(revision 24243)
+++ 	(revision )
@@ -1,57 +1,0 @@
--- Get list of warps that can be diffed, with any associated diff,
--- and the best stack to use as a template
--- Warps without an existing diff can be identified by a NULL diff_id
-SELECT
-    warpsToDiff.warp_id,
-    warpsToDiff.skycell_id,
-    warpsToDiff.tess_id,
-    warpsToDiff.filter,
-    warpsToDiff.good_frac,
-    warpsToDiff.diff_id,
-    current_stack_id,
-    best_stack_id,
-    exp_id
-FROM (
-    -- Get list of warps that can be diffed, with any associated diff
-    SELECT
-        warpSkyfile.warp_id,
-        warpSkyfile.skycell_id,
-        warpSkyfile.tess_id,
-        warpSkyfile.good_frac,
-        warpRun.label,
-        filter,
-        warpRun.label as warp_label,
-        diffInputs.diff_id,
-        diffInputs.stack2 AS current_stack_id,
-        rawExp.exp_id
-    FROM warpSkyfile
-    JOIN warpRun USING(warp_id)
-    JOIN fakeRun USING(fake_id)
-    JOIN camRun USING(cam_id)
-    JOIN chipRun USING(chip_id)
-    JOIN rawExp USING(exp_id)
-    -- Check if it has an associated diff
-    LEFT JOIN diffInputSkyfile AS diffInputs
-        ON diffInputs.warp1 = warpSkyfile.warp_id
-        AND diffInputs.skycell_id = warpSkyfile.skycell_id
-        AND diffInputs.stack2 IS NOT NULL
-    WHERE
-        warpSkyfile.fault = 0
-        AND warpSkyfile.ignored = 0
-    -- warpsToDiff WHERE hook %s
-    ) AS warpsToDiff
--- Get best stack as a function of skycell_id, filter
-JOIN (
-    SELECT
-        MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment
-        skycell_id,
-        filter
-    FROM stackRun
-    JOIN stackSumSkyfile USING(stack_id)
-    WHERE stackRun.state = 'full'
-        AND stackSumSkyfile.fault = 0
-    -- stacksForDiff WHERE hook %s
-    GROUP BY
-        skycell_id,
-        filter
-    ) AS stacksForDiff USING(skycell_id, filter)
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definebyquery_part1.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definebyquery_part1.sql	(revision 24243)
+++ 	(revision )
@@ -1,22 +1,0 @@
-    SELECT
-        exp_id,
-        warp_id,
-        rawExp.filter,
-        warpRun.label,
-        warpRun.tess_id,
-        count(skycell_id) as skycell_count
-    FROM warpRun
-    JOIN warpSkyfile USING(warp_id)
-    JOIN fakeRun USING(fake_id) 
-    JOIN camRun USING(cam_id) 
-    JOIN chipRun USING (chip_id) 
-    JOIN rawExp USING(exp_id)
-    LEFT JOIN diffRun USING(exp_id)
-
-    WHERE
-        warpSkyfile.ignored = 0
-        -- warp where hook %s
-        -- exp where hook %s
-        -- diff where hook %s
-
-    GROUP BY exp_id, warp_id DESC
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definebyquery_part2.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definebyquery_part2.sql	(revision 24243)
+++ 	(revision )
@@ -1,36 +1,0 @@
--- insert skycells to diff
-INSERT INTO skycellsToDiff
-    SELECT
-        0,                      -- diff_id
-        warpSkyfile.skycell_id,
-        warpSkyfile.warp_id,    -- warp1
-        NULL,                   -- stack1
-        NULL,                   -- warp2
-        max_stack_id,           -- stack2
-        warpSkyfile.tess_id,
-        0
-        FROM warpSkyfile
-        JOIN warpRun USING(warp_id)
-        LEFT JOIN (
-            SELECT
-                MAX(stack_id) AS max_stack_id, -- most recent stack, by virtue of auto-increment
-                stackRun.skycell_id,
-                stackRun.tess_id,
-                filter
-            FROM stackRun
-            JOIN stackSumSkyfile USING(stack_id)
-            WHERE stackSumSkyfile.fault = 0
-                AND stackRun.state = 'full'
-            -- stacks where hook %s
-            GROUP BY
-                skycell_id,
-                filter
-            ) as bestStacks
-        USING(skycell_id)
-        WHERE
-            warpSkyfile.warp_id = %lld
-            AND ignored = 0
-            AND filter = '%s'
-            AND warpSkyfile.tess_id = bestStacks.tess_id
-        -- (good fraction test goes here for example also above)
-        -- warp where hook %s
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definebyquery_temp_create.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definebyquery_temp_create.sql	(revision 24243)
+++ 	(revision )
@@ -1,10 +1,0 @@
-CREATE TEMPORARY TABLE skycellsToDiff (
-diff_id BIGINT,
-skycell_id VARCHAR(64),
-warp1 BIGINT,
-stack1 BIGINT,
-warp2 BIGINT,
-stack2 BIGINT,
-tess_id VARCHAR(64),
-diff_skycell_id BIGINT
-) ENGINE=MEMORY;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack.sql	(revision 24244)
@@ -0,0 +1,58 @@
+-- Get list of warps that can be diffed, with any associated diff,
+-- and the best stack to use as a template
+-- Warps without an existing diff can be identified by a NULL diff_id
+SELECT
+    warpsToDiff.warp_id,
+    warpsToDiff.skycell_id,
+    warpsToDiff.tess_id,
+    warpsToDiff.filter,
+    warpsToDiff.good_frac,
+    warpsToDiff.diff_id,
+    current_stack_id,
+    best_stack_id,
+    exp_id
+FROM (
+    -- Get list of warps that can be diffed, with any associated diff
+    SELECT
+        warpSkyfile.warp_id,
+        warpSkyfile.skycell_id,
+        warpSkyfile.tess_id,
+        warpSkyfile.good_frac,
+        warpRun.label,
+        filter,
+        warpRun.label as warp_label,
+        diffInputs.diff_id,
+        diffInputs.stack2 AS current_stack_id,
+        rawExp.exp_id
+    FROM warpSkyfile
+    JOIN warpRun USING(warp_id)
+    JOIN fakeRun USING(fake_id)
+    JOIN camRun USING(cam_id)
+    JOIN chipRun USING(chip_id)
+    JOIN rawExp USING(exp_id)
+    -- Check if it has an associated diff
+    LEFT JOIN diffInputSkyfile AS diffInputs
+        ON diffInputs.warp1 = warpSkyfile.warp_id
+        AND diffInputs.skycell_id = warpSkyfile.skycell_id
+        AND diffInputs.stack2 IS NOT NULL
+    WHERE
+        warpSkyfile.fault = 0
+        AND warpSkyfile.quality = 0
+    -- warpsToDiff WHERE hook %s
+    ) AS warpsToDiff
+-- Get best stack as a function of skycell_id, filter
+JOIN (
+    SELECT
+        MAX(stack_id) AS best_stack_id, -- most recent stack, by virtue of auto-increment
+        skycell_id,
+        filter
+    FROM stackRun
+    JOIN stackSumSkyfile USING(stack_id)
+    WHERE stackRun.state = 'full'
+        AND stackSumSkyfile.fault = 0
+        AND stackSumSkyfile.quality = 0
+    -- stacksForDiff WHERE hook %s
+    GROUP BY
+        skycell_id,
+        filter
+    ) AS stacksForDiff USING(skycell_id, filter)
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack_part1.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack_part1.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack_part1.sql	(revision 24244)
@@ -0,0 +1,33 @@
+SELECT
+    exp_id,
+    warp_id,
+    rawExp.filter,
+    warpRun.label,
+    warpRun.tess_id,
+    COUNT(skycell_id) as skycell_count
+FROM warpRun
+JOIN warpSkyfile USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+LEFT JOIN (
+    SELECT DISTINCT
+        diffRun.*,
+        exp_id
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    JOIN warpRun
+        ON warpRun.warp_id = diffInputSkyfile.warp1
+    JOIN fakeRun USING(fake_id)
+    JOIN camRun USING(cam_id)
+    JOIN chipRun USING(chip_id)
+    WHERE warp1 IS NOT NULL
+) AS diffExp USING(exp_id)
+WHERE
+    warpSkyfile.fault = 0
+    AND warpSkyfile.quality = 0
+    -- warp where hook %s
+    -- exp where hook %s
+    -- diff where hook %s
+GROUP BY exp_id, warp_id DESC
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack_part2.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack_part2.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack_part2.sql	(revision 24244)
@@ -0,0 +1,36 @@
+-- insert skycells to diff
+INSERT INTO skycellsToDiff
+SELECT
+    0,                      -- diff_id
+    warpSkyfile.skycell_id,
+    warpSkyfile.warp_id,    -- warp1
+    NULL,                   -- stack1
+    NULL,                   -- warp2
+    max_stack_id,           -- stack2
+    warpSkyfile.tess_id,
+    0
+FROM warpSkyfile
+JOIN warpRun USING(warp_id)
+LEFT JOIN (
+    SELECT
+        MAX(stack_id) AS max_stack_id, -- most recent stack, by virtue of auto-increment
+        stackRun.skycell_id,
+        stackRun.tess_id,
+        filter
+    FROM stackRun
+    JOIN stackSumSkyfile USING(stack_id)
+    WHERE stackSumSkyfile.fault = 0
+        AND stackSumSkyfile.quality = 0
+        AND stackRun.state = 'full'
+    -- stacks where hook %s
+    GROUP BY
+        skycell_id,
+        filter
+    ) as bestStacks USING(skycell_id)
+WHERE
+    warpSkyfile.warp_id = %lld
+    AND warpSkyfile.fault = 0
+    AND warpSkyfile.quality = 0
+    AND filter = '%s'
+    AND warpSkyfile.tess_id = bestStacks.tess_id
+-- warp where hook %s
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack_temp_create.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack_temp_create.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpstack_temp_create.sql	(revision 24244)
@@ -0,0 +1,10 @@
+CREATE TEMPORARY TABLE skycellsToDiff (
+diff_id BIGINT,
+skycell_id VARCHAR(64),
+warp1 BIGINT,
+stack1 BIGINT,
+warp2 BIGINT,
+stack2 BIGINT,
+tess_id VARCHAR(64),
+diff_skycell_id BIGINT
+) ENGINE=MEMORY;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_insert.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_insert.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_insert.sql	(revision 24244)
@@ -11,6 +11,6 @@
 FROM warpSkyfile AS inputWarpSkyfile
 JOIN warpSkyfile AS templateWarpSkyfile USING(skycell_id, tess_id)
-WHERE inputWarpSkyfile.ignored = 0
-    AND templateWarpSkyfile.ignored = 0
+WHERE inputWarpSkyfile.quality = 0
+    AND templateWarpSkyfile.quality = 0
     AND inputWarpSkyfile.warp_id = %s
     AND templateWarpSkyfile.warp_id = %s
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_select.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_select.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_select.sql	(revision 24244)
@@ -1,6 +1,7 @@
 SELECT
     inputWarpRun.warp_id AS input_warp_id,
+    inputWarpRun.tess_id AS tess_id,
     inputRawExp.exp_id AS input_exp_id,
-    inputWarpRun.tess_id AS tess_id,
+    -- The following trick pulls out the warp_id that has the smallest distance
     SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id
 FROM warpRun AS inputWarpRun
@@ -9,5 +10,5 @@
 JOIN chipRun AS inputChipRun USING(chip_id)
 JOIN rawExp AS inputRawExp USING(exp_id)
--- To find exposures that haven't been diffed:%s LEFT JOIN diffRun ON diffRun.exp_id = inputRawExp.exp_id
+-- To find exposures that haven't been diffed, insert newline here:%s LEFT JOIN diffs USING(exp_id)
 JOIN warpRun AS templateWarpRun
     ON templateWarpRun.warp_id != inputWarpRun.warp_id -- Don't use self as template!
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_temp_create.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_temp_create.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_temp_create.sql	(revision 24244)
@@ -0,0 +1,8 @@
+-- Exposures that have diffs
+-- Temporary table has useful indices that make future query faster!
+CREATE TEMPORARY TABLE diffs (
+    diff_id BIGINT,
+    exp_id BIGINT,
+    PRIMARY KEY(diff_id, exp_id),
+    KEY(exp_id)
+) ENGINE=MEMORY;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_temp_insert.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_temp_insert.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_temp_insert.sql	(revision 24244)
@@ -0,0 +1,32 @@
+-- Get list of exposures that have diffs
+-- Only interested in whole exposures (diffRun.exposure = 1)
+INSERT INTO diffs
+SELECT DISTINCT
+    diffWarps.diff_id,
+    exp_id
+FROM (
+    -- Forward diffs
+    SELECT
+        diffRun.diff_id,
+        warp1 AS warp_id
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp1 IS NOT NULL
+        AND diffRun.exposure = 1
+    -- WHERE hook %s
+    UNION
+    -- Backward diffs
+    SELECT
+        diffRun.diff_id,
+        warp2 AS warp_id
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp2 IS NOT NULL
+        AND diffRun.exposure = 1
+        AND diffRun.bothways = 1
+    -- WHERE hook %s
+    ) AS diffWarps
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_test.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_test.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_definewarpwarp_test.sql	(revision 24244)
@@ -0,0 +1,82 @@
+DROP TABLE IF EXISTS diffs;
+
+-- Create a temporary table with useful indices
+CREATE TEMPORARY TABLE diffs (
+    diff_id BIGINT,
+    exp_id BIGINT,
+    PRIMARY KEY(diff_id, exp_id),
+    KEY(exp_id)
+) ENGINE=MEMORY;
+
+
+-- Get list of exposures that have diffs
+INSERT INTO diffs
+SELECT DISTINCT
+    diffWarps.diff_id,
+    exp_id
+FROM (
+    -- Forward diffs
+    SELECT
+        diffRun.diff_id,
+        warp1 AS warp_id
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE warp1 IS NOT NULL
+    -- WHERE hook %s
+        AND diffRun.label = 'ThreePi.Run2.r.v0'
+    UNION
+    -- Backward diffs
+    SELECT
+        diffRun.diff_id,
+        warp2 AS warp_id
+    FROM diffRun
+    JOIN diffInputSkyfile
+        ON diffInputSkyfile.diff_id = diffRun.diff_id
+        AND diffRun.bothways = 1
+    WHERE warp2 IS NOT NULL
+    -- WHERE hook %s
+        AND diffRun.label = 'ThreePi.Run2.r.v0'
+    ) AS diffWarps
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+;
+
+
+
+-- EXPLAIN
+SELECT
+    inputWarpRun.warp_id AS input_warp_id,
+    inputWarpRun.tess_id AS tess_id,
+    inputRawExp.exp_id AS input_exp_id,
+    SUBSTRING_INDEX(GROUP_CONCAT(templateWarpRun.warp_id ORDER BY ABS(ASIN(SQRT(POW(SIN(0.5*(inputRawExp.decl - templateRawExp.decl)),2) + COS(inputRawExp.decl) * COS(templateRawExp.decl) * POW(SIN(0.5*(inputRawExp.ra - templateRawExp.ra)),2))))), ',', 1) AS template_warp_id
+FROM warpRun AS inputWarpRun
+JOIN fakeRun AS inputFakeRun USING(fake_id)
+JOIN camRun AS inputCamRun USING(cam_id)
+JOIN chipRun AS inputChipRun USING(chip_id)
+JOIN rawExp AS inputRawExp USING(exp_id)
+-- To find exposures that haven't been diffed, insert newline here:%s
+ LEFT JOIN diffs USING(exp_id)
+JOIN warpRun AS templateWarpRun
+    ON templateWarpRun.warp_id != inputWarpRun.warp_id -- Don't use self as template!
+    AND templateWarpRun.tess_id = inputWarpRun.tess_id -- Ensure using same tessellation
+JOIN fakeRun AS templateFakeRun
+    ON templateFakeRun.fake_id = templateWarpRun.fake_id
+JOIN camRun AS templateCamRun
+    ON templateCamRun.cam_id = templateFakeRun.cam_id
+JOIN chipRun AS templateChipRun
+    ON templateChipRun.chip_id = templateCamRun.chip_id
+JOIN rawExp AS templateRawExp
+    ON templateRawExp.exp_id = templateChipRun.exp_id
+    AND templateRawExp.filter = inputRawExp.filter
+-- WHERE hook %s
+WHERE (inputWarpRun.label = 'ThreePi.Run2.r.v0')
+    AND ((DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + COS(inputRawExp.decl)*COS(templateRawExp.decl)*POW(SIN(inputRawExp.ra - templateRawExp.ra),2)))) < 0.10000000 + 0.00000119))
+    AND ((TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)) > 0.00000000 - 0.00000119))
+    AND (templateWarpRun.label = 'ThreePi.Run2.r.v0')
+--    AND (diffs.diff_id IS NULL)
+    AND inputWarpRun.state = 'full'
+-- WHERE done
+GROUP BY input_warp_id
+;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_export_input_skyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_export_input_skyfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_export_input_skyfile.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT diffInputSkyfile.* FROM diffInputSkyfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_export_run.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_export_run.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_export_run.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT diffRun.* from diffRun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_export_skyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_export_skyfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_export_skyfile.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT diffSkyfile.* FROM diffSkyfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_pendingcleanuprun.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_pendingcleanuprun.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_pendingcleanuprun.sql	(revision 24244)
@@ -9,5 +9,5 @@
     USING(diff_id)
 JOIN warpSkyfile
-    ON  diffInputSkyfile.warp_id    = warpSkyfile.warp_id
+    ON  diffInputSkyfile.warp1    = warpSkyfile.warp_id
     AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
     AND diffInputSkyfile.tess_id    = warpSkyfile.tess_id
@@ -23,3 +23,3 @@
     USING(exp_id)
 WHERE
-    diffRun.state = 'goto_cleaned'
+    (diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged')
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_pendingcleanupskyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_pendingcleanupskyfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_pendingcleanupskyfile.sql	(revision 24244)
@@ -9,3 +9,7 @@
     USING(diff_id)
 WHERE
-    diffRun.state = 'goto_cleaned'
+   ((diffRun.state = 'goto_cleaned'  AND diffSkyfile.data_state = 'full')
+    OR
+    (diffRun.state = 'goto_scrubbed' AND diffSkyfile.data_state != 'scrubbed')
+    OR
+    (diffRun.state = 'goto_purged'   AND diffSkyfile.data_state != 'purged'))
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_skyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_skyfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_skyfile.sql	(revision 24244)
@@ -4,16 +4,16 @@
     diffRun.tess_id,
     diffRun.state,
+    diffRun.workdir,
     warp1,
     stack1,
     warp2,
-    stack2,
-    exp_id
+    stack2
 FROM diffRun
-JOIN diffSkyfile
-    USING(diff_id)
-JOIN diffInputSkyfile
-    USING (diff_id, skycell_id)
+JOIN diffSkyfile USING(diff_id)
+JOIN diffInputSkyfile USING(diff_id, skycell_id)
 JOIN warpRun
     ON warpRun.warp_id = diffInputSkyfile.warp1
-JOIN rawExp
-    USING(exp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_todiffskyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_todiffskyfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_todiffskyfile.sql	(revision 24244)
@@ -9,5 +9,6 @@
     diffRun.tess_id,
     diffRun.label,
-    diffRun.state
+    diffRun.state,
+    diffRun.bothways
 FROM diffRun
 JOIN diffInputSkyfile USING(diff_id)
@@ -63,5 +64,6 @@
     AND diffSkyfile.diff_id IS NULL)
     OR (diffRun.state = 'update'
-    AND diffSkyfile.fault = 0)
+    AND diffSkyfile.fault = 0
+    AND diffSkyfile.quality = 0)
     )
 -- Ensure input warps are available
@@ -69,18 +71,20 @@
     OR (warpRun.state = 'full'
     AND warpSkyfile.fault = 0
-    AND warpSkyfile.ignored = 0))
+    AND warpSkyfile.quality = 0))
 -- Ensure input stacks are available
     AND (diffInputSkyfile.stack1 IS NULL
     OR (stackRun.state = 'full'
-    AND stackSumSkyfile.fault = 0))
+    AND stackSumSkyfile.fault = 0
+    AND stackSumSkyfile.quality = 0))
 -- Ensure template warps are available
     AND (diffInputSkyfile.warp2 IS NULL
     OR (warpTemplateRun.state = 'full'
     AND warpTemplateSkyfile.fault = 0
-    AND warpTemplateSkyfile.ignored = 0))
+    AND warpTemplateSkyfile.quality = 0))
 -- Ensure template stacks are available
     AND (diffInputSkyfile.stack2 IS NULL
     OR (stackTemplateRun.state = 'full'
-    AND stackTemplateSkyfile.fault = 0))
+    AND stackTemplateSkyfile.fault = 0
+    AND stackTemplateSkyfile.quality = 0))
 
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_camera.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_camera.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_camera.sql	(revision 24244)
@@ -0,0 +1,21 @@
+SELECT
+    'camera' as stage,
+    camRun.cam_id as stage_id,
+    rawExp.exp_name as run_tag,
+    rawExp.obs_mode,
+    distTarget.target_id,
+    distTarget.clean
+FROM camRun 
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+JOIN distTarget ON distTarget.stage = 'camera'
+    AND rawExp.obs_mode = distTarget.obs_mode
+JOIN rcInterest USING(target_id)
+LEFT JOIN distRun ON distRun.stage = 'camera' 
+    AND camRun.cam_id = distRun.stage_id
+    AND distRun.clean = distTarget.clean
+    -- JOIN hook %s
+WHERE distTarget.state = 'enabled'
+    AND rcInterest.state = 'enabled'
+    AND distRun.dist_id IS NULL      -- no existing distRun 
+    AND ((camRun.state = 'full') OR (distTarget.clean AND camRun.state = 'cleaned'))
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_chip.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_chip.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_chip.sql	(revision 24244)
@@ -0,0 +1,20 @@
+SELECT
+    'chip' as stage,
+    chipRun.chip_id as stage_id,
+    rawExp.exp_name as run_tag,
+    rawExp.obs_mode,
+    distTarget.target_id,
+    distTarget.clean
+FROM chipRun
+JOIN rawExp USING(exp_id)
+JOIN distTarget ON distTarget.stage = 'chip'
+    AND rawExp.obs_mode = distTarget.obs_mode 
+JOIN rcInterest USING(target_id)
+LEFT JOIN distRun ON distRun.stage = 'chip' 
+    AND distRun.stage_id = chipRun.chip_id
+    AND distRun.clean = distTarget.clean
+    -- JOIN hook %s
+WHERE distTarget.state = 'enabled'
+    AND rcInterest.state = 'enabled'
+    AND distRun.dist_id IS NULL      -- no existing distRun 
+    AND ((chipRun.state = 'full') OR (distTarget.clean AND chipRun.state = 'cleaned'))
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_diff.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_diff.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_diff.sql	(revision 24244)
@@ -0,0 +1,26 @@
+SELECT DISTINCT
+    'diff' as stage,
+    diffRun.diff_id AS stage_id,
+    rawExp.exp_name as run_tag,
+    rawExp.obs_mode,
+    distTarget.target_id,
+    distTarget.clean
+FROM diffRun
+JOIN diffInputSkyfile using(diff_id)
+JOIN warpRun on diffInputSkyfile.warp1 = warpRun.warp_id
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+JOIN distTarget ON distTarget.stage = 'diff' AND rawExp.obs_mode = distTarget.obs_mode
+JOIN rcInterest USING(target_id)
+LEFT JOIN distRun ON distRun.stage = 'diff' AND (distRun.stage_id = diff_id)
+    AND distRun.clean = distTarget.clean
+    -- JOIN hook %s
+WHERE  distTarget.state = 'enabled'
+    AND rcInterest.state = 'enabled'
+    AND distRun.dist_id IS NULL
+    AND ((diffRun.state = 'full') OR (distTarget.clean AND diffRun.state = 'cleaned'))
+
+
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_fake.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_fake.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_fake.sql	(revision 24244)
@@ -0,0 +1,19 @@
+SELECT
+    'fake' as stage,
+    fakeRun.fake_id AS stage_id,
+    rawExp.exp_name as run_tag,
+    rawExp.obs_mode,
+    distTarget.target_id,
+    distTarget.clean
+FROM fakeRun
+JOIN camRun USING(cam_id) -- ON fakeRun.cam_id = camRun.cam_id
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+JOIN distTarget ON distTarget.stage = 'fake' AND rawExp.obs_mode = distTarget.obs_mode
+JOIN rcInterest USING(target_id)
+LEFT JOIN distRun ON distRun.stage = 'fake' AND (distRun.stage_id = fake_id)
+    AND distRun.clean = distTarget.clean
+WHERE  distTarget.state = 'enabled'
+    AND rcInterest.state = 'enabled'
+    AND distRun.dist_id IS NULL
+    AND ((fakeRun.state = 'full') OR (distTarget.clean AND fakeRun.state = 'cleaned'))
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_raw.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_raw.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_raw.sql	(revision 24244)
@@ -0,0 +1,18 @@
+SELECT DISTINCT     -- we use distinct to handle multiple chip runs. Distribution will find right chipRun
+    'raw' AS stage,
+    rawExp.exp_id AS stage_id,
+    rawExp.exp_name AS run_tag,
+    rawExp.obs_mode,
+    distTarget.target_id,
+    distTarget.clean
+FROM rawExp
+JOIN chipRun USING(exp_id)
+JOIN distTarget ON distTarget.obs_mode = rawExp.obs_mode AND distTarget.stage = 'raw'
+JOIN rcInterest USING(target_id)
+LEFT JOIN distRun ON distRun.stage = 'raw' AND distRun.stage_id = exp_id
+    AND distRun.clean = distTarget.clean
+    -- JOIN hook for magicked stuff %s
+WHERE distTarget.state = 'enabled'
+    AND rcInterest.state = 'enabled'
+    AND distRun.dist_id IS NULL      -- no existing distRun
+
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_select.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_select.sql	(revision 24243)
+++ 	(revision )
@@ -1,124 +1,0 @@
--- Maybe it would be better to split the query into multiple modes
--- the way it is outstanding rawExp's to bundle will block other stages
--- from being processed
-SELECT
-    stage,
-    stage_id,
-    obs_mode,
-    clean
-FROM (
-    SELECT DISTINCT     -- we use distinct to handle multiple chip runs. Distribution will find right one
-        'raw' AS stage,
-        rawExp.exp_id AS stage_id,
-        rawExp.obs_mode,
-        distTarget.clean
-    FROM rawExp
-    JOIN chipRun USING(exp_id)
-    JOIN distTarget ON distTarget.obs_mode = rawExp.obs_mode AND distTarget.stage = 'raw'
-    JOIN rcInterest USING(target_id)
-    LEFT JOIN distRun ON distRun.stage = 'raw' AND distRun.stage_id = exp_id
-    WHERE distTarget.state = 'enabled'
-        AND rcInterest.state = 'enabled'
-        AND chipRun.state = 'full'       -- Note: we need a completed chip stage magicDSRun because we
-                                         -- need the mask file to NAN masked pixels
-        -- to support cameras that don't need magic disttool adds the magicked restriction 
-        -- \nAND rawExp.magicked AND chipRun.magicked here:
-        -- raw magicked HOOK %s
-        AND distRun.dist_id IS NULL      -- no existing distRun 
-    -- raw where hook %s
-
-UNION
-    SELECT
-        'chip' as stage,
-        chipRun.chip_id as stage_id,
-        rawExp.obs_mode,
-        distTarget.clean
-    FROM chipRun
-    JOIN rawExp USING(exp_id)
-    JOIN distTarget ON distTarget.stage = 'chip' AND rawExp.obs_mode = distTarget.obs_mode 
-    JOIN rcInterest USING(target_id)
-    LEFT JOIN distRun ON distTarget.stage = 'chip' AND distRun.stage_id = chipRun.chip_id
-    WHERE distTarget.state = 'enabled'
-        AND rcInterest.state = 'enabled'
-        AND distRun.dist_id IS NULL      -- no existing distRun 
-        AND ((!distTarget.clean AND (chipRun.state = 'full')
-            -- If camera requires files to be magicked before distibution
-            -- the following gets added by disttool
-            AND chipRun.magicked 
-            -- chip magicked HOOK %s
-           ) 
-           OR (distTarget.clean AND chipRun.state = 'cleaned'))
-    -- chip where hook %s
-
-UNION
-    SELECT
-        'cam' as stage,
-        camRun.cam_id as stage_id,
-        rawExp.obs_mode,
-        distTarget.clean
-    FROM camRun 
-    JOIN chipRun USING(chip_id)
-    JOIN rawExp USING(exp_id)
-    JOIN distTarget ON distTarget.stage = 'cam' AND rawExp.obs_mode = distTarget.obs_mode
-    JOIN rcInterest USING(target_id)
-    LEFT JOIN distRun ON distRun.stage = 'cam' AND camRun.cam_id = distRun.stage_id
-    WHERE distTarget.state = 'enabled'
-        AND rcInterest.state = 'enabled'
-        AND distRun.dist_id IS NULL      -- no existing distRun 
-        AND ((!distTarget.clean  AND camRun.state = 'full'
-            -- need magicked chip run because camera mask are destreaked by chip destreaking
-            -- the following line
-            -- AND chipRun.magicked
-            -- cam magicked HOOK %s
-             )
-            OR (distTarget.clean AND camRun.state = 'cleaned')
-        )
-    -- cam where hook %s
-
-UNION
-    SELECT
-        'fake' as stage,
-        fakeRun.fake_id as stage_id,
-        rawExp.obs_mode,
-        distTarget.clean
-    FROM fakeRun
-    JOIN camRun USING(cam_id)
-    JOIN chipRun USING(chip_id)
-    JOIN rawExp USING(exp_id)
-    JOIN distTarget ON rawExp.obs_mode = distTarget.obs_mode AND distTarget.stage = 'fake'
-    JOIN rcInterest USING(target_id)
-    LEFT JOIN distRun ON distRun.stage = 'fake' AND camRun.cam_id = distRun.stage_id
-    WHERE distTarget.state = 'enabled'
-        AND rcInterest.state = 'enabled'
-        AND distRun.dist_id IS NULL      -- no existing distRun 
-        AND ((fakeRun.state = 'full') OR (distTarget.clean AND fakeRun.state = 'cleaned'))
-    -- fake where hook %s
-
-UNION
-    SELECT
-        'warp' as stage,
-        warpRun.warp_id AS stage_id,
-        rawExp.obs_mode,
-        distTarget.clean
-    FROM warpRun
-    JOIN fakeRun USING(fake_id)
-    JOIN camRun ON fakeRun.cam_id = camRun.cam_id
-    JOIN chipRun USING(chip_id)
-    JOIN rawExp USING(exp_id)
-    JOIN distTarget ON distTarget.stage = 'warp' AND rawExp.obs_mode = distTarget.obs_mode
-    JOIN rcInterest USING(target_id)
-    LEFT JOIN distRun ON distRun.stage = 'warp' AND (distRun.stage_id = warp_id)
-    WHERE  distTarget.state = 'enabled'
-        AND rcInterest.state = 'enabled'
-        AND distRun.dist_id IS NULL
-        AND ((warpRun.state = 'full' 
-            --  AND warpRun.magicked
-            -- warp magicked hook %s
-             ) 
-            OR (distTarget.clean AND warpRun.state = 'cleaned')
-        )
-        -- warp where hook %s
-) as foo
-
-
-
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_stack.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_stack.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_stack.sql	(revision 24244)
@@ -0,0 +1,24 @@
+SELECT DISTINCT
+    'stack' as stage,
+    stackRun.stack_id AS stage_id,
+    -- run tag in the form 'stack.$skycell_id.$stack_id'
+    CONCAT_WS('.', 'stack', stackRun.skycell_id, convert(stackRun.stack_id, CHAR)) as run_tag,
+    rawExp.obs_mode,
+    distTarget.target_id,
+    distTarget.clean
+FROM stackRun
+JOIN stackInputSkyfile using(stack_id)
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+JOIN distTarget ON distTarget.stage = 'stack' AND rawExp.obs_mode = distTarget.obs_mode
+JOIN rcInterest USING(target_id)
+LEFT JOIN distRun ON distRun.stage = 'stack' AND (distRun.stage_id = stack_id)
+    AND distRun.clean = distTarget.clean
+    -- JOIN hook %s
+WHERE  distTarget.state = 'enabled'
+    AND rcInterest.state = 'enabled'
+    AND distRun.dist_id IS NULL
+    AND ((stackRun.state = 'full') OR (distTarget.clean AND stackRun.state = 'cleaned'))
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_warp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_warp.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_definebyquery_warp.sql	(revision 24244)
@@ -0,0 +1,23 @@
+SELECT
+    'warp' as stage,
+    warpRun.warp_id AS stage_id,
+    rawExp.exp_name as run_tag,
+    rawExp.obs_mode,
+    distTarget.target_id,
+    distTarget.clean
+FROM warpRun
+JOIN fakeRun USING(fake_id)
+JOIN camRun ON fakeRun.cam_id = camRun.cam_id
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+JOIN distTarget ON distTarget.stage = 'warp' AND rawExp.obs_mode = distTarget.obs_mode
+JOIN rcInterest USING(target_id)
+LEFT JOIN distRun ON distRun.stage = 'warp' AND (distRun.stage_id = warp_id)
+    AND distRun.clean = distTarget.clean
+WHERE  distTarget.state = 'enabled'
+    AND rcInterest.state = 'enabled'
+    AND distRun.dist_id IS NULL
+    AND ((warpRun.state = 'full') OR (distTarget.clean AND warpRun.state = 'cleaned'))
+
+
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingcomponent.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingcomponent.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingcomponent.sql	(revision 24244)
@@ -1,3 +1,4 @@
 SELECT * FROM (
+    -- raw stage
 SELECT
     distRun.dist_id,
@@ -8,22 +9,26 @@
     clean,
     rawExp.camera,
-    outroot,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
     rawImfile.uri as path_base,         -- change this once rawImfile has path_base
     chipProcessedImfile.path_base as chip_path_base,
     NULL as state,
     NULL as data_state,
+    0 as quality,
+    distRun.no_magic,
     rawImfile.magicked
 FROM distRun
 JOIN rawExp ON exp_id = stage_id
-JOIN (                      -- find the last magicked chip run 
+JOIN (                      -- find the last satisfactory chip run 
+                            -- note this requires that a chip run have been completed even when 
+                            -- magic is not required for the run
     SELECT
         exp_id,
         MAX(chip_id) AS chip_id
     FROM chipRun
+    JOIN distRun ON stage_id = exp_id AND stage = 'raw'
     WHERE 
         chipRun.state = 'full'
         AND chipRun.exp_id = exp_id
-        --   AND chipRun.magicked
-        -- magicked hook 1 %s
+        AND (chipRun.magicked OR distRun.no_magic)
         GROUP BY exp_id
     ) AS bestChipRun 
@@ -37,8 +42,37 @@
 WHERE
     distRun.state = 'new'
+    AND distRun.clean = 0
     AND distRun.stage = 'raw'
     AND distComponent.dist_id IS NULL
-    -- if magicked add AND rawImfile.magicked here
+    AND (rawExp.magicked OR distRun.no_magic)
     -- where hook 1 %s
+UNION
+    -- raw stage clean (dbinfo only)
+SELECT
+    distRun.dist_id,
+    distRun.label,
+    stage,
+    stage_id,
+    'exposure' AS component,
+    clean,
+    rawExp.camera,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
+    NULL,
+    NULL,
+    NULL as state,
+    NULL as data_state,
+    0 as quality,
+    distRun.no_magic,
+    rawExp.magicked
+FROM distRun
+JOIN rawExp ON exp_id = stage_id
+LEFT JOIN distComponent 
+    ON distRun.dist_id = distComponent.dist_id 
+WHERE
+    distRun.state = 'new'
+    AND distRun.stage = 'raw'
+    AND distRun.clean
+    AND distComponent.dist_id IS NULL
+    -- where hook 2 %s
 
 -- chip stage
@@ -52,9 +86,11 @@
     clean,
     rawExp.camera,
-    outroot,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
     chipProcessedImfile.path_base,
     chipProcessedImfile.path_base as chip_path_base,
     chipRun.state,
     chipProcessedImfile.data_state,
+    chipProcessedImfile.quality,
+    distRun.no_magic,
     chipProcessedImfile.magicked
 FROM distRun
@@ -69,34 +105,41 @@
     AND distRun.stage = 'chip'
     AND distComponent.dist_id IS NULL
-    -- where hook 2 %s
-UNION
-SELECT
-    distRun.dist_id,
-    distRun.label,
-    stage,
-    stage_id,
-    chipProcessedImfile.class_id AS component,
-    clean,
-    rawExp.camera,
-    outroot,
+    AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
+    AND (chipRun.state = 'full' OR (distRun.clean AND chipRun.state = 'cleaned'))
+    -- where hook 3 %s
+UNION
+SELECT
+    distRun.dist_id,
+    distRun.label,
+    stage,
+    stage_id,
+    'exposure' AS component,
+--    chipProcessedImfile.class_id AS component,
+    clean,
+    rawExp.camera,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
     camProcessedExp.path_base,
-    chipProcessedImfile.path_base as chip_path_base,
+    NULL as chip_path_base,
     camRun.state,
     NULL,
-    0
+    camProcessedExp.quality,
+    distRun.no_magic,
+    chipRun.magicked
 FROM distRun
 JOIN camRun ON camRun.cam_id = distRun.stage_id
 JOIN camProcessedExp USING(cam_id)
 JOIN chipRun USING(chip_id)
-JOIN chipProcessedImfile USING(exp_id, chip_id)
-JOIN rawExp using(exp_id)
-LEFT JOIN distComponent 
-    ON distRun.dist_id = distComponent.dist_id 
-    AND chipProcessedImfile.class_id = distComponent.component
+-- JOIN chipProcessedImfile USING(exp_id, chip_id)
+JOIN rawExp using(exp_id)
+LEFT JOIN distComponent 
+    ON distRun.dist_id = distComponent.dist_id 
+--    AND chipProcessedImfile.class_id = distComponent.component
 WHERE
     distRun.state = 'new'
     AND distRun.stage = 'camera'
     AND distComponent.dist_id IS NULL
-    -- where hook 3 %s
+    AND (distRun.clean OR chipRun.magicked OR distRun.no_magic)
+    AND (camRun.state = 'full' OR (distRun.clean AND camRun.state = 'cleaned'))
+    -- where hook 4 %s
 UNION
 SELECT
@@ -108,9 +151,11 @@
     clean,
     rawExp.camera,
-    outroot,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
     fakeProcessedImfile.path_base,
     NULL,
     fakeRun.state,
     NULL,
+    0 as quality,
+    distRun.no_magic,
     0
 FROM distRun
@@ -127,5 +172,5 @@
     AND distRun.stage = 'fake'
     AND distComponent.dist_id IS NULL
-    -- where hook 4 %s
+    -- where hook 5 %s
 UNION
 SELECT
@@ -137,9 +182,11 @@
     clean,
     rawExp.camera,
-    outroot,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
     warpSkyfile.path_base,
     NULL as chip_path_base,
     warpRun.state,
     warpSkyfile.data_state,
+    warpSkyfile.quality,
+    distRun.no_magic,
     warpSkyfile.magicked
 FROM distRun
@@ -157,5 +204,7 @@
     AND distRun.stage = 'warp'
     AND distComponent.dist_id IS NULL
-    -- where hook 5 %s
+    AND (distRun.clean OR warpRun.magicked OR distRun.no_magic)
+    AND (warpRun.state = 'full' OR (distRun.clean AND warpRun.state = 'cleaned'))
+    -- where hook 6 %s
 UNION
 SELECT
@@ -167,5 +216,5 @@
     clean,
     rawExp.camera,
-    outroot,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
     diffSkyfile.path_base,
     NULL as chip_path_base,
@@ -174,9 +223,17 @@
     -- diffSkyfile.data_state,
     'full' AS data_state,
+    diffSkyfile.quality,
+    distRun.no_magic,
     diffSkyfile.magicked
 FROM distRun
 JOIN diffRun ON stage_id = diff_id
 JOIN diffSkyfile using(diff_id)
-JOIN rawExp using(exp_id)
+JOIN diffInputSkyfile USING(diff_id, skycell_id)
+JOIN warpRun
+    ON warpRun.warp_id = diffInputSkyfile.warp1
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
 LEFT JOIN distComponent 
     ON distRun.dist_id = distComponent.dist_id 
@@ -186,5 +243,7 @@
     AND distRun.stage = 'diff'
     AND distComponent.dist_id IS NULL
-    -- where hook 6 %s
+    AND (distRun.clean OR diffRun.magicked OR distRun.no_magic)
+    AND (diffRun.state = 'full' OR (distRun.clean AND diffRun.state = 'cleaned'))
+    -- where hook 7 %s
 UNION
 SELECT DISTINCT
@@ -196,5 +255,5 @@
     clean,
     rawExp.camera,
-    outroot,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
     stackSumSkyfile.path_base,
     NULL as chip_path_base,
@@ -203,4 +262,6 @@
     -- stackSumSkyfile.data_state,
     'full' AS data_state,
+    stackSumSkyfile.quality,
+    1 AS no_magic,
     0 AS magicked
 FROM distRun
@@ -232,4 +293,5 @@
     AND distRun.stage = 'stack'
     AND distComponent.dist_id IS NULL
-    -- where hook 7 %s
+    AND (stackRun.state = 'full' OR (distRun.clean AND stackRun.state = 'cleaned'))
+    -- where hook 8 %s
 ) as Foo
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingdest.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingdest.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingdest.sql	(revision 24244)
@@ -0,0 +1,12 @@
+SELECT DISTINCT
+    rcDestination.*,
+    count(fs_id) as pending_fs
+FROM rcDestination
+JOIN rcRun using(dest_id)
+WHERE rcRun.state = 'new'
+    AND rcDestination.state = 'enabled'
+    AND status_uri IS NOT NULL
+    -- where hook %s
+GROUP BY dest_id
+HAVING count(fs_id) > 0
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingfileset.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingfileset.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingfileset.sql	(revision 24244)
@@ -0,0 +1,18 @@
+SELECT
+    dist_id,
+    target_id,
+    distRun.stage,
+    stage_id,
+    CONCAT_WS('.', distRun.outroot, CONVERT(distRun.dist_id, CHAR)) as dist_dir,
+    rcDSProduct.name AS product_name,
+    rcDSProduct.prod_id,
+    rcDSProduct.dbname AS ds_dbname,
+    rcDSProduct.dbhost AS ds_dbhost
+FROM rcDestination 
+JOIN rcInterest USING(dest_id) 
+JOIN distTarget USING(target_id) 
+JOIN rcDSProduct USING(prod_id)
+JOIN distRun USING(target_id) 
+LEFT JOIN rcDSFileset USING(prod_id, dist_id)
+WHERE distRun.state = 'full'
+    AND rcDSFileset.fs_id IS NULL
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_processedcomponent.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_processedcomponent.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_processedcomponent.sql	(revision 24244)
@@ -1,2 +1,14 @@
-SELECT *
-FROM    distComponent
+SELECT 
+    dist_id,
+    target_id,
+    stage,
+    stage_id,
+    component,
+    CONCAT_WS('.', outroot, CONVERT(dist_id, CHAR)) as outdir,
+    bytes,
+    md5sum,
+    name,
+    distComponent.state,
+    distComponent.fault
+FROM    distRun
+JOIN distComponent USING(dist_id)
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_queuercrun.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_queuercrun.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_queuercrun.sql	(revision 24244)
@@ -0,0 +1,21 @@
+-- create rcRun's for all destinations that have filesets that they are interested in available
+INSERT INTO rcRun
+SELECT                  -- rows in this select must match rcRun
+    0,                  -- rc_id (auto-increment)
+    fs_id,
+    dest_id,
+    'new',
+    NULL,                -- status_fs
+    NULL,                -- registered (database sets this to CURRENT_TIMESTAMP)
+    0                    -- fault
+
+FROM rcDestination 
+JOIN rcInterest USING(dest_id) 
+JOIN distTarget USING(target_id) 
+JOIN rcDSProduct USING(prod_id)
+JOIN distRun USING(target_id, stage) 
+JOIN rcDSFileset USING(prod_id, dist_id)
+LEFT JOIN rcRun using(fs_id, dest_id)
+WHERE rcRun.rc_id IS NULL
+    AND rcDSFileset.state = 'full'
+    AND rcDSFileset.fault = 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_revertfileset.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_revertfileset.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_revertfileset.sql	(revision 24244)
@@ -0,0 +1,4 @@
+DELETE FROM rcDSFileset
+USING rcDSFileset, distRun
+WHERE distRun.dist_id = rcDSFileset.dist_id
+    AND rcDSFileset.fault != 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_revertrcrun.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_revertrcrun.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_revertrcrun.sql	(revision 24244)
@@ -0,0 +1,3 @@
+UPDATE rcRun SET fault = 0
+WHERE fault != 0
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_toadvance.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_toadvance.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_toadvance.sql	(revision 24244)
@@ -3,13 +3,17 @@
     stage,
     stage_id,
-    outroot
+    CONCAT_WS('.', outroot, CONVERT(dist_id, CHAR)) as outdir,
+    label,
+    clean
 FROM
     (
--- raw stage
-SELECT 
+-- raw stage not clean
+SELECT
     distRun.dist_id,
     stage,
     stage_id,
-    outroot
+    outroot,
+    label,
+    clean
     FROM distRun
     JOIN rawImfile ON stage_id = rawImfile.exp_id
@@ -19,4 +23,5 @@
     WHERE
         distRun.state = 'new'
+        AND distRun.clean = 0
         AND distRun.fault = 0
         AND distRun.stage = 'raw'
@@ -28,10 +33,31 @@
         AND SUM(distComponent.fault) = 0
 UNION
--- chip stage
-SELECT 
+-- clean distribution of raw files (dbinfo only) only 1 component
+SELECT
     distRun.dist_id,
     stage,
     stage_id,
-    outroot
+    outroot,
+    label,
+    clean
+    FROM distRun
+    LEFT JOIN distComponent
+        ON distRun.dist_id = distComponent.dist_id
+    WHERE
+        distRun.state = 'new'
+        AND distRun.clean
+        AND distRun.fault = 0
+        AND distRun.stage = 'raw'
+        AND distComponent.component IS NOT NULL
+        AND distComponent.fault = 0
+UNION
+-- chip stage
+SELECT
+    distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    label,
+    clean
     FROM distRun
     JOIN chipProcessedImfile ON stage_id = chipProcessedImfile.chip_id
@@ -50,10 +76,55 @@
         AND SUM(distComponent.fault) = 0
 UNION
--- warp stage
-SELECT 
+-- camera stage
+SELECT distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    distRun.label,
+    clean
+    FROM distRun
+    JOIN camRun ON stage_id = cam_id
+    JOIN chipRun USING(chip_id)
+    LEFT JOIN distComponent  USING(dist_id)
+    WHERE
+        distRun.state = 'new'
+        AND distRun.fault = 0
+        AND distComponent.fault = 0
+        AND distRun.stage = 'camera'
+--        AND ((chipRun.magicked AND camRun.state = 'full') OR distRun.no_magic)
+--        AND (camRun.state = 'full' OR (distRun.clean and camRun.state = 'cleaned'))
+UNION
+-- fake stage
+SELECT
     distRun.dist_id,
     stage,
     stage_id,
-    outroot
+    outroot,
+    label,
+    clean
+    FROM distRun
+    JOIN fakeProcessedImfile ON stage_id = fakeProcessedImfile.fake_id
+    LEFT JOIN distComponent
+        ON distComponent.dist_id = distRun.dist_id
+        AND distComponent.component = fakeProcessedImfile.class_id
+    WHERE
+        distRun.state = 'new'
+        AND distRun.fault = 0
+        AND distRun.stage = 'fake'
+    GROUP BY
+        dist_id,
+        fakeProcessedImfile.fake_id
+    HAVING
+        COUNT(fakeProcessedImfile.class_id) = COUNT(distComponent.component)
+        AND SUM(distComponent.fault) = 0
+UNION
+-- warp stage
+SELECT
+    distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    label,
+    clean
     FROM distRun
     JOIN warpSkyfile on stage_id = warp_id
@@ -66,5 +137,5 @@
         AND distRun.stage = 'warp'
 --        AND warpSkyfile.fault = 0
---        AND warpSkyfile.ignored = 0
+--        AND warpSkyfile.quality = 0
     GROUP BY
         distRun.dist_id,
@@ -75,12 +146,14 @@
 UNION
 -- diff stage
-SELECT DISTINCT
+SELECT
     distRun.dist_id,
     stage,
     stage_id,
-    outroot
+    outroot,
+    distRun.label,
+    clean
     FROM distRun
     JOIN diffSkyfile
-        ON distRun.dist_id = diffSkyfile.diff_id
+        ON stage_id = diffSkyfile.diff_id
     LEFT JOIN distComponent
         ON distRun.dist_id = distComponent.dist_id
@@ -99,9 +172,11 @@
 UNION
 -- stack stage
-SELECT 
+SELECT
     distRun.dist_id,
     stage,
     stage_id,
-    outroot
+    outroot,
+    label,
+    clean
     FROM distRun
     JOIN stackSumSkyfile on stage_id = stack_id
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_updatercrun.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_updatercrun.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_updatercrun.sql	(revision 24244)
@@ -0,0 +1,9 @@
+UPDATE rcRun
+JOIN rcDestination USING(dest_id)
+JOIN rcDSProduct USING(prod_id) 
+JOIN rcDSFileset using(prod_id, fs_id)
+SET
+    -- set hook %s
+WHERE
+    rcRun.state = 'new'
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/faketool_export_processed_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/faketool_export_processed_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/faketool_export_processed_imfile.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT fakeProcessedImfile.* FROM fakeProcessedImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/faketool_export_run.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/faketool_export_run.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/faketool_export_run.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT fakeRun.* FROM fakeRun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/faketool_processedimfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/faketool_processedimfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/faketool_processedimfile.sql	(revision 24244)
@@ -1,4 +1,5 @@
 SELECT DISTINCT
     fakeProcessedImfile.*,
+    fakeRun.workdir,
     rawExp.exp_tag,
     rawExp.exp_name,
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/flatcorr_completely_processed_chiprun.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/flatcorr_completely_processed_chiprun.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/flatcorr_completely_processed_chiprun.sql	(revision 24244)
@@ -14,5 +14,5 @@
         dvodb
     FROM
-        (SELECT 
+        (SELECT
             chipRun.*,
             rawImfile.class_id as rawimfile_class_id,
@@ -33,4 +33,5 @@
             COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id)
             AND SUM(chipProcessedImfile.fault) = 0
-        ) as Foo
+            AND SUM(chipProcessedImfile.quality > 0) != COUNT(*)
+       ) as Foo
     ) as Bar
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/flatcorr_pending.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/flatcorr_pending.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/flatcorr_pending.sql	(revision 24244)
@@ -1,3 +1,3 @@
-SELECT 
+SELECT
     *
 FROM
@@ -21,4 +21,4 @@
 HAVING
     COUNT(rawImfile.class_id) = COUNT(chipProcessedImfile.class_id)
-    AND SUM(chipProcessedImfile.fault) = 0
+    AND SUM(chipProcessedImfile.fault > 0) = 0
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_completed_runs.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_completed_runs.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_completed_runs.sql	(revision 24244)
@@ -4,5 +4,5 @@
     (
 -- raw stage
-SELECT 
+SELECT
     magicDSRun.magic_ds_id
     FROM magicDSRun
@@ -22,5 +22,5 @@
 UNION
 -- chip stage
-SELECT 
+SELECT
     magicDSRun.magic_ds_id
     FROM magicDSRun
@@ -40,5 +40,5 @@
 UNION
 -- warp stage
-SELECT 
+SELECT
     magicDSRun.magic_ds_id
     FROM magicDSRun
@@ -51,5 +51,5 @@
         AND magicDSRun.stage = 'warp'
         AND warpSkyfile.fault = 0
-        AND warpSkyfile.ignored = 0
+        AND warpSkyfile.quality = 0
     GROUP BY
         magicDSRun.magic_ds_id,
@@ -66,5 +66,5 @@
     JOIN magicInputSkyfile USING(magic_id)
     JOIN diffSkyfile
-        ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
+        ON magicRun.diff_id = diffSkyfile.diff_id
         AND magicInputSkyfile.node = diffSkyfile.skycell_id
     LEFT JOIN magicDSFile
@@ -75,4 +75,5 @@
         AND magicDSRun.stage = 'diff'
         AND diffSkyfile.fault = 0
+        AND diffSkyfile.quality = 0
     GROUP BY
         magicDSRun.magic_ds_id,
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_getrunids.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_getrunids.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_getrunids.sql	(revision 24244)
@@ -9,5 +9,5 @@
 JOIN magicInputSkyfile USING(magic_id)
 JOIN diffSkyfile 
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
+    ON magicRun.diff_id = diffSkyfile.diff_id
     AND magicInputSkyfile.node = diffSkyfile.skycell_id
 JOIN diffInputSkyfile
@@ -15,5 +15,6 @@
     AND diffInputSkyfile.skycell_id = diffSkyfile.skycell_id
 JOIN warpRun
-    ON diffInputSkyfile.warp1 = warp_id
+    ON (diffInputSkyfile.warp1 = warp_id AND !magicRun.inverse)
+    OR (diffInputSkyfile.warp2 = warp_id AND magicRun.inverse)
 JOIN fakeRun USING(fake_id)
 JOIN camRun USING(cam_id)
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_getskycells.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_getskycells.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_getskycells.sql	(revision 24244)
@@ -2,11 +2,10 @@
     diffSkyfile.diff_id,
     diffSkyfile.skycell_id,
-    diffSkyfile.uri,
     diffSkyfile.path_base
 FROM magicDSRun
 JOIN magicRun USING(magic_id)
-JOIN magicInputSkyfile USING(magic_id, diff_id)
+JOIN magicInputSkyfile USING(magic_id)
 JOIN diffSkyfile
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
+    ON magicRun.diff_id = diffSkyfile.diff_id
     AND magicInputSkyfile.node = diffSkyfile.skycell_id
 JOIN diffInputSkyfile
@@ -21,6 +20,7 @@
     ON warpSkyfile.warp_id = warpSkyCellMap.warp_id
     AND warpSkyfile.skycell_id = warpSkyCellMap.skycell_id
-    AND warpSkyfile.ignored = 0
+    AND warpSkyfile.quality = 0
 WHERE
     diffSkyfile.fault = 0
+    AND diffSkyfile.quality = 0
     AND magic_ds_id = %lld
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_todestreak.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_todestreak.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_todestreak.sql	(revision 24244)
@@ -5,4 +5,5 @@
     magicRun.magic_id,
     magicRun.exp_id,
+    magicDSRun.label,
     camera,
     magicMask.uri as streaks_uri,
@@ -12,4 +13,5 @@
     rawImfile.uri as uri,
     NULL as path_base,
+    magicRun.inverse,
     camProcessedExp.path_base as cam_path_base,
     outroot,
@@ -35,4 +37,5 @@
     magicDSRun.magic_id,
     chipRun.exp_id,
+    magicDSRun.label,
     camera,
     magicMask.uri as streaks_uri,
@@ -42,4 +45,5 @@
     chipProcessedImfile.uri,
     chipProcessedImfile.path_base,
+    magicRun.inverse,
     camProcessedExp.path_base as cam_path_base,
     outroot,
@@ -49,4 +53,5 @@
 FROM magicDSRun
 JOIN magicMask USING (magic_id)
+JOIN magicRun USING(magic_id)
 JOIN camProcessedExp USING(cam_id)
 JOIN chipRun ON chip_id = stage_id
@@ -61,4 +66,5 @@
     AND chipRun.state = 'full'
     AND chipProcessedImfile.fault = 0
+    AND chipProcessedImfile.quality = 0
     AND magicDSFile.component IS NULL
 UNION
@@ -68,4 +74,5 @@
     magicRun.magic_id,
     magicRun.exp_id,
+    magicDSRun.label,
     camera,
     magicMask.uri as streaks_uri,
@@ -75,4 +82,5 @@
     warpSkyfile.uri,
     warpSkyfile.path_base,
+    magicRun.inverse,
     NULL as cam_path_base,
     outroot,
@@ -95,5 +103,5 @@
     AND warpRun.state = 'full'
     AND warpSkyfile.fault = 0
-    AND warpSkyfile.ignored = 0
+    AND warpSkyfile.quality = 0
     AND magicDSFile.component IS NULL
 UNION
@@ -103,4 +111,5 @@
     magicRun.magic_id,
     magicRun.exp_id,
+    magicDSRun.label,
     rawExp.camera,
     magicMask.uri as streaks_uri,
@@ -108,6 +117,7 @@
     magicRun.diff_id as stage_id,
     diffSkyfile.skycell_id as component,
-    diffSkyfile.uri,
+    NULL AS uri,
     diffSkyfile.path_base,
+    magicRun.inverse,
     NULL as cam_path_base,
     outroot,
@@ -121,5 +131,5 @@
 JOIN magicInputSkyfile USING(magic_id)
 JOIN diffSkyfile
-    ON  magicInputSkyfile.diff_id = diffSkyfile.diff_id
+    ON  magicRun.diff_id = diffSkyfile.diff_id
     AND magicInputSkyfile.node = diffSkyfile.skycell_id
 LEFT JOIN magicDSFile
@@ -130,4 +140,5 @@
     AND magicDSRun.stage = 'diff'
     AND diffSkyfile.fault = 0
+    AND diffSkyfile.quality = 0
     AND magicDSFile.component IS NULL
 ) as Foo
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_chipprocessedimfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_chipprocessedimfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_chipprocessedimfile.sql	(revision 24244)
@@ -26,4 +26,5 @@
     chipRun.state = 'full'
     AND chipProcessedImfile.fault = 0
+    AND chipProcessedImfile.quality = 0
 --   AND magicRun.state = 'full'
 --   AND magicMask.fault = 0
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery.sql	(revision 24243)
+++ 	(revision )
@@ -1,112 +1,0 @@
--- This is a sequence of SQL for searching for exposures on which
--- magic may be run, and populating the run.  This file is provided as
--- a convenience, and is intended solely for playing around and
--- testing.  Updates here will not affect the operation of magictool.
--- Conversely, it may not be up to date with the actual queries used
--- by magictool.
-
---------------------------------------------------------------------
--- NOTE: THIS FILE IS NOT USED BY magictool
--- NOTE : this file contains examples of tess_id as a join qualifier which are wrong
---------------------------------------------------------------------
-
--- magictool_definebyquery_temp_create.sql
-CREATE TEMPORARY TABLE magicBestDiffs (
-exp_id BIGINT,
-skycell_id VARCHAR(64),
-tess_id VARCHAR(64),
-diff_id BIGINT,
-PRIMARY KEY(exp_id, skycell_id, tess_id)
-) ENGINE=MEMORY;
-
-
-
--- magictool_definebyquery_temp_insert.sql
--- List of best differences for each exposure
-INSERT INTO magicBestDiffs
-SELECT
-    rawExp.exp_id,
-    diffInputSkyfile.skycell_id,
-    diffInputSkyfile.tess_id,
-    MAX(diffSkyfile.diff_id) AS diff_id
-FROM rawExp
-JOIN chipRun USING(exp_id)
-JOIN camRun USING(chip_id)
-JOIN fakeRun USING(cam_id)
-JOIN warpRun USING(fake_id)
-JOIN warpSkyCellMap USING(warp_id)
-JOIN warpSkyfile USING(warp_id, skycell_id)
-JOIN diffInputSkyfile
-    ON diffInputSkyfile.warp_id = warpSkyfile.warp_id
-    AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
-    AND diffInputSkyfile.template = 0 -- selecting inputs only
-JOIN diffRun USING(diff_id)
-JOIN diffSkyfile USING(diff_id)
-WHERE
-    diffSkyfile.fault = 0
--- WHERE hook %s
-AND exp_id = 36475
-AND diffRun.label = 'magic_2008-11-12'
-GROUP BY
-    exp_id,
-    diffInputSkyfile.skycell_id,
-    diffInputSkyfile.tess_id
-;
-
-
--- magictool_definebyquery_select.sql
--- Get a list of exposures on which magic may be performed
-SELECT DISTINCT
-    exp_id,
-    filter,
-    num_todo,
-    num_done,
-    MAX(magic_id) AS best_magic_id
-FROM (
-    -- Number of skycells as a function of exposure
-    SELECT
-        exp_id,
-        filter,
-        COUNT(DISTINCT warpSkyfile.tess_id,warpSkyfile.skycell_id) AS num_todo
-    FROM rawExp
-    JOIN chipRun USING(exp_id)
-    JOIN camRun USING(chip_id)
-    JOIN fakeRun USING(cam_id)
-    JOIN warpRun USING(fake_id)
-    JOIN warpSkyCellMap USING(warp_id)
-    JOIN warpSkyfile USING(warp_id, skycell_id)
-    JOIN diffInputSkyfile USING(warp_id,skycell_id)
-    JOIN diffRun USING(diff_id)
-    WHERE
-        warpSkyfile.ignored = 0
-        -- magicSkycellNums WHERE hook %s
-AND exp_id = 36475
-AND diffRun.label = 'magic_2008-11-12'
-    GROUP BY
-        exp_id
-    ) AS magicSkycellNums
-JOIN (
-    -- Number of completed diffs for an exposure
-    SELECT
-        exp_id,
-        COUNT(diff_id) AS num_done
-    FROM magicBestDiffs
-    GROUP BY
-        exp_id
-    ) AS magicDiffNums USING(exp_id)
-LEFT JOIN magicRun USING(exp_id)
-;
-
-WHERE
-    magic_id IS NULL
-;
-
--- magictool_definebyquery_insert.sql
--- Insert the best list of diffs as magic inputs
-INSERT INTO magicInputSkyfile
-SELECT
-    @MAGIC_ID@, -- Update this with the appropriate magic_id
-    diff_id,
-    CONCAT(tess_id, '.', skycell_id) AS node
-FROM magicBestDiffs
-;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_insert.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_insert.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_insert.sql	(revision 24244)
@@ -3,5 +3,4 @@
 SELECT
     @MAGIC_ID@, -- Update this with the appropriate magic_id
-    diff_id,
     skycell_id
 FROM diffSkyfile
@@ -9,2 +8,3 @@
     diff_id = @DIFF_ID@ -- Update this with the appropriate diff_id
     AND fault = 0
+    AND quality = 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_select.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_select.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_select.sql	(revision 24244)
@@ -1,8 +1,36 @@
 -- Get a list of exposures on which magic may be performed
-SELECT 
+SELECT
     exp_id,
-    MAX(diffRun.diff_id) AS diff_id
-FROM diffRun
-JOIN rawExp USING(exp_id)
+    MAX(diffWarps.diff_id) AS diff_id,
+    -- The following trick pulls out the 'inverse' value for the maximum diff_id
+    CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id), ',', 1), UNSIGNED) AS inverse
+FROM (
+    -- Forward diffs
+    SELECT
+        diffRun.diff_id,
+        warp1 AS warp_id,
+        0 AS inverse
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp1 IS NOT NULL
+        AND diffRun.exposure = 1
+    -- WHERE hook %s
+    UNION
+    -- Backward diffs
+    SELECT
+        diffRun.diff_id,
+        warp2 AS warp_id,
+        1 AS inverse
+    FROM diffRun
+    JOIN diffInputSkyfile USING(diff_id)
+    WHERE diffInputSkyfile.warp2 IS NOT NULL
+        AND diffRun.exposure = 1
+        AND diffRun.bothways = 1
+    -- WHERE hook %s
+    ) AS diffWarps
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
 LEFT JOIN magicRun USING(exp_id)
 -- WHERE hook %s
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_select_test.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_select_test.sql	(revision 24243)
+++ 	(revision )
@@ -1,46 +1,0 @@
--- This is the combination of the two parts of the query to get a list
--- of exposures to be magic-ed
-
---------------------------------------------------------------------------------
--- THIS FILE IS INTENDED FOR TESTING ONLY!  IT IS NOT USED BY stacktool.
--- CHANGES SHOULD BE MADE TO magictool_definebyquery_select_part1.sql
--- AND magictool_definebyquery_select_part2.sql
---------------------------------------------------------------------------------
-
--- magictool_definebyquery_select_part1.sql
-SELECT
-    *
-FROM (
-    -- Number of skycells as a function of exposure
-    SELECT
-        exp_id,
-        filter,
-        COUNT(skycell_id) AS num_todo
-    FROM rawExp
-    JOIN chipRun USING(exp_id)
-    JOIN camRun USING(chip_id)
-    JOIN fakeRun USING(cam_id)
-    JOIN warpRun USING(fake_id)
-    JOIN warpSkyCellMap USING(warp_id)
-    JOIN warpSkyfile USING(warp_id, skycell_id)
-    WHERE
-        warpSkyfile.ignored = 0
-        AND warpRun.state = 'full'
-    -- INSERT HERE any additional restrictions (e.g., exp_id, warpSkyfile.good_frac)
--- magictool_definebyquery_select_part2.sql
-    GROUP BY
-        exp_id
-    ) AS magicSkycellNums
-JOIN (
-    -- Number of completed diffs for an exposure
-    SELECT
-        exp_id,
-        COUNT(diff_id) AS num_done
-    FROM magicBestDiffs
-    GROUP BY
-        exp_id
-    ) AS magicDiffNums USING(exp_id)
-LEFT JOIN magicRun USING(exp_id)
-WHERE
-    num_done = num_todo
-    AND magicRun.magic_id IS NULL
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_temp_create.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_temp_create.sql	(revision 24243)
+++ 	(revision )
@@ -1,7 +1,0 @@
-CREATE TEMPORARY TABLE magicBestDiffs (
-exp_id BIGINT,
-skycell_id VARCHAR(64),
-tess_id VARCHAR(64),
-diff_id BIGINT,
-PRIMARY KEY(exp_id, skycell_id, tess_id)
-) ENGINE=MEMORY;
Index: anches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_temp_insert.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_definebyquery_temp_insert.sql	(revision 24243)
+++ 	(revision )
@@ -1,27 +1,0 @@
--- List of best differences for each exposure
-INSERT INTO magicBestDiffs
-SELECT
-    rawExp.exp_id,
-    diffInputSkyfile.skycell_id,
-    diffInputSkyfile.tess_id,
-    MAX(diffSkyfile.diff_id) AS diff_id
-FROM rawExp
-JOIN chipRun USING(exp_id)
-JOIN camRun USING(chip_id)
-JOIN fakeRun USING(cam_id)
-JOIN warpRun USING(fake_id)
-JOIN warpSkyCellMap USING(warp_id)
-JOIN warpSkyfile USING(warp_id, skycell_id)
-JOIN diffInputSkyfile
-    ON diffInputSkyfile.warp1 = warpSkyfile.warp_id
-    AND diffInputSkyfile.skycell_id = warpSkyfile.skycell_id
-JOIN diffRun USING(diff_id)
-JOIN diffSkyfile USING(diff_id)
-WHERE
-    diffSkyfile.fault = 0
--- WHERE hook %s
-GROUP BY
-    exp_id,
-    diffInputSkyfile.skycell_id,
-    diffInputSkyfile.tess_id
-;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_diffskyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_diffskyfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_diffskyfile.sql	(revision 24244)
@@ -20,8 +20,9 @@
     ON warpSkyfile.warp_id = warpSkyCellMap.warp_id
     AND warpSkyfile.skycell_id = warpSkyCellMap.skycell_id
-    AND warpSkyfile.ignored = 0
+    AND warpSkyfile.quality = 0
 WHERE
     diffRun.state = 'full'
     AND diffSkyfile.fault = 0
+    AND diffSkyfile.quality = 0
 --   AND magicRun.state = 'full'
 --   AND magicMask.fault = 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_inputs.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_inputs.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_inputs.sql	(revision 24244)
@@ -1,36 +1,73 @@
-SELECT *
-FROM (
--- Single skycells
+-- Get input details for a magic node
 SELECT
     magicRun.magic_id,
-    magicRun.state,
-    magicInputSkyfile.node,
-    diffSkyfile.diff_id,
-    diffSkyfile.uri,
-    diffSkyfile.path_base,
-    diffSkyfile.fault
+    magicRun.inverse, -- Using the inverse subtraction?
+    magicRun.diff_id,
+    magicInputs.node,
+    -- Only one of diff_path_base and magic_path_base should be non-NULL
+    -- If diff_path_base is non-NULL, then only one of warp_path_base and stack_path_base should be non-NULL
+    magicInputs.diff_path_base, -- path_base for the diff (if any)
+    magicInputs.warp_path_base, -- path_base for the template warp (if any)
+    magicInputs.stack_path_base, -- path_base for the template stack (if any)
+    magicInputs.magic_path_base -- path_base for child nodes (if any)
 FROM magicRun
-JOIN magicInputSkyfile
-USING(magic_id)
-JOIN diffSkyfile
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
-UNION
--- Merged skycells
-SELECT
-    magicRun.magic_id,
-    magicRun.state,
-    magicTree.node,
-    0,   -- no diff_id
-    magicNodeResult.uri,
-    NULL, -- magicNodeResult doesn't have a path_base
-    magicNodeResult.fault
-FROM magicTree
-JOIN magicRun
-    USING(magic_id)
-JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.dep = magicNodeResult.node
-) as Foo
-WHERE
-    fault = 0
+JOIN (
+    -- Single skycells: have uri=NULL
+    SELECT
+        magic_id,
+        magicRun.diff_id,
+        skycell_id AS node,
+        diffSkyfile.path_base AS diff_path_base,
+        warpSkyfile.path_base AS warp_path_base,
+        stackSumSkyfile.path_base AS stack_path_base,
+        NULL AS magic_path_base
+    FROM magicRun
+    JOIN magicTree USING(magic_id)
+    JOIN magicInputSkyfile USING(magic_id, node)
+    JOIN diffRun USING(diff_id)
+    JOIN diffSkyfile
+        ON diffSkyfile.diff_id = magicRun.diff_id
+        AND diffSkyfile.skycell_id = magicInputSkyfile.node
+    JOIN (
+        -- Template for non-inverse
+        SELECT
+            magic_id,
+            skycell_id,
+            warp2 AS warp_id,
+            stack2 AS stack_id
+        FROM magicRun
+        JOIN diffInputSkyfile USING(diff_id)
+        WHERE magicRun.inverse = 0
+            -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s
+        UNION
+        -- Template for inverse
+        SELECT
+            magic_id,
+            skycell_id,
+            warp1 AS warp_id,
+            stack1 AS stack_id
+        FROM magicRun
+        JOIN diffInputSkyfile USING(diff_id)
+        WHERE magicRun.inverse = 1
+            -- WHERE hook (magicRun.magic_id, diffInputSkyfile.skycell_id) %s
+        ) AS diffTemplates USING(magic_id, skycell_id)
+    LEFT JOIN warpSkyfile USING(warp_id, skycell_id)
+    LEFT JOIN stackSumSkyfile USING(stack_id)
+    -- WHERE hook (magicRun.magic_id, magicTree.node) %s
+    UNION
+    -- Merged skycells: have diff_id=0, various_path_base=NULL
+    SELECT
+        magicTree.magic_id,
+        0 AS diff_id,
+        magicTree.dep,
+        NULL AS diff_path_base,
+        NULL AS warp_path_base,
+        NULL AS stack_path_base,
+        magicNodeResult.path_base AS magic_path_base
+    FROM magicTree
+    JOIN magicRun USING(magic_id)
+    JOIN magicNodeResult
+        ON magicTree.magic_id = magicNodeResult.magic_id
+        AND magicTree.dep = magicNodeResult.node
+    -- WHERE hook (magicRun.magic_id, magicTree.node) %s
+    ) AS magicInputs USING(magic_id)
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_inputskyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_inputskyfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_inputskyfile.sql	(revision 24244)
@@ -1,11 +1,11 @@
 SELECT
     magicInputSkyfile.*,
-    diffSkyfile.uri
+    magicRun.inverse,
+    diffSkyfile.path_base
 FROM magicRun
-JOIN magicInputSkyfile
-    USING(magic_id)
+JOIN magicInputSkyfile USING(magic_id)
 JOIN diffSkyfile
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
+    ON diffSkyfile.diff_id = magicRun.diff_id
+    AND diffSkyfile.skycell_id = magicInputSkyfile.node
 WHERE
     magicRun.state = 'new'
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_revertnode.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_revertnode.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_revertnode.sql	(revision 24244)
@@ -0,0 +1,4 @@
+DELETE magicNodeResult 
+FROM magicNodeResult 
+JOIN magicRun USING(magic_id) 
+WHERE magicNodeResult.fault != 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_toprocess_inputs.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_toprocess_inputs.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_toprocess_inputs.sql	(revision 24244)
@@ -1,23 +1,22 @@
 SELECT
     magicTree.*,
+    magicRun.workdir,
+    rawExp.exp_id,
     rawExp.camera,
-    rawExp.exp_id,
-    diffSkyfile.path_base,
-    magicRun.workdir,
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done
-FROM magicTree
-JOIN magicRun USING(magic_id)
+    magicNodeResult.magic_id IS TRUE AS done,
+    magicNodeResult.fault IS TRUE AS bad
+FROM magicRun
+JOIN magicTree USING(magic_id)
 JOIN magicInputSkyfile USING(magic_id, node)
-JOIN diffSkyfile 
-    ON magicInputSkyfile.diff_id = diffSkyfile.diff_id
-    AND magicInputSkyfile.node = diffSkyfile.skycell_id
 JOIN rawExp USING(exp_id)
-LEFT JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.node = magicNodeResult.node
+JOIN diffSkyfile -- only get nodes that match a skycell
+    ON diffSkyfile.diff_id = magicRun.diff_id
+    AND diffSkyfile.skycell_id = magicInputSkyfile.node
+LEFT JOIN magicNodeResult USING(magic_id, node)
 WHERE
     magicRun.state = 'new'
     AND magicNodeResult.magic_id IS NULL
     AND magicNodeResult.node IS NULL
+-- WHERE hook %s
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_toprocess_tree.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_toprocess_tree.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_toprocess_tree.sql	(revision 24244)
@@ -1,20 +1,18 @@
 SELECT
     magicTree.*,
-    exp_id,
-    camera,
     magicRun.workdir,
+    rawExp.exp_id,
+    rawExp.camera,
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.fault IS TRUE as bad
+    magicNodeResult.magic_id IS TRUE AS done,
+    magicNodeResult.fault IS TRUE AS bad
 FROM magicTree
 JOIN magicRun USING(magic_id)
 JOIN rawExp USING(exp_id)
-LEFT JOIN magicNodeResult
-    ON magicTree.magic_id = magicNodeResult.magic_id
-    AND magicTree.node = magicNodeResult.node
+LEFT JOIN magicNodeResult USING(magic_id, node)
 WHERE
     magicRun.state = 'new'
-    -- where hook %s
+-- WHERE hook %s
 ORDER BY
     magicRun.magic_id
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_totree.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_totree.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_totree.sql	(revision 24244)
@@ -6,6 +6,7 @@
     ra,
     decl,
-    tess_id
+    diffRun.tess_id
 FROM magicRun
+JOIN diffRun USING(diff_id)
 JOIN rawExp USING(exp_id)
 LEFT JOIN magicTree
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_warpskyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_warpskyfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_warpskyfile.sql	(revision 24244)
@@ -21,4 +21,5 @@
     warpRun.state = 'full'
     AND warpSkyfile.fault = 0
+    AND warpSkyfile.quality = 0
 --   AND magicRun.state = 'full'
 --   AND magicMask.fault = 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/pxadmin_create_tables.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/pxadmin_create_tables.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/pxadmin_create_tables.sql	(revision 24244)
@@ -18,6 +18,6 @@
     epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     PRIMARY KEY(exp_name, camera, telescope),
-    KEY(fault))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    KEY(fault)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE summitImfile (
@@ -34,7 +34,6 @@
     PRIMARY KEY(exp_name, camera, telescope, class, class_id),
     KEY(file_id),
-    FOREIGN KEY (exp_name, camera, telescope)
-        REFERENCES  summitExp(exp_name, camera, telescope))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(exp_name, camera, telescope) REFERENCES summitExp(exp_name, camera, telescope)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE pzDownloadExp (
@@ -46,7 +45,6 @@
     PRIMARY KEY(exp_name, camera, telescope),
     KEY(state),
-    FOREIGN KEY (exp_name, camera, telescope)
-        REFERENCES  summitExp(exp_name, camera, telescope))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(exp_name, camera, telescope) REFERENCES summitExp(exp_name, camera, telescope)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE pzDownloadImfile (
@@ -62,8 +60,7 @@
     PRIMARY KEY(exp_name, camera, telescope, class, class_id),
     KEY(fault),
-    FOREIGN KEY (exp_name, camera, telescope)
-        REFERENCES  pzDownloadExp(exp_name, camera, telescope),
-    FOREIGN KEY (exp_name, camera, telescope, class, class_id)
-        REFERENCES  summitImfile(exp_name, camera, telescope, class, class_id)) ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY (exp_name, camera, telescope) REFERENCES pzDownloadExp(exp_name, camera, telescope),
+    FOREIGN KEY(exp_name, camera, telescope, class, class_id) REFERENCES summitImfile(exp_name, camera, telescope, class, class_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE newExp (
@@ -98,7 +95,6 @@
     epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
     PRIMARY KEY(exp_id, tmp_class_id),
-    FOREIGN KEY (exp_id)
-        REFERENCES  newExp(exp_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(exp_id) REFERENCES newExp(exp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE rawExp (
@@ -172,7 +168,6 @@
     KEY(end_stage),
     KEY(fault),
-    FOREIGN KEY (exp_id)
-        REFERENCES  newExp(exp_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(exp_id) REFERENCES newExp(exp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE rawImfile (
@@ -237,4 +232,5 @@
     ignored TINYINT DEFAULT 0,
     hostname VARCHAR(64),
+    quality SMALLINT NOT NULL DEFAULT 0,
     fault SMALLINT NOT NULL,
     epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
@@ -245,10 +241,9 @@
     KEY(fault),
     KEY(raw_image_id),
+    KEY(quality),
+    KEY(dateobs),
     UNIQUE KEY(exp_id, tmp_class_id),
-    FOREIGN KEY (exp_id, tmp_class_id)
-        REFERENCES newImfile(exp_id, tmp_class_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
-
-CREATE TABLE guidePendingExp (guide_id BIGINT AUTO_INCREMENT, exp_id BIGINT, recipe VARCHAR(64), PRIMARY KEY(guide_id), KEY(guide_id), KEY(exp_id)) ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(exp_id, tmp_class_id) REFERENCES newImfile(exp_id, tmp_class_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE chipRun (
@@ -273,7 +268,6 @@
     KEY(end_stage),
     INDEX(chip_id, exp_id),
-    FOREIGN KEY (exp_id)
-        REFERENCES rawExp(exp_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE chipImfile (
@@ -283,6 +277,6 @@
     PRIMARY KEY(chip_id, class_id),
     KEY(chip_imfile_id),
-    FOREIGN KEY (chip_id) REFERENCES  chipRun(chip_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE chipProcessedImfile (
@@ -343,4 +337,5 @@
     n_cr INT,
     path_base VARCHAR(255),
+    quality SMALLINT NOT NULL DEFAULT 0,
     fault SMALLINT NOT NULL,
     magicked BIGINT,
@@ -348,14 +343,13 @@
     KEY(data_state),
     KEY(fault),
-    FOREIGN KEY (chip_id, exp_id)
-        REFERENCES  chipRun(chip_id, exp_id),
-    FOREIGN KEY (exp_id, class_id)
-        REFERENCES  rawImfile(exp_id, class_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    KEY(quality),
+    FOREIGN KEY(chip_id, exp_id) REFERENCES chipRun(chip_id, exp_id),
+    FOREIGN KEY(exp_id, class_id) REFERENCES rawImfile(exp_id, class_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE chipMask (
     label VARCHAR(64),
-    PRIMARY KEY(label))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    PRIMARY KEY(label)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE camRun (
@@ -380,7 +374,6 @@
     KEY(end_stage),
     INDEX(cam_id, chip_id),
-    FOREIGN KEY (chip_id)
-        REFERENCES chipRun(chip_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE camProcessedExp (
@@ -444,15 +437,16 @@
     n_astrom INT,
     path_base VARCHAR(255),
+    quality SMALLINT NOT NULL DEFAULT 0,
     fault SMALLINT NOT NULL,
     PRIMARY KEY(cam_id),
     KEY(fault),
-    FOREIGN KEY (cam_id)
-        REFERENCES camRun(cam_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    KEY(quality),
+    FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE camMask (
     label VARCHAR(64),
-    PRIMARY KEY(label))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    PRIMARY KEY(label)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE fakeRun (
@@ -475,7 +469,6 @@
     KEY(end_stage),
     INDEX(fake_id, cam_id),
-    FOREIGN KEY (cam_id)
-        REFERENCES camRun(cam_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE fakeProcessedImfile (
@@ -493,14 +486,12 @@
     PRIMARY KEY(fake_id, exp_id, class_id),
     KEY(fault),
-    FOREIGN KEY (fake_id)
-        REFERENCES fakeRun(fake_id),
-    FOREIGN KEY (exp_id, class_id)
-        REFERENCES rawImfile(exp_id, class_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(fake_id) REFERENCES fakeRun(fake_id),
+    FOREIGN KEY(exp_id, class_id) REFERENCES rawImfile(exp_id, class_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE fakeMask (
     label VARCHAR(64),
-    PRIMARY KEY(label))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    PRIMARY KEY(label)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE detRun (
@@ -535,5 +526,4 @@
     ref_det_id BIGINT,
     ref_iter INT,
-    -- parent INT, :: dropping this
     PRIMARY KEY(det_id),
     KEY(det_id),
@@ -543,21 +533,18 @@
     KEY(state),
     KEY(label),
-    -- KEY(parent), :: dropping this
-    INDEX(det_id, iteration))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    INDEX(det_id, iteration)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE detInputExp (
-det_id BIGINT,
-iteration INT,
-exp_id BIGINT,
-include TINYINT,
-PRIMARY KEY(det_id, iteration, exp_id),
-INDEX(det_id, exp_id),
-INDEX(det_id, iteration),
-FOREIGN KEY (det_id)
-REFERENCES  detRun(det_id),
-FOREIGN KEY (exp_id)
-REFERENCES  rawExp(exp_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    det_id BIGINT,
+    iteration INT,
+    exp_id BIGINT,
+    include TINYINT,
+    PRIMARY KEY(det_id, iteration, exp_id),
+    INDEX(det_id, exp_id),
+    INDEX(det_id, iteration),
+    FOREIGN KEY(det_id) REFERENCES detRun(det_id),
+    FOREIGN KEY(exp_id) REFERENCES rawExp(exp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE detProcessedImfile (
@@ -585,8 +572,6 @@
     INDEX(det_id, class_id),
     INDEX(det_id, exp_id),
-    FOREIGN KEY (det_id, exp_id)
-        REFERENCES  detInputExp(det_id, exp_id),
-    FOREIGN KEY (exp_id, class_id)
-        REFERENCES  rawImfile(exp_id, class_id)
+    FOREIGN KEY(det_id, exp_id) REFERENCES detInputExp(det_id, exp_id),
+    FOREIGN KEY(exp_id, class_id) REFERENCES rawImfile(exp_id, class_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -611,8 +596,6 @@
     PRIMARY KEY(det_id, exp_id),
     KEY(fault),
-    FOREIGN KEY (det_id, exp_id)
-        REFERENCES  detInputExp(det_id, exp_id),
-    FOREIGN KEY (det_id, exp_id)
-        REFERENCES  detProcessedImfile(det_id, exp_id)
+    FOREIGN KEY(det_id, exp_id) REFERENCES detInputExp(det_id, exp_id),
+    FOREIGN KEY(det_id, exp_id) REFERENCES detProcessedImfile(det_id, exp_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -635,8 +618,6 @@
     PRIMARY KEY(det_id, iteration, class_id),
     KEY(fault),
-    FOREIGN KEY (det_id, iteration)
-        REFERENCES  detInputExp(det_id, iteration),
-    FOREIGN KEY (det_id, class_id)
-        REFERENCES  detProcessedImfile(det_id, class_id)
+    FOREIGN KEY(det_id, iteration) REFERENCES detInputExp(det_id, iteration),
+    FOREIGN KEY(det_id, class_id) REFERENCES detProcessedImfile(det_id, class_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -650,8 +631,6 @@
     PRIMARY KEY(det_id, iteration, class_id),
     KEY(fault),
-    FOREIGN KEY (det_id, iteration)
-    REFERENCES  detInputExp(det_id, iteration),
-    FOREIGN KEY (det_id, iteration, class_id)
-    REFERENCES  detStackedImfile(det_id, iteration, class_id)
+    FOREIGN KEY(det_id, iteration) REFERENCES detInputExp(det_id, iteration),
+    FOREIGN KEY(det_id, iteration, class_id) REFERENCES  detStackedImfile(det_id, iteration, class_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -675,8 +654,6 @@
     KEY(fault),
     INDEX(det_id, iteration),
-    FOREIGN KEY (det_id)
-    REFERENCES  detInputExp(det_id),
-    FOREIGN KEY (det_id, iteration, class_id)
-    REFERENCES  detNormalizedStatImfile(det_id, iteration, class_id)
+    FOREIGN KEY(det_id) REFERENCES detInputExp(det_id),
+    FOREIGN KEY(det_id, iteration, class_id) REFERENCES detNormalizedStatImfile(det_id, iteration, class_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -698,8 +675,6 @@
     PRIMARY KEY(det_id, iteration),
     KEY(fault),
-    FOREIGN KEY (det_id, iteration)
-    REFERENCES  detInputExp(det_id, iteration),
-    FOREIGN KEY (det_id, iteration)
-    REFERENCES  detNormalizedImfile(det_id, iteration)
+    FOREIGN KEY(det_id, iteration) REFERENCES detInputExp(det_id, iteration),
+    FOREIGN KEY(det_id, iteration) REFERENCES detNormalizedImfile(det_id, iteration)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -736,8 +711,6 @@
     KEY(fault),
     INDEX(det_id, iteration, exp_id),
-    FOREIGN KEY (det_id, iteration, exp_id)
-    REFERENCES  detInputExp(det_id, iteration, exp_id),
-    FOREIGN KEY (det_id, exp_id, class_id)
-    REFERENCES  detProcessedImfile(det_id, exp_id, class_id)
+    FOREIGN KEY (det_id, iteration, exp_id) REFERENCES detInputExp(det_id, iteration, exp_id),
+    FOREIGN KEY (det_id, exp_id, class_id) REFERENCES detProcessedImfile(det_id, exp_id, class_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -771,8 +744,6 @@
        KEY(fault),
        INDEX(det_id, iteration),
-       FOREIGN KEY (det_id, iteration, exp_id)
-       REFERENCES  detInputExp(det_id, iteration, exp_id),
-       FOREIGN KEY (det_id, iteration, exp_id)
-       REFERENCES  detResidImfile(det_id, iteration, exp_id)
+       FOREIGN KEY(det_id, iteration, exp_id) REFERENCES detInputExp(det_id, iteration, exp_id),
+       FOREIGN KEY(det_id, iteration, exp_id) REFERENCES detResidImfile(det_id, iteration, exp_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -788,8 +759,6 @@
        PRIMARY KEY(det_id, iteration),
        KEY(fault),
-       FOREIGN KEY (det_id, iteration)
-         REFERENCES  detInputExp(det_id, iteration),
-       FOREIGN KEY (det_id, iteration)
-         REFERENCES  detResidExp(det_id, iteration)
+       FOREIGN KEY(det_id, iteration) REFERENCES detInputExp(det_id, iteration),
+       FOREIGN KEY(det_id, iteration) REFERENCES detResidExp(det_id, iteration)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -812,6 +781,5 @@
        PRIMARY KEY(det_id, iteration, class_id),
        KEY(fault),
-       FOREIGN KEY (det_id, iteration)
-         REFERENCES  detRun(det_id, iteration)
+       FOREIGN KEY(det_id, iteration) REFERENCES detRun(det_id, iteration)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -838,6 +806,5 @@
     KEY(end_stage),
     INDEX(warp_id, fake_id),
-    FOREIGN KEY (fake_id)
-        REFERENCES fakeRun(fake_id)
+    FOREIGN KEY(fake_id) REFERENCES fakeRun(fake_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -850,6 +817,5 @@
     PRIMARY KEY(warp_id, skycell_id, tess_id, class_id),
     KEY(fault),
-    FOREIGN KEY (warp_id)
-        REFERENCES warpRun(warp_id)
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -860,6 +826,6 @@
     PRIMARY KEY(warp_id, skycell_id),
     KEY(warp_skyfile_id),
-    FOREIGN KEY (warp_id) REFERENCES  warpRun(warp_id))
-ENGINE=innodb DEFAULT CHARSET=latin1;
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 
@@ -881,12 +847,12 @@
     ymin INT,
     ymax INT,
-    ignored TINYINT,
+    quality SMALLINT NOT NULL DEFAULT 0,
     fault SMALLINT,
     magicked TINYINT,
     PRIMARY KEY(warp_id, skycell_id, tess_id),
     KEY(good_frac),
-    KEY(ignored), KEY(fault),
-    FOREIGN KEY (warp_id, skycell_id, tess_id)
-        REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id)
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(warp_id, skycell_id, tess_id) REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -911,5 +877,6 @@
         KEY(state),
         KEY(skycell_id),
-        KEY(tess_id)
+        KEY(tess_id),
+        KEY(label)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -918,6 +885,6 @@
         warp_id BIGINT,
         PRIMARY KEY(stack_id, warp_id),
-        FOREIGN KEY (stack_id)  REFERENCES  stackRun(stack_id),
-        FOREIGN KEY (warp_id)  REFERENCES  warpSkyfile(warp_id)
+        FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id),
+        FOREIGN KEY(warp_id) REFERENCES warpSkyfile(warp_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -948,4 +915,5 @@
         hostname VARCHAR(64),
         good_frac FLOAT,
+        quality SMALLINT NOT NULL DEFAULT 0,
         fault SMALLINT,
         PRIMARY KEY(stack_id),
@@ -953,5 +921,6 @@
         KEY(good_frac),
         KEY(fault),
-        FOREIGN KEY (stack_id)  REFERENCES  stackRun(stack_id)
+        KEY(quality),
+        FOREIGN KEY(stack_id) REFERENCES stackRun(stack_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -965,5 +934,6 @@
         registered DATETIME,
         tess_id VARCHAR(64),
-        exp_id  BIGINT,
+        bothways TINYINT DEFAULT 0,
+        exposure TINYINT DEFAULT 0,
         magicked TINYINT,
         PRIMARY KEY(diff_id),
@@ -971,5 +941,5 @@
         KEY(state),
         KEY(tess_id),
-        FOREIGN KEY (exp_id) REFERENCES rawExp(exp_id)
+        KEY(label)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -991,9 +961,9 @@
         KEY(skycell_id),
         KEY(tess_id),
-        FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id),
-        FOREIGN KEY (warp1, skycell_id, tess_id)  REFERENCES  warpSkyfile(warp_id, skycell_id, tess_id),
-        FOREIGN KEY (warp2, skycell_id, tess_id)  REFERENCES  warpSkyfile(warp_id, skycell_id, tess_id),
-        FOREIGN KEY (stack1)  REFERENCES  stackSumSkyfile(stack_id),
-        FOREIGN KEY (stack2)  REFERENCES  stackSumSkyfile(stack_id)
+        FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id),
+        FOREIGN KEY(warp1, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id),
+        FOREIGN KEY(warp2, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id),
+        FOREIGN KEY(stack1) REFERENCES stackSumSkyfile(stack_id),
+        FOREIGN KEY(stack2) REFERENCES stackSumSkyfile(stack_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1001,6 +971,6 @@
         diff_id BIGINT,
         skycell_id VARCHAR(64),
-        uri VARCHAR(255),
         path_base VARCHAR(255),
+        data_state VARCHAR(64),
         bg DOUBLE,
         bg_stdev DOUBLE,
@@ -1015,4 +985,5 @@
         kernel_xy FLOAT,
         kernel_yy FLOAT,
+        deconv_max FLOAT,
         sources INT,
         dtime_diff FLOAT,
@@ -1022,4 +993,5 @@
         hostname VARCHAR(64),
         good_frac FLOAT,
+        quality SMALLINT NOT NULL DEFAULT 0,
         fault SMALLINT,
         magicked TINYINT,
@@ -1027,5 +999,6 @@
         KEY(good_frac),
         KEY(fault),
-        FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id)
+        KEY(quality),
+        FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1034,4 +1007,5 @@
         exp_id BIGINT,
         diff_id BIGINT,
+        inverse TINYINT NOT NULL DEFAULT 0,
         state VARCHAR(64),
         workdir VARCHAR(255),
@@ -1047,6 +1021,6 @@
         KEY(label),
         KEY(fault),
-        FOREIGN KEY (exp_id)  REFERENCES rawExp(exp_id),
-        FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id)
+        FOREIGN KEY(exp_id)  REFERENCES rawExp(exp_id),
+        FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1056,6 +1030,6 @@
         node VARCHAR(64),
         PRIMARY KEY(magic_id, diff_id, node),
-        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id),
-        FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id)
+        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id),
+        FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1068,5 +1042,5 @@
         KEY(dep),
         INDEX(magic_id, node),
-        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id)
+        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1074,9 +1048,9 @@
         magic_id BIGINT,
         node VARCHAR(64),
-        uri VARCHAR(255),
+        path_base VARCHAR(255),
         fault SMALLINT,
         PRIMARY KEY(magic_id, node),
-        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id),
-        FOREIGN KEY (magic_id, node)  REFERENCES  magicTree(magic_id, node),
+        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id),
+        FOREIGN KEY(magic_id, node) REFERENCES magicTree(magic_id, node),
         KEY(fault)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -1088,5 +1062,5 @@
         fault SMALLINT,
         PRIMARY KEY(magic_id),
-        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id),
+        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id),
         KEY(fault)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -1109,5 +1083,5 @@
         KEY(magic_id),
         KEY(label),
-        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id)
+        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1120,5 +1094,5 @@
     PRIMARY KEY(magic_ds_id, component),
     KEY(fault),
-    FOREIGN KEY (magic_ds_id) REFERENCES magicDSRun(magic_ds_id)
+    FOREIGN KEY(magic_ds_id) REFERENCES magicDSRun(magic_ds_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1139,5 +1113,5 @@
         KEY(cal_id),
         KEY(last_step),
-        FOREIGN KEY (cal_id)  REFERENCES  calDB(cal_id)
+        FOREIGN KEY(cal_id) REFERENCES calDB(cal_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1167,6 +1141,6 @@
         include TINYINT,
         PRIMARY KEY(corr_id, chip_id),
-        FOREIGN KEY (corr_id)  REFERENCES  flatcorrRun(corr_id),
-        FOREIGN KEY (chip_id)  REFERENCES  chipRun(chip_id)
+        FOREIGN KEY(corr_id) REFERENCES flatcorrRun(corr_id),
+        FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1177,7 +1151,7 @@
         include TINYINT,
         PRIMARY KEY(corr_id, cam_id),
-        FOREIGN KEY (corr_id)  REFERENCES  flatcorrRun(corr_id),
-        FOREIGN KEY (chip_id)  REFERENCES  chipRun(chip_id),
-        FOREIGN KEY (cam_id)  REFERENCES  camRun(cam_id)
+        FOREIGN KEY(corr_id) REFERENCES flatcorrRun(corr_id),
+        FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id),
+        FOREIGN KEY(cam_id) REFERENCES camRun(cam_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1231,9 +1205,9 @@
         PRIMARY KEY(job_id, req_id),
         KEY(job_id),
-        FOREIGN KEY (req_id)  REFERENCES  pstampRequest(req_id)
+        FOREIGN KEY(req_id) REFERENCES pstampRequest(req_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
 CREATE TABLE distTarget (
-    target_id   BIGINT AUTO_INCREMENT, 
+    target_id   BIGINT AUTO_INCREMENT,
     obs_mode    VARCHAR(64),
     stage       VARCHAR(64),
@@ -1252,4 +1226,5 @@
     outroot     VARCHAR(255),
     clean       TINYINT,
+    no_magic    TINYINT,
     state       VARCHAR(64),
     time_stamp  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
@@ -1262,5 +1237,5 @@
 
 CREATE TABLE distComponent (
-    dist_id     BIGINT, 
+    dist_id     BIGINT,
     component   VARCHAR(64),
     bytes       INT,
@@ -1280,5 +1255,4 @@
     dbname      VARCHAR(64),
     dbhost      VARCHAR(64),
-    prod_root   VARCHAR(255),
     PRIMARY KEY(prod_id)
 )  ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -1301,10 +1275,8 @@
     dest_id     BIGINT,
     target_id   BIGINT,
-    prod_id     BIGINT,
     state       VARCHAR(64),
     PRIMARY KEY(int_id),
     FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id),
-    FOREIGN KEY(target_id) REFERENCES distTarget(target_id),
-    FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id)
+    FOREIGN KEY(target_id) REFERENCES distTarget(target_id)
 )  ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1313,10 +1285,13 @@
     dist_id     BIGINT,
     prod_id     BIGINT,
-    name        VARCHAR(64),
+    name        VARCHAR(255),
     state       VARCHAR(64),
     registered  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-    PRIMARY KEY(fs_id),
+    fault       SMALLINT DEFAULT 0,
+    PRIMARY KEY(dist_id, prod_id),
+    KEY(fs_id),
     FOREIGN KEY(dist_id) REFERENCES distRun(dist_id),
     FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id)
+--    KEY(dist_id, prod_id)
 )  ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1326,5 +1301,7 @@
     dest_id     BIGINT,
     state       VARCHAR(64),
+    status_fs_name   VARCHAR(64),
     registered  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    fault       SMALLINT DEFAULT 0,
     PRIMARY KEY(rc_id),
     FOREIGN KEY(fs_id) REFERENCES rcDSFileset(fs_id),
@@ -1332,6 +1309,63 @@
 )  ENGINE=innodb DEFAULT CHARSET=latin1;
 
-
-
--- This comment line is here to avoid empty query error. 
+-- Sources from which to receive files
+CREATE TABLE receiveSource (
+    source_id BIGINT AUTO_INCREMENT, -- unique identifier
+    source VARCHAR(128) NOT NULL, -- source URI
+    product VARCHAR(64) NOT NULL, -- product of interest
+    workdir VARCHAR(255) NOT NULL, -- where to extract
+    comment VARCHAR(255),       -- for human memory
+    fileset_last VARCHAR(128),  -- last fileset seen
+    status_product VARCHAR(64), -- status data store product
+    ds_dbname VARCHAR(64),      -- status data store's database name
+    ds_dbhost VARCHAR(64),      -- status data store's host name
+    PRIMARY KEY(source_id),
+    KEY(source),
+    KEY(product),
+    KEY(comment)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Filesets to receive
+CREATE TABLE receiveFileset (
+    fileset_id BIGINT AUTO_INCREMENT, -- unique identifier
+    source_id BIGINT NOT NULL,  -- link to receiveSource
+    fileset VARCHAR(128) NOT NULL, -- fileset to receive
+    state VARCHAR(64), -- new or full
+    dirinfo VARCHAR(255), -- uri for directory info file for this run
+    dbinfo VARCHAR(255), -- uri for database dump file for this run
+    fault SMALLINT NOT NULL DEFAULT 0, -- Fault code
+    PRIMARY KEY(fileset_id),
+    KEY(source_id),
+    CONSTRAINT UNIQUE(source_id, fileset),
+    FOREIGN KEY(source_id) REFERENCES receiveSource(source_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Files to receive
+CREATE TABLE receiveFile (
+    file_id BIGINT AUTO_INCREMENT, -- unique identifier
+    fileset_id BIGINT NOT NULL,  -- link to receiveFileset
+    file VARCHAR(128) NOT NULL, -- file to receive
+    bytes BIGINT,
+    md5sum VARCHAR(255),
+    file_type VARCHAR(64),
+    component VARCHAR(64),
+    PRIMARY KEY(file_id),
+    KEY(fileset_id),
+    CONSTRAINT UNIQUE(fileset_id, file),
+    FOREIGN KEY(fileset_id) REFERENCES receiveFileset(fileset_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Result of receiving files
+CREATE TABLE receiveResult (
+    file_id BIGINT AUTO_INCREMENT, -- link to receiveFile
+    dtime_copy FLOAT,           -- Time to copy
+    dtime_extract FLOAT,        -- Time to extract
+    fault SMALLINT NOT NULL DEFAULT 0, -- Fault code
+    PRIMARY KEY(file_id),
+    KEY(fault),
+    FOREIGN KEY(file_id) REFERENCES receiveFile(file_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
+-- This comment line is here to avoid empty query error.
 -- Another way to avoid that problem is to omit the semicolon above but I think that is untidy.
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/pxadmin_drop_tables.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/pxadmin_drop_tables.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/pxadmin_drop_tables.sql	(revision 24244)
@@ -1,3 +1,4 @@
 SET FOREIGN_KEY_CHECKS=0;
+
 DROP TABLE IF EXISTS pzDataStore;
 DROP TABLE IF EXISTS summitExp;
@@ -9,5 +10,4 @@
 DROP TABLE IF EXISTS rawExp;
 DROP TABLE IF EXISTS rawImfile;
-DROP TABLE IF EXISTS guidePendingExp;
 DROP TABLE IF EXISTS chipRun;
 DROP TABLE IF EXISTS chipProcessedImfile;
@@ -48,5 +48,4 @@
 DROP TABLE IF EXISTS magicNodeResult;
 DROP TABLE IF EXISTS magicMask;
-DROP TABLE IF EXISTS magicSkyfileMask;
 DROP TABLE IF EXISTS magicDSRun;
 DROP TABLE IF EXISTS magicDSFile;
@@ -69,4 +68,8 @@
 DROP TABLE IF EXISTS rcDSFileset;
 DROP TABLE IF EXISTS rcRun;
+DROP TABLE IF EXISTS receiveSource;
+DROP TABLE IF EXISTS receiveFileset;
+DROP TABLE IF EXISTS receiveFile;
+DROP TABLE IF EXISTS receiveResult;
 
 SET FOREIGN_KEY_CHECKS=1
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/pztool_find_completed_exp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/pztool_find_completed_exp.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/pztool_find_completed_exp.sql	(revision 24244)
@@ -3,4 +3,5 @@
     camera,
     telescope,
+    dateobs,
     state
 FROM (
@@ -8,5 +9,6 @@
         pzDownloadImfile.*,
         pzDownloadExp.state,
-        summitExp.imfiles
+        summitExp.imfiles,
+        summitExp.dateobs
     FROM pzDownloadExp
     JOIN summitExp
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/rcserver_updatercrun.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/rcserver_updatercrun.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/rcserver_updatercrun.sql	(revision 24244)
@@ -0,0 +1,9 @@
+UPDATE rcRun
+JOIN rcDestination USING(dest_id)
+JOIN rcDSProduct USING(prod_id) 
+JOIN rcDSFileset using(prod_id, fs_id)
+SET
+rcRun.fault= 42, rcDestination.last_fileset = 'o4741g0236o.chip.14519.33.1'
+
+
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_addfileset.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_addfileset.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_addfileset.sql	(revision 24244)
@@ -0,0 +1,4 @@
+SELECT fileset_id
+FROM receiveFileset
+WHERE source_id = %s
+    AND fileset = '%s'
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_list.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_list.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_list.sql	(revision 24244)
@@ -0,0 +1,2 @@
+SELECT *
+FROM receiveSource
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_pendingfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_pendingfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_pendingfile.sql	(revision 24244)
@@ -0,0 +1,22 @@
+SELECT
+    receiveFile.file_id,
+    source,
+    product,
+    workdir,
+    dirinfo,
+    fileset,
+    fileset_id,
+    file,
+    bytes,
+    md5sum,
+    file_type,
+    component
+FROM receiveFile
+JOIN receiveFileset USING(fileset_id)
+JOIN receiveSource USING(source_id)
+LEFT JOIN receiveResult
+    ON receiveResult.file_id = receiveFile.file_id
+WHERE
+    ((receiveFileset.state = 'new') OR (receiveFileset.state = 'listed' AND receiveFile.component = 'dirinfo'))
+    AND receiveFileset.fault = 0
+    AND receiveResult.file_id IS NULL
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_pendingfileset.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_pendingfileset.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_pendingfileset.sql	(revision 24244)
@@ -0,0 +1,5 @@
+SELECT *
+FROM receiveFileset
+JOIN receiveSource USING(source_id)
+LEFT JOIN receiveFile USING(fileset_id)
+WHERE receiveFile.fileset_id IS NULL
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_revert.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_revert.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_revert.sql	(revision 24244)
@@ -0,0 +1,6 @@
+DELETE FROM receiveResult
+USING receiveResult, receiveFile, receiveFileset, receiveSource
+WHERE receiveResult.file_id = receiveFile.file_id
+    AND receiveFile.fileset_id = receiveFileset.fileset_id
+    AND receiveFileset.source_id = receiveSource.source_id
+    AND receiveResult.fault != 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_toadvance.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_toadvance.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/receivetool_toadvance.sql	(revision 24244)
@@ -0,0 +1,16 @@
+SELECT
+    receiveFileset.fileset_id,
+    receiveFileset.fileset,
+    receiveFileset.dbinfo,
+    receiveSource.status_product,
+    receiveSource.ds_dbname,
+    receiveSource.ds_dbhost
+FROM receiveFileset 
+    JOIN receiveSource USING(source_id)
+    JOIN receiveFile USING(fileset_id)
+    LEFT JOIN receiveResult USING(file_id)
+WHERE receiveFileset.state = 'new' 
+    AND receiveFileset.fault = 0
+GROUP BY fileset_id 
+HAVING COUNT(receiveFile.file_id) = COUNT(receiveResult.file_id) 
+    AND SUM(receiveResult.fault) = 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/regtool_export_exp.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/regtool_export_exp.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/regtool_export_exp.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT * FROM rawExp
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/regtool_export_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/regtool_export_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/regtool_export_imfile.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT * FROM rawImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_insert.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_insert.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_insert.sql	(revision 24244)
@@ -21,3 +21,3 @@
     AND rawExp.filter = '%s'
     AND warpSkyfile.fault = 0
-    AND warpSkyfile.ignored = 0
+    AND warpSkyfile.quality = 0
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_insert_random_part1.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_insert_random_part1.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_insert_random_part1.sql	(revision 24244)
@@ -28,5 +28,5 @@
         AND rawExp.filter = '%s' -- the result of the query is grouped by filter and inserted for one at a time
         AND warpSkyfile.fault = 0
-        AND warpSkyfile.ignored = 0
+        AND warpSkyfile.quality = 0
 -- Put additional constraints here
 -- stacktool_definebyquery_insert_random_part2.sql goes here
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_part1.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_part1.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_part1.sql	(revision 24244)
@@ -17,13 +17,13 @@
         COUNT(warpSkyfile.skycell_id) AS num_warp -- number of warps that can be stacked
     FROM warpRun
-        JOIN warpSkyfile USING(warp_id)
-        JOIN fakeRun USING(fake_id)
-        JOIN camRun USING(cam_id)
-        JOIN chipRun USING(chip_id)
-        JOIN rawExp USING(exp_id)
+    JOIN warpSkyfile USING(warp_id)
+    JOIN fakeRun USING(fake_id)
+    JOIN camRun USING(cam_id)
+    JOIN chipRun USING(chip_id)
+    JOIN rawExp USING(exp_id)
     WHERE
         warpRun.state = 'full'
-    AND warpSkyfile.ignored = 0
-    AND warpSkyfile.fault = 0
+        AND warpSkyfile.fault = 0
+        AND warpSkyfile.quality = 0
     -- Here should follow the SQL in stacktool_definebyquery_part2.sql,
     -- after any additional WHERE conditions have been added
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_test.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_test.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_definebyquery_test.sql	(revision 24244)
@@ -31,5 +31,5 @@
     WHERE
         warpRun.state = 'full'
-    AND warpSkyfile.ignored = 0
+    AND warpSkyfile.quality = 0
     AND warpSkyfile.fault = 0
     -- Any additional selection on warps/exposures goes here
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_export_input_skyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_export_input_skyfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_export_input_skyfile.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT stackInputSkyfile.* FROM stackInputSkyfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_export_run.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_export_run.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_export_run.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT stackRun.* from stackRun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_export_sum_skyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_export_sum_skyfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_export_sum_skyfile.sql	(revision 24244)
@@ -0,0 +1,1 @@
+SELECT stackSumSkyfile.* FROM stackSumSkyfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_find_complete_warps.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_find_complete_warps.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_find_complete_warps.sql	(revision 24244)
@@ -6,5 +6,5 @@
     COUNT(stackRun.stack_id) AS num_extant
 FROM warpRun
-JOIN warpSkyfile 
+JOIN warpSkyfile
     USING(warp_id)
 JOIN fakeRun
@@ -23,5 +23,6 @@
 WHERE
     warpRun.state = 'full'
-    AND warpSkyfile.ignored = 0
+    AND warpSkyfile.fault = 0
+    AND warpSkyfile.quality = 0
 GROUP BY
     warpSkyfile.skycell_id, stackRun.stack_id
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_pendingcleanuprun.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_pendingcleanuprun.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_pendingcleanuprun.sql	(revision 24244)
@@ -27,3 +27,3 @@
      USING (exp_id)
 WHERE
-    stackRun.state = 'goto_cleaned'
+    (stackRun.state = 'goto_cleaned' OR stackRun.state = 'goto_scrubbed' OR stackRun.state = 'goto_purged')
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_pendingcleanupskyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_pendingcleanupskyfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_pendingcleanupskyfile.sql	(revision 24244)
@@ -4,8 +4,9 @@
     stackRun.workdir,
     stackRun.dvodb,
-    stackRun.tess_id
+    stackRun.tess_id,
+    stackRun.skycell_id
 FROM stackRun
 JOIN stackSumSkyfile
     USING(stack_id)
 WHERE
-    stackRun.state = 'goto_cleaned'
+    (stackRun.state = 'goto_cleaned' OR stackRun.state = 'goto_scrubbed' OR stackRun.state = 'goto_purged')
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_sumskyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_sumskyfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_sumskyfile.sql	(revision 24244)
@@ -2,4 +2,6 @@
     stackSumSkyfile.*,
     stackRun.state,
+    stackRun.skycell_id,
+    stackRun.workdir,
     (SELECT rawExp.camera FROM 
         stackInputSkyfile 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_tosum.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_tosum.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/stacktool_tosum.sql	(revision 24244)
@@ -12,7 +12,4 @@
     USING(stack_id)
 WHERE
-    ((stackRun.state = 'new'
-        AND stackSumSkyfile.stack_id IS NULL)
-    OR
-    (stackRun.state = 'update'
-        AND stackSumSkyfile.fault = 0))
+    ((stackRun.state = 'new' AND stackSumSkyfile.stack_id IS NULL)
+    OR (stackRun.state = 'update' AND stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0))
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_definebyquery.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_definebyquery.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_definebyquery.sql	(revision 24244)
@@ -1,37 +1,32 @@
-SELECT
-    *
-FROM
-    (SELECT DISTINCT
-        fakeRun.*,
-        chipRun.chip_id,
-        rawExp.camera,
-        rawExp.telescope,
-        rawExp.dateobs,
-        rawExp.exp_tag,
-        rawExp.exp_name,
-        rawExp.exp_type,
-        rawExp.filelevel,
-        rawExp.filter,
-        rawExp.airmass,
-        rawExp.ra,
-        rawExp.decl,
-        rawExp.exp_time,
-        rawExp.sat_pixel_frac,
-        rawExp.bg,
-        rawExp.bg_stdev,
-        rawExp.bg_mean_stdev,
-        rawExp.alt,
-        rawExp.az,
-        rawExp.ccd_temp,
-        rawExp.posang,
-        rawExp.object,
-        rawExp.sun_angle
-    FROM fakeRun
-    JOIN camRun
-        using(cam_id)
-    JOIN chipRun
-        using(chip_id)
-    JOIN rawExp
-        using(exp_id)
-    WHERE
-        fakeRun.state = 'full') as Foo
+SELECT DISTINCT
+    fakeRun.*,
+    chipRun.chip_id,
+    rawExp.camera,
+    rawExp.telescope,
+    rawExp.dateobs,
+    rawExp.exp_id,
+    rawExp.exp_tag,
+    rawExp.exp_name,
+    rawExp.exp_type,
+    rawExp.filelevel,
+    rawExp.filter,
+    rawExp.airmass,
+    rawExp.ra,
+    rawExp.decl,
+    rawExp.exp_time,
+    rawExp.sat_pixel_frac,
+    rawExp.bg,
+    rawExp.bg_stdev,
+    rawExp.bg_mean_stdev,
+    rawExp.alt,
+    rawExp.az,
+    rawExp.ccd_temp,
+    rawExp.posang,
+    rawExp.object,
+    rawExp.sun_angle
+FROM fakeRun
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+WHERE camRun.state = 'full'
+AND chipRun.state = 'full'
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_imfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_imfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    warpImfile.*
+FROM warpImfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_run.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_run.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_run.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    warpRun.*
+FROM warpRun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_skycell_map.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_skycell_map.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_skycell_map.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    warpSkyCellMap.*
+FROM warpSkyCellMap
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_skyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_skyfile.sql	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_export_skyfile.sql	(revision 24244)
@@ -0,0 +1,3 @@
+SELECT
+    warpSkyfile.*
+FROM warpSkyfile
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_finished_run_select.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_finished_run_select.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_finished_run_select.sql	(revision 24244)
@@ -5,4 +5,5 @@
    (SELECT DISTINCT
        warpRun.warp_id,
+       warpRun.label as label,
        warpSkyCellMap.warp_id as foo,
        warpSkyfile.warp_id as bar,
@@ -19,4 +20,4 @@
    HAVING
        COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)
-       AND SUM(warpSkyfile.fault) = 0
+       AND SUM(warpSkyfile.fault > 0) = 0
  ) as Foo
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_imfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_imfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_imfile.sql	(revision 24244)
@@ -26,3 +26,5 @@
     AND camRun.state = 'full'
     AND chipRun.state = 'full'
+    AND chipProcessedImfile.quality = 0
+    and camProcessedExp.quality = 0
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_pendingcleanupskyfile.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_pendingcleanupskyfile.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_pendingcleanupskyfile.sql	(revision 24244)
@@ -12,5 +12,5 @@
    ((warpRun.state = 'goto_cleaned'  AND warpSkyfile.data_state = 'full')
     OR
-    (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state = 'full')
+    (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state != 'scrubbed')
     OR
     (warpRun.state = 'goto_purged'   AND warpSkyfile.data_state != 'purged'))
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_revertwarped_delete.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_revertwarped_delete.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_revertwarped_delete.sql	(revision 24244)
@@ -6,3 +6,5 @@
     AND camRun.chip_id = chipRun.chip_id
     AND chipRun.exp_id = rawExp.exp_id
+    AND warpRun.state = 'new'
     AND warpSkyfile.fault != 0
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_warped.sql
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_warped.sql	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_warped.sql	(revision 24244)
@@ -2,4 +2,5 @@
     warpSkyfile.*,
     warpRun.state,
+    warpRun.workdir,
     warpImfile.warp_skyfile_id,
     rawExp.exp_id,
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/Makefile.am	(revision 24244)
@@ -19,5 +19,6 @@
 	regtool \
 	stacktool \
-	warptool
+	warptool \
+	receivetool
 
 
@@ -212,4 +213,11 @@
     pxinjectConfig.c
 
+
+receivetool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+receivetool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+receivetool_SOURCES = \
+    receivetool.c \
+    receivetoolConfig.c
+
 clean-local:
 	-rm -f TAGS
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtool.c	(revision 24244)
@@ -109,9 +109,9 @@
     psMetadata *where = psMetadataAlloc();
     pxcamGetSearchArgs (config, where);
-    PXOPT_COPY_STR(config->args, where, "-label",     "chipRun.label",     "==");
-    PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
-
-    if (!psListLength(where->list)
-        && !psMetadataLookupBool(NULL, config->args, "-all")) {
+    pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "==");
+
+    if (!psListLength(where->list) &&
+        !psMetadataLookupBool(NULL, config->args, "-all")) {
         psFree(where);
         psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
@@ -137,6 +137,6 @@
     // use psDBGenerateWhereSQL because the SQL yields an intermediate table
     if (where && psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereSQL(where, NULL);
-        psStringAppend(&query, "%s", whereClause);
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
@@ -175,11 +175,32 @@
     // old values in place (i.e., passing the values through).
 
-    // loop over our list of chipRun rows
+    // if end_stage is warp (or NULL), check for valid tess_id
     for (long i = 0; i < psArrayLength(output); i++) {
         psMetadata *md = output->data[i];
 
-        chipRunRow *row = chipRunObjectFromMetadata(md);
+        bool status;
+        char *end_stage = psMetadataLookupStr(&status, md, "end_stage");
+	if (end_stage && strcasecmp(end_stage, "warp")) continue;
+
+        char *raw_tess_id   = psMetadataLookupStr(&status, md, "tess_id");
+	if (raw_tess_id || tess_id) continue;
+
+        char *label  = psMetadataLookupStr(&status, md, "label");
+        psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id");
+
+        if (!status) {
+	    psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id);
+            psFree(output);
+            return false;
+        }
+    }
+
+    // loop over our list of camRun rows
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        camRunRow *row = camRunObjectFromMetadata(md);
         if (!row) {
-            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun");
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun");
             psFree(output);
             return false;
@@ -247,5 +268,5 @@
 
     if (state) {
-        // set chipRun.state to state
+        // set camRun.state to state
         if (!pxcamRunSetStateByQuery(config, where, state)) {
             psFree(where);
@@ -255,5 +276,5 @@
 
     if (label) {
-        // set chipRun.label to label
+        // set camRun.label to label
         if (!pxcamRunSetLabelByQuery(config, where, label)) {
             psFree(where);
@@ -275,5 +296,5 @@
     pxcamGetSearchArgs (config, where);
     PXOPT_COPY_S64(config->args, where, "-cam_id",    "camRun.cam_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label",     "camRun.label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "==");
     PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "==");
 
@@ -289,6 +310,6 @@
     // use psDBGenerateWhereSQL because the SQL yields an intermediate table
     if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereSQL(where, NULL);
-        psStringAppend(&query, "%s", whereClause);
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
@@ -339,10 +360,8 @@
     pxcamGetSearchArgs (config, where);
     PXOPT_COPY_S64(config->args, where, "-cam_id",    "camRun.cam_id",                "==");
-    PXOPT_COPY_STR(config->args, where, "-label",     "camRun.label",                 "==");
+    pxAddLabelSearchArgs (config, where, "-label",    "camRun.label",                 "==");
     PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction",             "==");
-    PXOPT_COPY_S64(config->args, where, "-chip_id",   "chipRun.chip_id",              "==");
-    PXOPT_COPY_STR(config->args, where, "-class_id",  "chipProcessedImfile.class_id", "==");
-
-    // XXX is this used? PXOPT_COPY_STR(config->args, where, "-class",    "class",         "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id",   "camRun.chip_id",              "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id",  "camProcessedExp.class_id", "==");
 
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
@@ -356,6 +375,6 @@
     // use psDBGenerateWhereSQL because the SQL yields an intermediate table
     if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereSQL(where, NULL);
-        psStringAppend(&query, "%s", whereClause);
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
@@ -381,5 +400,5 @@
 
     // negate simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "chipProcessedImfile", !simple)) {
+    if (!ippdbPrintMetadatas(stdout, output, "camProcessedExp", !simple)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print array");
         psFree(output);
@@ -470,9 +489,10 @@
 
     PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
 
     // generate restrictions
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-cam_id",   "cam_id",   "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",   "camRun.cam_id",   "==");
 
     psString query = pxDataGet("camtool_find_pendingexp.sql");
@@ -484,6 +504,6 @@
     // use psDBGenerateWhereSQL because the SQL yields an intermediate table
     if (psListLength(where->list)) {
-        psString whereClaus = psDBGenerateWhereSQL(where, NULL);
-        psStringAppend(&query, "%s", whereClaus);
+        psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClaus);
         psFree(whereClaus);
     }
@@ -535,15 +555,15 @@
         zpt_lq,
         zpt_uq,
-	fwhm_major,
-	fwhm_major_lq,
-	fwhm_major_uq,
-	fwhm_minor,
-	fwhm_minor_lq,
-	fwhm_minor_uq,
-
-	iq_fwhm_major,
-	iq_fwhm_major_err,
-	iq_fwhm_minor,
-	iq_fwhm_minor_err,
+        fwhm_major,
+        fwhm_major_lq,
+        fwhm_major_uq,
+        fwhm_minor,
+        fwhm_minor_lq,
+        fwhm_minor_uq,
+
+        iq_fwhm_major,
+        iq_fwhm_major_err,
+        iq_fwhm_minor,
+        iq_fwhm_minor_err,
 
         iq_m2,
@@ -578,5 +598,6 @@
         n_astrom,
         path_base,
-        code
+        fault,
+        quality
         );
 
@@ -601,7 +622,7 @@
 
     // NULL for end_stage means go as far as possible
-    // EAM : skip here if code != 0
+    // EAM : skip here if fault != 0
     // Also, we can run fake even if tess_id is not defined
-    if (code || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam"))) {
+    if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "cam"))) {
         psFree(row);
         psFree(pendingRow);
@@ -656,6 +677,13 @@
     pxcamGetSearchArgs (config, where);
     PXOPT_COPY_S64(config->args, where, "-cam_id",    "camRun.cam_id",    "==");
-    PXOPT_COPY_STR(config->args, where, "-label",     "camRun.label",     "==");
+    pxAddLabelSearchArgs (config, where, "-label",    "camRun.label",     "==");
     PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "==");
+
+    if (!psListLength(where->list) &&
+        !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required");
+        return false;
+    }
 
     psString query = pxDataGet("camtool_find_processedexp.sql");
@@ -669,14 +697,24 @@
         psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
+    } 
+
+    // we either add AND (condition) or WHERE (condition):
+    if (where->list && faulted) {
+        // list only faulted rows
+        psStringAppend(&query, " %s", " AND camProcessedExp.fault != 0");
+    } 
+    if (where->list && !faulted) {
+        // don't list faulted rows
+        psStringAppend(&query, " %s", " AND camProcessedExp.fault = 0");
+    }
+    if (!where->list && faulted) {
+        // list only faulted rows
+        psStringAppend(&query, " %s", " WHERE camProcessedExp.fault != 0");
+    } 
+    if (!where->list && !faulted) {
+        // don't list faulted rows
+        psStringAppend(&query, " %s", " WHERE camProcessedExp.fault = 0");
     }
     psFree(where);
-
-    if (faulted) {
-        // list only faulted rows
-        psStringAppend(&query, " %s", "AND camProcessedExp.fault != 0");
-    } else {
-        // don't list faulted rows
-        psStringAppend(&query, " %s", "AND camProcessedExp.fault = 0");
-    }
 
     // order by cam_id so that the postage stamp parser can easliy find the 'latest' astrometry
@@ -730,5 +768,5 @@
     PXOPT_COPY_STR(config->args, where, "-label",     "camRun.label",          "==");
     PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction",      "==");
-    PXOPT_COPY_S16(config->args, where, "-code",      "camProcessedExp.fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "camProcessedExp.fault", "==");
 
     if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
@@ -818,5 +856,5 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
 
     psMetadata *where = psMetadataAlloc();
@@ -826,5 +864,5 @@
     PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
 
-    if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, code)) {
+    if (!pxSetFaultCode(config->dbh, "camProcessedExp", where, fault)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
         psFree (where);
@@ -915,5 +953,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "==");
 
     psString query = pxDataGet("camtool_pendingcleanuprun.sql");
@@ -980,5 +1018,5 @@
         PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "==");
     }
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
     psString query = pxDataGet("camtool_pendingcleanupexp.sql");
@@ -1099,5 +1137,5 @@
     char sqlFilename[80];
   } ExportTable;
-  
+
   int numExportTables = 2;
 
@@ -1107,4 +1145,5 @@
   PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
   PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
 
   FILE *f = fopen (outfile, "w");
@@ -1155,7 +1194,17 @@
     }
     if (!psArrayLength(output)) {
-      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psError(PS_ERR_UNKNOWN, true, "no rows found");
       psFree(output);
-      return true;
+      return false;
+    }
+
+    if (clean) {
+        if (!strcmp(tables[i].tableName, "camRun")) {
+            if (!pxSetStateCleaned("camRun", "state", output)) {
+                psFree(output);
+                psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
+                return false;
+            }
+        }
     }
 
@@ -1179,5 +1228,5 @@
 
   PS_ASSERT_PTR_NON_NULL(config, NULL);
-  
+
   PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
 
@@ -1190,5 +1239,5 @@
   psAssert (item, "entry not in input?");
   psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-  
+
   psMetadataItem *entry = psListGet (item->data.list, 0);
   assert (entry);
@@ -1199,6 +1248,6 @@
   // fprintf (stdout, "---- cam run ----\n");
   // psMetadataPrint (stderr, entry->data.md, 1);
-  
-  item = psMetadataLookup (input, "camProcessedImfile");
+
+  item = psMetadataLookup (input, "camProcessedExp");
   psAssert (item, "entry not in input?");
   psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtoolConfig.c	(revision 24244)
@@ -52,5 +52,5 @@
     psMetadata *definebyqueryArgs = psMetadataAlloc();
     pxcamSetSearchArgs(definebyqueryArgs);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",              0, "search by chipRun label", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",          0, "search by chipRun reduction class", NULL);
 
@@ -82,5 +82,5 @@
     pxcamSetSearchArgs(pendingexpArgs);
     psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id",            0, "search by cam_id", 0);
-    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label",             0, "search by camRun label", NULL);
+    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL);
     psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction",         0, "search by camRun reduction class", NULL);
     psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit",             0, "limit result set to N items", 0);
@@ -91,5 +91,5 @@
     pxcamSetSearchArgs(pendingimfileArgs);
     psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-cam_id",   0,            "search by camtool ID", 0);
-    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label",    0,            "search by camRun label", NULL);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL);
     psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-reduction",0,            "search by camRun reduction class", NULL);
     psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0,            "search by class ID", NULL);
@@ -164,5 +164,6 @@
 
     psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", NULL);
-    psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
+    psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
     psMetadataAddBool(addprocessedexpArgs, PS_LIST_TAIL, "-faulted",  0,            "only return imfiles with a fault status set", false);
 
@@ -171,10 +172,11 @@
     pxcamSetSearchArgs(processedexpArgs);
     psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id",   0,            "search by cam_id", 0);
-    psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label",    0,            "search by camRun label", NULL);
+    psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL);
     psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0,            "search by camRun reduction class", NULL);
 
-    psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
-    psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0,            "use the simple output format", false);
-    psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted",  0,            "only return imfiles with a fault status set", false);
+    psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit",    0,            "limit result set to N items", 0);
+    psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all",     0,            "list everything without restriction", false);
+    psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
+    psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0,            "only return imfiles with a fault status set", false);
 
     // -revertprocessedexp
@@ -190,4 +192,5 @@
 
     psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all",  0,            "allow everything to be queued without search terms", false);
+    psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updateprocessedexp
@@ -198,5 +201,5 @@
     psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class",  0,            "search by class", NULL);
     psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
-    psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-code",  0,            "set fault code (required)", INT16_MAX);
+    psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code (required)", INT16_MAX);
 
     // -block
@@ -215,5 +218,5 @@
     // XXX allow full search options?
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -222,5 +225,5 @@
     // XXX allow full search options?
     psMetadata *pendingcleanupexpArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-cam_id", 0,            "search by camera ID", 0);
     psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-exp_id",                 0,            "search by exp_id", NULL);
@@ -239,4 +242,5 @@
     psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
     psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean",  0,          "export tables as cleaned", false);
 
     // -importrun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptool.c	(revision 24244)
@@ -125,5 +125,5 @@
     psMetadata *where = psMetadataAlloc();
     pxchipGetSearchArgs (config, where); // rawExp only
-    PXOPT_COPY_STR(config->args, where, "-label", "rawExp.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE");
 
     // psListLength(where->list) is at least 1 because exp_type defaults to "object"
@@ -198,18 +198,75 @@
     }
 
+    // if end_stage is warp (or NULL), check for valid tess_id
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        bool status;
+        char *end_stage = psMetadataLookupStr(&status, md, "end_stage");
+        if (end_stage && strcasecmp(end_stage, "warp")) continue;
+
+        char *raw_tess_id   = psMetadataLookupStr(&status, md, "tess_id");
+        if (raw_tess_id || tess_id) continue;
+
+        char *label  = psMetadataLookupStr(&status, md, "label");
+        psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id");
+
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id);
+            psFree(output);
+            return false;
+        }
+    }
+
+
+# define GET_VALUE(PTYPE,CTYPE,VALUE,NAME) 				\
+    PTYPE VALUE;							\
+    { bool status;							\
+	VALUE = psMetadataLookup##CTYPE(&status, md, NAME);		\
+	if (!status) {							\
+	    psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", NAME); \
+	    psFree(output);						\
+	    return false;						\
+	} }
+
     // loop over our list of exp_ids
     for (long i = 0; i < psArrayLength(output); i++) {
         psMetadata *md = output->data[i];
 
-        bool status;
-        psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_id");
+        rawExpRow *row = rawExpObjectFromMetadata(md);
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun");
             psFree(output);
             return false;
         }
-        //
+
+        GET_VALUE (psS64,    S64, exp_id,      	 "exp_id");
+        GET_VALUE (psString, Str, raw_workdir, 	 "workdir");
+        GET_VALUE (psString, Str, raw_label,   	 "label");
+        GET_VALUE (psString, Str, raw_reduction, "reduction");
+        // GET_VALUE (psString, Str, raw_expgroup,  "expgroup");
+        GET_VALUE (psString, Str, raw_dvodb,     "dvodb");
+        GET_VALUE (psString, Str, raw_tess_id,   "tess_id");
+        GET_VALUE (psString, Str, raw_end_stage, "end_stage");
+
+        if (!row->exp_id) {
+            psError(PS_ERR_UNKNOWN, false, "failed to find value for exp_id");
+            psFree(output);
+            return false;
+        }
+
         // queue the exp
-        if (!pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) {
+        if (!pxchipQueueByExpTag(config, 
+				 exp_id, 
+				 workdir     ? workdir   : raw_workdir,
+				 label       ? label     : raw_label,
+				 reduction   ? reduction : raw_reduction,
+				 // expgroup    ? expgroup  : raw_expgroup,
+				 // XXX how does expgroup get defined?
+				 expgroup,
+				 dvodb       ? dvodb     : raw_dvodb,
+				 tess_id     ? tess_id   : raw_tess_id,
+				 end_stage   ? end_stage : raw_end_stage
+				 )) {
             if (!psDBRollback(config->dbh)) {
                 psError(PS_ERR_UNKNOWN, false, "database error");
@@ -239,9 +296,10 @@
     pxchipGetSearchArgs (config, where); // rawExp, chipRun
     PXOPT_COPY_S64(config->args,  where, "-chip_id", "chipRun.chip_id", "==");
+    // we only allow a single label to match (do not use pxAddLabelSearchArgs here)
     PXOPT_COPY_STR(config->args,  where, "-label",   "chipRun.label",   "==");
     PXOPT_COPY_STR(config->args,  where, "-state",   "chipRun.state",   "==");
 
-    if (!psListLength(where->list)
-        && !psMetadataLookupBool(NULL, config->args, "-all")) {
+    if (!psListLength(where->list) &&
+        !psMetadataLookupBool(NULL, config->args, "-all")) {
         psFree(where);
         where = NULL;
@@ -291,5 +349,5 @@
     pxchipGetSearchArgs (config, where); //chipRun, rawExp
     PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "==");
 
     psString query = pxDataGet("chiptool_pendingimfile.sql");
@@ -409,14 +467,15 @@
     PXOPT_LOOKUP_F32(dtime_script, config->args,   "-dtime_script", false, false);
     PXOPT_LOOKUP_STR(hostname, config->args,       "-hostname", false, false);
-    PXOPT_LOOKUP_F32(n_stars, config->args,    	   "-n_stars", false, false);
-    PXOPT_LOOKUP_F32(n_psfstars, config->args,	   "-n_psfstars", false, false);
-    PXOPT_LOOKUP_F32(n_iqstars, config->args, 	   "-n_iqstars", false, false);
-    PXOPT_LOOKUP_F32(n_extended, config->args, 	   "-n_extended", false, false);
-    PXOPT_LOOKUP_F32(n_cr, config->args,       	   "-n_cr", false, false);
-    PXOPT_LOOKUP_STR(path_base, config->args,  	   "-path_base", false, false);
-    PXOPT_LOOKUP_BOOL(magicked, config->args,  	   "-magicked", false);
+    PXOPT_LOOKUP_F32(n_stars, config->args,        "-n_stars", false, false);
+    PXOPT_LOOKUP_F32(n_psfstars, config->args,     "-n_psfstars", false, false);
+    PXOPT_LOOKUP_F32(n_iqstars, config->args,      "-n_iqstars", false, false);
+    PXOPT_LOOKUP_F32(n_extended, config->args,     "-n_extended", false, false);
+    PXOPT_LOOKUP_F32(n_cr, config->args,           "-n_cr", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args,      "-path_base", false, false);
+    PXOPT_LOOKUP_BOOL(magicked, config->args,      "-magicked", false);
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
 
     if (!psDBTransaction(config->dbh)) {
@@ -449,33 +508,33 @@
                                    fwhm_minor_uq,
 
-				   iq_fwhm_major,
-				   iq_fwhm_major_err,
-				   iq_fwhm_minor,
-				   iq_fwhm_minor_err,
-
-				   iq_m2,
-				   iq_m2_err,
-				   iq_m2_lq,
-				   iq_m2_uq,
-
-				   iq_m2c,
-				   iq_m2c_err,
-				   iq_m2c_lq,
-				   iq_m2c_uq,
-
-				   iq_m2s,
-				   iq_m2s_err,
-				   iq_m2s_lq,
-				   iq_m2s_uq,
-
-				   iq_m3,
-				   iq_m3_err,
-				   iq_m3_lq,
-				   iq_m3_uq,
-
-				   iq_m4,
-				   iq_m4_err,
-				   iq_m4_lq,
-				   iq_m4_uq,
+                                   iq_fwhm_major,
+                                   iq_fwhm_major_err,
+                                   iq_fwhm_minor,
+                                   iq_fwhm_minor_err,
+
+                                   iq_m2,
+                                   iq_m2_err,
+                                   iq_m2_lq,
+                                   iq_m2_uq,
+
+                                   iq_m2c,
+                                   iq_m2c_err,
+                                   iq_m2c_lq,
+                                   iq_m2c_uq,
+
+                                   iq_m2s,
+                                   iq_m2s_err,
+                                   iq_m2s_lq,
+                                   iq_m2s_uq,
+
+                                   iq_m3,
+                                   iq_m3_err,
+                                   iq_m3_lq,
+                                   iq_m3_uq,
+
+                                   iq_m4,
+                                   iq_m4_err,
+                                   iq_m4_lq,
+                                   iq_m4_uq,
 
                                    dtime_detrend,
@@ -490,5 +549,6 @@
                                    n_cr,
                                    path_base,
-                                   code,
+                                   fault,
+                                   quality,
                                    magicked
             )) {
@@ -540,6 +600,13 @@
     PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "==");
     PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE");
     PXOPT_COPY_S32(config->args, where, "-magicked", "chipRun.magicked", "==");
+
+    if (!psListLength(where->list) &&
+        !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required");
+        return false;
+    }
 
     psString query = pxDataGet("chiptool_processedimfile.sql");
@@ -609,7 +676,8 @@
     PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "LIKE");
+    // require a single label 
+    PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "==");
     PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "chipProcessedImfile.fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "chipProcessedImfile.fault", "==");
 
     if (!psListLength(where->list)
@@ -652,7 +720,7 @@
     PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
-    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);
-
-    if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, code)) {
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
+
+    if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
         return false;
@@ -835,5 +903,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
     psString query = pxDataGet("chiptool_pendingcleanuprun.sql");
@@ -899,5 +967,5 @@
         PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
     }
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
     psString query = pxDataGet("chiptool_pendingcleanupimfile.sql");
@@ -1027,5 +1095,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
     PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
 
@@ -1088,5 +1156,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
     // look for completed chipPendingExp
@@ -1273,12 +1341,14 @@
     char sqlFilename[80];
   } ExportTable;
-  
+
   int numExportTables = 3;
 
   PS_ASSERT_PTR_NON_NULL(config, NULL);
 
-  PXOPT_LOOKUP_S64(det_id, config->args, "-chip_id", true,  false);
+  PXOPT_LOOKUP_S64(dummy, config->args, "-chip_id", true,  false);
   PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
   PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
+
 
   FILE *f = fopen (outfile, "w");
@@ -1331,7 +1401,21 @@
     }
     if (!psArrayLength(output)) {
-      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psTrace("chiptool", PS_LOG_INFO, "no rows found");
       psFree(output);
-      return true;
+      return false;
+    }
+
+    if (clean) {
+        bool success = true; 
+        if (!strcmp(tables[i].tableName, "chipRun")) {
+            success = pxSetStateCleaned("chipRun", "state", output);
+        } else if (!strcmp(tables[i].tableName, "chipProcessedImfile")) {
+            success = pxSetStateCleaned("chipProcessedImfile", "data_state", output);
+        }
+        if (!success) {
+            psFree(output);
+            psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
+            return false;
+        }
     }
 
@@ -1352,68 +1436,49 @@
 bool importrunMode(pxConfig *config)
 {
-  unsigned int nFail;
-  
-  int numImportTables = 2;
- 
-  char tables[2] [80] = {"chipImfile", "chipProcessedImfile"};
-
-  PS_ASSERT_PTR_NON_NULL(config, NULL);
-  
-  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
-
-  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
-
-  fprintf (stdout, "---- input ----\n");
-  psMetadataPrint (stderr, input, 1);
-
-  psMetadataItem *item = psMetadataLookup (input, "chipRun");
-  psAssert (item, "entry not in input?");
-  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-
-  psMetadataItem *entry = psListGet (item->data.list, 0);
-  assert (entry);
-  assert (entry->type == PS_DATA_METADATA);
-  chipRunRow *chipRun = chipRunObjectFromMetadata (entry->data.md);
-  chipRunInsertObject (config->dbh, chipRun);
-
-  // fprintf (stdout, "---- chip run ----\n");
-  // psMetadataPrint (stderr, entry->data.md, 1);
-  
-  for (int i = 0; i < numImportTables; i++) {
-    psMetadataItem *item = psMetadataLookup (input, tables[i]);
-    psAssert (item, "entry not in input?");
-    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-  
-    switch (i) {
-      case 0:
-        for (int i = 0; i < item->data.list->n; i++) {
-          psMetadataItem *entry = psListGet (item->data.list, i);
-          assert (entry);
-          assert (entry->type == PS_DATA_METADATA);
-          chipImfileRow *chipImfile = chipImfileObjectFromMetadata (entry->data.md);
-          chipImfileInsertObject (config->dbh, chipImfile);
-
-          // fprintf (stdout, "---- row %d ----\n", i);
-          // psMetadataPrint (stderr, entry->data.md, 1);
-        }
-        break;
-        
-      case 1:
-        for (int i = 0; i < item->data.list->n; i++) {
-          psMetadataItem *entry = psListGet (item->data.list, i);
-          assert (entry);
-          assert (entry->type == PS_DATA_METADATA);
-          chipProcessedImfileRow *chipProcessedImfile = chipProcessedImfileObjectFromMetadata (entry->data.md);
-          chipProcessedImfileInsertObject (config->dbh, chipProcessedImfile);
-
-          // fprintf (stdout, "---- row %d ----\n", i);
-          // psMetadataPrint (stderr, entry->data.md, 1);
-        }
-        break;
-    }
-  }
-
-  return true;
-}
-
-
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false);
+    unsigned int nFail = 0;               // Number of failed lines
+    psMetadata *input = psMetadataConfigRead(NULL, &nFail, infile, false);
+    if (nFail) {
+        psError(PS_ERR_IO, false, "%d failed lines in input", nFail);
+        psFree(input);
+        return false;
+    }
+
+    psVector *identifiers = psVectorAllocEmpty(16, PS_TYPE_U64); // Identifiers inserted
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL);       // Iterator
+    psMetadataItem *item;               // Item from iteration
+    while ((item = psMetadataGetAndIncrement(iter))) {
+        PXMIRROR_PRIMARY(item, "chipRun", chipRunRow, chipRunObjectFromMetadata, identifiers, chip_id,
+                         chipRunInsertObject, config->dbh,
+                         { psFree(iter); psFree(identifiers); psFree(input); });
+
+        PXMIRROR_OTHER(item, "chipImfile", chipImfileRow, chipImfileObjectFromMetadata, identifiers, chip_id,
+                       chipImfileInsertObject, config->dbh,
+                       { psFree(iter); psFree(identifiers); psFree(input); });
+
+        PXMIRROR_OTHER(item, "chipProcessedImfile", chipProcessedImfileRow,
+                       chipProcessedImfileObjectFromMetadata, identifiers, chip_id,
+                       chipProcessedImfileInsertObject, config->dbh,
+                       { psFree(iter); psFree(identifiers); psFree(input); });
+    }
+    psFree(iter);
+    psFree(input);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psLogMsg("chiptool", PS_LOG_INFO, "%ld chipRuns added", identifiers->n);
+    psFree(identifiers);
+
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptoolConfig.c	(revision 24244)
@@ -47,5 +47,5 @@
     psMetadata *definebyqueryArgs = psMetadataAlloc();
     pxchipSetSearchArgs (definebyqueryArgs);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",              0, "search by rawExp label (LIKE comparison)", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by rawExp label (LIKE comparison)", NULL);
 
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_type",          PS_META_REPLACE, "search by exp_type", "object");
@@ -75,5 +75,5 @@
     psMetadata *pendingimfileArgs = psMetadataAlloc();
     psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-chip_id",  0,            "search by chip ID", 0);
-    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label",  0, "search by label", 0);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by label", 0);
     pxchipSetSearchArgs(pendingimfileArgs);
     psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -146,17 +146,19 @@
 
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base",  0,            "define base output location", NULL);
-    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
+    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
     psMetadataAddBool(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", false);
 
     // -processedimfile
     psMetadata *processedimfileArgs = psMetadataAlloc();
+    pxchipSetSearchArgs(processedimfileArgs);
     psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-chip_id",  0,         "search by  chip ID", 0);
     psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-chip_imfile_id",  0,  "search by chip_file_id", 0);
     psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-class_id",           0, "search by class ID", NULL);
     psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-reduction",          0, "search by reduction class", NULL);
-    psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-label",              0, "search by chipRun label (LIKE comparison)", NULL);
+    psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL);
     psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "search by magicked status", false);
-    pxchipSetSearchArgs(processedimfileArgs);
-    psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit",  0,           "limit result set to N items", 0);
+    psMetadataAddU64(processedimfileArgs,  PS_LIST_TAIL, "-limit",  0,           "limit result set to N items", 0);
+    psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all",  0,            "list everything without search terms", false);
     psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted",  0,        "only return imfiles with a fault status set", false);
     psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple",  0,         "use the simple output format", false);
@@ -170,5 +172,5 @@
     pxchipSetSearchArgs(revertprocessedimfileArgs);
     psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all",  0,            "allow everything to be queued without search terms", false);
-    psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updateprocessedimfile
@@ -176,11 +178,5 @@
     psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-chip_id",  0,            "search by chip ID", 0);
     psMetadataAddStr(updateprocessedimfileArgs,  PS_LIST_TAIL, "-class_id",           0, "search by class ID", NULL);
-    psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code (required)", 0);
-
-    // -advanceexp
-    psMetadata *advanceexpArgs = psMetadataAlloc();
-    psMetadataAddS64(advanceexpArgs, PS_LIST_TAIL, "-chip_id",  0,          "search by chip ID", 0);
-    psMetadataAddStr(advanceexpArgs, PS_LIST_TAIL, "-label",  0,            "advance exposures for specified label", NULL);
-    psMetadataAddU64(advanceexpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code (required)", 0);
 
     // -block
@@ -206,5 +202,5 @@
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -212,5 +208,5 @@
     // -pendingcleanupimfile
     psMetadata *pendingcleanupimfileArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-chip_id", 0,          "search by chip ID", 0);
     psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id",                 0,            "search by exp_id", NULL);
@@ -226,8 +222,14 @@
     // -run
     psMetadata *runArgs = psMetadataAlloc();
-    psMetadataAddStr(runArgs, PS_LIST_TAIL, "-label",  0,       "search by label", NULL);
+    psMetadataAddStr(runArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     psMetadataAddBool(runArgs, PS_LIST_TAIL, "-simple",  0,     "use the simple output format", false);
     psMetadataAddU64(runArgs, PS_LIST_TAIL, "-limit",  0,       "limit result set to N items", 0);
     psMetadataAddStr(runArgs, PS_LIST_TAIL, "-state", 0,        "search by state (required)", NULL);
+
+    // -advanceexp
+    psMetadata *advanceexpArgs = psMetadataAlloc();
+    psMetadataAddS64(advanceexpArgs, PS_LIST_TAIL, "-chip_id",  0,          "search by chip ID", 0);
+    psMetadataAddStr(advanceexpArgs, PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "advance exposures for specified label", NULL);
+    psMetadataAddU64(advanceexpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 
     // -tocleanedimfile
@@ -251,4 +253,5 @@
     psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-chip_id", 0,          "export this chip ID (required)", 0);
     psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean",  0,          "mark tables as cleaned", false);
     psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/detselect.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/detselect.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/detselect.c	(revision 24244)
@@ -71,4 +71,26 @@
 }
 
+# define PXOPT_COPY_NULLTEST_F32(from, to, oldname, newname, comment) \
+{ \
+    bool status = false; \
+    psF32 var = psMetadataLookupF32(&status, from, oldname); \
+    if (!status) { \
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for " oldname); \
+        return false; \
+    } \
+    if (!isnan(var)) { \
+        if (!psMetadataAddF32(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, comment, var)) { \
+            psError(PS_ERR_UNKNOWN, false, "failed to add item " newname); \
+            psFree(to); \
+            return false; \
+        } \
+        if (!psMetadataAddTime(to, PS_LIST_TAIL, newname, PS_META_DUPLICATE_OK, "==", NULL)) { \
+            psError(PS_ERR_UNKNOWN, false, "failed to add NULL test " newname); \
+            psFree(to); \
+            return false; \
+        } \
+    } \
+}
+
 static bool searchMode(pxConfig *config)
 {
@@ -89,14 +111,14 @@
 
     // airmass_min  < airmass  < airmass_max
-    PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_min", "<=");
-    PXOPT_COPY_F32(config->args, where, "-airmass", "airmass_max", ">=");
+    PXOPT_COPY_NULLTEST_F32(config->args, where, "-airmass", "airmass_min", "<=");
+    PXOPT_COPY_NULLTEST_F32(config->args, where, "-airmass", "airmass_max", ">=");
 
     // exp_time_min < exp_time < exp_time_max
-    PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_min", "<=");
-    PXOPT_COPY_F32(config->args, where, "-exp_time", "exp_time_max", ">=");
+    PXOPT_COPY_NULLTEST_F32(config->args, where, "-exp_time", "exp_time_min", "<=");
+    PXOPT_COPY_NULLTEST_F32(config->args, where, "-exp_time", "exp_time_max", ">=");
 
     // ccd_temp_min < ccd_temp < ccd_temp_max
-    PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_min", "<=");
-    PXOPT_COPY_F32(config->args, where, "-ccd_temp", "ccd_temp_max", ">=");
+    PXOPT_COPY_NULLTEST_F32(config->args, where, "-ccd_temp", "ccd_temp_min", "<=");
+    PXOPT_COPY_NULLTEST_F32(config->args, where, "-ccd_temp", "ccd_temp_max", ">=");
 
     PXOPT_COPY_F64(config->args, where, "-posang", "posang_min", "<=");
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool.c	(revision 24244)
@@ -2034,4 +2034,5 @@
 
   ExportTable tables [] = {
+    {"detRun", "dettool_export_run.sql"},
     {"detInputExp", "dettool_export_input_exp.sql"},
     {"detNormalizedExp", "dettool_export_normalized_exp.sql"},
@@ -2043,5 +2044,4 @@
     {"detResidExp", "dettool_export_resid_exp.sql"},
     {"detResidImfile", "dettool_export_resid_imfile.sql"},
-    {"detRun", "dettool_export_run.sql"},
     {"detRunSummary", "dettool_export_run_summary.sql"},
     {"detStackedImfile", "dettool_export_stacked_imfile.sql"},
@@ -2108,7 +2108,7 @@
   
   char tables[11] [80] = {"detInputExp", "detNormalizedExp",
-    "detNormalizedStatImfile", "detProcessedExp", "detProcessedImfile",
-    "detRegisteredImfile", "detResidExp", "detResidImfile",
-    "detNormalizedImfile", "detRunSummary", "detStackedImfile"};
+    "detNormalizedImfile", "detNormalizedStatImfile", "detProcessedExp",
+    "detProcessedImfile", "detRegisteredImfile", "detResidExp",
+    "detResidImfile", "detRunSummary", "detStackedImfile"};
 
   PS_ASSERT_PTR_NON_NULL(config, NULL);
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettoolConfig.c	(revision 24244)
@@ -254,5 +254,5 @@
     psMetadataAddS64(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "define exp ID (required)", 0);
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id",  0,            "define class ID (required)", NULL);
-    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri",  0,            "define URI", NULL);
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-recip",  0,            "define recipe", NULL);
@@ -287,5 +287,5 @@
     psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
     psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
-    psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updateprocessedimfile
@@ -334,5 +334,5 @@
     psMetadataAddF64(addprocessedexpArgs, PS_LIST_TAIL, "-user_5",  0,            "define user statistic (5)", NAN);
     psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base",  0,            "define base output location", NULL);
-    psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
 
     // -proccessedexp
@@ -348,5 +348,5 @@
     psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-det_id",  0,            "search by detrend ID (required)", 0);
     psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
-    psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updateprocessedexp
@@ -380,5 +380,5 @@
     psMetadataAddS32(addstackedArgs, PS_LIST_TAIL, "-iteration",  0,            "define iteration number (required)", 0);
     psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-class_id",  0,            "define class ID (required)", NULL);
-    psMetadataAddS16(addstackedArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addstackedArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-uri",  0,            "define URI", NULL);
     psMetadataAddStr(addstackedArgs, PS_LIST_TAIL, "-recip",  0,            "define recipe", NULL);
@@ -407,5 +407,5 @@
     psMetadataAddS32(revertstackedArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
     psMetadataAddStr(revertstackedArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
-    psMetadataAddS16(revertstackedArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertstackedArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updatestacked
@@ -443,5 +443,5 @@
     psMetadataAddStr(addnormstatArgs, PS_LIST_TAIL, "-class_id",  0,            "define class ID (required)", NULL);
     psMetadataAddF32(addnormstatArgs, PS_LIST_TAIL, "-norm",  0,            "define normal value (required)", NAN);
-    psMetadataAddS16(addnormstatArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addnormstatArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
 
     // -normalizedstat
@@ -459,5 +459,5 @@
     psMetadataAddS32(revertnormalizedstatArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
     psMetadataAddStr(revertnormalizedstatArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
-    psMetadataAddS16(revertnormalizedstatArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertnormalizedstatArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updatenormalizedstat
@@ -494,5 +494,5 @@
     psMetadataAddS32(addnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0,            "define iteration number", 0);
     psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0,            "define class ID (required)", NULL);
-    psMetadataAddS16(addnormalizedimfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addnormalizedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddStr(addnormalizedimfileArgs, PS_LIST_TAIL, "-uri", 0,            "define URI", NULL);
     psMetadataAddF64(addnormalizedimfileArgs, PS_LIST_TAIL, "-bg", 0,            "define exposure background", NAN);
@@ -521,5 +521,5 @@
     psMetadataAddS32(revertnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
     psMetadataAddStr(revertnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0,            "search by class ID", NULL);
-    psMetadataAddS16(revertnormalizedimfileArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertnormalizedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updatenormalizedimfile
@@ -555,5 +555,5 @@
     psMetadataAddS64(addnormalizedexpArgs, PS_LIST_TAIL, "-det_id",  0,            "define detrend ID (required)", 0);
     psMetadataAddS32(addnormalizedexpArgs, PS_LIST_TAIL, "-iteration",  0,            "define iteration number", 0);
-    psMetadataAddS16(addnormalizedexpArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addnormalizedexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddStr(addnormalizedexpArgs, PS_LIST_TAIL, "-recip",  0,            "search for recipe", NULL);
     psMetadataAddF64(addnormalizedexpArgs, PS_LIST_TAIL, "-bg",  0,            "define exposure background", NAN);
@@ -580,5 +580,5 @@
     psMetadataAddS64(revertnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID (required)", 0);
     psMetadataAddS32(revertnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
-    psMetadataAddS16(revertnormalizedexpArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertnormalizedexpArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updatenormalizedexp
@@ -616,5 +616,5 @@
     psMetadataAddS64(addresidimfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "define detrend ID (required)", 0);
     psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-class_id",  0,          "define class ID (required)", NULL);
-    psMetadataAddS16(addresidimfileArgs, PS_LIST_TAIL, "-code",  0,              "set fault code", 0);
+    psMetadataAddS16(addresidimfileArgs, PS_LIST_TAIL, "-fault",  0,              "set fault code", 0);
     psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-uri",  0,               "define resid file URI", NULL);
     psMetadataAddStr(addresidimfileArgs, PS_LIST_TAIL, "-recip",  0,             "define recipe", NULL);
@@ -658,5 +658,5 @@
     psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-exp_id",  0,         "search by detrend ID", 0);
     psMetadataAddStr(revertresidimfileArgs, PS_LIST_TAIL, "-class_id",  0,       "search for class ID", NULL);
-    psMetadataAddS16(revertresidimfileArgs, PS_LIST_TAIL, "-code",  0,           "search by fault code", 0);
+    psMetadataAddS16(revertresidimfileArgs, PS_LIST_TAIL, "-fault",  0,           "search by fault code", 0);
 
     // -updateresidimfile
@@ -696,5 +696,5 @@
     psMetadataAddS32(addresidexpArgs, PS_LIST_TAIL, "-iteration",  0,            "define iteration number", 0);
     psMetadataAddS64(addresidexpArgs, PS_LIST_TAIL, "-exp_id",  0,            "define detrend ID (required)", 0);
-    psMetadataAddS16(addresidexpArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addresidexpArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddStr(addresidexpArgs, PS_LIST_TAIL, "-recip",  0,            "define recipe", NULL);
     psMetadataAddF64(addresidexpArgs, PS_LIST_TAIL, "-bg",  0,            "define exposure background", NAN);
@@ -734,5 +734,5 @@
     psMetadataAddS32(revertresidexpArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
     psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by detrend ID", 0);
-    psMetadataAddS16(revertresidexpArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertresidexpArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updateresidexp
@@ -777,5 +777,5 @@
     psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_stdev",  0,            "define exposure background stdev", NAN);
     psMetadataAddF64(adddetrunsummaryArgs, PS_LIST_TAIL, "-bg_mean_stdev",  0,            "define exposure background mean stdev", NAN);
-    psMetadataAddS16(adddetrunsummaryArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(adddetrunsummaryArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddBool(adddetrunsummaryArgs, PS_LIST_TAIL, "-accept",  0,            "declare that this detrun iteration is accepted as a master", false);
     psMetadataAddBool(adddetrunsummaryArgs, PS_LIST_TAIL, "-again",  0,            "start a new iteration of this detrend run", false);
@@ -794,5 +794,5 @@
     psMetadataAddS64(revertdetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID (required)", 0);
     psMetadataAddS32(revertdetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
-    psMetadataAddS16(revertdetrunsummaryArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertdetrunsummaryArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -updatedetrunsummary
@@ -886,9 +886,9 @@
     PXOPT_ADD_MODE("-input",           "", DETTOOL_MODE_INPUT,         inputArgs);
 
-    PXOPT_ADD_MODE("-toprocessedimfile", "",  		  DETTOOL_MODE_TOPROCESSEDIMFILE, toprocessedimfileArgs);
-    PXOPT_ADD_MODE("-addprocessedimfile", "", 		  DETTOOL_MODE_ADDPROCESSEDIMFILE,  addprocessedimfileArgs);
-    PXOPT_ADD_MODE("-processedimfile", "",    		  DETTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs);
-    PXOPT_ADD_MODE("-revertprocessedimfile", "", 	  DETTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs);
-    PXOPT_ADD_MODE("-updateprocessedimfile", "", 	  DETTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs);
+    PXOPT_ADD_MODE("-toprocessedimfile", "",              DETTOOL_MODE_TOPROCESSEDIMFILE, toprocessedimfileArgs);
+    PXOPT_ADD_MODE("-addprocessedimfile", "",             DETTOOL_MODE_ADDPROCESSEDIMFILE,  addprocessedimfileArgs);
+    PXOPT_ADD_MODE("-processedimfile", "",                DETTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs);
+    PXOPT_ADD_MODE("-revertprocessedimfile", "",          DETTOOL_MODE_REVERTPROCESSEDIMFILE, revertprocessedimfileArgs);
+    PXOPT_ADD_MODE("-updateprocessedimfile", "",          DETTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileArgs);
     PXOPT_ADD_MODE("-pendingcleanup_processedimfile", "", DETTOOL_MODE_PENDINGCLEANUP_PROCESSEDIMFILE, pendingcleanup_processedimfileArgs);
     PXOPT_ADD_MODE("-donecleanup_processedimfile", "",    DETTOOL_MODE_DONECLEANUP_PROCESSEDIMFILE, donecleanup_processedimfileArgs);
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_detrunsummary.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_detrunsummary.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_detrunsummary.c	(revision 24244)
@@ -101,5 +101,5 @@
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false);
 
@@ -107,10 +107,10 @@
             det_id,
             iteration,
-	    "full",
+            "full",
             bg,
             bg_stdev,
             bg_mean_stdev,
             accept,
-            code
+            fault
         );
 }
@@ -120,5 +120,5 @@
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
-    
+
     // build a query to search by det_id, iteration, exp_id
     psMetadata *where = psMetadataAlloc();
@@ -129,6 +129,6 @@
     PXOPT_LOOKUP_BOOL(again, config->args, "-again", false);
 
-    // The values supplied as arguments on the command (eg, -bg) are parsed 
-    // by mdToDetRunSummary below. 
+    // The values supplied as arguments on the command (eg, -bg) are parsed
+    // by mdToDetRunSummary below.
     // XXX why is there ever more than one?
 
@@ -141,6 +141,6 @@
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-	psStringAppend(&query, " WHERE %s", whereClause);
-	psFree(whereClause);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
     }
     psFree(where);
@@ -199,6 +199,6 @@
     psFree(output);
 
-    // XXX this logic does not deal with the case of -code being set
-    // XXX it should be an error for -again and -code to both be set
+    // XXX this logic does not deal with the case of -fault being set
+    // XXX it should be an error for -again and -fault to both be set
     if (again) {
         if (!startNewIteration(config, det_id)) {
@@ -306,5 +306,5 @@
     PXOPT_COPY_S64(config->args, where, "-det_id",    "det_id", "==");
     PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "==");
-    PXOPT_COPY_STR(config->args, where, "-code",      "fault", "==");
+    PXOPT_COPY_STR(config->args, where, "-fault",      "fault", "==");
 
     psString query = pxDataGet("dettool_revertdetrunsummary.sql");
@@ -340,5 +340,5 @@
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
-    
+
     PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required
     PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false);
@@ -355,5 +355,5 @@
     }
 
-    char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; 
+    char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64;
     if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_normalizedexp.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_normalizedexp.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_normalizedexp.c	(revision 24244)
@@ -72,9 +72,9 @@
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
-    
+
     PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required
     PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false);
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (fault == 0), false); // Required if fault == 0
     PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
     PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
@@ -117,20 +117,20 @@
     // insert the new row into the detProcessedImfile table
     if (!detNormalizedExpInsert(
-	    config->dbh,
-	    det_id,
-	    iteration,
-	    recipe,
-	    bg,
-	    bg_stdev,
-	    bg_mean_stdev,
-	    user_1,
-	    user_2,
-	    user_3,
-	    user_4,
-	    user_5,
-	    path_base,
-	    "full",
-	    code
-	    )) {
+            config->dbh,
+            det_id,
+            iteration,
+            recipe,
+            bg,
+            bg_stdev,
+            bg_mean_stdev,
+            user_1,
+            user_2,
+            user_3,
+            user_4,
+            user_5,
+            path_base,
+            "full",
+            fault
+            )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -221,5 +221,5 @@
     PXOPT_COPY_S64(config->args, where, "-det_id",    "det_id", "==");
     PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",      "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
 
     psString query = pxDataGet("dettool_revertnormalizedexp.sql");
@@ -260,5 +260,5 @@
 
     if (!setNormExpDataState(config, det_id, iteration, data_state)) {
-	return false;
+        return false;
     }
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_normalizedimfile.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_normalizedimfile.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_normalizedimfile.c	(revision 24244)
@@ -80,8 +80,8 @@
     PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-
-    // Required if code == 0
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
+    // Required if fault == 0
+    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false);
 
     // optional
@@ -97,21 +97,21 @@
 
     if (!detNormalizedImfileInsert(
-	    config->dbh,
-	    det_id,
-	    iteration,
-	    class_id,
-	    uri,
-	    bg,
-	    bg_stdev,
-	    bg_mean_stdev,
-	    user_1,
-	    user_2,
-	    user_3,
-	    user_4,
-	    user_5,
-	    path_base,
-	    "full",
-	    code
-	    )) {
+            config->dbh,
+            det_id,
+            iteration,
+            class_id,
+            uri,
+            bg,
+            bg_stdev,
+            bg_mean_stdev,
+            user_1,
+            user_2,
+            user_3,
+            user_4,
+            user_5,
+            path_base,
+            "full",
+            fault
+            )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -201,5 +201,5 @@
     PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id",  "class_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",      "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
 
     psString query = pxDataGet("dettool_revertnormalizedimfile.sql");
@@ -241,5 +241,5 @@
 
     if (!setNormImfileDataState(config, det_id, iteration, class_id, data_state)) {
-	return false;
+        return false;
     }
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_normalizedstat.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_normalizedstat.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_normalizedstat.c	(revision 24244)
@@ -83,17 +83,17 @@
     // default
     PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false);
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(code, config->args, "-fault", false, false);
 
     detNormalizedStatImfileRow *row = detNormalizedStatImfileRowAlloc
-	(det_id,
-	 iteration,
-	 class_id,
-	 norm,
-	 "full",
-	 code);
-									 
+        (det_id,
+         iteration,
+         class_id,
+         norm,
+         "full",
+         code);
+
     if (!detNormalizedStatImfileInsertObject(config->dbh, row)) {
-	psError(PS_ERR_UNKNOWN, false, "database error");
-	return false;
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
     }
 
@@ -181,5 +181,5 @@
     PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id",  "class_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",      "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
 
     psString query = pxDataGet("dettool_revertnormalizedstat.sql");
@@ -221,5 +221,5 @@
 
     if (!setNormStatImfileDataState(config, det_id, iteration, class_id, data_state)) {
-	return false;
+        return false;
     }
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_processedexp.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_processedexp.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_processedexp.c	(revision 24244)
@@ -80,8 +80,8 @@
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-
-    // Required if code == 0
-    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
+    // Required if fault == 0
+    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (fault == 0), false);
 
     // optional
@@ -141,6 +141,6 @@
         user_5,
         path_base,
-	"full",
-        code
+        "full",
+        fault
     );
 
@@ -231,5 +231,5 @@
     PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",   "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",   "fault", "==");
 
     psString query = pxDataGet("dettool_revertprocessedexp.sql");
@@ -269,5 +269,5 @@
     PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false);
     if (!setProcessedExpDataState(config, det_id, exp_id, data_state)) {
-	return false;
+        return false;
     }
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_processedimfile.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_processedimfile.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_processedimfile.c	(revision 24244)
@@ -92,9 +92,9 @@
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-
-    // Required if code == 0
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false);
-    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
+    // Required if fault == 0
+    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false);
+    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (fault == 0), false);
 
     // optional
@@ -144,6 +144,6 @@
         user_5,
         path_base,
-	"full",
-        code
+        "full",
+        fault
     );
     psFree(rawImfiles);
@@ -190,5 +190,5 @@
         psStringAppend(&query, " %s", whereClause);
         psFree(whereClause);
-	hasWhere = true;
+        hasWhere = true;
     }
     psFree (where);
@@ -196,21 +196,21 @@
     // restrict search to included imfiles
     if (included) {
-	if (hasWhere) {
-	    psStringAppend(&query, " AND detInputExp.include = 1");
-	} else {
-	    psStringAppend(&query, " WHERE detInputExp.include = 1");
-	}
-	hasWhere = true;
+        if (hasWhere) {
+            psStringAppend(&query, " AND detInputExp.include = 1");
+        } else {
+            psStringAppend(&query, " WHERE detInputExp.include = 1");
+        }
+        hasWhere = true;
     }
 
     if (hasWhere) {
-	psStringAppend(&query, " AND");
+        psStringAppend(&query, " AND");
     } else {
-	psStringAppend(&query, " WHERE");
+        psStringAppend(&query, " WHERE");
     }
 
     if (faulted) {
         // list only faulted rows
-	psStringAppend(&query, " %s", " detProcessedImfile.fault != 0");
+        psStringAppend(&query, " %s", " detProcessedImfile.fault != 0");
     } else {
         // don't list faulted rows
@@ -263,5 +263,5 @@
     PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
 
     psString query = pxDataGet("dettool_revertprocessedimfile.sql");
@@ -303,5 +303,5 @@
 
     if (!setProcessedImfileDataState(config, det_id, exp_id, class_id, data_state)) {
-	return false;
+        return false;
     }
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_residexp.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_residexp.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_residexp.c	(revision 24244)
@@ -95,6 +95,6 @@
     PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false);
     PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); // required
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (fault == 0), false); // Required if fault == 0
     PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
     PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
@@ -124,7 +124,7 @@
 
     if (psListLength(where->list)) {
-	psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-	psStringAppend(&query, " WHERE %s", whereClause);
-	psFree(whereClause);
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
     }
     psFree(where);
@@ -151,5 +151,5 @@
 
     if (!detResidExpInsert(
-	    config->dbh,
+            config->dbh,
             det_id,
             iteration,
@@ -174,7 +174,7 @@
             user_5,
             path_base,
-	    "full",
+            "full",
             !reject,
-            code
+            fault
         )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -225,5 +225,5 @@
     if (reject) {
         psStringAppend(&query, " %s", "AND detResidExp.accept != 0");
-    } 
+    }
 
     // treat limit == 0 as "no limit"
@@ -273,5 +273,5 @@
     PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id",    "exp_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",      "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
 
     psString query = pxDataGet("dettool_revertresidexp.sql");
@@ -330,5 +330,5 @@
     PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false);
     if (data_state) {
-	if (!isValidDataState (data_state)) return false;
+        if (!isValidDataState (data_state)) return false;
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_residimfile.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_residimfile.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_residimfile.c	(revision 24244)
@@ -75,5 +75,5 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    
+
     PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true, false); // required
     PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", false, false);
@@ -84,7 +84,7 @@
     PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false); // required
     PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); // required
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false); // Required if code == 0
-    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false); // Required if code == 0
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false); // Required if fault == 0
+    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (fault == 0), false); // Required if fault == 0
     PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
     PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
@@ -107,5 +107,5 @@
 
     if (!detResidImfileInsert(
-	    config->dbh,
+            config->dbh,
             det_id,
             iteration,
@@ -134,6 +134,6 @@
             user_5,
             path_base,
-	    "full",
-            code
+            "full",
+            fault
     )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -174,5 +174,5 @@
         psStringAppend(&query, " %s", whereClause);
         psFree(whereClause);
-	hasWhere = true;
+        hasWhere = true;
     }
     psFree(where);
@@ -180,16 +180,16 @@
     // restrict search to included imfiles
     if (included) {
-	if (hasWhere) {
-	    psStringAppend(&query, " AND detInputExp.include = 1");
-	} else {
-	    psStringAppend(&query, " WHERE detInputExp.include = 1");
-	}
-	hasWhere = true;
+        if (hasWhere) {
+            psStringAppend(&query, " AND detInputExp.include = 1");
+        } else {
+            psStringAppend(&query, " WHERE detInputExp.include = 1");
+        }
+        hasWhere = true;
     }
 
     if (hasWhere) {
-	psStringAppend(&query, " AND");
+        psStringAppend(&query, " AND");
     } else {
-	psStringAppend(&query, " WHERE");
+        psStringAppend(&query, " WHERE");
     }
 
@@ -249,5 +249,5 @@
     PXOPT_COPY_S64(config->args, where, "-exp_id",  "exp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id",  "class_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",      "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
 
     psString query = pxDataGet("dettool_revertresidimfile.sql");
@@ -289,5 +289,5 @@
     PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", true, false);
     if (!setResidImfileDataState(config, det_id, iteration, exp_id, class_id, data_state)) {
-	return false;
+        return false;
     }
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_stack.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_stack.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_stack.c	(revision 24244)
@@ -127,9 +127,9 @@
     PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-
-    // Required if code == 0
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false);
-    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (code == 0), false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
+    // Required if fault == 0
+    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (fault == 0), false);
+    PXOPT_LOOKUP_STR(recipe, config->args, "-recip", (fault == 0), false);
 
     // optional
@@ -188,6 +188,6 @@
             user_4,
             user_5,
-	    "full",
-            code
+            "full",
+            fault
         );
 
@@ -286,5 +286,5 @@
     PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id",  "class_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",      "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
 
     psString query = pxDataGet("dettool_revertstacked.sql");
@@ -326,5 +326,5 @@
 
     if (!setStackedImfileDataState(config, det_id, iteration, class_id, data_state)) {
-	return false;
+        return false;
     }
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.c	(revision 24244)
@@ -40,5 +40,5 @@
 static bool revertdiffskyfileMode(pxConfig *config);
 static bool definepoprunMode(pxConfig *config);
-static bool definebyqueryMode(pxConfig *config);
+static bool definewarpstackMode(pxConfig *config);
 static bool definewarpwarpMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
@@ -79,5 +79,5 @@
         MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
-        MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY,         definebyqueryMode);
+        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,         definewarpstackMode);
         MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
@@ -116,7 +116,8 @@
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
     PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
+    PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
+    PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
 
     // default
@@ -133,15 +134,16 @@
             registered,
             tess_id,
-            exp_id,
+            bothways,
+            exposure,
             false
     );
     if (!run) {
         psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object");
-        return true;
+        return false;
     }
     if (!diffRunInsertObject(config->dbh, run)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(run);
-        return true;
+        return false;
     }
 
@@ -150,7 +152,7 @@
 
     if (!diffRunPrintObject(stdout, run, !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print object");
-            psFree(run);
-            return false;
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(run);
+        return false;
     }
 
@@ -412,5 +414,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffRun.diff_id", "==");
-    PXOPT_COPY_STR(config->args, where,  "-label", "diffRun.label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -485,7 +487,7 @@
     PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required
     PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false);
-    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (code == 0), false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (fault == 0), false);
 
     // optional
@@ -506,4 +508,5 @@
     PXOPT_LOOKUP_F32(kernel_xy, config->args, "-kernel_xy", false, false);
     PXOPT_LOOKUP_F32(kernel_yy, config->args, "-kernel_yy", false, false);
+    PXOPT_LOOKUP_F32(deconv_max, config->args, "-deconv_max", false, false);
     PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false);
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
@@ -519,6 +522,6 @@
                            diff_id,
                            skycell_id,
-                           uri,
                            path_base,
+                           "full",
                            bg,
                            bg_stdev,
@@ -533,4 +536,5 @@
                            kernel_xy,
                            kernel_yy,
+                           deconv_max,
                            sources,
                            dtime_diff,
@@ -540,5 +544,6 @@
                            hostname,
                            good_frac,
-                           code,
+                           fault,
+                           quality,
                            magicked
           )) {
@@ -578,5 +583,5 @@
     PXOPT_COPY_S64(config->args, where,  "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
     PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "diffSkyfile.fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
@@ -655,5 +660,5 @@
     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffSkyfile.diff_id", "==");
     PXOPT_COPY_STR(config->args, where,  "-label", "diffRun.label", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",     "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",     "fault", "==");
 
     if (!psDBTransaction(config->dbh)) {
@@ -690,5 +695,5 @@
         psFree(query);
 
-        psLogMsg("diftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh));
+        psLogMsg("difftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh));
     }
 
@@ -766,5 +771,4 @@
                          psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
                          psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
-                         psS64 exp_id, // exposure id for input_warp_id (if defined)
                          pxConfig *config // Configuration
                          )
@@ -797,5 +801,6 @@
             registered,
             tess_id,
-            exp_id,
+            false,
+            false,
             false       // magicked
     );
@@ -877,5 +882,4 @@
     PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
     PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
-    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
@@ -897,9 +901,4 @@
         psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                 "No input has been defined (-input_stack_id or -input_warp_id)");
-        return false;
-    }
-    if (input_warp_id && !exp_id) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                "-exp_id is required with -input_warp_id.");
         return false;
     }
@@ -912,5 +911,4 @@
                       template_warp_id ? template_warp_id : PS_MAX_S64,
                       template_stack_id ? template_stack_id : PS_MAX_S64,
-                      exp_id ? exp_id : PS_MAX_S64,
                       config)) {
         psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
@@ -930,5 +928,5 @@
 
 
-static bool definebyqueryMode(pxConfig *config)
+static bool definewarpstackMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -955,7 +953,8 @@
     PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
     PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
     // find all things to queue
-    psString query = pxDataGet("difftool_definebyquery_part1.sql");
+    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -1052,6 +1051,17 @@
     }
 
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
     // create temporary table
-    query = pxDataGet("difftool_definebyquery_temp_create.sql");
+    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1065,5 +1075,5 @@
     query = NULL;
 
-    psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql");
+    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");
 
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
@@ -1178,5 +1188,6 @@
                 registered,
                 tess_id,
-                exp_id,
+                false,                  // bothways
+                true,                   // exposure
                 false       // magicked
         );
@@ -1269,9 +1280,23 @@
     PXOPT_COPY_S64(config->args, selectWhere, "-exp_id", "inputRawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, selectWhere, "-filter", "inputRawExp.filter", "==");
+    PXOPT_COPY_STR(config->args, selectWhere, "-obs_mode", "inputRawExp.obs_mode", "==");
+    PXOPT_COPY_STR(config->args, selectWhere, "-obs_mode", "templateRawExp.obs_mode", "==");
     PXOPT_COPY_STR(config->args, selectWhere, "-input_label", "inputWarpRun.label", "==");
     PXOPT_COPY_STR(config->args, selectWhere, "-template_label", "templateWarpRun.label", "==");
+    PXOPT_COPY_F32(config->args, selectWhere, "-rotdiff", "ABS(inputRawExp.posang - templateRawExp.posang)", "<=");
     PXOPT_COPY_F32(config->args, selectWhere, "-timediff",
                    "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<=");
-    PXOPT_COPY_F32(config->args, selectWhere, "-rotdiff", "ABS(inputRawExp.posang - templateRawExp.posang)", "<=");
+
+    // Haversine formula for great circle distance
+    PXOPT_COPY_F32(config->args, selectWhere, "-distance",
+                   "DEGREES(2*ASIN(SQRT(POW(SIN(inputRawExp.decl - templateRawExp.decl),2) + "
+                   "COS(inputRawExp.decl)*COS(templateRawExp.decl)*"
+                   "POW(SIN(inputRawExp.ra - templateRawExp.ra),2))))", "<=");
+
+    PXOPT_LOOKUP_BOOL(backwards, config->args, "-backwards", false);
+
+    psMetadataAddF32(selectWhere, PS_LIST_TAIL,
+                     "TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs))",
+                     PS_META_DUPLICATE_OK, backwards ? "<" : ">", 0.0);
 
     // Restrictions for inserting skycells
@@ -1289,9 +1314,78 @@
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        return false;
+    }
+
+    if (!rerun) {
+        // Need to build table of exposures with diffs
+        psString tempCreate = pxDataGet("difftool_definewarpwarp_temp_create.sql"); // Create temp table SQL
+        if (!tempCreate) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        if (!p_psDBRunQuery(config->dbh, tempCreate)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to create temp table: %s", tempCreate);
+            psFree(tempCreate);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(tempCreate);
+
+        psString tempInsert = pxDataGet("difftool_definewarpwarp_temp_insert.sql"); // Insert to temp table
+        if (!tempInsert) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psString where = psStringCopy(""); // WHERE for insertion
+        if (label) {
+            psStringAppend(&where, "\nAND diffRun.label = '%s'", label);
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, tempInsert, where, where)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to insert into temp table: %s", tempInsert);
+            psFree(tempInsert);
+            psFree(where);
+            psFree(selectWhere);
+            psFree(insertWhere);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(where);
+        psFree(tempInsert);
+    }
+
+    // Get list of warps to diff
     psString select = pxDataGet("difftool_definewarpwarp_select.sql");
     if (!select) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psFree(selectWhere);
+        psFree(insertWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
     }
@@ -1306,22 +1400,17 @@
     psFree(selectWhere);
 
+    if (!available) {
+        psStringAppend(&whereClause,
+                       "\n%s inputWarpRun.state = 'full'"
+                       "AND templateWarpRun.state = 'full'",
+                       whereClause ? "AND" : "WHERE");
+    }
+
     if (!rerun) {
-        psStringAppend(&whereClause, "\n%s diffRun.diff_id IS NULL", whereClause ? "AND" : "WHERE");
-    }
-
-    if (!available) {
-        psStringAppend(&whereClause, "\n%s inputWarpRun.state = 'full'", whereClause ? "AND" : "WHERE");
-    }
-
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(select);
-        psFree(whereClause);
-        psFree(insertWhere);
-        return false;
+        psStringAppend(&whereClause, "\n%s diffs.diff_id IS NULL", whereClause ? "AND" : "WHERE");
     }
 
     if (!p_psDBRunQueryF(config->dbh, select,
-                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffRun?
+                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffs?
                          whereClause)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause);
@@ -1368,9 +1457,20 @@
     }
 
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, results, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(results);
+            return false;
+        }
+        psFree(results);
+        return true;
+    }
+
     psString insert = pxDataGet("difftool_definewarpwarp_insert.sql"); // Insertion for each new run
 
     if (psListLength(insertWhere->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(insertWhere, NULL);
-        psStringAppend(&insert, "\n%s", whereClause);
+        psStringAppend(&insert, "\nAND %s", whereClause);
         psFree(whereClause);
     }
@@ -1382,9 +1482,8 @@
         psMetadata *row = results->data[i]; // Result row from query
 
-        psS64 exp_id = psMetadataLookupS64(NULL, row, "input_exp_id");
         psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id");
-        psS64 template_id = psMetadataLookupS64(NULL, row, "template_warp_id");
+        const char *template = psMetadataLookupStr(NULL, row, "template_warp_id");
         const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id");
-        if (!exp_id || !input_id || !template_id || !tess_id) {
+        if (!input_id || !template || !tess_id) {
             psError(PXTOOLS_ERR_PROG, false, "Identifiers not found");
             psFree(list);
@@ -1398,5 +1497,5 @@
 
         diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered,
-                                          tess_id, exp_id, false); // Run to insert
+                                          tess_id, true, true, false); // Run to insert
         if (!diffRunInsertObject(config->dbh, run)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1413,13 +1512,11 @@
 
         // Convert identifiers to string, for insertion into query
-        psString diff = NULL, input = NULL, template = NULL; // String versions of identifiers
+        psString diff = NULL, input = NULL; // String versions of identifiers
         psStringAppend(&diff, "%" PRId64, run->diff_id);
         psStringAppend(&input, "%" PRId64, input_id);
-        psStringAppend(&template, "%" PRId64, template_id);
 
         if (!p_psDBRunQueryF(config->dbh, insert, diff, input, template, input, template)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(input);
-            psFree(template);
             psFree(diff);
             psFree(run);
@@ -1433,5 +1530,4 @@
         }
         psFree(input);
-        psFree(template);
         psFree(diff);
 
@@ -1482,5 +1578,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
 
     psString query = pxDataGet("difftool_pendingcleanuprun.sql");
@@ -1547,5 +1643,5 @@
         PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
     }
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
 
     psString query = pxDataGet("difftool_pendingcleanupskyfile.sql");
@@ -1662,10 +1758,10 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
 
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-diff_id",   "diff_id",   "==");
 
-    if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, code)) {
+    if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
         psFree (where);
@@ -1737,4 +1833,5 @@
   PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
   PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
 
   FILE *f = fopen (outfile, "w");
@@ -1782,11 +1879,27 @@
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
-      psError(PS_ERR_UNKNOWN, false, "database error");
       return false;
     }
     if (!psArrayLength(output)) {
-      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psError(PS_ERR_UNKNOWN, true, "no rows found");
       psFree(output);
-      return true;
+      return false;
+    }
+
+    if (clean) {
+        bool success = true;
+        if (!strcmp(tables[i].tableName, "diffRun")) {
+            success = pxSetStateCleaned("diffRun", "state", output);
+#ifdef notyet
+        // diffSkyfile doesn't have dataState yet
+        } else if (!strcmp(tables[i].tableName, "diffSkyfile")) {
+            success = pxSetStateCleaned("diffSkyfile", "data_state", output);
+#endif
+        }
+        if (!success) {
+            psFree(output);
+            psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
+            return false;
+        }
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.h	(revision 24244)
@@ -34,5 +34,5 @@
     DIFFTOOL_MODE_REVERTDIFFSKYFILE,
     DIFFTOOL_MODE_DEFINEPOPRUN,
-    DIFFTOOL_MODE_DEFINEBYQUERY,
+    DIFFTOOL_MODE_DEFINEWARPSTACK,
     DIFFTOOL_MODE_DEFINEWARPWARP,
     DIFFTOOL_MODE_PENDINGCLEANUPRUN,
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftoolConfig.c	(revision 24244)
@@ -49,4 +49,6 @@
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0,            "define workdir (required)", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id",  0,            "define tessellation ID (required)", NULL);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways",  0,            "do the subtraction both ways?", false);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure",  0,            "subtraction for entire exposure?", false);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label",  0,            "define label", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction",  0,            "define reduction class", NULL);
@@ -80,5 +82,5 @@
     psMetadata *todiffskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(todiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,            "search by diff ID", 0);
-    psMetadataAddStr(todiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0);
+    psMetadataAddStr(todiffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", 0);
     psMetadataAddU64(todiffskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(todiffskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
@@ -88,6 +90,6 @@
     psMetadataAddS64(adddiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,            "define warp ID (required)", 0);
     psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,       "define skycell of file (required)", 0);
-    psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
-    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-uri", 0,            "define URI of file", 0);
+    psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
+    psMetadataAddS16(adddiffskyfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
     psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-path_base", 0,            "define base output location", 0);
     psMetadataAddF64(adddiffskyfileArgs, PS_LIST_TAIL, "-bg",  0,            "define exposure background", NAN);
@@ -107,4 +109,5 @@
     psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-kernel_xy",  0, "define kernel xy moment", NAN);
     psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-kernel_yy",  0, "define kernel yy moment", NAN);
+    psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-deconv_max",  0, "define maximum deconvolution fraction", NAN);
     psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-sources",  0,   "define number of sources", 0);
     psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0,   "define hostname", 0);
@@ -123,5 +126,5 @@
     psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
-    psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-code",  0,            "deifine fault code", 0);
+    psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "define fault code", 0);
 
     // -revertdiffskyfile
@@ -129,5 +132,5 @@
     psMetadataAddS64(revertdiffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,            "search by diff ID", 0);
     psMetadataAddStr(revertdiffskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
-    psMetadataAddS16(revertdiffskyfileArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertdiffskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -definepoprun
@@ -140,5 +143,4 @@
     psMetadataAddStr(definepoprunArgs, PS_LIST_TAIL, "-reduction",  0,            "define reduction class", NULL);
     psMetadataAddBool(definepoprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
-    psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-exp_id", 0,              "define exposure ID for template", 0);
     psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_warp_id", 0,            "define warp ID for template", 0);
     psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-template_stack_id", 0,            "define stack ID for template", 0);
@@ -146,22 +148,23 @@
     psMetadataAddS64(definepoprunArgs, PS_LIST_TAIL, "-input_stack_id", 0,            "define stack ID for input", 0);
 
-    // -definebyquery
-    psMetadata *definebyqueryArgs = psMetadataAlloc();
-    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
-    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
-    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",  0, "define reduction class", NULL);
-    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    // -definewarpstack
+    psMetadata *definewarpstackArgs = psMetadataAlloc();
+    psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
+    psMetadataAddS64(definewarpstackArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-stack_label", 0, "search by stack label", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL);
+    psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);
+    psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-reduction",  0, "define reduction class", NULL);
+    psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false);
 
     // -definewarpwarp
@@ -170,5 +173,8 @@
     psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
     psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
+    psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-distance", 0, "limit distance between input and template (deg)", NAN);
+    psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL);
     psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-timediff", 0, "limit time difference between input and template", NAN);
+    psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-backwards", 0, "Template comes after input?", false);
     psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-rotdiff", 0, "limit rotator difference between input and template", NAN);
     psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-input_label", 0, "search by warp label for input", NULL);
@@ -182,8 +188,9 @@
     psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false);
     psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-pretend", 0, "list results but to not queue", false);
 
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -191,5 +198,5 @@
     // -pendingcleanupskyfile
     psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,          "search by diff ID", 0);
     psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
@@ -204,5 +211,5 @@
     psMetadata *updatediffskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(updatediffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,      "define diff ID (required)", 0);
-    psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-code", 0,         "set fault code (required)", 0);
+    psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0,         "set fault code (required)", 0);
 
     // -exportrun
@@ -211,4 +218,5 @@
     psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
     psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean",  0,          "mark tables as cleaned", false);
 
     // -importrun
@@ -232,5 +240,5 @@
     PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs);
     PXOPT_ADD_MODE("-definepoprun",     "", DIFFTOOL_MODE_DEFINEPOPRUN,      definepoprunArgs);
-    PXOPT_ADD_MODE("-definebyquery",    "", DIFFTOOL_MODE_DEFINEBYQUERY,     definebyqueryArgs);
+    PXOPT_ADD_MODE("-definewarpstack",  "", DIFFTOOL_MODE_DEFINEWARPSTACK,   definewarpstackArgs);
     PXOPT_ADD_MODE("-definewarpwarp",   "", DIFFTOOL_MODE_DEFINEWARPWARP,    definewarpwarpArgs);
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttool.c	(revision 24244)
@@ -39,4 +39,24 @@
 static bool processedcomponentMode(pxConfig *config);
 static bool toadvanceMode(pxConfig *config);
+static bool pendingfilesetMode(pxConfig *config);
+static bool addfilesetMode(pxConfig *config);
+static bool revertfilesetMode(pxConfig *config);
+static bool queuercrunMode(pxConfig *config);
+static bool updatercrunMode(pxConfig *config);
+static bool revertrcrunMode(pxConfig *config);
+static bool pendingdestMode(pxConfig *config);
+
+static bool definetargetMode(pxConfig *config);
+static bool updatetargetMode(pxConfig *config);
+static bool listtargetMode(pxConfig *config);
+
+static bool definedsproductMode(pxConfig *config);
+static bool updatedsproductMode(pxConfig *config);
+
+static bool definedestinationMode(pxConfig *config);
+static bool updatedestinationMode(pxConfig *config);
+
+static bool defineinterestMode(pxConfig *config);
+static bool updateinterestMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -67,4 +87,20 @@
         MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode);
         MODECASE(DISTTOOL_MODE_TOADVANCE, toadvanceMode);
+        MODECASE(DISTTOOL_MODE_PENDINGFILESET, pendingfilesetMode);
+        MODECASE(DISTTOOL_MODE_ADDFILESET, addfilesetMode);
+        MODECASE(DISTTOOL_MODE_REVERTFILESET, revertfilesetMode);
+        MODECASE(DISTTOOL_MODE_QUEUERCRUN, queuercrunMode);
+        MODECASE(DISTTOOL_MODE_UPDATERCRUN, updatercrunMode);
+        MODECASE(DISTTOOL_MODE_REVERTRCRUN, revertrcrunMode);
+        MODECASE(DISTTOOL_MODE_PENDINGDEST, pendingdestMode);
+        MODECASE(DISTTOOL_MODE_DEFINETARGET, definetargetMode);
+        MODECASE(DISTTOOL_MODE_UPDATETARGET, updatetargetMode);
+        MODECASE(DISTTOOL_MODE_LISTTARGET, listtargetMode);
+        MODECASE(DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductMode);
+        MODECASE(DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductMode);
+        MODECASE(DISTTOOL_MODE_DEFINEDESTINATION, definedestinationMode);
+        MODECASE(DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationMode);
+        MODECASE(DISTTOOL_MODE_DEFINEINTEREST, defineinterestMode);
+        MODECASE(DISTTOOL_MODE_UPDATEINTEREST, updateinterestMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -92,8 +128,13 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+    // required
     PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
     PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id",  true, false);
     PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false);
+    PXOPT_LOOKUP_S64(target_id, config->args, "-target_id",  true, false);
+
+    // optional
     PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
+    PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false);
     PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
 
@@ -101,12 +142,21 @@
     // XXX: all of the following concerns will be managed properly by definebyquery
 
-    // TODO: check that stage_id actually exists for stage
-    // in magicdstool we queue off of a magic_id so the stage_id, exp_id, and cam_id get looked up 
+    // TODO: should we check that stage_id actually exists for stage
+    // in magicdstool we queue off of a magic_id so the stage_id, exp_id, and cam_id get looked up
     // when the run is queued
-    // Should we also check here that the run is full or cleaned and that all of the images
-    // are magicked. We need to do that at run time as well since the run and magic state can
-    // change between the time that it is queued.
-
-    if (!distRunInsert(config->dbh, 0, stage, stage_id, set_label, outroot, clean, "new", NULL, 0)) {
+
+    if (!distRunInsert(config->dbh,
+            0,          // dist_id
+            target_id,
+            stage,
+            stage_id,
+            set_label,
+            outroot,
+            clean,
+            no_magic,
+            "new",
+            NULL,       // time_stamp
+            0           // fault
+            )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -118,6 +168,262 @@
 static bool definebyqueryMode(pxConfig *config)
 {
-    psError(PS_ERR_PROGRAMMING, true, "-definebyquery is not implemented yet");
-    return false;
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_STR(stage, config->args,     "-stage", true, false);
+    PXOPT_LOOKUP_STR(workdir, config->args,   "-workdir", true, false);
+
+    // optional
+    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id",  false, false);
+    PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false);
+    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    
+    PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false);
+
+    // select arguments
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-target_id", "distTarget.target_id",    "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id",    "rawExp.exp_id",           "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id",   "chipRun.chip_id",         "==");
+    PXOPT_COPY_STR(config->args, where, "-obs_mode",  "rawExp.obs_mode",         "==");
+
+    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
+
+    psString query = NULL;
+    psString magicRunType = NULL;
+    psString runJoinStr = NULL;
+    if (!strcmp(stage, "raw")) {
+        magicRunType = "rawExp";
+        runJoinStr = "rawExp.exp_id";
+        query = pxDataGet("disttool_definebyquery_raw.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+    } else if (!strcmp(stage, "chip")) {
+        magicRunType = "chipRun";
+        runJoinStr = "chipRun.chip_id";
+        query = pxDataGet("disttool_definebyquery_chip.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        if (label) {
+            psStringAppend(&query, " AND (chipRun.label = '%s')", label);
+        }
+    } else if (!strcmp(stage, "camera")) {
+        magicRunType = "chipRun";    // This is used below to set the magicked business
+        query = pxDataGet("disttool_definebyquery_camera.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        if (label) {
+            psStringAppend(&query, " AND (camRun.label = '%s')", label);
+        }
+    } else if (!strcmp(stage, "fake")) {
+        magicRunType = "fakeRun";
+        query = pxDataGet("disttool_definebyquery_fake.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        if (label) {
+            psStringAppend(&query, " AND (fakeRun.label = '%s')", label);
+        }
+        // fake stage doesn't require magic
+        no_magic = true;
+    } else if (!strcmp(stage, "warp")) {
+        magicRunType = "warpRun";
+        runJoinStr = "warpRun.warp_id";
+        query = pxDataGet("disttool_definebyquery_warp.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        if (label) {
+            psStringAppend(&query, " AND (warpRun.label = '%s')", label);
+        }
+
+    } else if (!strcmp(stage, "diff")) {
+        magicRunType = "diffRun";
+        runJoinStr = "diffRun.diff_id";
+        query = pxDataGet("disttool_definebyquery_diff.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        if (label) {
+            psStringAppend(&query, " AND (diffRun.label = '%s')", label);
+        }
+
+    } else if (!strcmp(stage, "stack")) {
+        magicRunType = "stackRun";
+        query = pxDataGet("disttool_definebyquery_stack.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        if (label) {
+            psStringAppend(&query, " AND (stackRun.label = '%s')", label);
+        }
+        // stack stage doesn't require magic (perhaps let the script do this?
+        no_magic = true;
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "unknown value for stage: %s", stage);
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    psString joinHook = NULL;
+
+    if (!no_magic) {
+        psStringAppend(&query, " AND (distTarget.clean OR %s.magicked)", magicRunType);
+
+        // is selecting by magic_ds_id really interesting?
+        if (magic_ds_id) {
+            if (strcmp(stage, "camera")) {
+                // stage other than camera
+                if (!runJoinStr) {
+                    psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage);
+                    psFree(query);
+                    return false;
+                }
+                psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage"
+                                              " AND magicDSRun.stage_id = %s", runJoinStr);
+            } else {
+                // camera masks are magicked when the chipRun is magicked
+                // XXX: This is confusing. Is it dangerous?
+                // Maybe I should add a magicked bit to camRun. Note this isn't
+                psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = 'chip'"
+                                              " AND magicDSRun.stage_id = chipRun.chip_id");
+            }
+            psStringAppend(&query, " AND (magicDSRun.state = 'full' AND magicDSRun.re_place AND (magic_ds_id = %" PRId64 "))", magic_ds_id);
+        }
+    }
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, joinHook ? joinHook : "")) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+    psFree(joinHook);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("disttool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+
+    if (dry_run) {
+        if (!ippdbPrintMetadatas(stdout, output, "newdistRuns", true)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psArray *list = psArrayAllocEmpty(limit);
+    for (long i=0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+        psString stage = psMetadataLookupStr(NULL, md, "stage");
+        psString run_tag = psMetadataLookupStr(NULL, md, "run_tag");
+        psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id");
+        psS64 target_id = psMetadataLookupS64(NULL, md, "target_id");
+        bool clean = psMetadataLookupBool(NULL, md, "clean");
+
+        psString outroot = NULL;
+        psStringAppend(&outroot, "%s/%s/%s", workdir, run_tag, stage); 
+
+        distRunRow *row = distRunRowAlloc(
+                0,      // dist_id
+                target_id,
+                stage,
+                stage_id,
+                set_label,
+                outroot,
+                clean,
+                no_magic,
+                "new",
+                NULL,   // time_stamp
+                0       // fault
+                );
+
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to allocate distRunRow");
+            psFree(outroot);
+            psFree(output);
+            return false;
+        }
+        if (!distRunInsertObject(config->dbh, row)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(outroot);
+            psFree(output);
+            return false;
+        }
+        psFree(outroot);
+        psS64 dist_id = psDBLastInsertID(config->dbh);
+        row->dist_id = dist_id;
+        psArrayAdd(list, list->n, row);
+        psFree(row);
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!distRunPrintObjects(stdout, list, !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(list);
+        return false;
+    }
+
+    psFree(list);
+    psFree(output);
+
+    return true;
 }
 
@@ -138,8 +444,8 @@
     PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
     PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-
-    if ((!state) && (!label) && (!code)) {
-        psError(PXTOOLS_ERR_DATA, false, "parameters (-code or -set_state or -set_label) are required");
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
+    if ((!state) && (!label) && (!fault)) {
+        psError(PXTOOLS_ERR_DATA, false, "parameters (-fault or -set_state or -set_label) are required");
         psFree(where);
         return false;
@@ -156,6 +462,6 @@
     }
 
-    if (code) {
-        psStringAppend(&query, " , fault = %d", code);
+    if (fault) {
+        psStringAppend(&query, " , fault = %d", fault);
     }
 
@@ -182,5 +488,5 @@
     PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
     PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "distComponent.fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
 
     // It might be useful to be able to query by the parameters of the underlying runs
@@ -296,5 +602,4 @@
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-    PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false);
 
     // look for "inputs" that need to processed
@@ -319,6 +624,11 @@
     }
 
-    psString    chip_magic = "";
+    // the query has where hooks for each stage.
+    // right now we aren't using them.
+    // XXX: I think that I want to change the query from a union of selects on the various
+    // stages to separate queries. As it is pending data at the later stages of the pipline
+    // will get blocked by pending earlier stages
     psString    raw_where = "";
+    psString    raw_clean_where = "";
     psString    chip_where = "";
     psString    camera_where = "";
@@ -328,18 +638,8 @@
     psString    stack_where = "";
 
-    if (need_magic) {
-        chip_magic = psStringCopy("\nAND chipRun.magicked");
-        raw_where  = psStringCopy("\nAND rawExp.magicked");
-        chip_where = psStringCopy("\nAND (distRun.clean OR chipRun.magicked)");
-        // chipRun must be magicked to allow release camera level masks
-        camera_where = psStringCopy("\nAND (distRun.clean OR chipRun.magicked)");
-        warp_where = psStringCopy("\nAND (distRun.clean OR warpRun.magicked)");
-        diff_where = psStringCopy("\nAND (distRun.clean OR diffRun.magicked)");
-    }
-
     if (!p_psDBRunQueryF(config->dbh,
             query,
-            chip_magic,
             raw_where,
+            raw_clean_where,
             chip_where,
             camera_where,
@@ -397,6 +697,6 @@
     // unless fault code is set require filename, bytes, and md5sum
     bool require_fileinfo = false;
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-    if (!code) {
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    if (!fault) {
         require_fileinfo = true;
     }
@@ -405,5 +705,5 @@
     PXOPT_LOOKUP_STR(name, config->args, "-name", require_fileinfo, false);
 
-    if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", name, code)) {
+    if (!distComponentInsert(config->dbh, dist_id, component, bytes, md5sum, "full", name, fault)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -419,4 +719,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -555,2 +856,848 @@
     return true;
 }
+
+static bool pendingfilesetMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // look for "inputs" that need to processed
+    psString query = pxDataGet("disttool_pendingfileset.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("disttool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "pendingfileset", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+static bool addfilesetMode(pxConfig *config)
+{
+
+    // required values
+    PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false);
+    PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false);
+
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
+    // unless fault code is set require name
+    PXOPT_LOOKUP_STR(name, config->args, "-name", fault == 0, false);
+
+    if (!rcDSFilesetInsert(config->dbh, 
+            0,          // fs_id
+            dist_id,
+            prod_id,
+            name,
+            "full",
+            fault)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+static bool revertfilesetMode(pxConfig *config)
+{
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-dist_id", "rcDSFileset.dist_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");;
+    PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "rcDSFileset.fault", "==");
+
+    // It might be useful to be able to query by the parameters of the underlying runs
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = pxDataGet("disttool_revertfileset.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+
+    int numUpdated = psDBAffectedRows(config->dbh);
+
+    psLogMsg("disttool", PS_LOG_INFO, "deleted %d rcDSFilesets", numUpdated);
+
+    return true;
+}
+
+static bool pendingdestMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-dest_id", "dist_id", "==");
+//     PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // look for "inputs" that need to processed
+    psString query = pxDataGet("disttool_pendingdest.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString whereString = NULL;
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        // where string gets added by hook, not at the end of the query
+        psStringAppend(&whereString, "\n AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(whereString);
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("disttool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "pendingfileset", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+static bool queuercrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-dist_id",  "dist_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-dest_id",  "dest_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-prod_id",  "prod_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-target_id","target_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fs_id",    "fs_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-stage",    "stage", "==");;
+    PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label",    "label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+
+    psString query = pxDataGet("disttool_queuercrun.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    long numUpdated = psDBAffectedRows(config->dbh);
+
+    psLogMsg("disttool", PS_LOG_INFO, "Inserted %ld rcRuns", numUpdated);
+
+    return true;
+}
+
+
+static bool updatercrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_LOOKUP_S64(rc_id, config->args, "-rc_id", false, false);
+    PXOPT_LOOKUP_STR(fs_name, config->args, "-fs_name", false, false);
+    PXOPT_LOOKUP_STR(status_fs_name, config->args, "-status_fs_name", false, false);
+    PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", false, false);
+
+    // We either need rc_id or (dest_id and fs_name) to identifiy the rcRun
+    if ((!rc_id) && !(dest_id && fs_name)) {
+        psError(PXTOOLS_ERR_DATA, true, "either -rc_id or (-fs_name and -dest_id) are required");
+        return false;
+    }
+
+    // now that we have done the argument checking
+    PXOPT_COPY_S64(config->args, where, "-rc_id", "rc_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-fs_name", "rcDSFileset.name", "==");
+    PXOPT_COPY_S64(config->args, where, "-dest_id", "rcRun.dest_id", "==");
+
+    if (!psListLength(where->list)) {
+        // this can't happen because we checked above
+        psFree(where);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        return false;
+    }
+
+
+    PXOPT_LOOKUP_STR(last_fileset, config->args, "-set_last_fileset", false, false);
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
+    if (!state && (fault < 0)) {
+        psError(PXTOOLS_ERR_DATA, false, "parameters (-fault or -set_state) are required");
+        psFree(where);
+        return false;
+    }
+
+    psString query = pxDataGet("disttool_updatercrun.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString setString = psStringCopy("\n");
+    psString separator = "";
+    if (state) {
+        psStringAppend(&setString, " rcRun.state = '%s'", state);
+        separator = ",";
+    }
+
+    // default for fault is -1, so set it if it's zero or higher. This allows clearing fault
+    // without forcing revert first
+    if (fault >= 0) {
+        psStringAppend(&setString, "%s rcRun.fault = %d", separator, fault);
+    }
+
+    if (status_fs_name) {
+        psStringAppend(&setString, ", rcRun.status_fs_name = '%s'", status_fs_name);
+    }
+
+    if (last_fileset) {
+        psStringAppend(&setString, ", rcDestination.last_fileset = '%s'", last_fileset);
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, setString)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(setString);
+        psFree(query);
+        return false;
+    }
+    psFree(setString);
+    psFree(query);
+
+    long numUpdated = psDBAffectedRows(config->dbh);
+
+    psLogMsg("disttool", PS_LOG_INFO, "Updated %ld rows", numUpdated);
+
+    return true;
+}
+
+static bool revertrcrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-rc_id", "rc_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fault", "fault", "==");
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = pxDataGet("disttool_revertrcrun.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+
+    long numUpdated = psDBAffectedRows(config->dbh);
+
+    psLogMsg("disttool", PS_LOG_INFO, "Updated %ld rcRuns", numUpdated);
+
+    return true;
+}
+
+static bool definetargetMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_STR(obs_mode, config->args, "-obs_mode", true, false);
+    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
+
+    // optional
+    PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
+    PXOPT_LOOKUP_STR(comment, config->args, "-comment", false, false);
+
+    distTargetRow *row = distTargetRowAlloc(
+            0,          // target_id
+            obs_mode,
+            stage,
+            clean,
+            state ? state : "enabled",
+            comment
+            );
+            
+    if (!row) {
+        psError(PS_ERR_UNKNOWN, false, "failed to allocate distTarget object");
+        return false;
+    }
+   if (!distTargetInsertObject(config->dbh, row)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(row);
+        return false;
+    }
+
+    // get the assigned target_id
+    row->target_id = psDBLastInsertID(config->dbh);
+
+    if (!distTargetPrintObject(stdout, row, true)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print object");
+            psFree(row);
+            return false;
+    }
+
+    psFree(row);
+
+    return true;
+}
+static bool updatetargetMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "==");
+    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
+
+    psString query = psStringCopy("UPDATE distTarget SET state = '%s'");
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
+        psFree(where);
+        psFree(query);
+        return false;
+    }
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, state)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
+static bool listtargetMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "==");
+    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
+
+    PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
+    PXOPT_LOOKUP_BOOL(full, config->args, "-full", false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    if (clean && full) {
+        psError(PS_ERR_UNKNOWN, false, "can't select both -clean and -full");
+        return false;
+    }
+
+    psString query = psStringCopy("SELECT * FROM distTarget");
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+        if (clean) {
+            psStringAppend(&query, " (AND clean)");
+        } else if (full) {
+            psStringAppend(&query, " (AND !clean)");
+        }
+    } else if (clean) {
+        psStringAppend(&query, " WHERE clean");
+    } else if (full) {
+        psStringAppend(&query, " WHERE !clean");
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("disttool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (!ippdbPrintMetadatas(stdout, output, "distTarget", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+static bool definedsproductMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_STR(name, config->args,   "-name", true, false);
+    PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false);
+    PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false);
+
+    // XXX: should we insure that these names do not contatin any whitespace?
+
+    rcDSProductRow *row = rcDSProductRowAlloc(
+            0,          // prod_id
+            name,
+            dbname,
+            dbhost
+            );
+            
+    if (!row) {
+        psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDSProduct object");
+        return false;
+    }
+   if (!rcDSProductInsertObject(config->dbh, row)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(row);
+        return false;
+    }
+
+    // get the assigned target_id
+    row->prod_id = psDBLastInsertID(config->dbh);
+
+    if (!rcDSProductPrintObject(stdout, row, true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(row);
+        return false;
+    }
+
+    psFree(row);
+
+    return true;
+}
+static bool updatedsproductMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "==");
+
+    PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", false, false);
+    PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", false, false);
+
+    if (!(dbname || dbhost)) {
+        psError(PS_ERR_UNKNOWN, true, "one or more of dbname or dbhost is required");
+        psFree(where);
+        return false;
+    }
+    psString query = psStringCopy("UPDATE rcDSProduct SET");
+    psString sep = "";
+    if (dbname) {
+        psStringAppend(&query, " dbname = '%s'", dbname);
+        sep = ",";
+    }
+    if (dbhost) {
+        psStringAppend(&query, " %s dbhost = '%s'", sep, dbhost);
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
+        psFree(where);
+        psFree(query);
+        return false;
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
+static bool definedestinationMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(prod_id, config->args,      "-prod_id", true, false);
+    PXOPT_LOOKUP_STR(name, config->args,         "-name", true, false);
+
+    // optional
+    PXOPT_LOOKUP_STR(status_uri, config->args,   "-status_uri", false, false);
+    PXOPT_LOOKUP_STR(comment, config->args,      "-comment", false, false);
+    PXOPT_LOOKUP_STR(last_fileset, config->args, "-last_fileset", false, false);
+    PXOPT_LOOKUP_STR(state, config->args,        "-set_state", false, false);
+
+    // XXX: should we insure that these names do not contatin any whitespace?
+
+    rcDestinationRow *row = rcDestinationRowAlloc(
+            0,          // dest_id
+            prod_id,
+            name,
+            status_uri,
+            comment,
+            last_fileset,
+            state ? state : "enabled"
+            );
+            
+    if (!row) {
+        psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDestination object");
+        return false;
+    }
+   if (!rcDestinationInsertObject(config->dbh, row)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(row);
+        return false;
+    }
+
+    // get the assigned target_id
+    row->dest_id = psDBLastInsertID(config->dbh);
+
+    if (!rcDestinationPrintObject(stdout, row, true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(row);
+        return false;
+    }
+
+    psFree(row);
+
+    return true;
+}
+
+static bool updatedestinationMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "==");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
+#ifdef ALLOW_UPDATE_LAST_FILESET
+    PXOPT_LOOKUP_STR(last_fileset, config->args, "-set_last_fileset", false, false);
+    if (!(state || last_fileset)) {
+        psError(PS_ERR_UNKNOWN, true, "one or more of -set_state or -set_last_fileset is required");
+# else 
+    if (!state) {
+#endif
+        psFree(where);
+        return false;
+    }
+    psString query = psStringCopy("UPDATE rcDestination SET");
+    psString sep = "";
+    if (state) {
+        psStringAppend(&query, " state = '%s'", state);
+        sep = ",";
+    }
+#ifdef ALLOW_UPDATE_LAST_FILESET
+    // last_fileset normally gets set by updatercrunMode
+    // Allowing it to be set here might cause problems
+    // especially since we are allowing selection by prod_id
+    if (last_fileset) {
+        psStringAppend(&query, " %s last_fileset = '%s'", sep, last_fileset);
+    }
+#endif
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
+        psFree(where);
+        psFree(query);
+        return false;
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
+static bool defineinterestMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(dest_id, config->args,      "-dest_id", true, false);
+    PXOPT_LOOKUP_S64(target_id, config->args,    "-target_id", true, false);
+
+    // optional
+    PXOPT_LOOKUP_STR(state, config->args,        "-set_state", false, false);
+
+    // XXX: should we insure that these names do not contatin any whitespace?
+
+    rcInterestRow *row = rcInterestRowAlloc(
+            0,          // int_id
+            dest_id,
+            target_id,
+            state ? state : "enabled"
+            );
+            
+    if (!row) {
+        psError(PS_ERR_UNKNOWN, false, "failed to allocate rcInterest object");
+        return false;
+    }
+   if (!rcInterestInsertObject(config->dbh, row)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(row);
+        return false;
+    }
+
+    // get the assigned target_id
+    row->int_id = psDBLastInsertID(config->dbh);
+
+    if (!rcInterestPrintObject(stdout, row, true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(row);
+        return false;
+    }
+
+    psFree(row);
+
+    return true;
+}
+
+static bool updateinterestMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-int_id",    "int_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-dest_id",   "dest_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "==");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
+    if (!state) {
+        psError(PS_ERR_UNKNOWN, true, "-set_state is required");
+        psFree(where);
+        return false;
+    }
+    psString query = NULL;
+    psStringAppend(&query, "UPDATE rcInterest SET state = '%s'", state);
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
+        psFree(where);
+        psFree(query);
+        return false;
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttool.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttool.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttool.h	(revision 24244)
@@ -33,4 +33,20 @@
     DISTTOOL_MODE_PROCESSEDCOMPONENT,
     DISTTOOL_MODE_TOADVANCE,
+    DISTTOOL_MODE_PENDINGFILESET,
+    DISTTOOL_MODE_ADDFILESET,
+    DISTTOOL_MODE_REVERTFILESET,
+    DISTTOOL_MODE_QUEUERCRUN,
+    DISTTOOL_MODE_UPDATERCRUN,
+    DISTTOOL_MODE_REVERTRCRUN,
+    DISTTOOL_MODE_PENDINGDEST,
+    DISTTOOL_MODE_DEFINEDESTINATION,
+    DISTTOOL_MODE_UPDATEDESTINATION,
+    DISTTOOL_MODE_DEFINEDSPRODUCT,
+    DISTTOOL_MODE_UPDATEDSPRODUCT,
+    DISTTOOL_MODE_DEFINETARGET,
+    DISTTOOL_MODE_UPDATETARGET,
+    DISTTOOL_MODE_LISTTARGET,
+    DISTTOOL_MODE_DEFINEINTEREST,
+    DISTTOOL_MODE_UPDATEINTEREST,
 } disttoolMode;
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttoolConfig.c	(revision 24244)
@@ -43,15 +43,38 @@
     }
 
+    // -definebyquery
+    psMetadata *definebyqueryArgs = psMetadataAlloc();
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stage",     0, "define stage for bundle (required)", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir",   0, "define workdir (required)", NULL);
+
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",    0, "define label for run", NULL);
+
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be selected", false);
+
+    // select args
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-target_id",     0, "define target_id", 0); 
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id",        0, "define exp_id", 0); 
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id",       0, "define chip_id", 0); 
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_ds_id",   0, "define chip_id", 0); 
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",         0, "select by label", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-obs_mode",      0, "select by obs_mode", NULL);
+
+    psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
     // -definerun
     psMetadata *definerunArgs = psMetadataAlloc();
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage",         0, "define stage for bundle (required)", NULL);
-    psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id (required)", 0); 
+    psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id (required)", 0);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot",  0, "define output destination (required)", NULL);
+    psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0); 
     psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-clean", 0,   "build clean distribution bundle", false);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label",    0, "define label for run", NULL);
-    
+
     // -updaterun
     psMetadata *updaterunArgs = psMetadataAlloc();
-    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-dist_id",  0, "define dist_id", 0); 
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-dist_id",  0, "define dist_id", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "new value for label", NULL);
@@ -59,21 +82,20 @@
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state",     0, "value for state", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",     0, "limit updates to label", NULL);
-    psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-code",      0, "define fault code", 0); 
-    
+    psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-fault",      0, "define fault code", 0);
+
     // -revertrun
     psMetadata *revertrunArgs = psMetadataAlloc();
-    psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); 
+    psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
     psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-stage",    0, "define stage", NULL);
-    psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); 
+    psMetadataAddS64(revertrunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0);
     psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-state",    0, "define state", NULL);
     psMetadataAddStr(revertrunArgs, PS_LIST_TAIL, "-label",    0, "define label", NULL);
-    psMetadataAddS16(revertrunArgs, PS_LIST_TAIL, "-code", 0, "define fault code", 0); 
+    psMetadataAddS16(revertrunArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
     psMetadataAddBool(revertrunArgs, PS_LIST_TAIL, "-all",    0, "revert all faulted runs", NULL);
-    
+
     // -pendingcomponent
     psMetadata *pendingcomponentArgs = psMetadataAlloc();
-    psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); 
+    psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
     psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage",    0, "limit results to runs for stage", NULL);
-    psMetadataAddBool(pendingcomponentArgs, PS_LIST_TAIL, "-need_magic", 0, "magic is needed", false);
     psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label",    0, "limit results to label", NULL);
     psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
@@ -83,14 +105,14 @@
     // -addprocessedcomponent
     psMetadata *addprocessedcomponentArgs = psMetadataAlloc();
-    psMetadataAddS64(addprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); 
+    psMetadataAddS64(addprocessedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
     psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component (required)", NULL);
     psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL);
-    psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0); 
+    psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-bytes", 0, "define file size", 0);
     psMetadataAddStr(addprocessedcomponentArgs, PS_LIST_TAIL, "-md5sum", 0, "define stage for bundle", NULL);
-    psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-code", 0, "define fault code", 0); 
+    psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
 
     // -processedcomponent
     psMetadata *processedcomponentArgs = psMetadataAlloc();
-    psMetadataAddS64(processedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); 
+    psMetadataAddS64(processedcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
     psMetadataAddU64(processedcomponentArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
     psMetadataAddBool(processedcomponentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
@@ -98,11 +120,151 @@
     // -toadvance
     psMetadata *toadvanceArgs = psMetadataAlloc();
-    psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); 
-    psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
+    psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
+    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label",   0, "limit updates to label", NULL);
+    psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit",   0,  "limit result set to N items", 0);
     psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -pendingfileset
+    psMetadata *pendingfilesetArgs = psMetadataAlloc();
+    psMetadataAddS64(pendingfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
+    psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label",   0, "limit results to label", NULL);
+    psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-stage",   0, "limit results to runs for stage", NULL);
+    psMetadataAddU64(pendingfilesetArgs, PS_LIST_TAIL, "-limit",   0,  "limit result set to N items", 0);
+    psMetadataAddBool(pendingfilesetArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -addfileset
+    psMetadata *addfilesetArgs = psMetadataAlloc();
+    psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
+    psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0);
+    psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-name",    0, "define file name", NULL);
+    psMetadataAddS32(addfilesetArgs, PS_LIST_TAIL, "-fault",   0, "define fault code", 0);
+
+    // -pendingdest
+    psMetadata *pendingdestArgs = psMetadataAlloc();
+    psMetadataAddS64(pendingdestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0);
+    psMetadataAddU64(pendingdestArgs, PS_LIST_TAIL, "-limit",   0,  "limit result set to N items", 0);
+    psMetadataAddBool(pendingdestArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddStr(pendingdestArgs, PS_LIST_TAIL, "-label",   0, "limit results to label", NULL);
+
+    // -revertfileset
+    psMetadata *revertfilesetArgs = psMetadataAlloc();
+    psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-fs_id",   0, "define fs_id", 0);
+    psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
+    psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define dist_id", 0);
+    psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-stage",   0, "define stage", NULL);
+    psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0);
+    psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-state",   0, "define state", NULL);
+    psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-label",   0, "define label", NULL);
+    psMetadataAddS16(revertfilesetArgs, PS_LIST_TAIL, "-fault",   0, "define fault code", 0);
+    psMetadataAddBool(revertfilesetArgs, PS_LIST_TAIL, "-all",    0, "revert all faulted runs", NULL);
+
+    // -queuercrun
+    psMetadata *queuercrunArgs = psMetadataAlloc();
+    psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dist_id",   0, "define dist_id", 0);
+    psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dest_id",   0, "define dest_id", 0);
+    psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-prod_id",   0, "define prod_id", 0);
+    psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0);
+    psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-fs_id",     0, "define fs_id", 0);
+    psMetadataAddStr(queuercrunArgs, PS_LIST_TAIL, "-stage",     0, "define stage", NULL);
+    psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-stage_id",  0, "define stage_id", 0);
+    psMetadataAddStr(queuercrunArgs, PS_LIST_TAIL, "-state",     0, "define state", NULL);
+    psMetadataAddStr(queuercrunArgs, PS_LIST_TAIL, "-label",     0, "define label", NULL);
+    psMetadataAddU64(queuercrunArgs, PS_LIST_TAIL, "-limit",     0, "limit number of runs queued to N", 0);
+
+    // -updatercrun
+    psMetadata *updatercrunArgs = psMetadataAlloc();
+    psMetadataAddS64(updatercrunArgs, PS_LIST_TAIL, "-rc_id",    0, "define rc_id", 0);
+    psMetadataAddS64(updatercrunArgs, PS_LIST_TAIL, "-dest_id",  0, "define dest_id", 0);
+    psMetadataAddStr(updatercrunArgs, PS_LIST_TAIL, "-fs_name",  0, "define fileset name", NULL);
+    psMetadataAddStr(updatercrunArgs, PS_LIST_TAIL, "-status_fs_name",  0, "define status fileset name", NULL);
+    psMetadataAddStr(updatercrunArgs, PS_LIST_TAIL, "-set_state",0, "new value for state", NULL);
+    psMetadataAddStr(updatercrunArgs, PS_LIST_TAIL, "-set_last_fileset",0, "new value for last_fileset", NULL);
+    psMetadataAddS16(updatercrunArgs, PS_LIST_TAIL, "-fault",    0, "define fault code", -1);
+
+    // -revertrcrun
+    psMetadata *revertrcrunArgs = psMetadataAlloc();
+    psMetadataAddS64(revertrcrunArgs, PS_LIST_TAIL,  "-rc_id",   0, "define rc_id", 0);
+    psMetadataAddS64(revertrcrunArgs, PS_LIST_TAIL,  "-fs_id",   0, "define fs_id", 0);
+    psMetadataAddS64(revertrcrunArgs, PS_LIST_TAIL,  "-dest_id", 0, "define dest_id", 0);
+    psMetadataAddS16(revertrcrunArgs, PS_LIST_TAIL,  "-fault",   0, "define fault code", 0);
+    psMetadataAddBool(revertrcrunArgs, PS_LIST_TAIL, "-all",     0, "revert all faulted runs", NULL);
+
+    // -definedsproduct
+    psMetadata *definedsproductArgs = psMetadataAlloc();
+    psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-name",  0, "define product name", NULL);
+    psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL);
+    psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL);
+
+    // -updatedsproduct
+    // does this mode make sense?
+    psMetadata *updatedsproductArgs = psMetadataAlloc();
+    psMetadataAddS64(updatedsproductArgs, PS_LIST_TAIL, "-prod_id",   0, "select by prod_id", 0);
+    // can't select by name because it isn't necssarily unique
+    psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL);
+    psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL);
+
+    // -definedestination
+    psMetadata *definedestinationArgs = psMetadataAlloc();
+    psMetadataAddS64(definedestinationArgs, PS_LIST_TAIL, "-prod_id",     0, "define prod_id (required)", 0);
+    psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-name",        0, "define destination name (required)", NULL);
+    psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-status_uri",  0, "define status_uri", NULL);
+    psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-comment",     0, "define comment", NULL);
+    psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-last_fileset",0, "define last_fileset", NULL);
+    psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-set_state",   0, "define state", NULL);
+
+    // -updatedestination
+    psMetadata *updatedestinationArgs = psMetadataAlloc();
+    psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-dest_id",     0, "define dest_id", 0);
+    psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-prod_id",     0, "define prod_id", 0);
+    psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-name",        0, "define destination name", NULL);
+    psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-status_uri",  0, "define status_uri", NULL);
+    psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-comment",     0, "define comment", NULL);
+//  last_fileset gets updated by -updatercrun
+//  psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_last_fileset",0, "define last_fileset", NULL);
+    psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_state",   0, "define state", NULL);
+
+    // -definetarget
+    psMetadata *definetargetArgs = psMetadataAlloc();
+    psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-obs_mode",  0, "define obs_mode (required)", NULL);
+    psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-stage",     0, "define stage (required)", NULL);
+    psMetadataAddBool(definetargetArgs, PS_LIST_TAIL,"-clean",     0, "define clean", false);
+    psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL);
+    psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-comment",   0, "define comment", NULL);
+
+    // -updatetarget
+    psMetadata *updatetargetArgs = psMetadataAlloc();
+    psMetadataAddS64(updatetargetArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0);
+    psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-obs_mode",  0, "define obs_mode", NULL);
+    psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-stage",     0, "define stage", NULL);
+    psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL);
+
+    // -listtarget
+    psMetadata *listtargetArgs = psMetadataAlloc();
+    psMetadataAddS64(listtargetArgs, PS_LIST_TAIL, "-target_id", 0, "list target with target_id", 0);
+    psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-obs_mode",  0, "list targets for obs_mode", NULL);
+    psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-stage",     0, "list targets for stage", NULL);
+    psMetadataAddBool(listtargetArgs, PS_LIST_TAIL,"-clean",     0, "list clean targets", false);
+    psMetadataAddBool(listtargetArgs, PS_LIST_TAIL,"-full",      0, "list full targets", false);
+    psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-state",     0, "list tarets in state", NULL);
+    psMetadataAddU64(listtargetArgs, PS_LIST_TAIL, "-limit",     0, "limit number of targets listed to N", 0);
+    psMetadataAddBool(listtargetArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+
+    // -defineinterest
+    psMetadata *defineinterestArgs = psMetadataAlloc();
+    psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id",   0, "define dest_id (required)", 0);
+    psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0);
+    psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL);
+
+    // -updateinterest
+    psMetadata *updateinterestArgs = psMetadataAlloc();
+    psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-int_id",    0, "define int_id", 0);
+    psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-dest_id",   0, "define dest_id", 0);
+    psMetadataAddS64(updateinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0);
+    psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL);
 
     psMetadata *argSets = psMetadataAlloc();
     psMetadata *modes = psMetadataAlloc();
 
+    PXOPT_ADD_MODE("-definebyquery",    "", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs);
     PXOPT_ADD_MODE("-definerun",    "", DISTTOOL_MODE_DEFINERUN, definerunArgs);
     PXOPT_ADD_MODE("-updaterun",    "", DISTTOOL_MODE_UPDATERUN, updaterunArgs);
@@ -110,6 +272,29 @@
     PXOPT_ADD_MODE("-pendingcomponent",       "", DISTTOOL_MODE_PENDINGCOMPONENT,    pendingcomponentArgs);
     PXOPT_ADD_MODE("-addprocessedcomponent",      "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs);
-    PXOPT_ADD_MODE("-processedcomponent",      "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs);
-    PXOPT_ADD_MODE("-toadvance",      "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs);
+    PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs);
+    PXOPT_ADD_MODE("-toadvance",          "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs);
+    PXOPT_ADD_MODE("-pendingfileset",     "", DISTTOOL_MODE_PENDINGFILESET, pendingfilesetArgs);
+    PXOPT_ADD_MODE("-addfileset",         "", DISTTOOL_MODE_ADDFILESET, addfilesetArgs);
+    PXOPT_ADD_MODE("-revertfileset",      "", DISTTOOL_MODE_REVERTFILESET, revertfilesetArgs);
+    PXOPT_ADD_MODE("-queuercrun",         "", DISTTOOL_MODE_QUEUERCRUN, queuercrunArgs);
+    PXOPT_ADD_MODE("-updatercrun",        "", DISTTOOL_MODE_UPDATERCRUN, updatercrunArgs);
+    PXOPT_ADD_MODE("-revertrcrun",        "", DISTTOOL_MODE_REVERTRCRUN, revertrcrunArgs);
+    PXOPT_ADD_MODE("-pendingdest",        "", DISTTOOL_MODE_PENDINGDEST, pendingdestArgs);
+
+    PXOPT_ADD_MODE("-definedsproduct",    "", DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductArgs);
+    PXOPT_ADD_MODE("-updatedsproduct",    "", DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductArgs);
+//  PXOPT_ADD_MODE("-listdsproduct",      "", DISTTOOL_MODE_LISTDSPRODUCT, updatedsproductArgs);
+
+    PXOPT_ADD_MODE("-definedestination",  "", DISTTOOL_MODE_DEFINEDESTINATION, definedestinationArgs);
+    PXOPT_ADD_MODE("-updatedestination",  "", DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationArgs);
+//  PXOPT_ADD_MODE("-listdestination",    "", DISTTOOL_MODE_LISTDESTINATION, listdestinationArgs);
+
+    PXOPT_ADD_MODE("-definetarget",       "", DISTTOOL_MODE_DEFINETARGET, definetargetArgs);
+    PXOPT_ADD_MODE("-updatetarget",       "", DISTTOOL_MODE_UPDATETARGET, updatetargetArgs);
+    PXOPT_ADD_MODE("-listtarget",         "", DISTTOOL_MODE_LISTTARGET, listtargetArgs);
+
+    PXOPT_ADD_MODE("-defineinterest",     "", DISTTOOL_MODE_DEFINEINTEREST, defineinterestArgs);
+    PXOPT_ADD_MODE("-updateinterest",     "", DISTTOOL_MODE_UPDATEINTEREST, updateinterestArgs);
+//  PXOPT_ADD_MODE("-listinterest",       "", DISTTOOL_MODE_LISTINTEREST, listinterestArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/faketool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/faketool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/faketool.c	(revision 24244)
@@ -128,5 +128,5 @@
     PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<=");
     PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
     PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "==");
     PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "==");
@@ -227,4 +227,25 @@
         psFree(output);
         return false;
+    }
+
+    // if end_stage is warp (or NULL), check for valid tess_id
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        bool status;
+        char *end_stage = psMetadataLookupStr(&status, md, "end_stage");
+	if (end_stage && strcasecmp(end_stage, "warp")) continue;
+
+        char *raw_tess_id   = psMetadataLookupStr(&status, md, "tess_id");
+	if (raw_tess_id || tess_id) continue;
+
+        char *label  = psMetadataLookupStr(&status, md, "label");
+        psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id");
+
+        if (!status) {
+	    psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id);
+            psFree(output);
+            return false;
+        }
     }
 
@@ -355,6 +376,6 @@
 
     psMetadata *where = psMetadataAlloc();
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
     PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
     PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "==");
     PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
@@ -421,5 +442,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "==");
@@ -496,5 +517,5 @@
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args,        "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args,        "-fault", false, false);
 
     if (!psDBTransaction(config->dbh)) {
@@ -513,5 +534,5 @@
                                    path_base,
                                    "full",
-                                   code,
+                                   fault,
                                    NULL         // epoch
             )) {
@@ -542,5 +563,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id", "rawImfile.class_id", "==");
@@ -687,5 +708,5 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
 
     psMetadata *where = psMetadataAlloc();
@@ -693,5 +714,5 @@
     PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
 
-    if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", where, code)) {
+    if (!pxSetFaultCode(config->dbh, "fakeProcessedImfile", where, fault)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
         psFree(where);
@@ -874,5 +895,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "fakeRun.label", "==");
 
     psString query = pxDataGet("faketool_pendingcleanuprun.sql");
@@ -939,5 +960,5 @@
         PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
     }
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
     psString query = pxDataGet("faketool_pendingcleanupimfile.sql");
@@ -1058,5 +1079,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -1238,5 +1259,5 @@
 
   int numExportTables = 2;
-  
+
   PS_ASSERT_PTR_NON_NULL(config, NULL);
 
@@ -1244,4 +1265,5 @@
   PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
   PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
 
   FILE *f = fopen (outfile, "w");
@@ -1292,7 +1314,17 @@
     }
     if (!psArrayLength(output)) {
-      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psError(PS_ERR_UNKNOWN, true, "no rows found");
       psFree(output);
-      return true;
+      return false;
+    }
+
+    if (clean) {
+        if (!strcmp(tables[i].tableName, "fakeRun")) {
+            if(!pxSetStateCleaned("fakeRun", "state", output)) {
+                psFree(output);
+                psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
+                return false;
+            }
+        }
     }
 
@@ -1316,5 +1348,5 @@
 
   PS_ASSERT_PTR_NON_NULL(config, NULL);
-  
+
   PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/faketoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/faketoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/faketoolConfig.c	(revision 24244)
@@ -54,5 +54,5 @@
     psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-dateobs_end", 0,            "search for exposures by time (<)", NULL);
     psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_tag",  0,            "search by exp_tag", NULL);
-    psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-label",  0,            "search by label", NULL);
+    psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-exp_type",  0,            "search by exp_type", "object");
     psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-filelevel",  0,            "search by filelevel", NULL);
@@ -148,5 +148,5 @@
     psMetadata *pendingexpArgs = psMetadataAlloc();
     psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-fake_id", 0,            "search by fake ID", 0);
-    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
+    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0,            "search by camtool ID", 0);
     psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-chip_id", 0,            "search by chiptool ID", 0);
@@ -157,5 +157,5 @@
     psMetadata *pendingimfileArgs = psMetadataAlloc();
     psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-fake_id",  0,            "search by fake ID", 0);
-    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
     psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
@@ -175,5 +175,5 @@
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-hostname",  0,            "define hostname", NULL);
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base",  0,            "define base output location", NULL);
-    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
 
     // -processedimfile
@@ -193,5 +193,5 @@
     psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
     psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
-    psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code (required)", 0);
+    psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code (required)", 0);
 
     // -revertprocessedimfile
@@ -239,5 +239,5 @@
 
     psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all",  0,            "allow everything to be queued without search terms", false);
-    psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
 
@@ -245,5 +245,5 @@
     psMetadata *advanceexpArgs = psMetadataAlloc();
     psMetadataAddS64(advanceexpArgs, PS_LIST_TAIL, "-fake_id", 0,      "search by fake ID", 0);
-    psMetadataAddStr(advanceexpArgs, PS_LIST_TAIL, "-label",  0,       "search by label ", NULL);
+    psMetadataAddStr(advanceexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label ", NULL);
     psMetadataAddU64(advanceexpArgs, PS_LIST_TAIL, "-limit",  0,       "limit exposures to advance to N items", 0);
 
@@ -270,5 +270,5 @@
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanuprunArgs,  PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanuprunArgs,  PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     psMetadataAddU64(pendingcleanuprunArgs,  PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -276,5 +276,5 @@
     // -pendingcleanupimfile
     psMetadata *pendingcleanupimfileArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-fake_id", 0,          "search by chip ID", 0);
     psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
@@ -306,4 +306,5 @@
     psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
     psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean",  0,          "mark tables as cleaned", false);
 
     // -importrun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/flatcorr.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/flatcorr.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/flatcorr.c	(revision 24244)
@@ -535,4 +535,7 @@
     PXOPT_LOOKUP_BOOL(limit,   config->args, "-limit",   false);
 
+    psMetadata *where = psMetadataAlloc();
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+
     psString query = pxDataGet("flatcorr_pendingprocess.sql");
     if (!query) {
@@ -540,4 +543,11 @@
 	return false;
     }
+
+    if (where && psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
 
     // treat limit == 0 as "no limit"
@@ -761,9 +771,9 @@
 
   psMetadata *where = psMetadataAlloc();
-  PXOPT_COPY_S64(config->args, where, "-coor_id", "corr_id", "==");
+  PXOPT_COPY_S64(config->args, where, "-corr_id", "corr_id", "==");
 
   ExportTable tables [] = {
-    {"flatcorrRun", "flatcorrtool_export_run.sql"},
-    {"flatcorrCamLink", "flatcorrtool_export_cam_link.sql"},
+    {"flatcorrRun", "flatcorr_export_run.sql"},
+    {"flatcorrCamLink", "flatcorr_export_cam_link.sql"},
     {"flatcorrChipLink", "flatcorr_export_chip_link.sql"},
   };
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/flatcorrConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/flatcorrConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/flatcorrConfig.c	(revision 24244)
@@ -101,4 +101,5 @@
     psMetadataAddU64 (pendingprocessArgs, PS_LIST_TAIL, "-limit",  0, "limit result set to N items", 0);
     psMetadataAddBool(pendingprocessArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddStr(pendingprocessArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by label", NULL);
 
     // -addprocess (XXX need to add fault and stats)
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magicdstool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magicdstool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magicdstool.c	(revision 24244)
@@ -398,5 +398,5 @@
     PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -599,10 +599,10 @@
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     PXOPT_LOOKUP_STR(backup_path_base, config->args, "-backup_path_base", false, false);
     PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false);
     PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false);
 
-    if (setmagicked && (code != 0)) {
+    if (setmagicked && (fault != 0)) {
         psError(PS_ERR_UNKNOWN, true, " cannot setmagicked for faulted file");
         return false;
@@ -625,5 +625,5 @@
     }
 
-    if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, code)) {
+    if (!magicDSFileInsert(config->dbh, magic_ds_id, component, backup_path_base, recovery_path_base, fault)) {
             // rollback
         if (!psDBRollback(config->dbh)) {
@@ -781,5 +781,5 @@
     PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
     PXOPT_COPY_STR(config->args, where, "-component", "component", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
 
     psString query = psStringCopy("DELETE FROM magicDSFile WHERE fault != 0");
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magicdstoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magicdstoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magicdstoolConfig.c	(revision 24244)
@@ -88,5 +88,5 @@
     psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
     psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
-    psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL);
+    psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
     psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
@@ -99,5 +99,5 @@
     psMetadataAddStr(adddestreakedfileArgs, PS_LIST_TAIL, "-recovery_path_base", 0, "define recovery pixels URI", NULL);
     psMetadataAddBool(adddestreakedfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false);
-    psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0);
+    psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
 
     // -revertdestreakedfile
@@ -105,5 +105,5 @@
     psMetadataAddS64(revertdestreakedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0);
     psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL);
-    psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0);
+    psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
 
     // -getskycells
@@ -125,5 +125,5 @@
     PXOPT_ADD_MODE("-updaterun",           "update state of magic de-streak run",
                     MAGICDSTOOL_MODE_UPDATERUN,         updaterunArgs);
-    PXOPT_ADD_MODE("-todestreak",          "show pending files",    
+    PXOPT_ADD_MODE("-todestreak",          "show pending files",
                     MAGICDSTOOL_MODE_TODESTREAK,       todestreakArgs);
     PXOPT_ADD_MODE("-adddestreakedfile",   "add a de-streaked file",
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magictool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magictool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magictool.c	(revision 24244)
@@ -127,9 +127,12 @@
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
-
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    psMetadata *diffWhere = psMetadataAlloc(); // WHERE conditions for diffRuns
+    PXOPT_COPY_STR(config->args, diffWhere, "-diff_label", "diffRun.label", "==");
+    PXOPT_COPY_S64(config->args, diffWhere, "-diff_id", "diff_id", "==");
+
+    psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else
+    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "==");
 
     // Get list of exposures ready to magic
@@ -146,36 +149,46 @@
         PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
 
-        psString queryWhere = NULL;     // WHERE conditions for entire query
+        psString diffWhereStr = NULL;   // WHERE conditions for diffRuns
+        psString queryWhereStr = NULL;  // WHERE conditions for entire query
         if (!available) {
-            psStringAppend(&queryWhere, " \nWHERE diffRun.state = 'full'");
-        } else {
-            // what if no skycells for the diff run completed?
-        }
+            psStringAppend(&diffWhereStr, "\nAND diffRun.state = 'full'");
+        }
+        // what if no skycells for the diff run completed?
+
         if (!rerun) {
-            const char *newWhere = " magic_id IS NULL"; // String to add
-            if (queryWhere) {
-                psStringAppend(&queryWhere, " AND %s", newWhere);
-            } else {
-                psStringAppend(&queryWhere, "\nWHERE %s", newWhere);
-            }
-        }
+            psStringAppend(&queryWhereStr, "\n%s magic_id IS NULL", queryWhereStr ? "AND" : "WHERE");
+        }
+
         // now add the user specified qualifiers
-        if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&queryWhere, "\n%s %s", queryWhere == NULL ? "WHERE" : "AND", whereClause);
+        if (psListLength(diffWhere->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(diffWhere, NULL);
+            psStringAppend(&diffWhereStr, "\nAND %s", whereClause);
             psFree(whereClause);
         }
-        psFree(where);
-        if (!queryWhere) {
-            psStringAppend(&queryWhere, " ");
-        }
-
-        if (!p_psDBRunQueryF(config->dbh, query, queryWhere)) {
+        if (psListLength(queryWhere->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(queryWhere, NULL);
+            psStringAppend(&queryWhereStr, "\n%s %s", queryWhereStr ? "AND" : "WHERE", whereClause);
+            psFree(whereClause);
+        }
+        psFree(diffWhere);
+        psFree(queryWhere);
+
+        // Ensure the WHERE strings have something
+        if (!diffWhereStr) {
+            diffWhereStr = psStringCopy("");
+        }
+        if (!queryWhereStr) {
+            queryWhereStr = psStringCopy("");
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, query, diffWhereStr, diffWhereStr, queryWhereStr)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(queryWhere);
+            psFree(diffWhereStr);
+            psFree(queryWhereStr);
             psFree(query);
             return false;
         }
-        psFree(queryWhere);
+        psFree(diffWhereStr);
+        psFree(queryWhereStr);
         psFree(query);
     }
@@ -201,6 +214,18 @@
     }
 
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+
     // Parse the list of exposures ready to magic
-
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -215,7 +240,9 @@
         psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
         psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier
+        bool inverse = psMetadataLookupU64(NULL, row, "inverse"); // Inverse subtraction? Note types!
 
         // create a new magicRun for this group
-        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "new", workdir, "dirty", label, dvodb, registered, 0);
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label,
+                                            dvodb, registered, 0);
         if (!run) {
             psAbort("failed to alloc magicRun object");
@@ -296,4 +323,5 @@
 
     // optional
+    PXOPT_LOOKUP_BOOL(inverse, config->args, "-inverse", false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
@@ -305,4 +333,5 @@
             exp_id,
             diff_id ? diff_id : PS_MAX_S64,
+            inverse,
             "reg",      // state
             workdir,
@@ -363,5 +392,4 @@
     // required
     PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
     PXOPT_LOOKUP_STR(node, config->args, "-node", true, false);
 
@@ -369,5 +397,4 @@
             config->dbh,
             magic_id,
-            diff_id,
             node
     );
@@ -457,5 +484,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -535,9 +562,9 @@
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-
-    if (code > 0) {
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+
+    if (fault > 0) {
         char *query = "UPDATE magicRun SET fault = %d, state = 'full' WHERE magic_id = %" PRId64;
-        if (!p_psDBRunQueryF(config->dbh, query, code, magic_id)) {
+        if (!p_psDBRunQueryF(config->dbh, query, fault, magic_id)) {
             psError(PS_ERR_UNKNOWN, false,
                     "failed to set fault for magic_id %" PRId64, magic_id);
@@ -561,7 +588,7 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "fault", "==");
-
-    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'run' WHERE fault != 0");
+    PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
+
+    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'new' WHERE fault != 0");
 
     if (psListLength(where->list)) {
@@ -584,7 +611,13 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-node", "node", "==");
+    // Regrettably, there are multiple WHERE hooks which call the same things different names
+    psMetadata *templatesWhere = psMetadataAlloc(); // WHERE for selecting template
+    psMetadata *magicWhere = psMetadataAlloc();     // WHERE for selecting magic runs
+
+    PXOPT_COPY_S64(config->args, templatesWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, templatesWhere, "-node", "diffInputSkyfile.skycell_id", "==");
+
+    PXOPT_COPY_S64(config->args, magicWhere, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_STR(config->args, magicWhere, "-node", "magicTree.node", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -597,10 +630,20 @@
     }
 
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+    psString templatesWhereStr = psStringCopy(""); // WHERE for selecting template
+    psString magicWhereStr = psStringCopy("");     // WHERE for selecting magic runs
+
+    if (psListLength(templatesWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(templatesWhere, NULL);
+        psStringAppend(&templatesWhereStr, "\nAND %s", whereClause);
         psFree(whereClause);
     }
-    psFree(where);
+    psFree(templatesWhere);
+
+    if (psListLength(magicWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(magicWhere, NULL);
+        psStringAppend(&magicWhereStr, "\nWHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(magicWhere);
 
     // treat limit == 0 as "no limit"
@@ -611,9 +654,15 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+    if (!p_psDBRunQueryF(config->dbh, query,
+                         templatesWhereStr, templatesWhereStr,
+                         magicWhereStr, magicWhereStr)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(templatesWhereStr);
+        psFree(magicWhereStr);
         psFree(query);
         return false;
     }
+    psFree(templatesWhereStr);
+    psFree(magicWhereStr);
     psFree(query);
 
@@ -704,9 +753,9 @@
 
             bool status = false;
-            psS32 done = psMetadataLookupS32(&status, data, "done");
+            psS64 done = psMetadataLookupS64(&status, data, "done");
             if (!status) {
                 psAbort("failed to lookup value for done column");
             }
-            psS16 bad = psMetadataLookupS16(&status, data, "bad");
+            psS64 bad = psMetadataLookupS64(&status, data, "bad");
             if (!status) {
                 psAbort("failed to lookup value for bad column");
@@ -738,8 +787,9 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
 
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 
     // look for "inputs" that need to processed
@@ -750,13 +800,22 @@
     }
 
-    psString whereClause = NULL;
+    psString whereString = NULL;
     if (psListLength(where->list)) {
-        whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&whereString, "\nAND %s", whereClause);
+        psFree(whereClause);
     }
     psFree(where);
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, whereString, whereString)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(whereString);
         psFree(query);
         return false;
@@ -782,5 +841,21 @@
     }
 
+    if (limit) {
+        if (psArrayLength(output) >= limit) {
+            // we've found enough (note that limit was applied to the query so '> limit' won't happen)
+            // negative simple so the default is true
+            if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to print array");
+                psFree(output);
+                return false;
+            }
+            psFree(output);
+            return true;
+        }
+    }
+
     // look for tree nodes that need to be processed
+    // XXX: This gets all nodes from all magicRuns that are in 'new'state
+    // That doens't seem particularly efficient
     query = pxDataGet("magictool_toprocess_tree.sql");
     if (!query) {
@@ -789,17 +864,12 @@
     }
 
-    if (whereClause) {
-        psString new  = NULL;
-        psStringAppend(&new, "\n AND %s", whereClause);
-        psFree(whereClause);
-        whereClause = new;
-    }
-    if (!p_psDBRunQueryF(config->dbh, query, whereClause ? whereClause :  "")) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(whereClause);
+
+    if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString :  "")) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(whereString);
         psFree(query);
         return false;
     }
-    psFree(whereClause);
+    psFree(whereString);
     psFree(query);
 
@@ -828,4 +898,7 @@
     while (index <  psArrayLength(magicTree)) {
         bool status;
+        if (limit && (psArrayLength(output) >= limit)) {
+            break;
+        }
         psS64 current_magic_id = psMetadataLookupS64(&status, magicTree->data[index], "magic_id");
         if (!status) {
@@ -879,6 +952,14 @@
 
     }
+    psFree(magicTree);
 
     if (psArrayLength(output)) {
+        if (limit && (limit < psArrayLength(output))) {
+            // truncate the array
+            long arrayLength = psArrayLength(output);
+            for (long i = arrayLength - 1; i >= limit; i--) {
+                psArrayRemoveIndex(output, i);
+            }
+        }
         // negative simple so the default is true
         if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
@@ -889,5 +970,4 @@
     }
 
-    psFree(magicTree);
     psFree(output);
 
@@ -905,14 +985,14 @@
 
     // optional
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
 
     if (!magicNodeResultInsert(config->dbh,
                                magic_id,
                                node,
-                               uri,
-                               code
+                               path_base,
+                               fault
         )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -930,7 +1010,12 @@
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     PXOPT_COPY_STR(config->args, where, "-node", "node", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "fault", "==");
-
-    psString query = psStringCopy("DELETE FROM magicNodeResult WHERE fault != 0");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "magicNodeResult.fault", "==");
+
+    psString query = pxDataGet("magictool_revertnode.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
 
     if (psListLength(where->list)) {
@@ -938,4 +1023,8 @@
         psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
+    } else {
+        psError(PS_ERR_UNKNOWN, false, "search parameters are required");
+        psFree(where);
+        return false;
     }
     psFree(where);
@@ -945,4 +1034,8 @@
         return false;
     }
+
+    psS32 numUpdated = psDBAffectedRows(config->dbh);
+    psLogMsg("magictool", PS_LOG_INFO, "Updated %d magic nodes", numUpdated);
+
     return true;
 }
@@ -1024,5 +1117,5 @@
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
 
     if (!psDBTransaction(config->dbh)) {
@@ -1035,5 +1128,5 @@
                          uri,
                          streaks,
-                         code
+                         fault
         )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1081,5 +1174,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
 
     if (!psDBTransaction(config->dbh)) {
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magictoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magictoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/magictoolConfig.c	(revision 24244)
@@ -53,4 +53,5 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb",       0, "define dvodb", NULL);
     psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id", 0, "search diff_id", 0);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL);
@@ -58,4 +59,5 @@
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "generate new run even if existing?", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "list results but do not queue", false);
 
     // -definerun
@@ -65,4 +67,5 @@
     // following argument is for compatability
     psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff_id", 0);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-inverse", 0, "use the inverse subtraction?", false);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL);
@@ -78,5 +81,4 @@
     psMetadata *addinputskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
-    psMetadataAddS64(addinputskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", 0);
     psMetadataAddStr(addinputskyfileArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL);
 
@@ -93,5 +95,5 @@
     psMetadataAddS64(totreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
     psMetadataAddU64(totreeArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
-    psMetadataAddStr(totreeArgs, PS_LIST_TAIL, "-label",       0, "define label", NULL);
+    psMetadataAddStr(totreeArgs, PS_LIST_TAIL, "-label",    PS_META_DUPLICATE_OK, "define label", NULL);
     psMetadataAddBool(totreeArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
@@ -100,10 +102,10 @@
     psMetadataAddS64(inputtreeArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
     psMetadataAddStr(inputtreeArgs, PS_LIST_TAIL, "-dep_file", 0, "order of operations dep. file", NULL);
-    psMetadataAddS16(inputtreeArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0);
+    psMetadataAddS16(inputtreeArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
 
     // -reverttree
     psMetadata *reverttreeArgs = psMetadataAlloc();
     psMetadataAddS64(reverttreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0);
-    psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0);
+    psMetadataAddS16(reverttreeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
 
     // -inputs
@@ -118,5 +120,5 @@
     psMetadataAddS64(toprocessArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
     psMetadataAddU64(toprocessArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
-    psMetadataAddStr(toprocessArgs, PS_LIST_TAIL, "-label",       0, "define label", NULL);
+    psMetadataAddStr(toprocessArgs, PS_LIST_TAIL, "-label",       PS_META_DUPLICATE_OK, "define label", NULL);
     psMetadataAddBool(toprocessArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
@@ -125,6 +127,6 @@
     psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
     psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL);
-    psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL);
-    psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0);
+    psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base", NULL);
+    psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
 
     // -revertnode
@@ -132,5 +134,6 @@
     psMetadataAddS64(revertnodeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0);
     psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-node", 0, "search by node name", NULL);
-    psMetadataAddS16(revertnodeArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0);
+    psMetadataAddStr(revertnodeArgs, PS_LIST_TAIL, "-label",0, "search by label", NULL);
+    psMetadataAddS16(revertnodeArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
 
     // -tomask
@@ -144,10 +147,10 @@
     psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL);
     psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks", 0, "define number of streaks", 0);
-    psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0);
+    psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
 
     // -revertmask
     psMetadata *revertmaskArgs = psMetadataAlloc();
     psMetadataAddS64(revertmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", 0);
-    psMetadataAddS16(revertmaskArgs, PS_LIST_TAIL, "-code", 0, "search by fault code", 0);
+    psMetadataAddS16(revertmaskArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
 
     // -mask
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtools.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtools.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtools.c	(revision 24244)
@@ -29,5 +29,5 @@
 {
     PS_ASSERT_PTR_NON_NULL(state, false);
-    
+
     if (!strcmp(state, "new")) return true;
     if (!strcmp(state, "reg")) return true;
@@ -50,2 +50,59 @@
 // 'scrubbed' is a virtual state equivalent to cleaned, but allows files to be removed
 // even if the config files is missing
+
+
+// change the value for tableName.columName from 'full' to 'cleaned' if necessary
+bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows)
+{
+    for (long i = 0; i < psArrayLength(rows); i++) {
+        psMetadata *row = rows->data[i];
+        psString state = psMetadataLookupStr(NULL, row, columnName);
+        if (!state) {
+            psError(PS_ERR_PROGRAMMING, false, "%s not found in row %ld of table %s",
+		    columnName, i, tableName);
+            return false;
+        }
+        if (!strcmp("full", state)) {
+            // change full to cleaned
+            psMetadataAddStr(row, PS_LIST_TAIL, columnName, PS_META_REPLACE, "", "cleaned");
+        } else if (strcmp("cleaned", state)) {
+            // if state isn't cleaned or full we can't set it to cleaned
+            psError(PS_ERR_PROGRAMMING, true, "%s with state %s may not be exported cleaned",
+		    tableName, state);
+            return false;
+        }
+    }
+    return true;
+}
+
+// XXX verify data type?
+bool pxAddLabelSearchArgs (pxConfig *config, psMetadata *where, char *name, char *field, char *op) {
+
+    psMetadataItem *item = psMetadataLookup(config->args, name);
+    if (!item) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", name);
+        return false;
+    }
+    psAssert (item->type == PS_DATA_METADATA_MULTI, "%s should be a multi container", name);
+    psAssert (item->data.list->n, "%s should at least have a place-holder", name);
+    psMetadataItem *entry = (psMetadataItem *)item->data.list->head->data;
+    psAssert (entry, "%s should at least have a place-holder", name);
+    if (entry->data.str) {
+	psListIterator *iter = psListIteratorAlloc (item->data.list, PS_LIST_HEAD, true);
+	psMetadataItem *item = NULL;
+	while ((item = psListGetAndIncrement(iter))) {
+	    // need to change the name and comment
+	    psFree (item->name);
+	    item->name = psStringCopy (field);
+	    psFree (item->comment);
+	    item->comment = psStringCopy (op);
+	    if (!psMetadataAddItem(where, item, PS_LIST_TAIL, PS_META_DUPLICATE_OK)) {
+		psError(PS_ERR_UNKNOWN, false, "failed to add item %s", field);
+		psFree(where);
+		return false;
+	    }
+	}
+	psFree (iter);
+    }
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtools.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtools.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtools.h	(revision 24244)
@@ -50,4 +50,6 @@
 
 bool pxIsValidState(const char *state);
+bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows);
+bool pxAddLabelSearchArgs (pxConfig *config, psMetadata *where, char *field, char *name, char *op);
 
 bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code);
@@ -478,3 +480,85 @@
 }
 
+
+/// Add a primary item to the mirror database
+///
+/// ITEM: Item to add (if it is of the correct type)
+/// NAME: Name for table, to match item name
+/// ROWTYPE: Type of the row (type returned by PARSEFUNC)
+/// PARSEFUNC: Function to parse ITEM and return a ROWTYPE
+/// IDENTIFIERS: Vector (of type U64) to store identifiers
+/// IDNAME: Element of the row with U64 identifier
+/// INSERTFUNC: Function to insert a row into the database
+/// DATABASE: Database handle
+/// CLEANUP: Operation(s) to perform to cleanup in case of an error
+#define PXMIRROR_PRIMARY(ITEM, NAME, ROWTYPE, PARSEFUNC, IDENTIFIERS, IDNAME, INSERTFUNC, DATABASE, CLEANUP) \
+    if (strcmp((ITEM)->name, NAME) == 0) { \
+        if ((ITEM)->type != PS_DATA_METADATA) { \
+            psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Entry %s is not of type METADATA", (ITEM)->name); \
+            CLEANUP; \
+            return false; \
+        } \
+        ROWTYPE *row = PARSEFUNC((ITEM)->data.md); /* Row to insert */ \
+        if (!row) { \
+            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate %s row from metadata", (ITEM)->name); \
+            CLEANUP; \
+            return false; \
+        } \
+        if (!INSERTFUNC(DATABASE, row)) { \
+            psError(PS_ERR_UNKNOWN, false, "Unable to add %s %" PRIu64, (ITEM)->name, row->IDNAME); \
+            psFree(row); \
+            CLEANUP; \
+            return false; \
+        } \
+        psVectorAppend((IDENTIFIERS), row->IDNAME); \
+        psFree(row); \
+    }
+
+/// Add a dependent item to the mirror database
+///
+/// ITEM: Item to add (if it is of the correct type)
+/// NAME: Name for table, to match item name
+/// ROWTYPE: Type of the row (type returned by PARSEFUNC)
+/// PARSEFUNC: Function to parse ITEM and return a ROWTYPE
+/// IDENTIFIERS: Vector (of type U64) to check identifiers
+/// IDNAME: Element of the row with U64 identifier
+/// INSERTFUNC: Function to insert a row into the database
+/// DATABASE: Database handle
+/// CLEANUP: Operation(s) to perform to cleanup in case of an error
+#define PXMIRROR_OTHER(ITEM, NAME, ROWTYPE, PARSEFUNC, IDENTIFIERS, IDNAME, INSERTFUNC, DATABASE, CLEANUP) \
+    if (strcmp((ITEM)->name, NAME) == 0) { \
+        if ((ITEM)->type != PS_DATA_METADATA) { \
+            psError(PS_ERR_BAD_PARAMETER_TYPE, true, "Entry %s is not of type METADATA", (ITEM)->name); \
+            CLEANUP; \
+            return false; \
+        } \
+        ROWTYPE *row = PARSEFUNC(item->data.md); /* Row to insert */ \
+        if (!row) { \
+            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate %s row from metadata", (ITEM)->name); \
+            CLEANUP; \
+            return false; \
+        } \
+        bool found = false;         /* Found the identifier? */ \
+        for (int i = 0; i < (IDENTIFIERS)->n && !found; i++) { \
+            if (row->IDNAME == (IDENTIFIERS)->data.U64[i]) { \
+                found = true; \
+                break; \
+            } \
+        } \
+        if (!found) { \
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Identifier not found for %s %" PRIu64, \
+                    (ITEM)->name, row->IDNAME); \
+            psFree(row); \
+            CLEANUP; \
+            return false; \
+        } \
+        if (!INSERTFUNC(DATABASE, row)) { \
+            psError(PS_ERR_UNKNOWN, false, "Unable to add %s %" PRIu64, (ITEM)->name, row->IDNAME); \
+            psFree(row); \
+            CLEANUP; \
+            return false; \
+        } \
+        psFree(row); \
+    }
+
 #endif // PXTOOLS_H
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtoolsErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtoolsErrorCodes.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtoolsErrorCodes.dat	(revision 24244)
@@ -2,6 +2,6 @@
 # This file is used to generate pxtoolsErrorCodes.h
 #
-BASE = 1100		First value we use; lower values belong to psLib
-UNKNOWN			Unknown PM error code
+BASE = 9000		First value we use; lower values belong to psLib
+UNKNOWN			Unknown ippTools error code
 ARGUMENTS		Incorrect arguments
 SYS			System error
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtoolsErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtoolsErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtoolsErrorCodes.h.in	(revision 24244)
@@ -9,5 +9,5 @@
  */
 typedef enum {
-    PXTOOLS_ERR_BASE = 512,
+    PXTOOLS_ERR_BASE = 9000,
     PXTOOLS_ERR_${ErrorCode},
     PXTOOLS_ERR_NERROR
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxwarp.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxwarp.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxwarp.c	(revision 24244)
@@ -28,92 +28,4 @@
 #include "pxtools.h"
 #include "pxwarp.h"
-
-bool pxwarpSetSearchArgs (psMetadata *md) {
-    psMetadataAddS64(md, PS_LIST_TAIL, "-fake_id",            0, "search by fake_id", 0);
-    psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id",             0, "search by cam_id", 0);
-    psMetadataAddS64(md, PS_LIST_TAIL, "-chip_id",            0, "search by chip_id", 0);
-    psMetadataAddS64(md, PS_LIST_TAIL, "-exp_id",             0, "search by exp_id", 0);
-    psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name",           0, "search by exp_name", NULL);
-    psMetadataAddStr(md, PS_LIST_TAIL, "-inst",               0, "search for camera", NULL);
-    psMetadataAddStr(md, PS_LIST_TAIL, "-telescope",          0, "search for telescope", NULL);
-    psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin",     0, "search for exposures by time (>=)", NULL);
-    psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end",       0, "search for exposures by time (<)", NULL);
-    psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag",            0, "search by exp_tag", NULL);
-    psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type",           0, "search by exp_type", NULL);
-    psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel",          0, "search by filelevel", NULL);
-    psMetadataAddStr(md, PS_LIST_TAIL, "-filter",             0, "search for filter", NULL);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min",        0, "search by min airmass", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max",        0, "search by max airmass", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min",             0, "search by min", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max",             0, "search by max", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min",           0, "search by min", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max",           0, "search by max", NAN);
-    psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min",       0, "search by min", NAN);
-    psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max",       0, "search by max", NAN);
-    psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN);
-    psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min",             0, "search by max", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max",             0, "search by max", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min",       0, "search by max", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max",       0, "search by max", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min",  0, "search by max", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max",  0, "search by max", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min",            0, "search by min", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max",            0, "search by max", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-az_min",             0, "search by min", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-az_max",             0, "search by max", NAN);
-    psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min",       0, "search by min ccd tempature", NAN);
-    psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max",       0, "search by max ccd tempature", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min",         0, "search by min rotator position angle", NAN);
-    psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max",         0, "search by max rotator position angle", NAN);
-    psMetadataAddStr(md, PS_LIST_TAIL, "-object",             0, "search by exposure object", NULL);
-    psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min",         0, "search by min solar angle", NAN);
-    psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max",         0, "search by max solar angle", NAN);
-    return true;
-}
-
-bool pxwarpGetSearchArgs (pxConfig *config, psMetadata *where) {
-    PXOPT_COPY_S64(config->args, where, "-fake_id",            "fakeRun.fake_id",       "==");
-    PXOPT_COPY_S64(config->args, where, "-cam_id",             "camRun.cam_id",         "==");
-    PXOPT_COPY_S64(config->args, where, "-chip_id",            "chipRun.chip_id",       "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id",             "newExp.exp_id",         "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name",           "rawExp.exp_name",       "==");
-    PXOPT_COPY_STR(config->args, where, "-inst",               "rawExp.camera",         "==");
-    PXOPT_COPY_STR(config->args, where, "-telescope",          "rawExp.telescope",      "==");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin",     "rawExp.dateobs",        ">=");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",       "rawExp.dateobs",        "<=");
-    PXOPT_COPY_STR(config->args, where, "-exp_tag",            "rawExp.exp_tag",        "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_type",           "rawExp.exp_type",       "==");
-    PXOPT_COPY_STR(config->args, where, "-filelevel",          "rawExp.filelevel",      "==");
-    PXOPT_COPY_STR(config->args, where, "-filter",             "rawExp.filter",         "==");
-    PXOPT_COPY_F64(config->args, where, "-airmass_min",        "rawExp.airmass",        ">=");
-    PXOPT_COPY_F64(config->args, where, "-airmass_max",        "rawExp.airmass",        "<");
-    PXOPT_COPY_F64(config->args, where, "-ra_min",             "rawExp.ra",             ">=");
-    PXOPT_COPY_F64(config->args, where, "-ra_max",             "rawExp.ra",             "<");
-    PXOPT_COPY_F64(config->args, where, "-decl_min",           "rawExp.decl",           ">=");
-    PXOPT_COPY_F64(config->args, where, "-decl_max",           "rawExp.decl",           "<");
-    PXOPT_COPY_F32(config->args, where, "-exp_time_min",       "rawExp.exp_time",       ">=");
-    PXOPT_COPY_F32(config->args, where, "-exp_time_max",       "rawExp.exp_time",       "<");
-    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">=");
-    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<");
-    PXOPT_COPY_F64(config->args, where, "-bg_min",             "rawExp.bg",             ">=");
-    PXOPT_COPY_F64(config->args, where, "-bg_max",             "rawExp.bg",             "<");
-    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min",       "rawExp.bg_stdev",       ">=");
-    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max",       "rawExp.bg_stdev",       "<");
-    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min",  "rawExp.bg_mean_stdev",  ">=");
-    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max",  "rawExp.bg_mean_stdev",  "<");
-    PXOPT_COPY_F64(config->args, where, "-alt_min",            "rawExp.alt",            ">=");
-    PXOPT_COPY_F64(config->args, where, "-alt_max",            "rawExp.alt",            "<");
-    PXOPT_COPY_F64(config->args, where, "-az_min",             "rawExp.az",             ">=");
-    PXOPT_COPY_F64(config->args, where, "-az_max",             "rawExp.az",             "<");
-    PXOPT_COPY_F32(config->args, where, "-ccd_temp_min",       "rawExp.ccd_temp",       ">=");
-    PXOPT_COPY_F32(config->args, where, "-ccd_temp_max",       "rawExp.ccd_temp",       "<");
-    PXOPT_COPY_F64(config->args, where, "-posang_min",         "rawExp.posang",         ">=");
-    PXOPT_COPY_F64(config->args, where, "-posang_max",         "rawExp.posang",         "<");
-    PXOPT_COPY_STR(config->args, where, "-object",             "rawExp.object",         "==");
-    PXOPT_COPY_F32(config->args, where, "-solang_min",         "rawExp.solang",         ">=");
-    PXOPT_COPY_F32(config->args, where, "-solang_max",         "rawExp.solang",         "<");
-    return true;
-}
 
 bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state)
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxwarp.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxwarp.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxwarp.h	(revision 24244)
@@ -25,7 +25,4 @@
 #include "pxtools.h"
 
-bool pxwarpGetSearchArgs (pxConfig *config, psMetadata *where);
-bool pxwarpSetSearchArgs (psMetadata *md);
-
 bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state);
 bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state);
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pztool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pztool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pztool.c	(revision 24244)
@@ -338,5 +338,5 @@
         // a lot simplier than a complicated scheme (tried that) to attempt to
         // request on the right number of rows for each camera
-        
+
         // treat limit == 0 as "no limit"
         if (limit) {
@@ -403,5 +403,5 @@
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
 
     // start a transaction early so it will contain any row level locks
@@ -410,5 +410,5 @@
         return false;
     }
-    
+
     // query to get an excluse lock on this exposure in
     // pzDownloadExp
@@ -421,5 +421,5 @@
         PXOPT_COPY_STR(config->args, where,  "-inst", "camera", "==");
         PXOPT_COPY_STR(config->args, where,  "-telescope", "telescope", "==");
-        
+
         if (psListLength(where->list)) {
             psString whereClause = psDBGenerateWhereSQL(where, NULL);
@@ -455,5 +455,5 @@
             class_id,
             uri,
-            code,
+            fault,
             NULL,    // epoch
             hostname
@@ -605,5 +605,5 @@
             psError(PS_ERR_UNKNOWN, false, "database error");
             return false;
-        } 
+        }
 
         // sanity check: we should have inserted at least one row
@@ -675,5 +675,5 @@
             // init
             long counter = 0,   // the total number of elements zipped so far
-            i = 0,              // which array in the set 
+            i = 0,              // which array in the set
             index = 0;          // the depth into each array
             // test
@@ -782,10 +782,10 @@
     PXOPT_COPY_STR(config->args, where,  "-class_id", "class_id", "==");
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);
-
-    if (!pxSetFaultCode(config->dbh, "pzDownloadImfile", where, code)) {
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
+
+    if (!pxSetFaultCode(config->dbh, "pzDownloadImfile", where, fault)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
-	psFree (where);
-	return false;
+        psFree (where);
+        return false;
     }
     psFree(where);
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pztoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pztoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/pztoolConfig.c	(revision 24244)
@@ -45,8 +45,8 @@
     // -adddatastore
     psMetadata *adddatastoreArgs = psMetadataAlloc();
-    psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL); 
-    psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL); 
+    psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL);
+    psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL);
     psMetadataAddStr(adddatastoreArgs, PS_LIST_TAIL, "-uri", 0,            "define storage uri", NULL);
-    
+
     // -datastore
     psMetadata *datastoreArgs = psMetadataAlloc();
@@ -55,16 +55,16 @@
     // -seen
     psMetadata *seenArgs = psMetadataAlloc();
-    psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL); 
-    psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL); 
-    psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL); 
-    psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_type", 0,            "define exposure type", NULL); 
+    psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL);
+    psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL);
+    psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL);
+    psMetadataAddStr(seenArgs, PS_LIST_TAIL, "-exp_type", 0,            "define exposure type", NULL);
     psMetadataAddBool(seenArgs, PS_LIST_TAIL, "-simple", 0,            "use the simple output format", false);
-    
+
     // -pendingexp
     psMetadata *pendingexpArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL); 
-    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL); 
-    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL); 
-    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_type", 0,            "define exposure type", NULL); 
+    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL);
+    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL);
+    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL);
+    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-exp_type", 0,            "define exposure type", NULL);
     psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-desc", 0,            "sort ouput in descending format", false);
     psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -73,8 +73,8 @@
     // -pendingimfile
     psMetadata *pendingimfileArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL); 
-    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL); 
-    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL); 
-    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_type", 0,            "define exposure type", NULL); 
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-exp_type", 0,            "define exposure type", NULL);
     psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-desc", 0,            "sort ouput in descending format", false);
     psMetadataAddU64(pendingimfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -83,7 +83,7 @@
     // -copydone
     psMetadata *copydoneArgs = psMetadataAlloc();
-    psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID (required)", NULL); 
-    psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID (required)", NULL); 
-    psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID (required)", NULL); 
+    psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID (required)", NULL);
+    psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID (required)", NULL);
+    psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID (required)", NULL);
     psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-class", 0,            "define class", NULL);
     psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-class_id", 0,            "define class_id", NULL);
@@ -95,5 +95,5 @@
     psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-label",  0,        "define the label for the chip stage", NULL);
     psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-hostname",  0,     "define the host that copied the image", NULL);
-    psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0,     "lock pzDownImfile rows while advancing an exposure", false);
     // XXX: remove this once advance is fixed
@@ -102,7 +102,7 @@
     // -copied
     psMetadata *copiedArgs = psMetadataAlloc();
-    psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL); 
-    psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL); 
-    psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL); 
+    psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL);
+    psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL);
+    psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL);
     psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-class", 0,            "define class", NULL);
     psMetadataAddStr(copiedArgs, PS_LIST_TAIL, "-class_id", 0,            "define class_id", NULL);
@@ -113,19 +113,19 @@
     // -updatecopied
     psMetadata *updatecopiedArgs = psMetadataAlloc();
-    psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID (required)", NULL); 
-    psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID (required)", NULL); 
-    psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID (required)", NULL); 
+    psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID (required)", NULL);
+    psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID (required)", NULL);
+    psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID (required)", NULL);
     psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-class", 0,            "define class", NULL);
     psMetadataAddStr(updatecopiedArgs, PS_LIST_TAIL, "-class_id", 0,            "define class_id", NULL);
-    psMetadataAddS16(updatecopiedArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(updatecopiedArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -revertcopied
     psMetadata *revertcopiedArgs = psMetadataAlloc();
-    psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID (required)", NULL); 
-    psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID (required)", NULL); 
-    psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID (required)", NULL); 
+    psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID (required)", NULL);
+    psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID (required)", NULL);
+    psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID (required)", NULL);
     psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-class", 0,            "define class", NULL);
     psMetadataAddStr(revertcopiedArgs, PS_LIST_TAIL, "-class_id", 0,            "define class_id", NULL);
-    psMetadataAddS16(revertcopiedArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertcopiedArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -clearcommonfaults
@@ -134,7 +134,7 @@
     // -toadvance
     psMetadata *toadvanceArgs = psMetadataAlloc();
-    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-exp_name", 0,      "define exposure ID", NULL); 
-    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-inst", 0,          "define camera ID", NULL); 
-    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-telescope", 0,     "define telescope ID", NULL); 
+    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-exp_name", 0,      "define exposure ID", NULL);
+    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-inst", 0,          "define camera ID", NULL);
+    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-telescope", 0,     "define telescope ID", NULL);
     psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label",  0,        "define the label for the chip stage", NULL);
     psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple",  0,      "use the simple output format", false);
@@ -143,7 +143,7 @@
     // -advance
     psMetadata *advanceArgs = psMetadataAlloc();
-    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-exp_name", 0,   "define exposure ID (required)", NULL); 
-    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-inst", 0,       "define camera ID (required)", NULL); 
-    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-telescope", 0,  "define telescope ID (required)", NULL); 
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-exp_name", 0,   "define exposure ID (required)", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-inst", 0,       "define camera ID (required)", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-telescope", 0,  "define telescope ID (required)", NULL);
     psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-workdir",  0,   "define the \"default\" workdir for this exposure (required)", NULL);
     psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-dvodb",  0,     "define the dvodb for the next processing step", NULL);
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/receivetool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/receivetool.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/receivetool.c	(revision 24244)
@@ -0,0 +1,635 @@
+/*
+ * receivetool.c
+ *
+ * Copyright (C) 2008
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * program; see the file COPYING. If not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <inttypes.h>
+
+#include "pxtools.h"
+#include "pxdata.h"
+#include "receivetool.h"
+
+static bool definesourceMode(pxConfig *config);
+static bool listMode(pxConfig *config);
+static bool addfilesetMode(pxConfig *config);
+static bool updatelastMode(pxConfig *config);
+static bool pendingfilesetMode(pxConfig *config);
+static bool updatefilesetMode(pxConfig *config);
+static bool addfileMode(pxConfig *config);
+static bool pendingfileMode(pxConfig *config);
+static bool addresultMode(pxConfig *config);
+static bool toadvanceMode(pxConfig *config);
+static bool revertMode(pxConfig *config);
+
+# define MODECASE(caseName, func) \
+    case caseName: \
+    if (!func(config)) { \
+        goto FAIL; \
+    } \
+    break;
+
+
+int main(int argc, char **argv)
+{
+    psLibInit(NULL);
+
+    pxConfig *config = receivetoolConfig(NULL, argc, argv);
+    if (!config) {
+        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
+        goto FAIL;
+    }
+
+    switch (config->mode) {
+        MODECASE(RECEIVETOOL_MODE_DEFINESOURCE, definesourceMode);
+        MODECASE(RECEIVETOOL_MODE_LIST, listMode);
+        MODECASE(RECEIVETOOL_MODE_ADDFILESET, addfilesetMode);
+        MODECASE(RECEIVETOOL_MODE_UPDATELAST, updatelastMode);
+        MODECASE(RECEIVETOOL_MODE_PENDINGFILESET, pendingfilesetMode);
+        MODECASE(RECEIVETOOL_MODE_UPDATEFILESET, updatefilesetMode);
+        MODECASE(RECEIVETOOL_MODE_TOADVANCE, toadvanceMode);
+        MODECASE(RECEIVETOOL_MODE_ADDFILE, addfileMode);
+        MODECASE(RECEIVETOOL_MODE_PENDINGFILE, pendingfileMode);
+        MODECASE(RECEIVETOOL_MODE_ADDRESULT, addresultMode);
+        MODECASE(RECEIVETOOL_MODE_REVERT, revertMode);
+      default:
+        psAbort("invalid option (this should not happen)");
+    }
+
+    psFree(config);
+    pmConfigDone();
+    psLibFinalize();
+
+    exit(EXIT_SUCCESS);
+
+FAIL:
+    psErrorStackPrint(stderr, "\n");
+    int exit_status = pxerrorGetExitStatus();
+
+    psFree(config);
+    pmConfigDone();
+    psLibFinalize();
+
+    exit(exit_status);
+}
+
+static bool definesourceMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_STR(source, config->args, "-source", true, false);
+    PXOPT_LOOKUP_STR(product, config->args, "-product",  true, false);
+    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
+
+    // optional
+    PXOPT_LOOKUP_STR(comment, config->args, "-comment",  false, false);
+    PXOPT_LOOKUP_STR(last, config->args, "-last",  false, false);
+    PXOPT_LOOKUP_STR(status_product, config->args, "-status_product",  false, false);
+    PXOPT_LOOKUP_STR(ds_dbname, config->args, "-ds_dbname",  false, false);
+    PXOPT_LOOKUP_STR(ds_dbhost, config->args, "-ds_dbhost",  false, false);
+
+    if (!receiveSourceInsert(config->dbh, 0, source, product, workdir, comment, last, status_product, ds_dbname, ds_dbhost)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool listMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc(); // WHERE conditions
+
+    // required
+    PXOPT_COPY_STR(config->args, where, "-source", "receiveSource.source", "==");
+    PXOPT_COPY_STR(config->args, where, "-product", "receiveSource.product", "==");
+    PXOPT_COPY_S64(config->args, where, "-comment", "receiveSource.comment", "LIKE");
+
+    // optional
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("receivetool_list.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("receivetool", PS_LOG_INFO, "No rows found");
+        psFree(output);
+        return true;
+    }
+    if (!ippdbPrintMetadatas(stdout, output, "receiveSource", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "Failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool addfilesetMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(source_id, config->args, "-src_id", true, false);
+    psMetadataItem *filesets = psMetadataLookup(config->args, "-fileset");
+    if (!filesets) {
+        psError(PS_ERR_UNKNOWN, true, "-fileset is required");
+        return false;
+    }
+
+    psString query = pxDataGet("receivetool_addfileset.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement");
+        return false;
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+
+    psString source_id_str = NULL;      // source_id as a string
+    psStringAppend(&source_id_str, "%" PRId64, source_id);
+
+    psListIterator *iter = psListIteratorAlloc(filesets->data.list, PS_LIST_HEAD, false); // Iterator
+    psMetadataItem *item = NULL;        // Item from iteration
+    while ((item = psListGetAndIncrement(iter))) {
+        psAssert(item && item->data.V && item->type == PS_DATA_STRING, "Argument is bad");
+        const char *fileset = item->data.str; // Fileset name
+
+        if (!p_psDBRunQueryF(config->dbh, query, source_id_str, fileset)) {
+            psError(PS_ERR_UNKNOWN, false, "Database error");
+            psFree(source_id_str);
+            psFree(query);
+            psFree(iter);
+            if (!psDBTransaction(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "Database error");
+            }
+            return false;
+        }
+
+        psArray *output = p_psDBFetchResult(config->dbh); // Output of query
+        if (!output) {
+            psError(PS_ERR_UNKNOWN, false, "Database error");
+            psFree(source_id_str);
+            psFree(query);
+            psFree(iter);
+            return false;
+        }
+        if (psArrayLength(output) > 0) {
+            psTrace("receivetool", PS_LOG_INFO, "Fileset %s is already present", fileset);
+            psFree(output);
+            continue;
+        }
+        psFree(output);
+
+        if (!receiveFilesetInsert(config->dbh, 0, source_id, fileset, "reg", NULL, NULL, 0)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to add fileset");
+            psFree(source_id_str);
+            psFree(query);
+            psFree(iter);
+            if (!psDBTransaction(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "Database error");
+            }
+            return false;
+        }
+    }
+    psFree(iter);
+    psFree(query);
+    psFree(source_id_str);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool updatelastMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(source_id, config->args, "-src_id", true, false);
+    PXOPT_LOOKUP_STR(fileset, config->args, "-fileset",  true, false);
+
+    psString query = NULL;              // Query to execute
+    psStringAppend(&query, "UPDATE receiveSource SET fileset_last = \'%s\' WHERE source_id = %" PRId64,
+                   fileset, source_id);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
+
+static bool pendingfilesetMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc(); // WHERE conditions
+
+    // required
+    PXOPT_COPY_STR(config->args, where, "-source", "receiveSource.source", "==");
+    PXOPT_COPY_STR(config->args, where, "-product", "receiveSource.product", "==");
+    PXOPT_COPY_STR(config->args, where, "-comment", "receiveSource.comment", "LIKE");
+
+    // optional
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("receivetool_pendingfileset.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("receivetool", PS_LOG_INFO, "No rows found");
+        psFree(output);
+        return true;
+    }
+    if (!ippdbPrintMetadatas(stdout, output, "receiveFileset", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "Failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool addfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(fileset_id, config->args, "-fileset_id", true, false);
+    PXOPT_LOOKUP_STR(file_list, config->args, "-file_list", true, false);
+
+    unsigned int numBad;
+    psMetadata *files = psMetadataConfigRead(NULL, &numBad, file_list, false);
+    if (!files) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to cleanly read MDC file with file list.");
+        return false;
+    }
+
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+
+    psMetadataIterator *iter = psMetadataIteratorAlloc(files, PS_LIST_HEAD, NULL); // Iterator
+    psMetadataItem *item;
+    while ((item = psMetadataGetAndIncrement(iter))) {
+        psMetadata *md = item->data.md;
+        psString file = psMetadataLookupStr(NULL, md, "file");
+        psS64    bytes = psMetadataLookupS64(NULL, md, "bytes");
+        psString md5sum = psMetadataLookupStr(NULL, md, "md5sum");
+        psString file_type = psMetadataLookupStr(NULL, md, "file_type");
+        psString component = psMetadataLookupStr(NULL, md, "component");
+        
+        if (!file) {
+            psError(PS_ERR_UNKNOWN, false, "failed to find value for file");
+            return false;
+        }
+        if (!component) {
+            psError(PS_ERR_UNKNOWN, false, "failed to find value for component");
+            return false;
+        }
+
+        if (!receiveFileInsert(config->dbh, 0, fileset_id, file, bytes, md5sum, file_type, component)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to add file");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "Database error");
+                return false;
+            }
+            psFree(iter);
+            return false;
+        }
+    }
+
+    psFree(iter);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool pendingfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc(); // WHERE conditions
+
+    // required
+    PXOPT_COPY_STR(config->args, where, "-source", "receiveSource.source", "==");
+    PXOPT_COPY_STR(config->args, where, "-product", "receiveSource.product", "==");
+    PXOPT_COPY_STR(config->args, where, "-comment", "receiveSource.comment", "LIKE");
+    PXOPT_COPY_S64(config->args, where, "-fileset_id", "receiveFile.fileset_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-file_id", "receiveFile.file_id", "==");
+
+    // optional
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("receivetool_pendingfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("receivetool", PS_LOG_INFO, "No rows found");
+        psFree(output);
+        return true;
+    }
+    if (!ippdbPrintMetadatas(stdout, output, "receiveFile", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "Failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool addresultMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(file_id, config->args, "-file_id", true, false);
+
+    // optional
+    PXOPT_LOOKUP_F32(dtime_copy, config->args, "-dtime_copy", false, false);
+    PXOPT_LOOKUP_F32(dtime_extract, config->args, "-dtime_extract", false, false);
+    PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false);
+
+    if (!receiveResultInsert(config->dbh, file_id, dtime_copy, dtime_extract, fault)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool revertMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc(); // WHERE conditions
+
+    PXOPT_COPY_S64(config->args, where, "-fileset_id", "receiveResult.fileset_id", "==");
+    PXOPT_COPY_S32(config->args, where, "-fault", "receiveResult.fault", "==");
+    PXOPT_COPY_STR(config->args, where, "-source", "receiveSource.source", "==");
+    PXOPT_COPY_STR(config->args, where, "-product", "receiveSource.product", "==");
+    PXOPT_COPY_STR(config->args, where, "-comment", "receiveSource.comment", "LIKE");
+
+    psString query = pxDataGet("receivetool_revert.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+static bool toadvanceMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc(); // WHERE conditions
+
+    // optional
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("receivetool_toadvance.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "Failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("receivetool", PS_LOG_INFO, "No rows found");
+        psFree(output);
+        return true;
+    }
+    if (!ippdbPrintMetadatas(stdout, output, "toadvanceFilesets", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "Failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool updatefilesetMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(fileset_id, config->args, "-fileset_id", true, false);
+
+    // to chanage
+    PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_STR(destdir, config->args, "-destdir", false, false);
+    PXOPT_LOOKUP_STR(dirinfo, config->args, "-dirinfo", false, false);
+    PXOPT_LOOKUP_STR(dbinfo, config->args, "-dbinfo", false, false);
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
+
+    if (!fault && !dirinfo &&!dbinfo && !state) {
+        psError(PS_ERR_UNKNOWN, true, "at least one of -fault, -dirinfo, -dbinfo, -set_state are required");
+        return false;
+    }
+
+    psString query = NULL;              // Query to execute
+    psStringAppend(&query, "UPDATE receiveFileset SET ");
+    
+    psString sep = "";
+    if (fault) {
+        psStringAppend(&query, "%s fault = %d", sep, fault);
+        sep = ",";
+    }
+    if (dirinfo) {
+        psStringAppend(&query, "%s dirinfo = '%s'", sep, dirinfo);
+        sep = ",";
+    }
+    if (dbinfo) {
+        psStringAppend(&query, "%s dbinfo = '%s'", sep, dbinfo);
+        sep = ",";
+    }
+    if (state) {
+        psStringAppend(&query, "%s state = '%s'", sep, state);
+        sep = ",";
+    }
+    
+    psStringAppend(&query, " WHERE fileset_id = %" PRId64, fileset_id);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "Database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/receivetool.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/receivetool.h	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/receivetool.h	(revision 24244)
@@ -0,0 +1,42 @@
+/*
+ * receivetool.h
+ *
+ * Copyright (C) 2008 IfA
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * program; see the file COPYING. If not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef RECEIVETOOL_H
+#define RECEIVETOOL_H 1
+
+#include "pxtools.h"
+
+typedef enum {
+    RECEIVETOOL_MODE_NONE      = 0x0,
+    RECEIVETOOL_MODE_DEFINESOURCE,
+    RECEIVETOOL_MODE_LIST,
+    RECEIVETOOL_MODE_ADDFILESET,
+    RECEIVETOOL_MODE_UPDATELAST,
+    RECEIVETOOL_MODE_PENDINGFILESET,
+    RECEIVETOOL_MODE_UPDATEFILESET,
+    RECEIVETOOL_MODE_TOADVANCE,
+    RECEIVETOOL_MODE_ADDFILE,
+    RECEIVETOOL_MODE_PENDINGFILE,
+    RECEIVETOOL_MODE_ADDRESULT,
+    RECEIVETOOL_MODE_REVERT,
+} receivetoolMode;
+
+pxConfig *receivetoolConfig(pxConfig *config, int argc, char **argv);
+
+#endif // RECEIVETOOL_H
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/receivetoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/receivetoolConfig.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/receivetoolConfig.c	(revision 24244)
@@ -0,0 +1,170 @@
+/*
+ * receivetoolConfig.c
+ *
+ * Copyright (C) 2008
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * program; see the file COPYING. If not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <psmodules.h>
+
+#include "pxtools.h"
+#include "receivetool.h"
+
+pxConfig *receivetoolConfig(pxConfig *config, int argc, char **argv)
+{
+    if (!config) {
+        config = pxConfigAlloc();
+    }
+
+    pmConfigReadParamsSet(false);
+
+    // setup site config
+    config->modules = pmConfigRead(&argc, argv, NULL);
+    if (!config->modules) {
+        psError(PS_ERR_UNKNOWN, false, "Can't find site configuration!");
+        psFree(config);
+        return NULL;
+    }
+
+    // -definesource
+    psMetadata *definesourceArgs = psMetadataAlloc();
+    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-source", 0, "define source (required)", NULL);
+    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-product", 0, "define product (required)", NULL);
+    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-workdir",   0, "define workdir (required)", NULL);
+    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL);
+    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-last", 0, "define last fileset", NULL);
+    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-status_product", 0, "define status_product", NULL);
+    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-ds_dbname", 0, "define ds_dbname", NULL);
+    psMetadataAddStr(definesourceArgs, PS_LIST_TAIL, "-ds_dbhost", 0, "define ds_dbhost", NULL);
+
+    // -list
+    psMetadata *listArgs = psMetadataAlloc();
+    psMetadataAddStr(listArgs, PS_LIST_TAIL, "-source", 0, "search on source", NULL);
+    psMetadataAddStr(listArgs, PS_LIST_TAIL, "-product", 0, "search on product", NULL);
+    psMetadataAddStr(listArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL);
+    psMetadataAddU64(listArgs, PS_LIST_TAIL, "-limit",  0, "limit result set", 0);
+    psMetadataAddBool(listArgs, PS_LIST_TAIL, "-simple",  0, "use simple output format?", false);
+
+    // -addfileset
+    psMetadata *addfilesetArgs = psMetadataAlloc();
+    psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-src_id", 0, "define source_id (required)", 0);
+    psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-fileset", PS_META_DUPLICATE_OK, "define fileset (multiple OK, required)", NULL);
+
+    // -updatelast
+    psMetadata *updatelastArgs = psMetadataAlloc();
+    psMetadataAddS64(updatelastArgs, PS_LIST_TAIL, "-src_id", 0, "define source_id (required)", 0);
+    psMetadataAddStr(updatelastArgs, PS_LIST_TAIL, "-fileset", 0, "define last fileset (required)", NULL);
+
+    // -pendingfileset
+    psMetadata *pendingfilesetArgs = psMetadataAlloc();
+    psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-source", 0, "search on source", NULL);
+    psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-product", 0, "search on product", NULL);
+    psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL);
+    psMetadataAddBool(pendingfilesetArgs, PS_LIST_TAIL, "-simple",  0, "use simple output format?", false);
+    psMetadataAddU64(pendingfilesetArgs, PS_LIST_TAIL, "-limit",  0, "limit result set", 0);
+    psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL);
+
+    // -updatefileset
+    psMetadata *updatefilesetArgs = psMetadataAlloc();
+    psMetadataAddS64(updatefilesetArgs, PS_LIST_TAIL, "-fileset_id", 0, "define fileset_id (required)", 0);
+    psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL);
+    psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-destdir", 0, "define destdir", NULL);
+    psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-dirinfo", 0, "define dirinfo", NULL);
+    psMetadataAddStr(updatefilesetArgs, PS_LIST_TAIL, "-dbinfo", 0, "define dbinfo", NULL);
+    psMetadataAddS32(updatefilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
+
+    // -addfile
+    psMetadata *addfileArgs = psMetadataAlloc();
+    psMetadataAddS64(addfileArgs, PS_LIST_TAIL, "-fileset_id", 0, "define fileset_id (required)", 0);
+    psMetadataAddStr(addfileArgs, PS_LIST_TAIL, "-file_list", 0, "define file list (required)", NULL);
+
+    // -pendingfile
+    psMetadata *pendingfileArgs = psMetadataAlloc();
+    psMetadataAddStr(pendingfileArgs, PS_LIST_TAIL, "-source", 0, "search on source", NULL);
+    psMetadataAddStr(pendingfileArgs, PS_LIST_TAIL, "-product", 0, "search on product", NULL);
+    psMetadataAddStr(pendingfileArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL);
+    psMetadataAddS64(pendingfileArgs, PS_LIST_TAIL, "-fileset_id", 0, "search on fileset_id", 0);
+    psMetadataAddS64(pendingfileArgs, PS_LIST_TAIL, "-file_id", 0, "search on file_id", 0);
+    psMetadataAddU64(pendingfileArgs, PS_LIST_TAIL, "-limit",  0, "limit result set", 0);
+    psMetadataAddBool(pendingfileArgs, PS_LIST_TAIL, "-simple",  0, "use simple output format?", false);
+    psMetadataAddStr(pendingfileArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL);
+
+    // -addresult
+    psMetadata *addresultArgs = psMetadataAlloc();
+    psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-file_id", 0, "define receive_id (required)", 0);
+    psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "path_base for component", NULL);
+    psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_copy", 0, "define time to copy", NAN);
+    psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_extract", 0, "define time to extract", NAN);
+    psMetadataAddS32(addresultArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
+
+    // -revert
+    psMetadata *revertArgs = psMetadataAlloc();
+    psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-file_id", 0, "search on file_id", 0);
+    psMetadataAddS64(revertArgs, PS_LIST_TAIL, "-fileset_id", 0, "search on fileset_id", 0);
+    psMetadataAddS32(revertArgs, PS_LIST_TAIL, "-fault", 0, "search on fault code", 0);
+    psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-source", 0, "search on source", NULL);
+    psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-product", 0, "search on product", NULL);
+    psMetadataAddStr(revertArgs, PS_LIST_TAIL, "-comment", 0, "search on comment (LIKE)", NULL);
+
+    // -toadvance
+    psMetadata *toadvanceArgs = psMetadataAlloc();
+    psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-fileset_id", 0, "search on fileset_id", 0);
+    psMetadataAddS64(toadvanceArgs, PS_LIST_TAIL, "-source_id", 0, "search on source_id", 0);
+    psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit",  0, "limit result set", 0);
+    psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple",  0, "use simple output format?", false);
+    // label is not used but pantasks requires it
+    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", 0, "search on label", NULL);
+
+    psMetadata *argSets = psMetadataAlloc();
+    psMetadata *modes = psMetadataAlloc();
+
+    PXOPT_ADD_MODE("-definesource", "", RECEIVETOOL_MODE_DEFINESOURCE, definesourceArgs);
+    PXOPT_ADD_MODE("-list", "", RECEIVETOOL_MODE_LIST, listArgs);
+    PXOPT_ADD_MODE("-addfileset", "", RECEIVETOOL_MODE_ADDFILESET, addfilesetArgs);
+    PXOPT_ADD_MODE("-updatelast", "", RECEIVETOOL_MODE_UPDATELAST, updatelastArgs);
+    PXOPT_ADD_MODE("-updatefileset", "", RECEIVETOOL_MODE_UPDATEFILESET, updatefilesetArgs);
+    PXOPT_ADD_MODE("-pendingfileset", "", RECEIVETOOL_MODE_PENDINGFILESET, pendingfilesetArgs);
+    PXOPT_ADD_MODE("-toadvance", "", RECEIVETOOL_MODE_TOADVANCE, toadvanceArgs);
+    PXOPT_ADD_MODE("-addfile", "", RECEIVETOOL_MODE_ADDFILE, addfileArgs);
+    PXOPT_ADD_MODE("-pendingfile", "", RECEIVETOOL_MODE_PENDINGFILE, pendingfileArgs);
+    PXOPT_ADD_MODE("-addresult", "", RECEIVETOOL_MODE_ADDRESULT, addresultArgs);
+    PXOPT_ADD_MODE("-revert", "", RECEIVETOOL_MODE_REVERT, revertArgs);
+
+    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {
+        psError(PS_ERR_UNKNOWN, true, "option parsing failed");
+        psFree(argSets);
+        psFree(modes);
+        psFree(config);
+        return NULL;
+    }
+
+    psFree(argSets);
+    psFree(modes);
+
+    // define database handle, if used
+    // do this last so we don't setup a connection before CLI options are validated
+    config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules));
+    if (!config->dbh) {
+        psError(PS_ERR_UNKNOWN, false, "Can't configure database");
+        psFree(config);
+        return NULL;
+    }
+
+    return config;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtool.c	(revision 24244)
@@ -78,6 +78,6 @@
         MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP,    updateprocessedexpMode);
         MODECASE(REGTOOL_MODE_CLEARDUPEXP,           cleardupexpMode);
-        MODECASE(CHIPTOOL_MODE_EXPORTRUN,            exportrunMode);
-        MODECASE(CHIPTOOL_MODE_IMPORTRUN,            importrunMode);
+        MODECASE(REGTOOL_MODE_EXPORTRUN,             exportrunMode);
+        MODECASE(REGTOOL_MODE_IMPORTRUN,             importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -238,5 +238,7 @@
     PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false);
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
 
     if (!rawImfileInsert(
@@ -302,5 +304,6 @@
         ignored,
         hostname,
-        code,
+        fault,
+        quality,
         NULL,
         0
@@ -408,5 +411,5 @@
     PXOPT_COPY_STR(config->args, where,  "-tmp_class_id", "tmp_class_id", "==");
     PXOPT_COPY_STR(config->args, where,  "-class_id",     "class_id", "==");
-    PXOPT_COPY_S16(config->args, where,  "-code",         "fault", "==");
+    PXOPT_COPY_S16(config->args, where,  "-fault",         "fault", "==");
     PXOPT_COPY_S64(config->args, where,  "-exp_id_begin", "exp_id", ">=");
     PXOPT_COPY_S64(config->args, where,  "-exp_id_end", "exp_id", "<=");
@@ -452,13 +455,13 @@
     PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault",   false, false);
     PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false);
 
-    if ((code == INT16_MAX) && !isfinite(user_1)) { 
-        psError(PS_ERR_UNKNOWN, false, "one of -code or -user_1 must be selected");
-        return false;
-    }
-    if ((code != INT16_MAX) && isfinite(user_1)) { 
-        psError(PS_ERR_UNKNOWN, false, "only one of -code or -user_1 must be selected");
+    if ((fault == INT16_MAX) && !isfinite(user_1)) { 
+        psError(PS_ERR_UNKNOWN, false, "one of -fault or -user_1 must be selected");
+        return false;
+    }
+    if ((fault != INT16_MAX) && isfinite(user_1)) { 
+        psError(PS_ERR_UNKNOWN, false, "only one of -fault or -user_1 must be selected");
         return false;
     }
@@ -468,7 +471,7 @@
     PXOPT_COPY_STR(config->args, where,  "-class_id",     "class_id", "==");
 
-    if (code != INT16_MAX) {
+    if (fault != INT16_MAX) {
 	// this is fairly dangerous : can set all if the where is not set...
-	if (!pxSetFaultCode(config->dbh, "rawImfile", where, code)) {
+	if (!pxSetFaultCode(config->dbh, "rawImfile", where, fault)) {
 	    psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
 	    psFree (where);
@@ -644,5 +647,5 @@
 
     // default
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
 
     psString query = pxDataGet("regtool_pendingexp.sql");
@@ -801,5 +804,5 @@
         moon_phase,
         hostname,
-        code,
+        fault,
         NULL,
         0
@@ -995,5 +998,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where,  "-exp_id",       "exp_id", "==");
-    PXOPT_COPY_S16(config->args, where,  "-code",         "fault", "==");
+    PXOPT_COPY_S16(config->args, where,  "-fault",         "fault", "==");
     PXOPT_COPY_S64(config->args, where,  "-exp_id_begin", "exp_id", ">=");
     PXOPT_COPY_S64(config->args, where,  "-exp_id_end", "exp_id", "<=");
@@ -1039,7 +1042,7 @@
     PXOPT_COPY_S64(config->args, where,  "-exp_id",       "exp_id", "==");
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);
-
-    if (!pxSetFaultCode(config->dbh, "rawExp", where, code)) {
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
+
+    if (!pxSetFaultCode(config->dbh, "rawExp", where, fault)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
         psFree(where);
@@ -1161,5 +1164,5 @@
     char sqlFilename[80];
   } ExportTable;
-  
+
   int numExportTables = 2;
 
@@ -1180,6 +1183,6 @@
 
   ExportTable tables [] = {
-    {"rawExp", "regtool_export_raw__exp.sql"},
-    {"rawImfile", "regtool_export_raw_imfile.sql"},
+    {"rawExp", "regtool_export_exp.sql"},
+    {"rawImfile", "regtool_export_imfile.sql"},
   };
 
@@ -1241,5 +1244,5 @@
 
   PS_ASSERT_PTR_NON_NULL(config, NULL);
-  
+
   PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
 
@@ -1252,5 +1255,5 @@
   psAssert (item, "entry not in input?");
   psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-  
+
   psMetadataItem *entry = psListGet (item->data.list, 0);
   assert (entry);
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtoolConfig.c	(revision 24244)
@@ -122,5 +122,6 @@
     ADD_OPT(Time, addprocessedimfileArgs, "-dateobs",        "define observation time",         NULL);
     ADD_OPT(Str,  addprocessedimfileArgs, "-hostname",       "define host name",                NULL);
-    ADD_OPT(S16,  addprocessedimfileArgs, "-code",           "set fault code",                  0);
+    ADD_OPT(S16,  addprocessedimfileArgs, "-fault",           "set fault code",                  0);
+    ADD_OPT(S16,  addprocessedimfileArgs, "-quality",        "set quality flag", 0);
 
     // -processedimfile
@@ -140,5 +141,5 @@
     ADD_OPT(Str, revertprocessedimfileArgs, "-tmp_class_id",  "searcy by temp. class ID", NULL);
     ADD_OPT(Str, revertprocessedimfileArgs, "-class_id",      "search by class ID", NULL);
-    ADD_OPT(S16, revertprocessedimfileArgs, "-code",          "search by fault code", 0);
+    ADD_OPT(S16, revertprocessedimfileArgs, "-fault",          "search by fault code", 0);
     ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id_begin",  "search by exposure ID", 0);
     ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id_end",    "search by exposure ID", 0);
@@ -149,5 +150,5 @@
     ADD_OPT(Str, updateprocessedimfileArgs, "-class_id",      "search by class ID", NULL);
     ADD_OPT(F64, updateprocessedimfileArgs, "-user_1",        "set user stat (1)", NAN);
-    ADD_OPT(S16, updateprocessedimfileArgs, "-code",          "set fault code", INT16_MAX);
+    ADD_OPT(S16, updateprocessedimfileArgs, "-fault",          "set fault code", INT16_MAX);
 
     // -pendingexp
@@ -226,5 +227,5 @@
     ADD_OPT(Str,  addprocessedexpArgs, "-label",            "define label for chip stage (non-detrend data only)", NULL);
     ADD_OPT(Str,  addprocessedexpArgs, "-hostname",         "define host name", NULL);
-    ADD_OPT(S16,  addprocessedexpArgs, "-code",             "set fault code", 0);
+    ADD_OPT(S16,  addprocessedexpArgs, "-fault",             "set fault code", 0);
 
     // -processedexp
@@ -276,5 +277,5 @@
     psMetadata *revertprocessedexpArgs = psMetadataAlloc();
     psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id",   0,            "search by exposure ID", 0);
-    psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code",     0,            "search by fault code", 0);
+    psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-fault",     0,            "search by fault code", 0);
     psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id_begin",   0,      "search by exposure ID", 0);
     psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id_end",   0,      "search by exposure ID", 0);
@@ -283,5 +284,5 @@
     psMetadata *updatedprocessedexpArgs = psMetadataAlloc();
     psMetadataAddS64(updatedprocessedexpArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
-    psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-code",    0,            "set fault code (required)", INT16_MAX);
+    psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-fault",    0,            "set fault code (required)", INT16_MAX);
 
     // -exportrun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/stacktool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/stacktool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/stacktool.c	(revision 24244)
@@ -617,5 +617,5 @@
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -718,5 +718,6 @@
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
 
     if (!psDBTransaction(config->dbh)) {
@@ -754,5 +755,6 @@
                                hostname,
                                good_frac,
-                               code
+                               fault,
+                               quality
           )) {
         if (!psDBRollback(config->dbh)) {
@@ -862,5 +864,5 @@
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
     PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "==");
-    PXOPT_COPY_S16(config->args, where, "-code", "stackSumSkyfile.fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
 
     if (!psDBTransaction(config->dbh)) {
@@ -989,5 +991,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
 
     psString query = pxDataGet("stacktool_pendingcleanuprun.sql");
@@ -1053,5 +1055,5 @@
         PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
     }
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
 
     psString query = pxDataGet("stacktool_pendingcleanupskyfile.sql");
@@ -1169,10 +1171,10 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
 
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-stack_id",   "stack_id",   "==");
 
-    if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, code)) {
+    if (!pxSetFaultCode(config->dbh, "stackSumSkyfile", where, fault)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
         psFree (where);
@@ -1192,9 +1194,10 @@
 
   int numExportTables = 3;
-  
+
   PS_ASSERT_PTR_NON_NULL(config, NULL);
 
   PXOPT_LOOKUP_S64(det_id, config->args, "-stack_id", true,  false);
   PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
   PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
 
@@ -1247,8 +1250,26 @@
     }
     if (!psArrayLength(output)) {
-      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psError(PS_ERR_UNKNOWN, true, "no rows found");
       psFree(output);
-      return true;
-    }
+      return false;
+    }
+
+    if (clean) {
+        if (!strcmp(tables[i].tableName, "stackRun")) {
+            if (!pxSetStateCleaned("stackRun", "state", output)) {
+                psFree(output);
+                psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
+                return false;
+            }
+        }
+    }
+
+      // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
     psFree(output);
   }
@@ -1262,11 +1283,11 @@
 {
   unsigned int nFail;
-  
+
   int numImportTables = 2;
-  
+
   char tables[2] [80] = {"stackInputSkyfile", "stackSumSkyfile"};
 
   PS_ASSERT_PTR_NON_NULL(config, NULL);
-  
+
   PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
 
@@ -1293,5 +1314,5 @@
     psAssert (item, "entry not in input?");
     psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-  
+
     switch (i) {
       case 0:
@@ -1307,5 +1328,5 @@
         }
         break;
-        
+
       case 1:
         for (int i = 0; i < item->data.list->n; i++) {
@@ -1322,5 +1343,5 @@
     }
   }
-  
+
   return true;
 }
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/stacktoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/stacktoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/stacktoolConfig.c	(revision 24244)
@@ -114,5 +114,5 @@
     psMetadata *tosumArgs = psMetadataAlloc();
     psMetadataAddS64(tosumArgs, PS_LIST_TAIL, "-stack_id", 0,            "search by stack ID", 0);
-    psMetadataAddStr(tosumArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
+    psMetadataAddStr(tosumArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     psMetadataAddS64(tosumArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warp ID", 0);
     psMetadataAddU64(tosumArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -147,5 +147,6 @@
     psMetadataAddStr(addsumskyfileArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", 0);
     psMetadataAddF32(addsumskyfileArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
-    psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
+    psMetadataAddS16(addsumskyfileArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
 
     // -sumskyfile
@@ -162,9 +163,9 @@
     psMetadataAddS64(revertsumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,            "search by stack ID", 0);
     psMetadataAddStr(revertsumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0);
-    psMetadataAddS16(revertsumskyfileArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertsumskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
 
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -172,5 +173,5 @@
     // -pendingcleanupskyfile
     psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,          "search by stack ID", 0);
     psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
@@ -185,5 +186,5 @@
     psMetadata *updatesumskyfileArgs = psMetadataAlloc();
     psMetadataAddS64(updatesumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,            "define stack ID (required)", 0);
-    psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-code", 0,            "set fault code (required)", 0);
+    psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-fault", 0,            "set fault code (required)", 0);
 
     // -exportrun
@@ -192,4 +193,5 @@
     psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
     psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean",  0,          "mark tables as cleaned", false);
 
     // -importrun
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptool.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptool.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptool.c	(revision 24244)
@@ -30,5 +30,4 @@
 #include "pxtools.h"
 #include "warptool.h"
-#include "pxwarp.h"
 
 static psS64 definerunMode(pxConfig *config);
@@ -42,4 +41,5 @@
 static bool towarpedMode(pxConfig *config);
 static bool addwarpedMode(pxConfig *config);
+static bool advancerunMode(pxConfig *config);
 static bool warpedMode(pxConfig *config);
 static bool revertwarpedMode(pxConfig *config);
@@ -89,4 +89,5 @@
         MODECASE(WARPTOOL_MODE_TOWARPED,           towarpedMode);
         MODECASE(WARPTOOL_MODE_ADDWARPED,          addwarpedMode);
+        MODECASE(WARPTOOL_MODE_ADVANCERUN,         advancerunMode);
         MODECASE(WARPTOOL_MODE_WARPED,             warpedMode);
         MODECASE(WARPTOOL_MODE_REVERTWARPED,       revertwarpedMode);
@@ -135,5 +136,5 @@
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
-    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false);
+    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required (no default TESS)
     PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false);
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
@@ -191,7 +192,46 @@
 
     psMetadata *where = psMetadataAlloc();
-    pxwarpGetSearchArgs (config, where);
-    PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "==");
-    PXOPT_COPY_STR(config->args, where, "-label",     "fakeRun.label",     "==");
+    PXOPT_COPY_S64(config->args, where, "-fake_id",            "fakeRun.fake_id",       "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",             "camRun.cam_id",         "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id",            "chipRun.chip_id",       "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id",             "rawExp.exp_id",         "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name",           "rawExp.exp_name",       "==");
+    PXOPT_COPY_STR(config->args, where, "-inst",               "rawExp.camera",         "==");
+    PXOPT_COPY_STR(config->args, where, "-telescope",          "rawExp.telescope",      "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin",     "rawExp.dateobs",        ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",       "rawExp.dateobs",        "<=");
+    PXOPT_COPY_STR(config->args, where, "-exp_tag",            "rawExp.exp_tag",        "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_type",           "rawExp.exp_type",       "==");
+    PXOPT_COPY_STR(config->args, where, "-filelevel",          "rawExp.filelevel",      "==");
+    PXOPT_COPY_STR(config->args, where, "-filter",             "rawExp.filter",         "==");
+    PXOPT_COPY_F64(config->args, where, "-airmass_min",        "rawExp.airmass",        ">=");
+    PXOPT_COPY_F64(config->args, where, "-airmass_max",        "rawExp.airmass",        "<");
+    PXOPT_COPY_F64(config->args, where, "-ra_min",             "rawExp.ra",             ">=");
+    PXOPT_COPY_F64(config->args, where, "-ra_max",             "rawExp.ra",             "<");
+    PXOPT_COPY_F64(config->args, where, "-decl_min",           "rawExp.decl",           ">=");
+    PXOPT_COPY_F64(config->args, where, "-decl_max",           "rawExp.decl",           "<");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_min",       "rawExp.exp_time",       ">=");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_max",       "rawExp.exp_time",       "<");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_min",             "rawExp.bg",             ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_max",             "rawExp.bg",             "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min",       "rawExp.bg_stdev",       ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max",       "rawExp.bg_stdev",       "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min",  "rawExp.bg_mean_stdev",  ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max",  "rawExp.bg_mean_stdev",  "<");
+    PXOPT_COPY_F64(config->args, where, "-alt_min",            "rawExp.alt",            ">=");
+    PXOPT_COPY_F64(config->args, where, "-alt_max",            "rawExp.alt",            "<");
+    PXOPT_COPY_F64(config->args, where, "-az_min",             "rawExp.az",             ">=");
+    PXOPT_COPY_F64(config->args, where, "-az_max",             "rawExp.az",             "<");
+    PXOPT_COPY_F32(config->args, where, "-ccd_temp_min",       "rawExp.ccd_temp",       ">=");
+    PXOPT_COPY_F32(config->args, where, "-ccd_temp_max",       "rawExp.ccd_temp",       "<");
+    PXOPT_COPY_F64(config->args, where, "-posang_min",         "rawExp.posang",         ">=");
+    PXOPT_COPY_F64(config->args, where, "-posang_max",         "rawExp.posang",         "<");
+    PXOPT_COPY_STR(config->args, where, "-object",             "rawExp.object",         "==");
+    PXOPT_COPY_F32(config->args, where, "-solang_min",         "rawExp.solang",         ">=");
+    PXOPT_COPY_F32(config->args, where, "-solang_max",         "rawExp.solang",         "<");
+    PXOPT_COPY_STR(config->args, where, "-reduction",          "fakeRun.reduction",     "==");
+    pxAddLabelSearchArgs (config, where, "-label",             "fakeRun.label",         "==");
 
     if (!psListLength(where->list) &&
@@ -227,7 +267,7 @@
 
     // use psDBGenerateWhereSQL because the SQL yields an intermediate table
-    if (where && psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereSQL(where, NULL);
-        psStringAppend(&query, "%s", whereClause);
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "AND %s", whereClause);
         psFree(whereClause);
     }
@@ -256,4 +296,25 @@
     // data out so we have the option of changing these values or leaving the
     // old values in place (i.e., passing the values through).
+
+    // loop over our list of fakeRun rows to check the supplied and selected tess_id values:
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        fakeRunRow *row = fakeRunObjectFromMetadata(md);
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fakeRun");
+            psFree(output);
+            return false;
+        }
+
+	if (!tess_id  && !row->tess_id) {
+	    psError(PS_ERR_UNKNOWN, false, "cannot queue warp run without a defined tess id: label: %s, fake_id %" PRId64, row->label, row->fake_id);
+            psFree(output);
+            return false;
+	}
+
+        psFree(row);
+    }
+    psFree(output);
 
     // loop over our list of fakeRun rows
@@ -299,9 +360,8 @@
 
     psMetadata *where = psMetadataAlloc();
-    pxwarpGetSearchArgs (config, where);
     PXOPT_COPY_S64(config->args, where, "-warp_id",   "warpRun.warp_id",   "==");
     PXOPT_COPY_STR(config->args, where, "-reduction", "warpRun.reduction", "==");
-    PXOPT_COPY_STR(config->args, where, "-label",     "warpRun.label", 	   "==");
-    PXOPT_COPY_STR(config->args, where, "-state",     "warpRun.state", 	   "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "warpRun.label",     "==");
+    PXOPT_COPY_STR(config->args, where, "-state",     "warpRun.state",     "==");
 
     if (!psListLength(where->list)
@@ -498,5 +558,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -572,5 +632,5 @@
     PXOPT_LOOKUP_STR(mapfile, config->args, "-mapfile", false, false);
     PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false);
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
 
     if (!psDBTransaction(config->dbh)) {
@@ -579,5 +639,5 @@
     }
 
-    if (code == 0) {
+    if (fault == 0) {
         if (!parseAndInsertSkyCellMap(config, mapfile)) {
             psError(PS_ERR_UNKNOWN, false, "failed to inject mapfile: %s into the database", mapfile);
@@ -594,5 +654,5 @@
             NULL,   // tess_id
             NULL,   // class_id
-            code    // fault
+            fault    // fault
         );
     }
@@ -770,5 +830,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -859,9 +919,9 @@
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
-    PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false);
     PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false);
 
     // default values
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
 
     // we don't want to insert the last skyfile in a run but then not mark the
@@ -890,6 +950,6 @@
                            ymin,
                            ymax,
-                           !accept,
-                           code,
+                           fault,
+                           quality,
                            magicked
         )) {
@@ -901,12 +961,4 @@
     }
 
-    if (!warpCompletedRuns(config)) {
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
     // point of no return
     if (!psDBCommit(config->dbh)) {
@@ -914,4 +966,86 @@
         return false;
     }
+
+    return true;
+}
+
+static bool advancerunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+
+    psString query = pxDataGet("warptool_finished_run_select.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retrieve SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereSQL(where, NULL);
+        psStringAppend(&query, " %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("warptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    query = pxDataGet("warptool_finish_run.sql");
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+
+        bool status;
+        psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id");
+            psFree(output);
+            psFree(query);
+            return false;
+        }
+        psS32 magicked = psMetadataLookupS64(&status, row, "magicked");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to look up value for magicked");
+            psFree(output);
+            psFree(query);
+            return false;
+        }
+        if (!p_psDBRunQueryF(config->dbh, query, magicked, warp_id)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            psFree(query);
+            return false;
+        }
+
+        psS64 numUpdated = psDBAffectedRows(config->dbh);
+
+        if (numUpdated != 1) {
+            psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
+            psFree(query);
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(output);
+    psFree(query);
 
     return true;
@@ -1070,5 +1204,4 @@
 
     psMetadata *where = psMetadataAlloc();
-    pxwarpGetSearchArgs (config, where);
     PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
@@ -1076,5 +1209,5 @@
     PXOPT_COPY_STR(config->args, where, "-reduction",  "rawExp.reduction", "==");
     PXOPT_COPY_STR(config->args, where, "-label",      "warpRun.label", "==");
-    PXOPT_COPY_S16(config->args, where, "-code",       "warpSkyfile.fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",       "warpSkyfile.fault", "==");
 
     if (!psListLength(where->list)
@@ -1092,7 +1225,8 @@
 
 
+#ifdef notdef
+    int numUpdated = 0;                     // Number updated
+    {
     // Update state to 'new'
-    int numUpdated;                     // Number updated
-    {
         // This query is no longer necessary because we do not set warpRun's to full statte
         // if they have faulted skyfiles.
@@ -1140,4 +1274,6 @@
     psLogMsg("warptool", PS_LOG_INFO, "Updated %d warp runs", numUpdated);
 
+#endif // notdef
+
     // Delete product
     int numDeleted;                     // Number deleted
@@ -1261,5 +1397,5 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
 
     psString query = pxDataGet("warptool_pendingcleanuprun.sql");
@@ -1324,5 +1460,5 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
 
     psString query = pxDataGet("warptool_pendingcleanupskyfile.sql");
@@ -1524,12 +1660,12 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    // warp_id, skycell_id, code are required
+    // warp_id, skycell_id, fault are required
     PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
     PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
-    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
 
     psString query = pxDataGet("warptool_updateskyfile.sql");
 
-    if (!p_psDBRunQueryF(config->dbh, query, code, warp_id, skycell_id)) {
+    if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -1549,7 +1685,8 @@
     PS_ASSERT_PTR_NON_NULL(config, NULL);
 
-    PXOPT_LOOKUP_S64(det_id, config->args, "-warp_id", true,  false);
+    PXOPT_LOOKUP_S64(det_id,  config->args, "-warp_id", true,  false);
     PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
     PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+    PXOPT_LOOKUP_BOOL(clean,  config->args, "-clean", false);
 
     FILE *f = fopen (outfile, "w");
@@ -1566,9 +1703,10 @@
       {"warpImfile", "warptool_export_imfile.sql"},
       {"warpSkyfile", "warptool_export_skyfile.sql"},
-      {"warpSkyCellMap", "warptool_export_sky_cell_map.sql"},
+      {"warpSkyCellMap", "warptool_export_skycell_map.sql"},
     };
 
-
-    for (int i=0; i < sizeof(tables); i++) {
+    int numTables = sizeof(tables)/sizeof(tables[0]);
+
+    for (int i=0; i < numTables; i++) {
       psString query = pxDataGet(tables[i].sqlFilename);
       if (!query) {
@@ -1603,8 +1741,22 @@
       }
       if (!psArrayLength(output)) {
-        psTrace("regtool", PS_LOG_INFO, "no rows found");
+        psError(PS_ERR_UNKNOWN, true, "no rows found");
         psFree(output);
-        return true;
+        return false;
       }
+
+    if (clean) {
+        bool success = true; 
+        if (!strcmp(tables[i].tableName, "warpRun")) {
+            success = pxSetStateCleaned("warpRun", "state", output);
+        } else if (!strcmp(tables[i].tableName, "warpSkyfile")) {
+            success = pxSetStateCleaned("warpSkyfile", "data_state", output);
+        }
+        if (!success) {
+            psFree(output);
+            psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
+            return false;
+        }
+    }
 
       // we must write the export table in non-simple (true) format
@@ -1627,9 +1779,9 @@
 
   int numImportTables = 3;
-  
+
   char tables[3] [80] = {"warpImfile", "warpSkyfile", "warpSkyCellMap"};
 
   PS_ASSERT_PTR_NON_NULL(config, NULL);
-  
+
   PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
 
@@ -1642,5 +1794,5 @@
   psAssert (item, "entry not in input?");
   psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-  
+
   psMetadataItem *entry = psListGet (item->data.list, 0);
   assert (entry);
@@ -1656,7 +1808,7 @@
     psAssert (item, "entry not in input?");
     psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-    
+
     switch (i) {
-      case 0:  
+      case 0:
         for (int i = 0; i < item->data.list->n; i++) {
           entry = psListGet (item->data.list, i);
@@ -1670,6 +1822,6 @@
         }
         break;
-        
-      case 1:  
+
+      case 1:
         for (int i = 0; i < item->data.list->n; i++) {
           entry = psListGet (item->data.list, i);
@@ -1683,6 +1835,6 @@
         }
         break;
-        
-      case 2:  
+
+      case 2:
         for (int i = 0; i < item->data.list->n; i++) {
           entry = psListGet (item->data.list, i);
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptool.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptool.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptool.h	(revision 24244)
@@ -37,4 +37,5 @@
     WARPTOOL_MODE_TOWARPED,
     WARPTOOL_MODE_ADDWARPED,
+    WARPTOOL_MODE_ADVANCERUN,
     WARPTOOL_MODE_WARPED,
     WARPTOOL_MODE_REVERTWARPED,
Index: /branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptoolConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptoolConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptoolConfig.c	(revision 24244)
@@ -51,7 +51,46 @@
     // XXX need to allow multiple exp_ids
     psMetadata *definebyqueryArgs = psMetadataAlloc();
-    pxwarpSetSearchArgs (definebyqueryArgs);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-fake_id",            0, "search by fake_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id",             0, "search by cam_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id",            0, "search by chip_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id",             0, "search by exp_id", 0);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_name",           0, "search by exp_name", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-inst",               0, "search for camera", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-telescope",          0, "search for telescope", NULL);
+    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin",     0, "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-dateobs_end",       0, "search for exposures by time (<)", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_tag",            0, "search by exp_tag", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_type",           0, "search by exp_type", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filelevel",          0, "search by filelevel", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter",             0, "search for filter", NULL);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_min",        0, "search by min airmass", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_max",        0, "search by max airmass", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_min",             0, "search by min", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_max",             0, "search by max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_min",           0, "search by min", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_max",           0, "search by max", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min",       0, "search by min", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max",       0, "search by max", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_min",             0, "search by max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_max",             0, "search by max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min",       0, "search by max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max",       0, "search by max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min",  0, "search by max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max",  0, "search by max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_min",            0, "search by min", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_max",            0, "search by max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_min",             0, "search by min", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_max",             0, "search by max", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min",       0, "search by min ccd tempature", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max",       0, "search by max ccd tempature", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min",         0, "search by min rotator position angle", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max",         0, "search by max rotator position angle", NAN);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-object",             0, "search by exposure object", NULL);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_min",         0, "search by min solar angle", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_max",         0, "search by max solar angle", NAN);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",          0, "search by fakeRun reduction class", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",              0, "search on fakeRun label", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on fakeRun label", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_mode",           0, "define mode (warp, diff, stack, magic)", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir",        0, "define workdir", NULL);
@@ -83,5 +122,4 @@
     // XXX need to allow multiple exp_ids
     psMetadata *updaterunArgs = psMetadataAlloc();
-    pxwarpSetSearchArgs (updaterunArgs);
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0,    "search by warptool ID", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0,  "search by warpRun reduction class", NULL);
@@ -109,5 +147,5 @@
     psMetadata *tooverlapArgs = psMetadataAlloc();
     psMetadataAddS64(tooverlapArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warp ID", 0);
-    psMetadataAddStr(tooverlapArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
+    psMetadataAddStr(tooverlapArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     psMetadataAddU64(tooverlapArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(tooverlapArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
@@ -117,5 +155,5 @@
     psMetadataAddStr(addoverlapArgs, PS_LIST_TAIL, "-mapfile", 0,            "path to skycell <-> imfile mapping file", NULL);
     psMetadataAddS64(addoverlapArgs, PS_LIST_TAIL, "-warp_id",  0,            "set warp ID", 0);
-    psMetadataAddS16(addoverlapArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addoverlapArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
 
     // -scmap
@@ -131,5 +169,5 @@
     psMetadata *towarpedArgs = psMetadataAlloc();
     psMetadataAddS64(towarpedArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warptool ID", 0);
-    psMetadataAddStr(towarpedArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
+    psMetadataAddStr(towarpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
     psMetadataAddU64(towarpedArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(towarpedArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
@@ -152,7 +190,7 @@
     psMetadataAddStr(addwarpedArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", 0);
     psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
-    psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-accept",  0, "define if this skycell should be accepted", false);
     psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", false);
-    psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
+    psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-quality",  0,            "set quality", 0);
 
     // -warped
@@ -173,5 +211,4 @@
     // XXX need to allow multiple exp_ids
     psMetadata *revertwarpedArgs = psMetadataAlloc();
-    pxwarpSetSearchArgs(revertwarpedArgs); // XXX does this work here?
     psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warptool ID", 0);
     psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-skycell_id",  0,            "search by skycell ID", NULL);
@@ -179,6 +216,12 @@
     psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-reduction",  0,            "search by warpRun reduction class", NULL);
     psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-label",  0,            "search by warpRun label", NULL);
-    psMetadataAddS16(revertwarpedArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
+    psMetadataAddS16(revertwarpedArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
     psMetadataAddBool(revertwarpedArgs, PS_LIST_TAIL, "-all",  0,            "allow everything to be queued without search terms", false);
+
+    // -advancerun
+    psMetadata *advancerunArgs = psMetadataAlloc();
+    psMetadataAddS64(advancerunArgs, PS_LIST_TAIL, "-warp_id", 0,      "search by warp ID", 0);
+    psMetadataAddStr(advancerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label ", NULL);
+    psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit",  0,       "limit exposures to advance to N items", 0);
 
     // -block
@@ -196,5 +239,5 @@
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
@@ -202,5 +245,5 @@
     // -pendingcleanupskyfile
     psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc();
-    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
+    psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
     psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,          "search by warp ID (required)", 0);
     psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
@@ -232,5 +275,5 @@
     psMetadataAddS64(updateskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,    "warptool ID to update", 0);
     psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
-    psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-code",  0,      "new fault code", 0);
+    psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-fault",  0,      "new fault code", 0);
 
     // -exportrun
@@ -239,4 +282,5 @@
     psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
     psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean",  0,          "export run in cleaned state", false);
 
     // -importrun
@@ -259,4 +303,5 @@
     PXOPT_ADD_MODE("-towarped",        "", WARPTOOL_MODE_TOWARPED,       towarpedArgs);
     PXOPT_ADD_MODE("-addwarped",       "", WARPTOOL_MODE_ADDWARPED,      addwarpedArgs);
+    PXOPT_ADD_MODE("-advancerun",      "", WARPTOOL_MODE_ADVANCERUN,     advancerunArgs);
     PXOPT_ADD_MODE("-warped",          "", WARPTOOL_MODE_WARPED,         warpedArgs);
     PXOPT_ADD_MODE("-revertwarped",    "", WARPTOOL_MODE_REVERTWARPED,   revertwarpedArgs);
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/cfh12k/psastro.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/cfh12k/psastro.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/cfh12k/psastro.config	(revision 24244)
@@ -97,24 +97,24 @@
   FILTER   STR B
   ZEROPT   F32 26.00
-  PHOTCODE STR g
+  PHOTCODE STR g_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR V
   ZEROPT   F32 26.15
-  PHOTCODE STR g
+  PHOTCODE STR g_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR R
   ZEROPT   F32 26.19
-  PHOTCODE STR r
+  PHOTCODE STR r_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR I
   ZEROPT   F32 26.14
-  PHOTCODE STR i
+  PHOTCODE STR i_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR Z
   ZEROPT   F32 25.83
-  PHOTCODE STR z
+  PHOTCODE STR z_SYNTH
 END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/dvo.photcodes
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/dvo.photcodes	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/dvo.photcodes	(revision 24244)
@@ -27,10 +27,10 @@
 #                                                                                       astrometry  photometry     astr mask      phot mask
 # code name		    type  zero  airmass offset  c1    c2  slope  <color> eq     sys  scale  sys  scale     poor   bad     poor   bad 
-  1     g                    sec   0.000  0.000 0.000  1003  1004 0.0160     0  1051   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  2     r                    sec   0.000  0.000 0.000  1003  1004 0.0080     0  1052   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  3     i                    sec   0.000  0.000 0.000  1003  1004 0.0280     0  1053   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  4     z                    sec   0.000  0.000 0.000  1003  1004 0.1070     0  1054   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  5     y                    sec   0.000  0.000 0.000     -     - 0.0000     0  1054   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+  1     g                    sec   0.000  0.000 0.000    1     3  0.0000     0  1051   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  2     r                    sec   0.000  0.000 0.000    2     3  0.0000     0  1052   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  3     i                    sec   0.000  0.000 0.000    2     3  0.0000     0  1053   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  4     z                    sec   0.000  0.000 0.000    3     4  0.0000     0  1054   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  5     y                    sec   0.000  0.000 0.000    4     4  0.0000     0  1054   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
+
   1010  U_L92                ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1011  B_L92                ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -38,5 +38,5 @@
   1013  R_L92                ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1014  I_L92                ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   1020  U_STET               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1021  B_STET               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -44,5 +44,5 @@
   1023  R_STET               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1024  I_STET               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   1050  u_SDSS               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1051  g_SDSS               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -50,5 +50,5 @@
   1053  i_SDSS               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1054  z_SDSS               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   1055  U_SDSS               dep  25.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1056  G_SDSS               dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -90,7 +90,7 @@
   1088  SDSS.g.55            dep   0.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
   1089  SDSS.g.56            dep   0.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000
-													       																		
+
   1100  GSC                  ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   1000  USNO_BLUE            ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1001  USNO_RED             ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -98,5 +98,5 @@
   1003  USNO_F               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1004  USNO_N               ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   1110  USNO.098.RED         dep  20.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1111  USNO.098-0.RED70     dep  20.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -134,20 +134,20 @@
   1143  USNO.IVN.RG9         dep  20.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   1144  USNO.IVN.WR88A       dep  20.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   2020  TYCHO_B              ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   2021  TYCHO_V              ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   2011  2MASS_J              ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.050 0.000 1.000  0.000   0x0000 0x0000  0x0000 0x0000	
   2012  2MASS_H              ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.050 0.000 1.000  0.000   0x0000 0x0000  0x0000 0x0000	
   2013  2MASS_K              ref   0.000  0.000 0.000     -     - 0.0000     0     -   0.050 0.000 1.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
-# these were used for the synthetic photometry catalog based on 2mass					       										
-  3001  PS_g                 ref   0.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  3002  PS_r                 ref   0.000  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  3003  PS_i                 ref   0.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  3004  PS_z                 ref   0.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-  3005  PS_y                 ref   0.000  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
-### megacam photcodes:											       																
+
+# these were used for the synthetic photometry catalog based on 2mass
+  3001  SYNTH.g              ref   0.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  3002  SYNTH.r              ref   0.000  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  3003  SYNTH.i              ref   0.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  3004  SYNTH.z              ref   0.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+  3005  SYNTH.y              ref   0.000  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
+
+### megacam photcodes:
   100   MEGACAM.u.00         dep  25.280 -0.350 0.000     -     - 0.0000     0     -   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   101   MEGACAM.u.01         dep  25.280 -0.350 0.000     -     - 0.0000     0     -   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -190,5 +190,5 @@
   138   MEGACAM.u.38         dep  25.280 -0.350 0.000     -     - 0.0000     0     -   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   139   MEGACAM.u.39         dep  25.280 -0.350 0.000     -     - 0.0000     0     -   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
+
   200   MEGACAM.g.00         dep  26.460 -0.150 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   201   MEGACAM.g.01         dep  26.460 -0.150 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -231,5 +231,5 @@
   238   MEGACAM.g.38         dep  26.460 -0.150 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   239   MEGACAM.g.39         dep  26.460 -0.150 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
+
   300   MEGACAM.r.00         dep  25.978 -0.100 0.000     -     - 0.0000     0     2   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   301   MEGACAM.r.01         dep  25.978 -0.100 0.000     -     - 0.0000     0     2   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -272,5 +272,5 @@
   338   MEGACAM.r.38         dep  25.978 -0.100 0.000     -     - 0.0000     0     2   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   339   MEGACAM.r.39         dep  25.978 -0.100 0.000     -     - 0.0000     0     2   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
+
   400   MEGACAM.i.00         dep  25.744 -0.040 0.000     -     - 0.0000     0     3   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   401   MEGACAM.i.01         dep  25.744 -0.040 0.000     -     - 0.0000     0     3   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -313,5 +313,5 @@
   438   MEGACAM.i.38         dep  25.744 -0.040 0.000     -     - 0.0000     0     3   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   439   MEGACAM.i.39         dep  25.744 -0.040 0.000     -     - 0.0000     0     3   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
+
   500   MEGACAM.z.00         dep  24.800 -0.030 0.000     -     - 0.0000     0     4   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   501   MEGACAM.z.01         dep  24.800 -0.030 0.000     -     - 0.0000     0     4   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -354,6 +354,6 @@
   538   MEGACAM.z.38         dep  24.800 -0.030 0.000     -     - 0.0000     0     4   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   539   MEGACAM.z.39         dep  24.800 -0.030 0.000     -     - 0.0000     0     4   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
-### cfh12k photcodes											       																
+
+### cfh12k photcodes
   140   CFH12K.B.00          dep  26.000 -0.150 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   141   CFH12K.B.01          dep  26.000 -0.150 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -368,5 +368,5 @@
   150   CFH12K.B.10          dep  26.000 -0.150 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   151   CFH12K.B.11          dep  26.000 -0.150 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
+
   240   CFH12K.V.00          dep  26.150 -0.120 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   241   CFH12K.V.01          dep  26.150 -0.120 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -381,5 +381,5 @@
   250   CFH12K.V.10          dep  26.150 -0.120 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   251   CFH12K.V.11          dep  26.150 -0.120 0.000     -     - 0.0000     0     1   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
+
   340   CFH12K.R.00          dep  26.190 -0.090 0.000     -     - 0.0000     0     2   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   341   CFH12K.R.01          dep  26.190 -0.090 0.000     -     - 0.0000     0     2   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -394,5 +394,5 @@
   350   CFH12K.R.10          dep  26.190 -0.090 0.000     -     - 0.0000     0     2   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   351   CFH12K.R.11          dep  26.190 -0.090 0.000     -     - 0.0000     0     2   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
+
   440   CFH12K.I.00          dep  26.135 -0.040 0.000     -     - 0.0000     0     3   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   441   CFH12K.I.01          dep  26.135 -0.040 0.000     -     - 0.0000     0     3   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -407,5 +407,5 @@
   450   CFH12K.I.10          dep  26.135 -0.040 0.000     -     - 0.0000     0     3   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   451   CFH12K.I.11          dep  26.135 -0.040 0.000     -     - 0.0000     0     3   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
+
   540   CFH12K.Z.00          dep  25.830 -0.030 0.000     -     - 0.0000     0     4   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   541   CFH12K.Z.01          dep  25.830 -0.030 0.000     -     - 0.0000     0     4   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
@@ -420,5 +420,5 @@
   550   CFH12K.Z.10          dep  25.830 -0.030 0.000     -     - 0.0000     0     4   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
   551   CFH12K.Z.11          dep  25.830 -0.030 0.000     -     - 0.0000     0     4   0.000 0.000 1.000  0.000   0x0000 0x3888  0x0000 0x0000	
-													       																		
+
   600   CFH12K.Ha.00         dep  25.000  0.040 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   601   CFH12K.Ha.01         dep  25.000  0.040 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -433,5 +433,5 @@
   610   CFH12K.Ha.10         dep  25.000  0.040 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   611   CFH12K.Ha.11         dep  25.000  0.040 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   612   CFH12K.HaOff.00      dep  25.000  0.040 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   613   CFH12K.HaOff.01      dep  25.000  0.040 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -494,5 +494,5 @@
   692   CFH12K.B2F.10        dep  25.000  0.040 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   693   CFH12K.B2F.11        dep  25.000  0.040 0.000     -     - 0.0000     0     -   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   3200  ISP-Apogee-01.g      dep  19.250  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   3300  ISP-Apogee-01.r      dep  19.150  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -500,5 +500,5 @@
   3500  ISP-Apogee-01.z      dep  17.250  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   3600  ISP-Apogee-01.y      dep  14.150  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
+
   4100  SIMTEST.g.Chip       dep  25.000  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   4200  SIMTEST.r.Chip       dep  25.000  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -534,6 +534,6 @@
   4503  SIMMOSAIC.y.02       dep  25.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   4504  SIMMOSAIC.y.03       dep  25.000  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
-# 2007.10.02 : predicted g zero-points from 2006 NSF proposal						       											
+
+# 2007.10.02 : predicted g zero-points from 2006 NSF proposal
   10001 GPC1.g.XY01          dep  24.900  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   10002 GPC1.g.XY02          dep  24.900  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -596,6 +596,6 @@
   10075 GPC1.g.XY75          dep  24.900  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   10076 GPC1.g.XY76          dep  24.900  0.000 0.000     -     - 0.0000     0     1   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
-# 2007.10.02 : predicted r zero-points from 2006 NSF proposal						       											
+
+# 2007.10.02 : predicted r zero-points from 2006 NSF proposal
   10101 GPC1.r.XY01          dep  25.100  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   10102 GPC1.r.XY02          dep  25.100  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -658,6 +658,6 @@
   10175 GPC1.r.XY75          dep  25.100  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   10176 GPC1.r.XY76          dep  25.100  0.000 0.000     -     - 0.0000     0     2   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
-# 2007.10.02 : predicted i zero-points from 2006 NSF proposal						       											
+
+# 2007.10.02 : predicted i zero-points from 2006 NSF proposal
   10201 GPC1.i.XY01          dep  25.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   10202 GPC1.i.XY02          dep  25.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -720,6 +720,6 @@
   10275 GPC1.i.XY75          dep  25.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   10276 GPC1.i.XY76          dep  25.000  0.000 0.000     -     - 0.0000     0     3   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       																		
-# 2007.10.02 : predicted z zero-points from 2006 NSF proposal						       											
+
+# 2007.10.02 : predicted z zero-points from 2006 NSF proposal
   10301 GPC1.z.XY01          dep  24.600  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   10302 GPC1.z.XY02          dep  24.600  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
@@ -782,6 +782,6 @@
   10375 GPC1.z.XY75          dep  24.600  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   10376 GPC1.z.XY76          dep  24.600  0.000 0.000     -     - 0.0000     0     4   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
-													       										
-# 2007.10.02 : predicted y zero-points from 2006 NSF proposal						       											
+
+# 2007.10.02 : predicted y zero-points from 2006 NSF proposal
   10401 GPC1.y.XY01          dep  22.200  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
   10402 GPC1.y.XY02          dep  22.200  0.000 0.000     -     - 0.0000     0     5   0.000 0.000 0.000  0.000   0x0000 0x0000  0x0000 0x0000	
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/Makefile.am	(revision 24244)
@@ -14,4 +14,5 @@
 	format_mef.config \
 	format_relphot.config \
+	ghost.model.mdc \
 	ppImage.config \
         ppMerge.config \
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/camera.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/camera.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/camera.config	(revision 24244)
@@ -83,5 +83,7 @@
    r        STR   r.00000
    i        STR   i.00000
+   z        MULTI
    z        STR   z.00000
+   z        STR   ?.00000
    y        STR   y.00000
    w        STR   w.00000
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_20090220.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_20090220.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_20090220.config	(revision 24244)
@@ -267,5 +267,5 @@
           XY31  S32     19884
           XY32  S32     19884
-          XY33  S32     19884
+          XY33  S32     19884 # should be 19872
           XY34  S32     19884
           XY35  S32     19884
@@ -275,5 +275,5 @@
           XY41  S32     20041
           XY42  S32     20041
-          XY43  S32     20041
+          XY43  S32     20041 # should be 20029
           XY44  S32     20041
           XY45  S32     20041
@@ -331,5 +331,5 @@
           XY32  S32     15429
           XY33  S32     20572
-          XY34  S32     25715
+          XY34  S32     25715 # should be 25755
           XY35  S32     30858
           XY36  S32     36001
@@ -338,6 +338,6 @@
           XY41  S32     5449
           XY42  S32     10592
-          XY43  S32     15735
-          XY44  S32     20878
+          XY43  S32     15735 # should be 15707
+          XY44  S32     20878 # should be 20890
           XY45  S32     26021
           XY46  S32     31164
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_mef.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_mef.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_mef.config	(revision 24244)
@@ -186,8 +186,12 @@
  	CELL.XPARITY	STR	ATM1_1
 	CELL.YPARITY	STR	ATM2_2
-#	CELL.SATURATION	STR	SATURATE
 	FPA.EXPOSURE	STR	EXPREQ		# Requested exposure time, presumably camera exposure time
 	CELL.EXPOSURE	STR	EXPTIME		# Exposure time measured by shutter
 	CELL.DARKTIME	STR	DARKTIME	# Exposure time for camera
+	CELL.TIME	STR	MJD-OBS
+	CELL.GAIN	STR	GAIN
+	CELL.READNOISE	STR	RDNOISE
+	CELL.BAD	STR	LOWLEVEL
+	CELL.SATURATION	STR	SATLEVEL
 END
 
@@ -203,5 +207,5 @@
 	CHIP.XPARITY	S32	1
 	CHIP.YPARITY	S32	1
-	CHIP.X0.DEPEND		STR	CHIP.NAME
+	CHIP.X0.DEPEND	STR	CHIP.NAME
 	CHIP.X0		METADATA
           XY01  S32     4971
@@ -266,5 +270,5 @@
           XY76  S32     34954
 	END
-	CHIP.Y0.DEPEND		STR	CHIP.NAME
+	CHIP.Y0.DEPEND	STR	CHIP.NAME
 	CHIP.Y0		METADATA
           XY01  S32     10286
@@ -455,13 +459,8 @@
           XY76  S32     1
 	END
-	CELL.GAIN	F32	1.0
-	CELL.READNOISE	F32	0.0
 	CELL.READDIR	S32	1
-	CELL.BAD	S32	0
 	CELL.XSIZE	S32	590
 	CELL.YSIZE	S32	598
-	CELL.TIME	STR	MJD-OBS
 	CELL.TIMESYS	STR	TAI
-#	CELL.SATURATION	STR	60000
 END
 
@@ -478,6 +477,4 @@
 	CELL.TIME	STR	MJD
 	CELL.BINNING	STR	TOGETHER
-	# CELL.X0		STR	FORTRAN
-	# CELL.Y0		STR	FORTRAN
 END
  
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ghost.model.mdc
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ghost.model.mdc	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ghost.model.mdc	(revision 24244)
@@ -0,0 +1,60 @@
+GHOST.CENTER.X METADATA
+  NORDER_X S32 3
+  NORDER_Y S32 3
+  VAL_X00_Y00  F64 -4.421024e+01
+  VAL_X01_Y00  F64 1.216270e-02
+  VAL_X02_Y00  F64 -9.721643e-08
+  VAL_X03_Y00  F64 9.976554e-11
+  VAL_X00_Y01  F64 -1.762476e-03
+  VAL_X01_Y01  F64 1.247212e-07
+  VAL_X02_Y01  F64 3.629557e-11
+  VAL_X00_Y02  F64 -1.040174e-07
+  VAL_X01_Y02  F64 1.074674e-10
+  VAL_X00_Y03  F64 1.564112e-11
+  NELEMENTS  S32 10
+END
+
+GHOST.CENTER.Y METADATA
+  NORDER_X S32 3
+  NORDER_Y S32 3
+  VAL_X00_Y00  F64 -2.189470e+00
+  VAL_X01_Y00  F64 -4.186514e-03
+  VAL_X02_Y00  F64 1.131554e-07
+  VAL_X03_Y00  F64 1.415192e-11
+  VAL_X00_Y01  F64 1.569104e-02
+  VAL_X01_Y01  F64 -1.782801e-07
+  VAL_X02_Y01  F64 8.179602e-11
+  VAL_X00_Y02  F64 2.577055e-07
+  VAL_X01_Y02  F64 8.879423e-11
+  VAL_X00_Y03  F64 5.767429e-11
+  NELEMENTS  S32 10
+END
+
+GHOST.INNER.MAJOR METADATA
+  NORDER_X S32 1
+  VAL_X00  F64 3.926693e+01
+  VAL_X01  F64 5.325759e-03
+  NELEMENTS  S32 2
+END
+
+GHOST.INNER.MINOR METADATA
+  NORDER_X S32 1
+  VAL_X00  F64 5.287548e+01
+  VAL_X01  F64 -2.191669e-03
+  NELEMENTS  S32 2
+END
+
+GHOST.OUTER.MAJOR METADATA
+  NORDER_X S32 1
+  VAL_X00  F64 7.928722e+01
+  VAL_X01  F64 1.722181e-02
+  NELEMENTS  S32 2
+END
+
+GHOST.OUTER.MINOR METADATA
+  NORDER_X S32 1
+  VAL_X00  F64 1.314265e+02
+  VAL_X01  F64 -2.627153e-03
+  NELEMENTS  S32 2
+END
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppImage.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppImage.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppImage.config	(revision 24244)
@@ -18,4 +18,8 @@
 USE.DEBURNED.IMAGE      BOOL    TRUE           # use burntool-repaired image?
 
+# crosstalk measurements : only valid for GPC1 for now
+CROSSTALK.MEASURE  BOOL    FALSE          # Subtract model background?
+CROSSTALK.CORRECT  BOOL    FALSE           # Subtract model background?
+
 ## XXX use these local variations until we are building real detrend images
 
@@ -55,5 +59,5 @@
   BIAS             BOOL    FALSE           # Bias subtraction
   DARK             BOOL    TRUE            # Dark subtraction
-  REMNANCE         BOOL    TRUE            # Remnance masking
+  REMNANCE         BOOL    FALSE           # Remnance masking
   SHUTTER          BOOL    FALSE           # Shutter correction
   FLAT             BOOL    TRUE            # Flat-field normalisation
@@ -66,4 +70,5 @@
   ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
   BACKGROUND       BOOL    TRUE            # Subtract background?
+  CROSSTALK.MEASURE  BOOL  TRUE            # Subtract model background?
 END
 
@@ -120,20 +125,49 @@
   BASE.FITS        BOOL    TRUE            # Save base detrended image?
   BASE.MASK.FITS   BOOL    TRUE            # Save base detrended image?
-  BASE.VARIANCE.FITS BOOL    TRUE            # Save base detrended image?
-  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image? 
-  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image? 
-  CHIP.VARIANCE.FITS BOOL    FALSE           # Save chip-mosaic-ed image? 
-  OVERSCAN         BOOL    TRUE            # Overscan subtraction
-  BIAS             BOOL    FALSE           # Bias subtraction
-  DARK             BOOL    TRUE            # Dark subtraction
-  SHUTTER          BOOL    FALSE           # Shutter correction
-  FLAT             BOOL    FALSE           # Flat-field normalisation
-  MASK             BOOL    FALSE           # Mask bad pixels
-  FRINGE           BOOL    FALSE           # Fringe subtraction
-  PHOTOM           BOOL    FALSE           # Source identification and photometry
-  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
-  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
-  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
-  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+  BASE.VARIANCE.FITS BOOL  TRUE            # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL  FALSE           # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    FALSE           # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    FALSE           # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+END
+
+# Overscan, bias, dark -- search for DARK_PREMASK
+PPIMAGE_DARKMASK_PROCESS METADATA
+  BASE.FITS        BOOL    TRUE            # Save base detrended image?
+  BASE.MASK.FITS   BOOL    TRUE            # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL  TRUE            # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL  FALSE           # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    FALSE
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    FALSE           # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+
+  DETREND.CONSTRAINTS  METADATA
+    DARK METADATA
+      DETTYPE STR DARK_PREMASK
+      EXPTIME STR FPA.EXPOSURE
+    END
+  END   
 END
 
@@ -184,4 +218,5 @@
     DARK METADATA
       DETTYPE STR DARK_PREMASK
+      EXPTIME STR FPA.EXPOSURE
     END
   END   
@@ -210,4 +245,37 @@
 END
 
+# Overscan, bias, dark, shutter, flat-field -- use FLAT_PREMASK
+PPIMAGE_FLATMASK_PROCESS      METADATA
+  BASE.FITS        BOOL    TRUE            # Save base detrended image?
+  BASE.MASK.FITS   BOOL    TRUE            # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL  TRUE            # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL  FALSE           # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    FALSE           # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    TRUE            # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+
+  DETREND.CONSTRAINTS  METADATA
+    DARK METADATA
+      DETTYPE STR DARK_PREMASK
+      EXPTIME STR FPA.EXPOSURE
+    END
+    FLAT METADATA
+      DETTYPE STR FLAT_PREMASK
+      FILTER  STR FPA.FILTERID
+    END
+  END   
+END
+
 # Overscan, bias, dark, shutter, flat-field, fringe, photom, astrom
 PPIMAGE_DET_ONLY   METADATA
@@ -328,8 +396,9 @@
     END
     DARK METADATA
+      DETTYPE STR DARK_PREMASK
       EXPTIME STR FPA.EXPOSURE
     END
     FLAT METADATA
-      DETTYPE STR FLAT_RAW
+      DETTYPE STR FLAT_PREMASK
       FILTER  STR FPA.FILTERID
     END
@@ -559,2 +628,37 @@
 END
 
+# generate CTE map image
+PPIMAGE_CTEMAP     METADATA
+  BASE.FITS        BOOL    TRUE            # Save base image?
+  BASE.MASK.FITS   BOOL    TRUE            # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL  TRUE            # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL  FALSE           # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    FALSE           # Overscan subtraction
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    FALSE           # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    TRUE            # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+  CHECK.CTE        BOOL    TRUE            # measure CTE errors?
+
+  DETREND.CONSTRAINTS  METADATA
+    DARK METADATA
+      DETTYPE STR DARK_PREMASK
+      EXPTIME STR FPA.EXPOSURE
+    END
+    FLAT METADATA
+      DETTYPE STR FLAT_PREMASK
+      FILTER  STR FPA.FILTERID
+    END
+  END   
+END
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppMerge.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppMerge.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppMerge.config	(revision 24244)
@@ -1,14 +1,14 @@
 # Mask generation --- already included in default, above
 PPMERGE_DARKMASK METADATA
-  ITER           S32    3
-  MASK.SUSPECT	 F32	3.0		# Threshold for suspect pixels (sigma)
-  MASK.BAD	 F32	0.1		# Threshold for bad pixels 
-  MASK.MODE	 STR	FRACTION	# Meaning of threshold : value, sigma
-  SAMPLE         S32    100000          # Sampling factor for measuring the background
+  ITER               S32    3
+  MASK.SUSPECT.SIGMA F32	8.0		# Threshold for suspect pixels (sigma)
+  MASK.BAD	     F32	0.1		# Threshold for bad pixels 
+  MASK.MODE	     STR	FRACTION	# Meaning of threshold : value, sigma
+  SAMPLE             S32    100000          # Sampling factor for measuring the background
 END
 
 PPMERGE_FLATMASK METADATA
   ITER           S32    3
-  MASK.SUSPECT	 F32	3.0		# Threshold for suspect pixels (sigma)
+  MASK.SUSPECT.SIGMA F32	3.0		# Threshold for suspect pixels (sigma)
   MASK.BAD	 F32	0.1		# Threshold for bad pixels (sigma)
   MASK.MODE	 STR	FRACTION	# Threshold for bad pixels (sigma)
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppStack.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppStack.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppStack.config	(revision 24244)
@@ -3,7 +3,7 @@
 ITER		S32	1		# Number of rejection iterations
 COMBINE.REJ	F32	4.0		# Rejection threshold in combination (sigma)
-MASK.VAL	STR	MASK.VALUE,BAD.WARP	# Mask value of input bad pixels
+MASK.VAL	STR	MASK.VALUE,CONV.BAD	# Mask value of input bad pixels
 MASK.BAD	STR	BLANK		# Mask value to give bad pixels
-MASK.POOR	STR	POOR.WARP	# Mask value to give poor pixels
+MASK.POOR	STR	CONV.POOR	# Mask value to give poor pixels
 POOR.FRACTION	F32	0.1		# Maximum fraction of bad weight for poor pixels
 THRESHOLD.MASK	F32	0.8		# Threshold for mask deconvolution (0..1)
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/psastro.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/psastro.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/psastro.config	(revision 24244)
@@ -1,3 +1,6 @@
 	
+# save matched stars?
+PSASTRO.SAVE.REFMATCH      BOOL TRUE  # save refstar matches as table in output smf file
+
 # astrometry matching parameters
 
@@ -115,27 +118,73 @@
   FILTER   STR g
   ZEROPT   F32 24.9
-  PHOTCODE STR g
+  PHOTCODE STR g_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR r
   ZEROPT   F32 25.1
-  PHOTCODE STR r
+  PHOTCODE STR r_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR i
   ZEROPT   F32 25.0
-  PHOTCODE STR i
+  PHOTCODE STR i_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR z
   ZEROPT   F32 24.6
-  PHOTCODE STR z
+  PHOTCODE STR z_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR y
-  ZEROPT   F32 23.6
-  PHOTCODE STR y
+  ZEROPT   F32 22.2
+  PHOTCODE STR y_SYNTH
 END
 
 REFSTAR_MASK                    BOOL TRUE
+REFSTAR_MASK_MAX_MAG            F32 -15.0
+REFSTAR_MASK_SATSTAR_MAG_MAX    F32 -17.40
+REFSTAR_MASK_SATSTAR_MAG_SLOPE  F32  13.45
 REFSTAR_MASK_SATSTAR_POS_ZERO   F32  -0.779
+
+# Length = SLOPE*(MAG_MAX - InstMag)
+
+REFSTAR_MASK_SATSPIKE_MAG_SLOPE F32  52.00
+REFSTAR_MASK_SATSPIKE_MAG_MAX   F32 -16.20
+
+# pair 1 (longer)
+# REFSTAR_MASK_SATSPIKE_MAG_MAX1  F32 -16.20
+
+# pair 2 (shorter)
+# REFSTAR_MASK_SATSPIKE_MAG_MAX2  F32 -17.70
+
+REFSTAR_MASK_SATSPIKE_WIDTH     F32   8.00
+
+REFSTAR_MASK_BLEED              BOOL FALSE
+REFSTAR_MASK_BLEED_MAG_MAX      F32 -15.0
+REFSTAR_MASK_BLEED_MAG_SLOPE    F32   5.0
+
+PSASTRO.EXTRACT METADATA
+ EXTRACT_MAX_MAG                F32  -19.0
+ DVO.GETSTAR.MAX.RHO            F32  300.0
+END
+
+# ghost model file is relative to PATH (eg, ippconfig directory)
+GHOST_MODEL                     STR  gpc1/ghost.model.mdc
+GHOST_MAX_MAG                   F32 -19.0
+REFSTAR_COUNT_GHOSTS            BOOL TRUE
+REFSTAR_MASK_GHOST              BOOL TRUE
+
+# GHOST_CENTER_X_0         	F32   66.2
+# GHOST_CENTER_X_SLOPE         	F32   -0.02454
+# GHOST_CENTER_Y_0         	F32   16.6
+# GHOST_CENTER_Y_SLOPE         	F32   +0.01565
+# 
+# GHOST_INNER_MAJOR_0         	F32   32.9
+# GHOST_INNER_MAJOR_SLOPE         F32   +0.00612
+# GHOST_INNER_MINOR_0         	F32   58.7
+# GHOST_INNER_MINOR_SLOPE         F32   -0.00258
+# 
+# GHOST_OUTER_MAJOR_0         	F32   90.6
+# GHOST_OUTER_MAJOR_SLOPE         F32   +0.01548
+# GHOST_OUTER_MINOR_0         	F32  133.9
+# GHOST_OUTER_MINOR_SLOPE         F32   -0.00288
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/psphot.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/psphot.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/psphot.config	(revision 24244)
@@ -5,4 +5,7 @@
 SAVE.PSF	BOOL 	TRUE
 SAVE.PLOTS     	BOOL    TRUE
+
+# for testing:
+# BREAK_POINT STR BACKMDL
 
 BACKGROUND.XBIN	    S32  256            # size of background superpixels
@@ -16,6 +19,6 @@
 PEAKS_SMOOTH_SIGMA  F32   2.5            # smoothing kernel sigma in pixels
 PEAKS_NMAX          S32   5000           # on first pass, only keep NMAX peaks (0 == all)
-PEAKS_NSIGMA_LIMIT  F32   25.0            # peak significance threshold
-PEAKS_NSIGMA_LIMIT_2 F32  10.0             # peak significance threshold
+PEAKS_NSIGMA_LIMIT  F32   25.0           # peak significance threshold
+PEAKS_NSIGMA_LIMIT_2 F32   5.0           # peak significance threshold
 
 PSF_SN_LIM          F32  25              # minimum S/N for stars used for PSF model
@@ -32,14 +35,12 @@
 PSPHOT.CR.NSIGMA.SOFTEN             F32   0.0025            # Softening parameter for weights
 
-PSF.TREND.MODE                      STR   MAP
-# PSF.TREND.MODE                      STR   POLY_CHEB
-# PSF.TREND.MODE                      STR   POLY_ORD
-PSF.TREND.NX                        S32   1
-PSF.TREND.NY                        S32   1
+PSF.TREND.MODE                      STR   MAP # other options: POLY_CHEB, POLY_ORD
+PSF.TREND.NX                        S32   3
+PSF.TREND.NY                        S32   3
 
-MOMENTS_SN_MIN      F32   10.0
 EXT_MIN_SN           F32  50.0           # fit galaxies above this S/N limit
-FULL_FIT_SN_LIM      F32  50.0
-AP_MIN_SN            F32  50.0
+FULL_FIT_SN_LIM      F32  25.0
+MOMENTS_SN_MIN      F32    5.0
+AP_MIN_SN            F32   5.0
 
 OUTPUT.FORMAT        STR  PS1_V1
@@ -48,8 +49,6 @@
 PSF_MAX_CHI          F32  50.0		 # reject objects worse that this
 
-APTREND.ORDER.MAX    S32  1
-
 MEASURE.APTREND      BOOL TRUE           ### XXX for now, skip this (too many errors)
-# BREAK_POINT STR BACKMDL
+APTREND.ORDER.MAX    S32  3
 
 DIAGNOSTIC.PLOTS                    METADATA
@@ -61,5 +60,4 @@
 USE_FOOTPRINTS                      BOOL  TRUE       	  # use new pmFootprint peak packaging
 
-
 DIFF	METADATA
 	BACKGROUND.XBIN	    S32  32            # size of background superpixels
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/format_mef.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/format_mef.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/format_mef.config	(revision 24244)
@@ -98,5 +98,4 @@
 	FPA.OBJECT	STR	OBJECT
 	FPA.TIME	STR	MJD-OBS
-	FPA.TIMESYS	STR	TIMESYS
         FPA.RA          STR     RA
         FPA.DEC         STR     DEC
@@ -113,5 +112,4 @@
         CELL.READNOISE  STR     RDNOISE
 	CELL.TIME	STR	MJD-OBS
-	CELL.TIMESYS	STR	TIMESYS
         CELL.XBIN       STR     CCDBIN1
         CELL.YBIN       STR     CCDBIN2
@@ -123,4 +121,5 @@
 	FPA.INSTRUMENT		STR	MEGACAM		# this value is used in ipprc.config to find the recipes
 	FPA.DETECTOR	        STR	MEGACAM
+	FPA.TIMESYS		STR	UTC
 	CHIP.XSIZE		S32	2048
 	CHIP.YSIZE		S32	4612
@@ -135,4 +134,5 @@
 	CELL.YPARITY		S32	1
 	CELL.Y0			S32	0
+	CELL.TIMESYS		STR	UTC
 	CHIP.X0.DEPEND		STR	CHIP.NAME
 	CHIP.X0		METADATA
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/psastro.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/psastro.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/psastro.config	(revision 24244)
@@ -96,24 +96,24 @@
   FILTER   STR u
   ZEROPT   F32 25.28
-  PHOTCODE STR g
+  PHOTCODE STR g_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR g
   ZEROPT   F32 26.46
-  PHOTCODE STR g
+  PHOTCODE STR g_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR r
   ZEROPT   F32 25.98
-  PHOTCODE STR r
+  PHOTCODE STR r_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR i
   ZEROPT   F32 25.74
-  PHOTCODE STR i
+  PHOTCODE STR i_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR z
   ZEROPT   F32 24.80
-  PHOTCODE STR z
+  PHOTCODE STR z_SYNTH
 END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/psphot.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/psphot.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/psphot.config	(revision 24244)
@@ -24,5 +24,5 @@
 MOMENTS_SN_MIN      F32    2.0
 EXT_MIN_SN           F32  50.0           # fit galaxies above this S/N limit
-FULL_FIT_SN_LIM      F32  50.0
+FULL_FIT_SN_LIM      F32  10.0
 AP_MIN_SN            F32   2.0
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/Makefile.am	(revision 24244)
@@ -23,5 +23,6 @@
 	dvocorr.config \
 	jpeg.mdc \
-        ppStatsFromMetadata.config
+        ppStatsFromMetadata.config \
+	ppSkycell.config
 
 install_DATA = $(install_files)
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/addstar.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/addstar.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/addstar.config	(revision 24244)
@@ -4,5 +4,6 @@
 
 FLATCORR METADATA
-    IMAGES.ONLY  BOOL  FALSE
+    # IMAGES.ONLY  BOOL  FALSE
+    IMAGES.ONLY  BOOL  TRUE
 END
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-mef.mdc
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-mef.mdc	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-mef.mdc	(revision 24244)
@@ -126,4 +126,10 @@
 PPARITH.INPUT.IMAGE     INPUT    @FILES        CHIP       IMAGE
 PPARITH.INPUT.MASK      INPUT    @FILES        CHIP       MASK
+
+PPSKYCELL.IMAGE      	INPUT 	 @FILES        CHIP       IMAGE
+PPSKYCELL.MASK      	INPUT 	 @FILES        CHIP       MASK
+
+PPSIM.INPUT             INPUT    @FILES        FPA        IMAGE     
+PPSIM.REAL.SOURCES      INPUT    @FILES        CHIP       CMF       
 
 ### output file definitions
@@ -220,7 +226,12 @@
 PPSUB.OUTPUT.MASK       OUTPUT {OUTPUT}.mk.fits                  MASK      COMP_MASK  FPA        TRUE      NONE
 PPSUB.OUTPUT.VARIANCE   OUTPUT {OUTPUT}.wt.fits                  VARIANCE  COMP_WT    FPA        TRUE      NONE
+PPSUB.OUTPUT.SOURCES    OUTPUT {OUTPUT}.cmf                      CMF       NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.KERNELS    OUTPUT {OUTPUT}.subkernel                SUBKERNEL NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.JPEG1      OUTPUT {OUTPUT}.b1.jpg                   JPEG      NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.JPEG2      OUTPUT {OUTPUT}.b2.jpg                   JPEG      NONE       FPA        TRUE      NONE
+PPSUB.INVERSE           OUTPUT {OUTPUT}.inv.fits                 IMAGE     COMP_SUB   FPA        TRUE      NONE
+PPSUB.INVERSE.MASK      OUTPUT {OUTPUT}.inv.mask.fits            MASK      COMP_MASK  FPA        TRUE      NONE
+PPSUB.INVERSE.VARIANCE  OUTPUT {OUTPUT}.inv.wt.fits              VARIANCE  COMP_WT    FPA        TRUE      NONE
+PPSUB.INVERSE.SOURCES   OUTPUT {OUTPUT}.inv.cmf                  CMF       NONE       FPA        TRUE      NONE
 PPSUB.CONFIG            OUTPUT {OUTPUT}.ppSub.mdc                TEXT      NONE       FPA        TRUE      NONE
 PPSUB.INPUT.CONV        OUTPUT {OUTPUT}.inConv.fits              IMAGE     COMP_SUB   FPA        TRUE      NONE
@@ -246,9 +257,9 @@
 PPSTAMP.CHIP.MEF        OUTPUT {OUTPUT}.ch.fits                  IMAGE     NONE       CHIP       FALSE     MEF
                                                                                      
-PPSIM.OUTPUT.MEF        OUTPUT {OUTPUT}.fits                     IMAGE     NONE       CHIP       TRUE      MEF
-PPSIM.OUTPUT.SPL        OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     NONE       CHIP       TRUE      SPLIT
-PPSIM.FAKE.CHIP         OUTPUT {OUTPUT}.fake.fits                IMAGE     NONE       FPA        TRUE      SIMPLE
-PPSIM.FORCE.CHIP        OUTPUT {OUTPUT}.force.fits               IMAGE     NONE       FPA        TRUE      SIMPLE
-PPSIM.SOURCES           OUTPUT {OUTPUT}.cmf                      CMF       NONE       FPA        TRUE      MEF
+PPSIM.OUTPUT.MEF        OUTPUT {OUTPUT}.fits                     IMAGE     NONE       CHIP       TRUE      NONE
+PPSIM.OUTPUT.SPL        OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     NONE       CHIP       TRUE      NONE
+PPSIM.FAKE.CHIP         OUTPUT {OUTPUT}.fake.fits                IMAGE     NONE       FPA        TRUE      NONE
+PPSIM.FORCE.CHIP        OUTPUT {OUTPUT}.force.fits               IMAGE     NONE       FPA        TRUE      NONE
+PPSIM.SOURCES           OUTPUT {OUTPUT}.cmf                      CMF       NONE       FPA        TRUE      NONE
 PPSIM.FAKE.SOURCES      OUTPUT {OUTPUT}.fake.cmf                 CMF       NONE       FPA        TRUE      NONE
 PPSIM.FORCE.SOURCES     OUTPUT {OUTPUT}.force.cmf                CMF       NONE       FPA        TRUE      NONE
@@ -257,4 +268,7 @@
 PPARITH.OUTPUT.MASK     OUTPUT {OUTPUT}.fits                     MASK      COMP_MASK  CHIP       TRUE      NONE
                                                                                      
+PPSKYCELL.JPEG1    	OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg     JPEG      NONE       CHIP       TRUE      NONE
+PPSKYCELL.JPEG2    	OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg     JPEG      NONE       CHIP       TRUE      NONE
+
 LOG.IMFILE              OUTPUT {OUTPUT}.{CHIP.NAME}.log          TEXT      NONE       CHIP       TRUE      NONE
 LOG.EXP                 OUTPUT {OUTPUT}.log                      TEXT      NONE       FPA        TRUE      NONE
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-simple.mdc
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-simple.mdc	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-simple.mdc	(revision 24244)
@@ -90,4 +90,7 @@
 PPARITH.INPUT.MASK      INPUT    @FILES        CHIP       MASK
 
+PPSKYCELL.IMAGE      	INPUT 	 @FILES        CHIP       IMAGE
+PPSKYCELL.MASK      	INPUT 	 @FILES        CHIP       MASK
+
 PPSIM.INPUT             INPUT    @FILES        FPA        IMAGE     
 PPSIM.REAL.SOURCES      INPUT    @FILES        CHIP       CMF       
@@ -173,7 +176,12 @@
 PPSUB.OUTPUT.MASK       OUTPUT {OUTPUT}.mask.fits    MASK      NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.VARIANCE   OUTPUT {OUTPUT}.wt.fits      VARIANCE  NONE       FPA        TRUE      NONE
+PPSUB.OUTPUT.SOURCES    OUTPUT {OUTPUT}.cmf          CMF       NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.KERNELS    OUTPUT {OUTPUT}.subkernel    SUBKERNEL NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.JPEG1      OUTPUT {OUTPUT}.b1.jpg       JPEG      NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.JPEG2      OUTPUT {OUTPUT}.b2.jpg       JPEG      NONE       FPA        TRUE      NONE
+PPSUB.INVERSE           OUTPUT {OUTPUT}.inv.fits     IMAGE     NONE       FPA        TRUE      NONE
+PPSUB.INVERSE.MASK      OUTPUT {OUTPUT}.inv.mask.fits MASK     NONE       FPA        TRUE      NONE
+PPSUB.INVERSE.VARIANCE  OUTPUT {OUTPUT}.inv.wt.fits  VARIANCE  NONE       FPA        TRUE      NONE
+PPSUB.INVERSE.SOURCES   OUTPUT {OUTPUT}.inv.cmf      CMF       NONE       FPA        TRUE      NONE
 PPSUB.CONFIG            OUTPUT {OUTPUT}.ppSub.mdc    TEXT      NONE       FPA        TRUE      NONE
 PPSUB.INPUT.CONV        OUTPUT {OUTPUT}.inConv.fits     IMAGE  COMP_SUB   FPA        TRUE      NONE
@@ -208,4 +216,7 @@
 PPARITH.OUTPUT.MASK     OUTPUT {OUTPUT}.fits         MASK      NONE       FPA        TRUE      SIMPLE
 
+PPSKYCELL.JPEG1    	OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg JPEG NONE    CHIP       TRUE      NONE
+PPSKYCELL.JPEG2    	OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg JPEG NONE    CHIP       TRUE      NONE
+
 LOG.IMFILE              OUTPUT {OUTPUT}.imfile.log   TEXT      NONE       FPA        TRUE      NONE
 LOG.EXP                 OUTPUT {OUTPUT}.exp.log      TEXT      NONE       FPA        TRUE      NONE
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-split.mdc
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-split.mdc	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-split.mdc	(revision 24244)
@@ -98,4 +98,10 @@
 PPARITH.INPUT.IMAGE     INPUT    @FILES        CHIP       IMAGE
 PPARITH.INPUT.MASK      INPUT    @FILES        CHIP       MASK
+
+PPSKYCELL.IMAGE      	INPUT 	 @FILES        CHIP       IMAGE
+PPSKYCELL.MASK      	INPUT 	 @FILES        CHIP       MASK
+
+PPSIM.INPUT             INPUT    @FILES        CHIP       IMAGE     
+PPSIM.REAL.SOURCES      INPUT    @FILES        CHIP       CMF       
 
 ### output file definitions
@@ -186,13 +192,18 @@
 PPSUB.OUTPUT.MASK       OUTPUT {OUTPUT}.mask.fits                MASK      COMP_MASK  FPA        TRUE      NONE
 PPSUB.OUTPUT.VARIANCE   OUTPUT {OUTPUT}.wt.fits                  VARIANCE  COMP_WT    FPA        TRUE      NONE
+PPSUB.OUTPUT.SOURCES    OUTPUT {OUTPUT}.cmf                      CMF       NONE       FPA        TRUE      NONE
 PPSUB.OUTPUT.KERNELS    OUTPUT {OUTPUT}.subkernel                SUBKERNEL NONE       FPA        TRUE      NONE
-PPSUB.OUTPUT.JPEG1	OUTPUT {OUTPUT}.b1.jpg                   JPEG      NONE       FPA        TRUE      NONE
-PPSUB.OUTPUT.JPEG2	OUTPUT {OUTPUT}.b2.jpg                   JPEG      NONE       FPA        TRUE      NONE
+PPSUB.OUTPUT.JPEG1      OUTPUT {OUTPUT}.b1.jpg                   JPEG      NONE       FPA        TRUE      NONE
+PPSUB.OUTPUT.JPEG2      OUTPUT {OUTPUT}.b2.jpg                   JPEG      NONE       FPA        TRUE      NONE
+PPSUB.INVERSE           OUTPUT {OUTPUT}.inv.fits                 IMAGE     COMP_SUB   FPA        TRUE      NONE
+PPSUB.INVERSE.MASK      OUTPUT {OUTPUT}.inv.mask.fits            MASK      COMP_MASK  FPA        TRUE      NONE
+PPSUB.INVERSE.VARIANCE  OUTPUT {OUTPUT}.inv.wt.fits              VARIANCE  COMP_WT    FPA        TRUE      NONE
+PPSUB.INVERSE.SOURCES   OUTPUT {OUTPUT}.inv.cmf                  CMF       NONE       FPA        TRUE      NONE
 PPSUB.CONFIG            OUTPUT {OUTPUT}.ppSub.mdc                TEXT      NONE       FPA        TRUE      NONE
 PPSUB.INPUT.CONV        OUTPUT {OUTPUT}.inConv.fits              IMAGE     COMP_SUB   FPA        TRUE      NONE
 PPSUB.INPUT.CONV.MASK   OUTPUT {OUTPUT}.inConv.mk.fits           MASK      COMP_MASK  FPA        TRUE      NONE
 PPSUB.INPUT.CONV.VARIANCE OUTPUT {OUTPUT}.inConv.wt.fits         VARIANCE  COMP_WT    FPA        TRUE      NONE
-PPSUB.REF.CONV        	OUTPUT {OUTPUT}.refConv.fits             IMAGE     COMP_SUB   FPA        TRUE      NONE
-PPSUB.REF.CONV.MASK   	OUTPUT {OUTPUT}.refConv.mk.fits          MASK      COMP_MASK  FPA        TRUE      NONE
+PPSUB.REF.CONV          OUTPUT {OUTPUT}.refConv.fits             IMAGE     COMP_SUB   FPA        TRUE      NONE
+PPSUB.REF.CONV.MASK     OUTPUT {OUTPUT}.refConv.mk.fits          MASK      COMP_MASK  FPA        TRUE      NONE
 PPSUB.REF.CONV.VARIANCE OUTPUT {OUTPUT}.refConv.wt.fits          VARIANCE  COMP_WT    FPA        TRUE      NONE
 											        
@@ -211,14 +222,17 @@
 PPSTAMP.CHIP            OUTPUT {OUTPUT}.ch.fits                  IMAGE     NONE       CHIP       FALSE     MEF
 		        									        
-PPSIM.OUTPUT            OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     NONE       CHIP       TRUE      SPLIT
-PPSIM.FAKE.CHIP        	OUTPUT {OUTPUT}.fake.fits     		 IMAGE     NONE       FPA        TRUE      SIMPLE
-PPSIM.FORCE.CHIP       	OUTPUT {OUTPUT}.force.fits     		 IMAGE     NONE       FPA        TRUE      SIMPLE
-PPSIM.SOURCES           OUTPUT {OUTPUT}.{CHIP.NAME}.cmf          CMF       NONE       CHIP       TRUE      SPLIT
-PPSIM.FAKE.SOURCES     	OUTPUT {OUTPUT}.fake.cmf                 CMF       NONE       FPA        TRUE      NONE
-PPSIM.FORCE.SOURCES    	OUTPUT {OUTPUT}.force.cmf                CMF       NONE       FPA        TRUE      NONE
-											        
+PPSIM.OUTPUT            OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     NONE       CHIP       TRUE      NONE
+PPSIM.FAKE.CHIP        	OUTPUT {OUTPUT}.{CHIP.NAME}.fake.fits    IMAGE     NONE       CHIP       TRUE      NONE
+PPSIM.FORCE.CHIP       	OUTPUT {OUTPUT}.{CHIP.NAME}.force.fits   IMAGE     NONE       CHIP       TRUE      NONE
+PPSIM.SOURCES           OUTPUT {OUTPUT}.{CHIP.NAME}.cmf          CMF       NONE       CHIP       TRUE      NONE
+PPSIM.FAKE.SOURCES     	OUTPUT {OUTPUT}.{CHIP.NAME}.fake.cmf     CMF       NONE       CHIP       TRUE      NONE
+PPSIM.FORCE.SOURCES    	OUTPUT {OUTPUT}.{CHIP.NAME}.force.cmf    CMF       NONE       CHIP       TRUE      NONE
+
 PPARITH.OUTPUT.IMAGE   	OUTPUT {OUTPUT}.fits                     IMAGE     COMP_IMG   CHIP       TRUE      NONE
 PPARITH.OUTPUT.MASK    	OUTPUT {OUTPUT}.fits                     MASK      COMP_MASK  CHIP       TRUE      NONE
 		       										        
+PPSKYCELL.JPEG1    	OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg     JPEG      NONE       CHIP       TRUE      NONE
+PPSKYCELL.JPEG2    	OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg     JPEG      NONE       CHIP       TRUE      NONE
+
 LOG.IMFILE             	OUTPUT {OUTPUT}.{CHIP.NAME}.log          TEXT      NONE       CHIP       TRUE      NONE
 LOG.EXP                	OUTPUT {OUTPUT}.log                      TEXT      NONE       FPA        TRUE      NONE
@@ -226,4 +240,5 @@
 TRACE.IMFILE           	OUTPUT {OUTPUT}.{CHIP.NAME}.trace        TEXT      NONE       CHIP       TRUE      NONE
 TRACE.EXP              	OUTPUT {OUTPUT}.trace                    TEXT      NONE       FPA        TRUE      NONE
+
 
 # {FPA.OBS}
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/jpeg.mdc
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/jpeg.mdc	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/jpeg.mdc	(revision 24244)
@@ -6,9 +6,16 @@
 # * SCALE.MAX: Maximum for scaling
 
-PPIMAGE.JPEG1	METADATA
-	COLORMAP	STR	-greyscale
-	SCALE.MODE	STR	RANGE
-	SCALE.MIN	F32	 -5.0
-	SCALE.MAX	F32	+10.0
+PPIMAGE.JPEG1  	        METADATA
+	COLORMAP	STR	-greyscale
+	SCALE.MODE	STR	RANGE
+	SCALE.MIN	F32	 -5.0
+	SCALE.MAX	F32	+10.0
+END
+
+PPIMAGE.JPEG1.HEAT	METADATA
+	COLORMAP	STR	heat
+	SCALE.MODE	STR	FRACTION
+	SCALE.MIN	F32	0.95
+	SCALE.MAX	F32	1.05
 END
 
@@ -198,2 +205,18 @@
 	END
 END
+
+
+PPSKYCELL.JPEG1		METADATA
+	COLORMAP	STR	-greyscale
+	SCALE.MODE	STR	FRACTION
+	SCALE.MIN	F32	0.95
+	SCALE.MAX	F32	1.05
+END
+
+PPSKYCELL.JPEG2		METADATA
+	COLORMAP	STR	-greyscale
+	SCALE.MODE	STR	FRACTION
+	SCALE.MIN	F32	0.95
+	SCALE.MAX	F32	1.05
+END
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppImage.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppImage.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppImage.config	(revision 24244)
@@ -13,4 +13,6 @@
 MASK               BOOL    FALSE           # Mask bad pixels
 MASK.BUILD         BOOL    FALSE           # Build internal mask image
+MASK.SATURATED     BOOL    TRUE            # Mask the saturated pixels
+MASK.LOW           BOOL    TRUE            # Mask pixels below valid range
 VARIANCE.BUILD     BOOL    FALSE           # Build internal variance image
 FRINGE             BOOL    FALSE           # Fringe subtraction
@@ -35,4 +37,11 @@
 BIN1.JPEG          BOOL    FALSE           # Save 1st binned jpeg?
 BIN2.JPEG          BOOL    FALSE           # Save 2nd binned jpeg?
+
+# crosstalk measurements : only valid for GPC1 for now
+CROSSTALK.MEASURE  BOOL    FALSE           # Subtract model background?
+CROSSTALK.CORRECT  BOOL    FALSE           # Subtract model background?
+
+# apply the cell parity flips to the cells?
+APPLY.CELL.PARITY  BOOL    FALSE
 
 ## if we use multithreaded detrending, detrend this number of rows per thread
@@ -134,4 +143,27 @@
 END
 
+# Overscan subtraction only
+QUICKMOSAIC        METADATA
+  BASE.FITS        BOOL    FALSE           # Save base detrended image?
+  BASE.MASK.FITS   BOOL    FALSE           # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL  FALSE           # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL  FALSE           # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    FALSE           # Bias subtraction
+  DARK             BOOL    FALSE           # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    FALSE           # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS        BOOL    FALSE           # Save 1st binned chip image?
+  BIN2.FITS        BOOL    FALSE           # Save 2nd binned chip image?
+  FPA1.FITS        BOOL    TRUE            # Save 1st binned fpa image? 
+  FPA2.FITS        BOOL    TRUE            # Save 2nd binned fpa image? 
+END
 
 # No operation except rebinning
@@ -419,4 +451,33 @@
   BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
   BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+END
+
+# Overscan, bias, dark -- search for DARK_PREMASK
+PPIMAGE_DARKMASK_PROCESS METADATA
+  BASE.FITS        BOOL    TRUE            # Save base detrended image?
+  BASE.MASK.FITS   BOOL    TRUE            # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL    TRUE            # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL    FALSE           # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    TRUE            # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    FALSE           # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+
+  DETREND.CONSTRAINTS  METADATA
+    DARK METADATA
+      DETTYPE STR DARK_PREMASK
+      EXPTIME STR FPA.EXPOSURE
+    END
+  END   
 END
 
@@ -463,4 +524,5 @@
   BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?
   BIN2.FITS          BOOL    TRUE            # Save 2nd binned chip image?
+  MASK.SATURATED     BOOL    FALSE           # DO NOT Mask the saturated pixels
 
   DETREND.CONSTRAINTS  METADATA
@@ -491,4 +553,37 @@
   BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
   BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+END
+
+# Overscan, bias, dark, shutter, flat-field -- use FLAT_PREMASK
+PPIMAGE_FLATMASK_PROCESS      METADATA
+  BASE.FITS        BOOL    TRUE            # Save base detrended image?
+  BASE.MASK.FITS   BOOL    TRUE            # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL  TRUE            # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image? 
+  CHIP.VARIANCE.FITS BOOL  FALSE           # Save chip-mosaic-ed image? 
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    TRUE            # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    TRUE            # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+
+  DETREND.CONSTRAINTS  METADATA
+    DARK METADATA
+      DETTYPE STR DARK_PREMASK
+      EXPTIME STR FPA.EXPOSURE
+    END
+    FLAT METADATA
+      DETTYPE STR FLAT_PREMASK
+      FILTER  STR FPA.FILTERID
+    END
+  END   
 END
 
@@ -1444,4 +1539,15 @@
   BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
   CHECK.CTE        BOOL    TRUE            # measure CTE errors?
+
+  DETREND.CONSTRAINTS  METADATA
+    DARK METADATA
+      DETTYPE STR DARK_PREMASK
+      EXPTIME STR FPA.EXPOSURE
+    END
+    FLAT METADATA
+      DETTYPE STR FLAT_PREMASK
+      FILTER  STR FPA.FILTERID
+    END
+  END   
 END
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppMerge.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppMerge.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppMerge.config	(revision 24244)
@@ -1,50 +1,57 @@
 # Recipe configuration for ppMerge
 
-ROWS            S32     128		# Number of rows to read at once
-ELECTRONS       F32     100.0           # Minimum number of electrons for useful signal
-SAMPLE          S32     100000          # Sampling factor for measuring the background
-REJ		F32	3.0		# Rejection threshold (sigma)
-ITER		S32	2		# Number of rejection iterations
-FRACHIGH	F32	0.0		# Fraction of high pixels to reject immediately
-FRACLOW		F32	0.0		# Fraction of low pixels to reject immediately
-NKEEP		S32	5		# Minimum number of pixels in stack to keep
-VARIANCES	BOOL	FALSE		# Use image variances in combination?
-FRINGE.NUM	S32	10000		# Number of fringe regions
-FRINGE.SIZE	S32	5		# Half-size of fringe regions
-FRINGE.XSMOOTH	S32	5		# Number of smoothing regions in x
-FRINGE.YSMOOTH	S32	11		# Number of smoothing regions in y
-CTE.MIN         F32	0.2		# regions lower than this in the CTE image are masked
-SHUTTER.SIZE	S32	128		# Size for shutter measurement regions
-MASK.SUSPECT	F32	5.0		# Threshold for suspect pixels (sigma)
-MASK.BAD	F32	0.2		# Threshold for bad pixels
-MASK.MODE	STR	FRACTION	# Mode for identifying bad pixels in the suspect map
-MASK.CHIPSTATS	BOOL	TRUE		# Measure stats for masking by chip (otherwise by readout)?
-MASK.GROW	S32	0		# Grow bad pixels by this radius
-MASK.SET.VALUE	STR	FLAT		# set this bit in the output mask
+ROWS                S32     128             # Number of rows to read at once
+ELECTRONS           F32     100.0           # Minimum number of electrons for useful signal
+SAMPLE              S32     100000          # Sampling factor for measuring the background
+REJ                 F32     3.0             # Rejection threshold (sigma)
+ITER                S32     2               # Number of rejection iterations
+FRACHIGH            F32     0.0             # Fraction of high pixels to reject immediately
+FRACLOW             F32     0.0             # Fraction of low pixels to reject immediately
+NKEEP               S32     5               # Minimum number of pixels in stack to keep
+VARIANCES           BOOL    FALSE           # Use image variances in combination?
+FRINGE.NUM          S32     10000           # Number of fringe regions
+FRINGE.SIZE         S32     5               # Half-size of fringe regions
+FRINGE.XSMOOTH      S32     5               # Number of smoothing regions in x
+FRINGE.YSMOOTH      S32     11              # Number of smoothing regions in y
+CTE.MIN             F32     0.5             # regions lower than this in the CTE image are masked
+SHUTTER.SIZE        S32     128             # Size for shutter measurement regions
 
-COMBINE		STR	CLIPPED		# Statistic to use for combination
-MEAN		STR	ROBUST_MEDIAN	# Statistic to use to measure the mean
-STDEV		STR	ROBUST_STDEV	# Statistic to use to measure the stdev
+INPUT.MASKS.USE	BOOL	TRUE		# respect the input mask data?
 
-MASK.SMOOTH.SUSPECT BOOL TRUE           # smooth the suspect-pixel image before making mask
-MASK.SMOOTH.SCALE   F32  3.0            # sigma (pixels) of smoothing kernel
+COMBINE             STR     CLIPPED         # Statistic to use for combination
+MEAN                STR     ROBUST_MEDIAN   # Statistic to use to measure the mean
+STDEV               STR     ROBUST_STDEV    # Statistic to use to measure the stdev
 
-STATS.BY.CHIP   BOOL    TRUE            # measure stats for masking by chip (or by readout)
-MASK.GROW.NPIX  S32     3               # measure stats for masking by chip (or by readout)
+MASK.SUSPECT.MODE   STR     SIGMA           # how to identify suspect pixels: SIGMA, VALUE
+MASK.SUSPECT.MIN    F32    -100.0           # for MASK.SUSPECT.MODE == VALUE, below this is suspect
+MASK.SUSPECT.MAX    F32    +100.0           # for MASK.SUSPECT.MODE == VALUE, above this is suspect
+MASK.SUSPECT.SIGMA  F32     5.0             # Threshold for suspect pixels (sigma)
+
+MASK.SMOOTH.SUSPECT BOOL    TRUE            # smooth the suspect-pixel image before making mask
+MASK.SMOOTH.SCALE   F32     3.0             # sigma (pixels) of smoothing kernel
+
+MASK.BAD            F32     0.2             # Threshold for bad pixels
+MASK.MODE           STR     FRACTION        # Mode for identifying bad pixels in the suspect map
+MASK.CHIPSTATS      BOOL    TRUE            # Measure stats for masking by chip (otherwise by readout)?
+MASK.GROW           S32     0               # Grow bad pixels by this radius
+MASK.SET.VALUE      STR     FLAT            # set this bit in the output mask
+
+STATS.BY.CHIP       BOOL    TRUE            # measure stats for masking by chip (or by readout)
+MASK.GROW.NPIX      S32     3               # measure stats for masking by chip (or by readout)
 
 # Ordinates for fitting dark current
-DARK.ORDINATES	METADATA
-	CELL.DARKTIME	S32	1	# Traditional dark current term
+DARK.ORDINATES  METADATA
+        CELL.DARKTIME   S32     1       # Traditional dark current term
 END
-DARK.NORM	STR	NONE		# Dark normalisation concept
+DARK.NORM       STR     NONE            # Dark normalisation concept
 
 # Bias combination --- don't want min/max rejection
-PPMERGE_BIAS	METADATA
-	REJ		F32	3.0		# Rejection threshold (sigma)
-	ITER		S32	2		# Number of rejection iterations
-	FRACHIGH	F32	0.0		# Fraction of high pixels to reject immediately
-	FRACLOW		F32	0.0		# Fraction of low pixels to reject immediately
-	VARIANCES	BOOL	FALSE		# Use image variances?
-	COMBINE		STR	CLIPPED		# Statistic to use for combination: 
+PPMERGE_BIAS    METADATA
+        REJ             F32     3.0             # Rejection threshold (sigma)
+        ITER            S32     2               # Number of rejection iterations
+        FRACHIGH        F32     0.0             # Fraction of high pixels to reject immediately
+        FRACLOW         F32     0.0             # Fraction of low pixels to reject immediately
+        VARIANCES       BOOL    FALSE           # Use image variances?
+        COMBINE         STR     CLIPPED         # Statistic to use for combination: 
 END
 
@@ -52,45 +59,45 @@
 # Dark combination --- don't want min/max rejection
 # More aggressive clipping than bias, so as to remove CRs
-PPMERGE_DARK	METADATA
-	REJ		F32	3.0		# Rejection threshold (sigma)
-	ITER		S32	2		# Number of rejection iterations
-	FRACHIGH	F32	0.0		# Fraction of high pixels to reject immediately
-	FRACLOW		F32	0.0		# Fraction of low pixels to reject immediately
-	VARIANCES	BOOL	TRUE		# Use image variances?
-	COMBINE		STR	CLIPPED		# Statistic to use for combination: 
+PPMERGE_DARK    METADATA
+        REJ             F32     3.0             # Rejection threshold (sigma)
+        ITER            S32     2               # Number of rejection iterations
+        FRACHIGH        F32     0.0             # Fraction of high pixels to reject immediately
+        FRACLOW         F32     0.0             # Fraction of low pixels to reject immediately
+        VARIANCES       BOOL    TRUE            # Use image variances?
+        COMBINE         STR     CLIPPED         # Statistic to use for combination: 
 END
 
 # Flat combination --- use min/max rejection
-PPMERGE_FLAT	METADATA
-	REJ		F32	3.0		# Rejection threshold (sigma)
-	ITER		S32	1		# Number of rejection iterations
-	FRACHIGH	F32	0.3		# Fraction of high pixels to reject immediately
-	FRACLOW		F32	0.1		# Fraction of low pixels to reject immediately
-	NKEEP		S32	5		# Minimum number of pixels in stack to keep
-	VARIANCES	BOOL	TRUE		# Use image variances?
-	COMBINE		STR	MEAN		# Statistic to use for combination: 
+PPMERGE_FLAT    METADATA
+        REJ             F32     3.0             # Rejection threshold (sigma)
+        ITER            S32     1               # Number of rejection iterations
+        FRACHIGH        F32     0.3             # Fraction of high pixels to reject immediately
+        FRACLOW         F32     0.1             # Fraction of low pixels to reject immediately
+        NKEEP           S32     5               # Minimum number of pixels in stack to keep
+        VARIANCES       BOOL    TRUE            # Use image variances?
+        COMBINE         STR     MEAN            # Statistic to use for combination: 
 END
 
 
 # Fringe combination --- already included in default, above
-PPMERGE_FRINGE	METADATA
-	FRACHIGH	F32	0.1		# Fraction of high pixels to reject immediately
-	VARIANCES	BOOL	TRUE		# Use image variances?
+PPMERGE_FRINGE  METADATA
+        FRACHIGH        F32     0.1             # Fraction of high pixels to reject immediately
+        VARIANCES       BOOL    TRUE            # Use image variances?
 END
 
 # Mask generation --- already included in default, above
 PPMERGE_DARKMASK METADATA
-	ITER		S32	2		# Number of iterations
-	MASK.BAD	F32	0.2		# Threshold for bad pixels (sigma)
-	MASK.MODE	STR	FRACTION	# Mode for identifying bad pixels in the suspect map
-        MASK.SET.VALUE	STR	DARK		# set this bit in the output mask
+        ITER            S32     2               # Number of iterations
+        MASK.BAD        F32     0.2             # Threshold for bad pixels (sigma)
+        MASK.MODE       STR     FRACTION        # Mode for identifying bad pixels in the suspect map
+        MASK.SET.VALUE  STR     DARK            # set this bit in the output mask
 END
 
 # Mask generation --- already included in default, above
 PPMERGE_FLATMASK METADATA
-	ITER		S32	2		# Number of iterations
-	MASK.BAD	F32	0.2		# Threshold for bad pixels (sigma)
-	MASK.MODE	STR	FRACTION	# Mode for identifying bad pixels in the suspect map
-        MASK.SET.VALUE	STR	FLAT		# set this bit in the output mask
+        ITER            S32     2               # Number of iterations
+        MASK.BAD        F32     0.2             # Threshold for bad pixels (sigma)
+        MASK.MODE       STR     FRACTION        # Mode for identifying bad pixels in the suspect map
+        MASK.SET.VALUE  STR     FLAT            # set this bit in the output mask
 END
 
@@ -98,17 +105,18 @@
 # but it then sets a mask based on the resulting data values (using CTE.MIN)
 PPMERGE_CTEMASK METADATA
-	REJ		F32	3.0		# Rejection threshold (sigma)
-	ITER		S32	2		# Number of rejection iterations
-	FRACHIGH	F32	0.0		# Fraction of high pixels to reject immediately
-	FRACLOW		F32	0.0		# Fraction of low pixels to reject immediately
-	VARIANCES	BOOL	FALSE		# Use image variances?
-	COMBINE		STR	CLIPPED		# Statistic to use for combination: 
-        CTE.MIN         F32	0.2		# regions lower than this in the CTE image are masked
-        MASK.SET.VALUE	STR	CTE		# set this bit in the output mask
+        REJ             F32     3.0             # Rejection threshold (sigma)
+        ITER            S32     2               # Number of rejection iterations
+        FRACHIGH        F32     0.0             # Fraction of high pixels to reject immediately
+        FRACLOW         F32     0.0             # Fraction of low pixels to reject immediately
+        VARIANCES       BOOL    FALSE           # Use image variances?
+        COMBINE         STR     CLIPPED         # Statistic to use for combination: 
+        CTE.MIN         F32     0.5             # regions lower than this in the CTE image are masked
+        MASK.SET.VALUE  STR     CTE             # set this bit in the output mask
+	INPUT.MASKS.USE	BOOL	FALSE		# respect the input mask data?
 END
 
 # Shutter generation --- already included in default, above
-PPMERGE_SHUTTER	METADATA
-	REJ		F32	2.0		# Rejection threshold (sigma)
-	ITER		S32	1		# Number of rejection iterations
+PPMERGE_SHUTTER METADATA
+        REJ             F32     2.0             # Rejection threshold (sigma)
+        ITER            S32     1               # Number of rejection iterations
 END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSim.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSim.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSim.config	(revision 24244)
@@ -5,4 +5,6 @@
 
 IMAGE.TYPE      STR     BIAS            # default image type
+
+OBS_MODE        STR     NONE
 
 BIAS            BOOL    TRUE
@@ -86,25 +88,10 @@
 
 # filter-dependent parameters
-ZEROPTS MULTI
-
 ZEROPTS  METADATA
-  FILTER    STR  g
-  ZERO_PT   F32  24.0
-END
-ZEROPTS  METADATA
-  FILTER    STR  r
-  ZERO_PT   F32  25.15
-END
-ZEROPTS  METADATA
-  FILTER    STR  i
-  ZERO_PT   F32  25.00
-END
-ZEROPTS  METADATA
-  FILTER    STR  z
-  ZERO_PT   F32  24.50
-END
-ZEROPTS  METADATA
-  FILTER    STR  y
-  ZERO_PT   F32  24.00
+	g	F32	24.00
+	r	F32	25.15
+	i	F32	25.00
+	z	F32	24.50
+	y	F32	24.00
 END
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSkycell.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSkycell.config	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSkycell.config	(revision 24244)
@@ -0,0 +1,6 @@
+### Recipe file for ppSkycell
+
+MASKVAL		STR	MASK.VALUE	# Symbolic name of values to mask
+BIN1		S32	4		# Binning factor for output 1
+BIN2		S32	4		# Binning factor for output 2
+
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStack.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStack.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStack.config	(revision 24244)
@@ -4,5 +4,5 @@
 ITER		S32	1		# Number of rejection iterations
 COMBINE.REJ	F32	3.0		# Rejection threshold in combination (sigma)
-COMBINE.SYS	F32	0.08		# Relative systematic error in combination
+COMBINE.SYS	F32	0.03		# Relative systematic error in combination
 COMBINE.DISCARD	F32	0.2		# Discard fraction for Olympic weighted mean
 MASK.VAL	STR	MASK.VALUE,CONV.BAD	# Mask value of input bad pixels
@@ -11,5 +11,5 @@
 POOR.FRACTION	F32	0.01		# Maximum fraction of bad variance for poor pixels
 THRESHOLD.MASK	F32	0.5		# Threshold for mask deconvolution (0..1)
-DECONV.LIMIT	F32	1.03		# Deconvolution fraction for rejecting entire image
+DECONV.LIMIT	F32	5.0		# Deconvolution fraction for rejecting entire image
 IMAGE.REJ	F32	0.1		# Rejected pixel fraction threshold for rejecting entire image
 MATCH.REJ	F32	2.5		# Rejection threshold for chi^2 values from matching
@@ -53,11 +53,4 @@
 END
 	
-
-### The following is obsolete
-#SOURCE.RADIUS	F32	1.0		# Radius (pixels) for matching sources
-#SOURCE.MIN	S32	15		# Minimum number of sources for merging source lists
-#SOURCE.ITER	S32	2		# Number of rejection iterations for magnitude difference
-#SOURCE.REJ	F32	2.0		# Rejection limit (sigma) for magnitude difference
-
 PSF.INSTANCES	S32	15		# Number of instances for PSF generation
 PSF.RADIUS	F32	20.0		# Radius for PSF generation
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStats.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStats.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStats.config	(revision 24244)
@@ -7,6 +7,5 @@
 # Mask value to use for statistics.  this is only used for stand-alone
 # operations.  ppStats library calls pass a mask value as needed
-MASKVAL         STR     SAT,BAD 
-
+MASKVAL         STR     SAT,LOW 
 
 # Define the outputs as MULTI
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStatsFromMetadata.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStatsFromMetadata.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStatsFromMetadata.config	(revision 24244)
@@ -50,4 +50,5 @@
   ENTRY  VAL  ROBUST_MEDIAN    	  F32  ROBUST_STDEV      -bg_mean_stdev      
   ENTRY  VAL  ROBUST_STDEV     	  F32  RMS               -bg_stdev           
+  ENTRY  VAL  QUALITY             S32  CONSTANT          -quality             # Bad quality flag
 END
 
@@ -168,4 +169,5 @@
   ENTRY  VAL  IQ_M2S_L            F32  SAMPLE_MEAN      -iq_m2s_lq            
   ENTRY  VAL  IQ_M2S_U            F32  SAMPLE_MEAN      -iq_m2s_uq            
+  ENTRY  VAL  QUALITY             S32  CONSTANT          -quality             # Bad quality flag
 END
 
@@ -225,4 +227,6 @@
   ENTRY  VAL  iq_m2s              F32  UQ               -iq_m2s_uq            
   ENTRY  VAL  iq_m2s              F32  LQ               -iq_m2s_lq            
+
+  ENTRY  VAL  QUALITY             S32  CONSTANT          -quality             # Bad quality flag
 END
 
@@ -253,4 +257,5 @@
   ENTRY  VAL  RANGE.YMAX          S32  CONSTANT          -ymax          
   ENTRY  VAL  ACCEPT              BOOL CONSTANT          -accept
+  ENTRY  VAL  QUALITY             S32  CONSTANT          -quality             # Bad quality flag
 END
 
@@ -279,4 +284,5 @@
   ENTRY  VAL  NUM_SOURCES         S32  SUM               -sources                       
   ENTRY  VAL  GOOD_PIXEL_FRAC     F32  MEAN              -good_frac                     
+  ENTRY  VAL  QUALITY             S32  CONSTANT          -quality             # Bad quality flag
 END
 
@@ -284,21 +290,23 @@
   ENTRY MULTI
   TYPE   VAL  KEYWORD               TYPE STATISTIC         FLAG 
-  ENTRY  VAL  ROBUST_MEDIAN         F32  ROBUST_MEDIAN     -bg             
-  ENTRY  VAL  ROBUST_STDEV          F32  RMS               -bg_stdev       
-  ENTRY  VAL  TIME_SUB              F32  SUM               -dtime_diff     
-  ENTRY  VAL  TIME_MATCH            F32  SUM               -dtime_match    
-  ENTRY  VAL  TIME_PHOT             F32  SUM               -dtime_phot     
-  ENTRY  VAL  SUBTRACTION.STAMPS    S32  SAMPLE_MEAN       -stamps_num     
-  ENTRY  VAL  SUBTRACTION.DEV.MEAN  F32  SAMPLE_MEAN       -stamps_mean    
-  ENTRY  VAL  SUBTRACTION.DEV.RMS   F32  SAMPLE_MEAN       -stamps_rms     
-  ENTRY  VAL  SUBTRACTION.NORM      F32  SAMPLE_MEAN       -norm           
-  ENTRY  VAL  SUBTRACTION.BGDIFF    F32  SAMPLE_MEAN       -bg_diff        
-  ENTRY  VAL  SUBTRACTION.MX        F32  SAMPLE_MEAN       -kernel_x       
-  ENTRY  VAL  SUBTRACTION.MY        F32  SAMPLE_MEAN       -kernel_y       
-  ENTRY  VAL  SUBTRACTION.MXX       F32  SAMPLE_MEAN       -kernel_xx      
-  ENTRY  VAL  SUBTRACTION.MXY       F32  SAMPLE_MEAN       -kernel_xy      
-  ENTRY  VAL  SUBTRACTION.MYY       F32  SAMPLE_MEAN       -kernel_yy      
-  ENTRY  VAL  NUM_SOURCES           S32  SUM               -sources        
-  ENTRY  VAL  GOOD_PIXEL_FRAC       F32  MEAN              -good_frac      
+  ENTRY  VAL  ROBUST_MEDIAN          F32  ROBUST_MEDIAN     -bg             
+  ENTRY  VAL  ROBUST_STDEV           F32  RMS               -bg_stdev       
+  ENTRY  VAL  TIME_SUB               F32  SUM               -dtime_diff     
+  ENTRY  VAL  TIME_MATCH             F32  SUM               -dtime_match    
+  ENTRY  VAL  TIME_PHOT              F32  SUM               -dtime_phot     
+  ENTRY  VAL  SUBTRACTION.STAMPS     S32  SAMPLE_MEAN       -stamps_num     
+  ENTRY  VAL  SUBTRACTION.DEV.MEAN   F32  SAMPLE_MEAN       -stamps_mean    
+  ENTRY  VAL  SUBTRACTION.DEV.RMS    F32  SAMPLE_MEAN       -stamps_rms     
+  ENTRY  VAL  SUBTRACTION.NORM       F32  SAMPLE_MEAN       -norm           
+  ENTRY  VAL  SUBTRACTION.BGDIFF     F32  SAMPLE_MEAN       -bg_diff        
+  ENTRY  VAL  SUBTRACTION.MX         F32  SAMPLE_MEAN       -kernel_x       
+  ENTRY  VAL  SUBTRACTION.MY         F32  SAMPLE_MEAN       -kernel_y       
+  ENTRY  VAL  SUBTRACTION.MXX        F32  SAMPLE_MEAN       -kernel_xx      
+  ENTRY  VAL  SUBTRACTION.MXY        F32  SAMPLE_MEAN       -kernel_xy      
+  ENTRY  VAL  SUBTRACTION.MYY        F32  SAMPLE_MEAN       -kernel_yy      
+  ENTRY  VAL  SUBTRACTION.DECONV.MAX F32  SAMPLE_MEAN       -deconv_max
+  ENTRY  VAL  NUM_SOURCES            S32  SUM               -sources        
+  ENTRY  VAL  GOOD_PIXEL_FRAC        F32  MEAN              -good_frac      
+  ENTRY  VAL  QUALITY                S32  CONSTANT          -quality             # Bad quality flag
 END
 
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSub.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSub.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSub.config	(revision 24244)
@@ -14,5 +14,4 @@
 SYS		F32	0.05		# Relative systematic error in kernel
 
-# EAM : we now ignore these: BAD.WARP is always bad, POOR.WARP is always defined
 MASK.IN		STR	MASK.VALUE,CONV.BAD	# Mask value for input
 MASK.BAD        STR	CONV.BAD	# Mask value to give bad pixels
@@ -45,4 +44,5 @@
 INTERPOLATION	STR	LANCZOS3	# Interpolation mode for bad pixels
 
+INVERSE		BOOL	FALSE		# Generate inverse subtraction?
 PHOTOMETRY	BOOL	FALSE		# Perform photometry?
 
@@ -60,2 +60,8 @@
 DIFF	METADATA
 END
+
+# Difference of two warps
+WARPWARP	METADATA
+	INVERSE		BOOL	TRUE	# Generate inverse subtraction?
+	PHOTOMETRY	BOOL	TRUE	# Perform photometry?
+END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/psastro.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/psastro.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/psastro.config	(revision 24244)
@@ -1,5 +1,6 @@
 
-PSASTRO.SAVE.REFSTARS      BOOL FALSE
-PSASTRO.ONLY.REFSTARS      BOOL FALSE
+PSASTRO.SAVE.REFSTARS      BOOL FALSE  # save refstars in a separate FITS table
+PSASTRO.ONLY.REFSTARS      BOOL FALSE  # skip all but refstar matches
+PSASTRO.SAVE.REFMATCH      BOOL FALSE  # save refstar matches as table in output smf file
 
 # perform single-chip astrometry?
@@ -121,4 +122,11 @@
 REFSTAR_MASK_SATSPIKE_MAG_MAX   F32 -17.0
 REFSTAR_MASK_SATSPIKE_WIDTH     F32  10.0
+
+REFSTAR_MASK_GHOST              BOOL FALSE
+REFSTAR_COUNT_GHOSTS            BOOL FALSE
+GHOST_MODEL                     STR  NONE
+GHOST_MAX_MAG                   F32 -19.5
+
+REFSTAR_MASK_BLEED              BOOL FALSE
 REFSTAR_MASK_BLEED_MAG_MAX      F32 -15.0
 REFSTAR_MASK_BLEED_MAG_SLOPE    F32   5.0
@@ -167,2 +175,5 @@
 PHOTCODE.DATA METADATA
 END
+
+PSASTRO.EXTRACT METADATA
+END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/reductionClasses.mdc
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/reductionClasses.mdc	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/reductionClasses.mdc	(revision 24244)
@@ -97,5 +97,5 @@
 
 	# Generation of pixel masks from darks and flats
-	DARKMASK_PROCESS	STR	PPIMAGE_OBD
+	DARKMASK_PROCESS	STR	PPIMAGE_DARKMASK_PROCESS
 	DARKMASK_RESID		STR	PPIMAGE_M
 	DARKMASK_VERIFY		STR	PPIMAGE_M
@@ -104,5 +104,5 @@
 	DARKMASK_JPEG_RESID	STR	BIAS_RESID
 
-	FLATMASK_PROCESS	STR	PPIMAGE_OBDSF
+	FLATMASK_PROCESS	STR	PPIMAGE_FLATMASK_PROCESS
 	FLATMASK_RESID		STR	PPIMAGE_M
 	FLATMASK_VERIFY		STR	PPIMAGE_M
@@ -242,2 +242,11 @@
 	ADDSTAR		STR	ADDSTAR
 END
+
+# Intended for areas with high stellar density
+WARPWARP	METADATA
+	DIFF_PPSUB	STR	WARPWARP
+	DIFF_PSPHOT	STR	DIFF
+	JPEG_BIN1	STR	PPIMAGE_J1
+	JPEG_BIN2	STR	PPIMAGE_J2
+	ADDSTAR		STR	ADDSTAR
+END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/sdssmosaic/psastro.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/sdssmosaic/psastro.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/sdssmosaic/psastro.config	(revision 24244)
@@ -109,24 +109,25 @@
   FILTER   STR r
   ZEROPT   F32 24.0
-  PHOTCODE STR r
+  PHOTCODE STR r_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR i
   ZEROPT   F32 22.0
-  PHOTCODE STR i
+  PHOTCODE STR i_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR u
   ZEROPT   F32 23.8
-  PHOTCODE STR u
+  PHOTCODE STR g_SYNTH
+  # there is no u_SYNTH, so use g as a close substitute
 END
 PHOTCODE.DATA METADATA
   FILTER   STR z
   ZEROPT   F32 21.9
-  PHOTCODE STR z
+  PHOTCODE STR z_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR g
   ZEROPT   F32 24.4
-  PHOTCODE STR g
+  PHOTCODE STR g_SYNTH
 END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/sdssmosaic/psphot.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/sdssmosaic/psphot.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/sdssmosaic/psphot.config	(revision 24244)
@@ -10,4 +10,8 @@
 #SAVE.PLOTS     	BOOL    TRUE
 
+# for testing:
+# BREAK_POINT STR BACKMDL
+
+
 BACKGROUND.XBIN	    S32  256            # size of background superpixels
 BACKGROUND.YBIN	    S32  256            # size of background superpixels
@@ -20,6 +24,6 @@
 PEAKS_SMOOTH_SIGMA  F32   2.5            # smoothing kernel sigma in pixels
 PEAKS_NMAX          S32   5000           # on first pass, only keep NMAX peaks (0 == all)
-PEAKS_NSIGMA_LIMIT  F32   25.0            # peak significance threshold
-PEAKS_NSIGMA_LIMIT_2 F32  10.0             # peak significance threshold
+PEAKS_NSIGMA_LIMIT  F32   25.0           # peak significance threshold
+PEAKS_NSIGMA_LIMIT_2 F32   5.0           # peak significance threshold
 
 PSF_SN_LIM          F32  25              # minimum S/N for stars used for PSF model
@@ -28,7 +32,11 @@
 PSF_CLUMP_NY        S32   3               # selecting PSF stars
 PSF_MOMENTS_RADIUS  F32  12               # calculate initial source moments with this radius
+PSF_CLUMP_GRID_SCALE F32 2.5		# size of Sx,Sy image pixel
+
+PSF.RESIDUALS       BOOL  TRUE            # generate the residuals?
 
 # PSF model parameters : choose the PSF model desired
 PSF_MODEL           STR  PS_MODEL_PS1_V1
+PSPHOT.CR.NSIGMA.SOFTEN             F32   0.0025            # Softening parameter for weights
 
 # Use same default setting as in 2.6.1 (MAP is now global default)
@@ -37,16 +45,17 @@
 PSF.TREND.NY                        S32   0
 
-MOMENTS_SN_MIN      F32   30.0
 EXT_MIN_SN           F32  50.0           # fit galaxies above this S/N limit
-FULL_FIT_SN_LIM      F32  50.0
-AP_MIN_SN            F32  50.0
+FULL_FIT_SN_LIM      F32  25.0
+MOMENTS_SN_MIN      F32    5.0
+AP_MIN_SN            F32   5.0
 
-OUTPUT.FORMAT       STR PS1_DEV_1
+OUTPUT.FORMAT        STR  PS1_V1
 
 PSF_SHAPE_NSIGMA     F32  3.0		 # max significance for shape variation
 PSF_MAX_CHI          F32  50.0		 # reject objects worse that this
-APTREND.ORDER.MAX    S32  1
 
 MEASURE.APTREND      BOOL TRUE           ### XXX for now, skip this (too many errors)
+APTREND.ORDER.MAX    S32  3
+
 # BREAK_POINT STR PASS1
 
@@ -76,4 +85,5 @@
  PSF.TREND.NY		S32	0
  MEASURE.APTREND        BOOL  TRUE
+ APTREND.ORDER.MAX    	S32   3 # used to be 1
 END
 
@@ -83,3 +93,9 @@
  PSF.TREND.NY		S32	3
  MEASURE.APTREND        BOOL  TRUE
+ APTREND.ORDER.MAX    	S32   3 # used to be 1
 END
+
+DIFF	METADATA
+	BACKGROUND.XBIN	    S32  32            # size of background superpixels
+	BACKGROUND.YBIN	    S32  32            # size of background superpixels
+END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/format.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/format.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/format.config	(revision 24244)
@@ -40,4 +40,5 @@
 	FPA.DEC		STR	DEC
 	FPA.OBJECT	STR	OBJECT
+        FPA.OBS.MODE    STR     OBS_MODE
 	FPA.TIME	STR	DATE-OBS UTC-OBS	# Date and time
 	FPA.EXPOSURE	STR	EXPTIME
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/ppStack.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/ppStack.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/ppStack.config	(revision 24244)
@@ -1,1 +1,2 @@
 PSF.MODEL	STR	PS_MODEL_GAUSS	# Model for PSF generation
+DECONV.LIMIT	F32	1.5		# Deconvolution fraction for rejecting entire image
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/ppSub.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/ppSub.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/ppSub.config	(revision 24244)
@@ -1,27 +1,1 @@
 ### Recipe file for ppSub
-
-KERNEL.TYPE	STR	RINGS		# Kernel type to use (POIS|ISIS|SPAM|FRIES|GUNK|RINGS)
-KERNEL.SIZE     S32	15		# Kernel half-size (pixels)
-SPATIAL.ORDER   S32	1 		# Spatial polynomial order
-REGION.SIZE	F32	0		# Iso-kernel region size (pixels)
-STAMP.SPACING   F32	200		# Typical spacing between stamps (pixels)
-STAMP.FOOTPRINT S32	35		# Size of stamps (pixels)
-STAMP.THRESHOLD F32	0		# Flux threshold for stamps (ADU)
-ITER            S32	2		# Number of rejection iterations
-REJ             F32	2.		# Rejection level (std dev)
-MASK.IN		STR	MASK.VALUE	# Mask value for input
-MASK.BAD        STR	BLANK		# Mask value to give bad pixels
-MASK.POOR       STR	POOR.WARP	# Mask value to give poor pixels
-POOR.FRACTION	F32	0.1		# Maximum fraction of bad weight for poor pixels
-BADFRAC		F32	0.8		# Maximum fraction of bad pixels
-@ISIS.WIDTHS	F32	1 3 5		# Gaussian FWHMs for ISIS kernels
-@ISIS.ORDERS	S32	2 2 2		# Polynomial orders for ISIS kernels
-INNER		S32	5		# Inner half-size for SPAM and FRIES kernels
-RINGS.ORDER	S32	2		# Polynomial order for RINGS kernels
-PENALTY		F32	1.0		# Penalty for wideness
-
-### Modifications to use when stacking data
-STACK	METADATA
-	KERNEL.TYPE	STR	RINGS	# Kernel type to use (POIS|ISIS|SPAM|FRIES|GUNK|RINGS)
-	KERNEL.SIZE     S32     15      # Kernel half-size (pixels)
-END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/psastro.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/psastro.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/simtest/psastro.config	(revision 24244)
@@ -17,19 +17,19 @@
   FILTER   STR g
   ZEROPT   F32 24.0
-  PHOTCODE STR g
+  PHOTCODE STR g_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR r
   ZEROPT   F32  25.15
-  PHOTCODE STR r
+  PHOTCODE STR r_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR i
   ZEROPT   F32  25.00
-  PHOTCODE STR i
+  PHOTCODE STR i_SYNTH
 END
 PHOTCODE.DATA METADATA
   FILTER   STR z
   ZEROPT   F32  24.50
-  PHOTCODE STR z
+  PHOTCODE STR z_SYNTH
 END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/site.config.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/site.config.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/site.config.in	(revision 24244)
@@ -55,9 +55,14 @@
 DBPORT          S32     0                       # Database port (for psDBInit); 0 = default
 
+# DataStore Database configuration
+DS_DBSERVER     STR     XXX
+DS_DBNAME       STR     XXX
+DS_DBUSER       STR     XXX
+DS_DBPASSWORD   STR     XXX
+
+# root directory of the data store
+DATA_STORE_ROOT STR PATH
+
+PSTAMP_DATA_STORE_ROOT STR PATH
+
 # other basic values:
-# XXX is TIME still needed / used?
-# XXX use autoconf to put this in a known location?
-# TIME          STR     psTime.config   # Time configuration file
-
-PSTAMP_DATA_STORE_ROOT STR /data/ipp000.0/datastore/dsroot # Data store root directory
-
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/system.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/system.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/system.config	(revision 24244)
@@ -51,3 +51,4 @@
  	PPSTATS_METADATA STR		recipes/ppStatsFromMetadata.config # Image statistics
  	DVOCORR         STR		recipes/dvocorr.config # Image statistics
+	PPSKYCELL	STR		recipes/ppSkycell.config	# JPEGs of skycells
 END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/vysos5/format_flipped.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/vysos5/format_flipped.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/vysos5/format_flipped.config	(revision 24244)
@@ -22,7 +22,7 @@
 	amplifier	METADATA
 		CELL.TRIMSEC.SOURCE	STR	VALUE
-		CELL.TRIMSEC		STR	[1:4098,1:4087]
+		CELL.TRIMSEC		STR	[1:4098,1:4089]
 		CELL.BIASSEC.SOURCE	STR	VALUE
-		CELL.BIASSEC		STR	[1:4098,4089:4098]
+		CELL.BIASSEC		STR	[1:4098,4090:4098]
 	END
 END
Index: /branches/cnb_branches/cnb_branch_20090301/ippconfig/vysos5/ppImage.config
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ippconfig/vysos5/ppImage.config	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ippconfig/vysos5/ppImage.config	(revision 24244)
@@ -23,4 +23,7 @@
 ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
 
+# apply the cell parity flips to the cells?
+APPLY.CELL.PARITY  BOOL    TRUE
+
 # binned output image options
 BIN1.XBIN               S32     8
Index: /branches/cnb_branches/cnb_branch_20090301/magic/README
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/magic/README	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/magic/README	(revision 24244)
@@ -1,8 +1,9 @@
-To build The remove streaks portions of magic
+To build the DetectStreaks portion of magic
 
-tar zxf ~sydney/magic.tgz
-tar xzf ~sydney/ssa-core-cpp.tgz
+tar xf ~sydney/magic.tgz
+tar xf ~sydney/ssa-core-cpp.tgz
 
 cd ssa-core-cpp
+make distclean
 ./configure
 make
@@ -11,4 +12,6 @@
 make install
 
+# to build streaksremove and related tools
+
 cd ../remove/src
 make install
Index: /branches/cnb_branches/cnb_branch_20090301/magic/magic/makefile
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/magic/magic/makefile	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/magic/magic/makefile	(revision 24244)
@@ -4,5 +4,5 @@
 # XXX integrate with the build system
 #
-TARGETS = RemoveStreaks
+TARGETS = DetectStreaks
 SSA_SRC_DIR = ../ssa-core-cpp/src
 CFITSIO_DIR = ../cfitsio
@@ -12,6 +12,6 @@
 
 CXX = c++
+CXXFLAGS = -Wall -g
 CXXFLAGS = -Wall -O3 -DNDEBUG
-#CXXFLAGS = -Wall -g
 
 SRCS = $(TARGETS).cpp $(SSA_SRC_DIR)/math/Constants.cpp
@@ -24,9 +24,9 @@
 
 clean :
-	   - rm $(TARGETS)
+	   -rm -f $(TARGETS)
 
 DESTDIR=$(PSCONFDIR)/$(PSCONFIG)
 
 install:	$(TARGETS)
-	cp RemoveStreaks $(DESTDIR)/bin
-	chmod 755  $(DESTDIR)/bin/RemoveStreaks
+	cp DetectStreaks $(DESTDIR)/bin
+	chmod 755  $(DESTDIR)/bin/DetectStreaks
Index: /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/Makefile.simple
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/Makefile.simple	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/Makefile.simple	(revision 24244)
@@ -28,5 +28,6 @@
     streaksrelease.o
 
-STREAKSFLAGS=-DSTREAKS_COMPRESS_OUTPUT=1
+# STREAKSFLAGS=-DSTREAKS_COMPRESS_OUTPUT=1
+OPTFLAGS= -g -O2
 OPTFLAGS= -g
 CFLAGS=`psmodules-config --cflags` -std=gnu99 ${OPTFLAGS} ${STREAKSFLAGS}
Index: /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksio.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksio.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksio.c	(revision 24244)
@@ -6,4 +6,5 @@
 static void streakFilesFree(streakFiles *sf);
 static void freeImages(streakFiles *);
+static void setExciseValue(sFile *in);
 
 static nebServer *ourNebServer = NULL;
@@ -12,5 +13,5 @@
 // if remove is true the calling program is streaksremove and the recovery files are outputs
 // if false the recovery files are inputs
-streakFiles *openFiles(pmConfig *config, bool remove)
+streakFiles *openFiles(pmConfig *config, bool remove, char *program_name)
 {
     bool status;
@@ -20,4 +21,5 @@
 
     sf->config = config;
+    sf->program_name = basename(program_name);
 
     // error checking is done by sFileOpen. If a file can't be opened we just exit
@@ -471,4 +473,5 @@
     sf->inImage->numCols = readout->image->numCols;
     sf->inImage->numRows = readout->image->numRows;
+    setExciseValue(sf->inImage);
 }
 
@@ -545,11 +548,5 @@
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
-        if (in->image->type.type == PS_TYPE_U16) {
-            in->exciseValue = 65535;
-            psMetadataAddU16(in->header, PS_LIST_TAIL, "BLANK", 0, "", 65535);
-            psMetadataAddU16(in->header, PS_LIST_TAIL, "ZBLANK", 0, "", 65535);
-        } else {
-            in->exciseValue = NAN;
-        }
+        setExciseValue(in);
     }  else {
         if (stage != IPP_STAGE_RAW) {
@@ -613,4 +610,6 @@
 
 #ifdef STREAKS_COMPRESS_OUTPUT
+    // Paul says that I should be able to leave this blank
+    bitpix = 0;
     setFitsOptions(out, bitpix, bscale, bzero);
     setFitsOptions(rec, bitpix, bscale, bzero);
@@ -916,5 +915,4 @@
 deleteFile(sFile *sfile)
 {
-
     if (sfile->inNebulous) {
         nebServer *server = getNebServer(NULL);
@@ -970,5 +968,5 @@
 
 void
-setMaskedToNAN(streakFiles *sfiles, psU8 maskMask, bool printCounts)
+setMaskedToNAN(streakFiles *sfiles, psU32 maskMask, bool printCounts)
 {
         int maskedPixels = 0;
@@ -993,5 +991,5 @@
                 // but I want to get the counts
                 double imageVal  = psImageGet(image, x, y);
-                psU8 maskVal;
+                psU32 maskVal;
                 if (sfiles->stage == IPP_STAGE_RAW) {
                     int xChip, yChip;
@@ -1018,11 +1016,11 @@
         }
         if (printCounts) {
-            printf("time to NAN mask pixels: %f\n", psTimerClear("NAN_MASKED"));
+            psLogMsg(sfiles->program_name, PS_LOG_INFO, "time to NAN mask pixels: %f\n", psTimerClear("NAN_MASKED"));
             int totalPixels = image->numRows * image->numCols;
-            printf("pixels:        %10ld\n", totalPixels);
-            printf("masked pixels: %10ld %4.2f%%\n", maskedPixels, 100. * maskedPixels / totalPixels);
-            printf("nand pixels:   %10ld %4.2f%%\n", nandPixels, 100. * nandPixels / totalPixels);
+            psLogMsg(sfiles->program_name, PS_LOG_INFO, "pixels:        %10ld\n", totalPixels);
+            psLogMsg(sfiles->program_name, PS_LOG_INFO, "masked pixels: %10ld %4.2f%%\n", maskedPixels, 100. * maskedPixels / totalPixels);
+            psLogMsg(sfiles->program_name, PS_LOG_INFO, "nand pixels:   %10ld %4.2f%%\n", nandPixels, 100. * nandPixels / totalPixels);
             if (weight) {
-                printf("nand weights:  %10ld %4.2f%%\n", nandWeights, 100. * nandWeights / totalPixels);
+                psLogMsg(sfiles->program_name, PS_LOG_INFO, "nand weights:  %10ld %4.2f%%\n", nandWeights, 100. * nandWeights / totalPixels);
             }
         }
@@ -1035,2 +1033,47 @@
     ourNebServer = NULL;
 }
+static void
+setExciseValue(sFile *in)
+{
+    if (in->image->type.type == PS_TYPE_U16) {
+        in->exciseValue = 65535;
+        psMetadataAddU16(in->header, PS_LIST_TAIL, "BLANK", 0, "", 65535);
+        psMetadataAddU16(in->header, PS_LIST_TAIL, "ZBLANK", 0, "", 65535);
+    } else {
+        in->exciseValue = NAN;
+    }
+}
+
+void
+strkGetMaskValues(streakFiles *sfiles, psU32 *maskStreak, psU32 *maskMask)
+{
+    if (sfiles->inMask && sfiles->inMask->header) {
+        if (!pmConfigMaskReadHeader(sfiles->config, sfiles->inMask->header)) {
+            streaksExit("failed to read mask values from file", PS_EXIT_CONFIG_ERROR);
+        }
+    }
+
+    bool status;
+    psMetadata *masks = psMetadataLookupMetadata(&status, sfiles->config->recipes, "MASKS");
+    if (!status) {
+        psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n");
+        streaksExit("", PS_EXIT_CONFIG_ERROR);
+    }
+    *maskStreak = psMetadataLookupU32(&status, masks, "STREAK");
+    if (!status) {
+        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for STREAK in recipes\n");
+        streaksExit("", PS_EXIT_CONFIG_ERROR);
+    }
+
+    // optionally setting pixels with any mask bits execpt CONV.POOR to NAN
+    // check old name first
+    psU32 convPoor = (double) psMetadataLookupU32(&status, masks, "POOR.WARP");
+    if (!status) {
+        convPoor = (double) psMetadataLookupU32(&status, masks, "CONV.POOR");
+        if (!status) {
+            psError(PM_ERR_CONFIG, false, "failed to lookup mask value for CONV.POOR in recipes\n");
+            streaksExit("", PS_EXIT_CONFIG_ERROR);
+        }
+    }
+    *maskMask = ~convPoor;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksio.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksio.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksio.h	(revision 24244)
@@ -2,5 +2,5 @@
 #define STREAKS_IO_H 1
 
-streakFiles *openFiles(pmConfig *config, bool remove);
+streakFiles *openFiles(pmConfig *config, bool remove, char *program_name);
 
 sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect,
@@ -16,4 +16,5 @@
 void copyFitsOptions(sFile *out, sFile *rec, sFile *in);
 void setupImageRefs(sFile *out, sFile *recoveryOut, sFile *in, int extnum, bool exciseAll);
+void strkGetMaskValues(streakFiles *sfiles, psU32 *maskStreak, psU32 *maskMask);
 
 void writeImage(sFile *sfile, psString extname, int extnum);
@@ -25,5 +26,5 @@
 
 bool isExciseValue(double, double);
-void setMaskedToNAN(streakFiles *sfiles, psU8 maskMask, bool printCounts);
+void setMaskedToNAN(streakFiles *sfiles, psU32 maskMask, bool printCounts);
 
 bool replicateOutputs(streakFiles *sf);
Index: /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksrelease.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksrelease.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksrelease.c	(revision 24244)
@@ -37,5 +37,5 @@
 
     // Does true work here?
-    streakFiles *sfiles = openFiles(config, true);
+    streakFiles *sfiles = openFiles(config, true, argv[0]);
 
     if (sfiles->stage == IPP_STAGE_RAW) {
Index: /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksremove.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksremove.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksremove.c	(revision 24244)
@@ -23,22 +23,6 @@
     }
 
-    psMetadata *masks = psMetadataLookupMetadata(&status, config->recipes, "MASKS");
-    if (!status) {
-        psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n");
-        streaksExit("", PS_EXIT_CONFIG_ERROR);
-    }
-    double maskStreak = (double) psMetadataLookupU8(&status, masks, "STREAK");
-    if (!status) {
-        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for STREAK in recipes\n");
-        streaksExit("", PS_EXIT_CONFIG_ERROR);
-    }
-
-    // optionally setting pixels with any mask bits execpt POOR.WARP to NAN
-    psU8 poorWarp = (double) psMetadataLookupU8(&status, masks, "POOR.WARP");
-    if (!status) {
-        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for POOR.WARP in recipes\n");
-        streaksExit("", PS_EXIT_CONFIG_ERROR);
-    }
-    psU8 maskMask = ~poorWarp;
+    psU32 maskStreak = 0;
+    psU32 maskMask = 0;
 
     psString streaksFileName = psMetadataLookupStr(NULL, config->arguments, "STREAKS");
@@ -50,5 +34,5 @@
     }
 
-    streakFiles *sfiles = openFiles(config, true);
+    streakFiles *sfiles = openFiles(config, true, argv[0]);
     setupAstrometry(sfiles);
 
@@ -81,5 +65,5 @@
         }
         psF64 cwp_t = psTimerClear("COMPUTE_WARPED_PIXELS");
-        printf("time to compute warped pixels: %f\n", cwp_t);
+        psLogMsg("streaksremove", PS_LOG_INFO, "time to compute warped pixels: %f\n", cwp_t);
     }
     
@@ -114,4 +98,9 @@
         }
 
+        // now that we've read the input files, lookup the mask values
+        if (maskStreak == 0) {
+            strkGetMaskValues(sfiles, &maskStreak, &maskMask);
+        }
+
         totalPixels += sfiles->inImage->numRows * sfiles->inImage->numCols;
 
@@ -124,5 +113,5 @@
                                       sfiles->inImage->numCols, sfiles->inImage->numRows);
 
-            printf("time to get streak pixels: %f\n", psTimerClear("GET_STREAK_PIXELS"));
+            psLogMsg("streaksremove", PS_LOG_INFO, "time to get streak pixels: %f\n", psTimerClear("GET_STREAK_PIXELS"));
 
             
@@ -131,7 +120,9 @@
                     psTimerStart("EXCISE_NON_WARPED");
 
+                    // set non-warped pixels and variance to NAN, mask to maskStreak (since the pixel
+                    // is excised as part of the destreaking process)
                     exciseNonWarpedPixels(sfiles, maskStreak);
 
-                    printf("time to excise non warped pixels: %f\n", psTimerClear("EXCISE_NON_WARPED"));
+                    psLogMsg("streaksremove", PS_LOG_INFO, "time to excise non warped pixels: %f\n", psTimerClear("EXCISE_NON_WARPED"));
                 }
                 totalStreakPixels +=  psArrayLength(pixels);
@@ -149,5 +140,5 @@
                     }
                 }
-                printf("time to remove streak pixels: %f\n", psTimerClear("REMOVE_STREAKS"));
+                psLogMsg("streaksremove", PS_LOG_INFO, "time to remove streak pixels: %f\n", psTimerClear("REMOVE_STREAKS"));
 
                 if (nanForRelease) {
@@ -170,15 +161,15 @@
         writeImages(sfiles, exciseImageCube);
 
-        printf("time to process component %d: %f\n", sfiles->extnum, psTimerClear("PROCESS_COMPONENT"));
+        psLogMsg("streaksremove", PS_LOG_INFO, "time to process component %d: %f\n", sfiles->extnum, psTimerClear("PROCESS_COMPONENT"));
     } while (streakFilesNextExtension(sfiles));
 
     psFree(streaks);
 
-    printf("pixels: %ld streak pixels: %ld %4.2f%%\n", totalPixels, totalStreakPixels, 100. * totalStreakPixels / totalPixels);
+    psLogMsg("streaksremove", PS_LOG_INFO, "pixels: %ld streak pixels: %ld %4.2f%%\n", totalPixels, totalStreakPixels, 100. * totalStreakPixels / totalPixels);
 
     psTimerStart("CLOSE_IMAGES");
     // close all files
     closeImages(sfiles);
-    printf("time to close images: %f\n", psTimerClear("CLOSE_IMAGES"));
+    psLogMsg("streaksremove", PS_LOG_INFO, "time to close images: %f\n", psTimerClear("CLOSE_IMAGES"));
 
     // NOTE: from here on we can't just quit if something goes wrong.
@@ -227,5 +218,5 @@
     streaksNebulousCleanup(); 
     pmConfigDone();
-    printf("time to run streaksremove: %f\n", psTimerClear("STREAKSREMOVE"));
+    psLogMsg("streaksremove", PS_LOG_INFO, "time to run streaksremove: %f\n", psTimerClear("STREAKSREMOVE"));
     psLibFinalize();
 
@@ -648,4 +639,5 @@
             }
         }
+        // XXX: check this
         // Assumption: there are no mask and weight images for video cells
         return;
Index: /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksremove.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksremove.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksremove.h	(revision 24244)
@@ -72,4 +72,5 @@
     psVector    *tiles;
     double  transparentStreaks;
+    psString    program_name;
 } streakFiles;
 
Index: /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksreplace.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksreplace.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksreplace.c	(revision 24244)
@@ -33,5 +33,5 @@
     }
 
-    streakFiles *sfiles = openFiles(config, false);
+    streakFiles *sfiles = openFiles(config, false, argv[0]);
 
     if (sfiles->stage == IPP_STAGE_RAW) {
Index: /branches/cnb_branches/cnb_branch_20090301/ppArith/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppArith/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppArith/configure.ac	(revision 24244)
@@ -26,4 +26,6 @@
 CFLAGS="${CFLAGS} -Wall -Werror"
 
+IPP_VERSION
+
 AC_SUBST([PPARITH_CFLAGS])
 AC_SUBST([PPARITH_LIBS])
Index: /branches/cnb_branches/cnb_branch_20090301/ppArith/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppArith/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppArith/src/Makefile.am	(revision 24244)
@@ -1,15 +1,30 @@
 bin_PROGRAMS = ppArith
 
-# PPARITH_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-# PPARITH_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-# PPARITH_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
-# 
-# # Force recompilation of ppArithVersion.c, since it gets the version information
-# ppArithVersion.c: FORCE
-# 	touch ppArith.c
-# FORCE: ;
+if HAVE_SVNVERSION
+PPARITH_VERSION=`$(SVNVERSION) ..`
+else
+PPARITH_VERSION="UNKNOWN"
+endif
 
-ppArith_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PPARITH_CFLAGS) -DPPARITH_VERSION=$(SVN_VERSION) -DPPARITH_BRANCH=$(SVN_BRANCH) -DPPARITH_SOURCE=$(SVN_SOURCE)
-ppArith_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PPSTATS_LIBS)   $(PSPHOT_LIBS)   $(PPARITH_LIBS)
+if HAVE_SVN
+PPARITH_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPARITH_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PPARITH_BRANCH="UNKNOWN"
+PPARITH_SOURCE="UNKNOWN"
+endif
+
+# Force recompilation of ppArithVersion.c, since it gets the version information
+ppArithVersion.c: ppArithVersionDefinitions.h
+ppArithVersionDefinitions.h: ppArithVersionDefinitions.h.in FORCE
+	-$(RM) ppArithVersionDefinitions.h
+	$(SED) -e "s|@PPARITH_VERSION@|\"$(PPARITH_VERSION)\"|" -e "s|@PPARITH_BRANCH@|\"$(PPARITH_BRANCH)\"|" -e "s|@PPARITH_SOURCE@|\"$(PPARITH_SOURCE)\"|" ppArithVersionDefinitions.h.in > ppArithVersionDefinitions.h
+FORCE: ;
+
+BUILT_SOURCES = ppArithVersionDefinitions.h
+
+
+ppArith_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PPARITH_CFLAGS)
+ppArith_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PPSTATS_LIBS)   $(PPARITH_LIBS)
 
 ppArith_SOURCES =		\
Index: /branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArithVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArithVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArithVersion.c	(revision 24244)
@@ -21,4 +21,5 @@
 
 #include "ppArith.h"
+#include "ppArithVersionDefinitions.h"
 
 #ifndef PPARITH_VERSION
@@ -32,11 +33,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString ppArithVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PPARITH_BRANCH), xstr(PPARITH_VERSION));
+    psStringAppend(&value, "%s@%s", PPARITH_BRANCH, PPARITH_VERSION);
     return value;
 }
@@ -44,5 +42,5 @@
 psString ppArithSource(void)
 {
-    return psStringCopy(xstr(PPARITH_SOURCE));
+    return psStringCopy(PPARITH_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArithVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArithVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArithVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PPARITH_VERSION_DEFINITIONS_H
+#define PPARITH_VERSION_DEFINITIONS_H
+
+#define PPARITH_VERSION @PPARITH_VERSION@ // SVN version
+#define PPARITH_BRANCH  @PPARITH_BRANCH@  // SVN branch
+#define PPARITH_SOURCE  @PPARITH_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppConfigDump/src/ppConfigDump.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppConfigDump/src/ppConfigDump.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppConfigDump/src/ppConfigDump.c	(revision 24244)
@@ -49,5 +49,5 @@
 {
     psLibInit(NULL);
-    psTraceSetLevel("err", 10);
+    (void) psTraceSetLevel("err", 10);
 
     // load the site-wide configuration information
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/configure.ac	(revision 24244)
@@ -26,4 +26,6 @@
 CFLAGS="${CFLAGS=} -Wall -Werror"
 
+IPP_VERSION
+
 AC_SUBST([PPIMAGE_CFLAGS])
 AC_SUBST([PPIMAGE_LIBS])
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/Makefile.am	(revision 24244)
@@ -4,14 +4,28 @@
 	ppImage.h 
 
-#PPIMAGE_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-#PPIMAGE_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-#PPIMAGE_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+if HAVE_SVNVERSION
+PPIMAGE_VERSION=`$(SVNVERSION) ..`
+else
+PPIMAGE_VERSION="UNKNOWN"
+endif
+
+if HAVE_SVN
+PPIMAGE_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPIMAGE_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PPIMAGE_BRANCH="UNKNOWN"
+PPIMAGE_SOURCE="UNKNOWN"
+endif
 
 # Force recompilation of ppImageVersion.c, since it gets the version information
-# ppImageVersion.c: FORCE
-# 	touch ppImageVersion.c
-# FORCE: ;
+ppImageVersion.c: ppImageVersionDefinitions.h
+ppImageVersionDefinitions.h: ppImageVersionDefinitions.h.in FORCE
+	-$(RM) ppImageVersionDefinitions.h
+	$(SED) -e "s|@PPIMAGE_VERSION@|\"$(PPIMAGE_VERSION)\"|" -e "s|@PPIMAGE_BRANCH@|\"$(PPIMAGE_BRANCH)\"|" -e "s|@PPIMAGE_SOURCE@|\"$(PPIMAGE_SOURCE)\"|" ppImageVersionDefinitions.h.in > ppImageVersionDefinitions.h
+FORCE: ;
 
-ppImage_CFLAGS = $(PPIMAGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSASTRO_CFLAGS) $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPPIMAGE_VERSION=$(SVN_VERSION) -DPPIMAGE_BRANCH=$(SVN_BRANCH) -DPPIMAGE_SOURCE=$(SVN_SOURCE)
+BUILT_SOURCES = ppImageVersionDefinitions.h
+
+ppImage_CFLAGS = $(PPIMAGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSASTRO_CFLAGS) $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
 ppImage_LDFLAGS = $(PPIMAGE_LIBS) $(PSASTRO_LIBS) $(PPSTATS_LIBS) $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 ppImage_SOURCES = \
@@ -36,6 +50,10 @@
 	ppImageMetadataStats.c \
 	ppImageReplaceBackground.c \
+	ppImageMeasureCrosstalk.c \
+	ppImageCorrectCrosstalk.c \
 	ppImageDefineFile.c \
 	ppImageSetMaskBits.c \
+	ppImageParityFlip.c \
+	ppImageCheckCTE.c \
 	ppImageFileCheck.c \
 	ppImageVersion.c \
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImage.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImage.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImage.h	(revision 24244)
@@ -26,4 +26,6 @@
     bool doMaskBuild;                   // Build internal mask
     bool doVarianceBuild;               // Build internal variance map
+    bool doMaskSat;                     // mask saturated pixels
+    bool doMaskLow;                     // mask low pixels
     bool doMask;                        // Mask bad pixels
     bool doNonLin;                      // Non-linearity correction
@@ -40,4 +42,9 @@
     bool doAstromMosaic;                // full-mosaic Astrometry
     bool doStats;                       // call ppStats on the image
+    bool checkCTE;                      // measure pixel-based variance
+    bool applyParity;			// Apply Cell parities
+
+    bool doCrosstalkMeasure;            // measure crosstalk signal
+    bool doCrosstalkCorrect;            // apply crosstalk correction
 
     // output files requested
@@ -61,6 +68,7 @@
     psImageMaskType markValue;          // apply this bit-mask to choose masked bits
     psImageMaskType satMask;            // Mask value to give saturated pixels
-    psImageMaskType badMask;            // Mask value to give bad pixels
+    psImageMaskType lowMask;            // Mask value to give bad pixels
     psImageMaskType flatMask;           // Mask value to give bad flat pixels
+    psImageMaskType darkMask;           // Mask value to give bad dark pixels
     psImageMaskType blankMask;          // Mask value to give blank pixels
 
@@ -87,5 +95,5 @@
     float remnanceThresh;               // Threshold for remnance detection
 
-    char *normClass;			// class to use for per-class normalization 
+    char *normClass;                    // class to use for per-class normalization
 } ppImageOptions;
 
@@ -111,4 +119,7 @@
 bool ppImageSetMaskBits (pmConfig *config, ppImageOptions *options);
 
+// apply the cell flips to the input data before analysis
+bool ppImageParityFlip (pmConfig *config, const ppImageOptions *options, const pmFPAview *view);
+
 // Loop over the input
 bool ppImageLoop(pmConfig *config, ppImageOptions *options);
@@ -132,4 +143,6 @@
 bool ppImageDetrendFree(pmConfig *config, pmFPAview *view);
 bool ppImageFringeFree(pmConfig *config, pmFPAview *view);
+
+bool ppImageCheckCTE(pmConfig *config, ppImageOptions *options, pmFPAview *view);
 
 // Record which detrend file was used for the detrending
@@ -143,7 +156,7 @@
 bool ppImageRebinChip (pmConfig *config, pmFPAview *view, ppImageOptions *options, char *outName);
 
-bool ppImagePhotom (pmConfig *config, pmFPAview *view);
-bool ppImageAstrom (pmConfig *config);
-bool ppImageAddstar (pmConfig *config);
+bool ppImagePhotom(psMetadata *stats, pmConfig *config, pmFPAview *view);
+bool ppImageAstrom(pmConfig *config, psMetadata *stats);
+bool ppImageAddstar(pmConfig *config);
 
 // Subtract background from the chip-mosaicked image
@@ -180,4 +193,10 @@
 
 
+// measure the crosstalk signal
+bool ppImageMeasureCrosstalk(pmConfig *config, ppImageOptions *options, pmFPAview *view);
+
+// correct the crosstalk signal
+bool ppImageCorrectCrosstalk(pmConfig *config, ppImageOptions *options, pmFPAview *view);
+
 // Measure fringes
 bool ppImageDetrendFringeMeasure(pmReadout *readout, // Readout to measure
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageAstrom.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageAstrom.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageAstrom.c	(revision 24244)
@@ -7,5 +7,5 @@
 // this function is mostly equivalent to the top-level of psastro, with some
 // modifications since the data has already been loaded.
-bool ppImageAstrom (pmConfig *config) {
+bool ppImageAstrom (pmConfig *config, psMetadata *stats) {
 
     bool status;
@@ -27,5 +27,5 @@
     }
 
-    if (!psastroAnalysis (config)) {
+    if (!psastroAnalysis(config, stats)) {
         psError (PSASTRO_ERR_UNKNOWN, false, "failure in psastro analysis\n");
         return false;
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageCheckCTE.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageCheckCTE.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageCheckCTE.c	(revision 24244)
@@ -60,8 +60,6 @@
             fineRegion = psImageBinningSetFineRegion (binning, ruffRegion);
             fineRegion = psRegionForImage (image, fineRegion);
-            if (fineRegion.x0 >= image->numCols || fineRegion.x1 >= image->numCols ||
-                fineRegion.y0 >= image->numRows || fineRegion.y1 >= image->numRows) {
-                continue;
-            }
+            if (fineRegion.x0 >= image->numCols) continue;
+	    if (fineRegion.y0 >= image->numRows) continue;
 
             psImage *subset  = psImageSubset (image, fineRegion);
@@ -85,8 +83,13 @@
 	    float normVariance = PS_SQR(stats->robustStdev) / cellMedian;
 	    // float normVariance = PS_SQR(stats->sampleStdev) / cellMedian;
+	    // if (!isfinite(normVariance)) fprintf (stderr, "** normVariance is nan **\n");
 
 	    // apply resulting value to the input pixels
 	    for (int jy = fineRegion.y0; jy < fineRegion.y1; jy++) {
+	      if (jy < 0) continue;
+	      if (jy >= image->numRows) continue;
 	      for (int jx = fineRegion.x0; jx < fineRegion.x1; jx++) {
+		if (jx < 0) continue;
+		if (jx >= image->numCols) continue;
 		image->data.F32[jy][jx] = normVariance;
 	      }
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageCorrectCrosstalk.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageCorrectCrosstalk.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageCorrectCrosstalk.c	(revision 24244)
@@ -0,0 +1,18 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "ppImage.h"
+
+#define ESCAPE(MESSAGE) { \
+  psError(PS_ERR_UNKNOWN, false, MESSAGE); \
+  psFree(view); \
+  return false; \
+}
+
+// For the moment, this implementation is VERY GPC-specific
+
+bool ppImageCorrectCrosstalk(pmConfig *config, ppImageOptions *options, pmFPAview *view)
+{
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageDetrendReadout.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageDetrendReadout.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageDetrendReadout.c	(revision 24244)
@@ -17,5 +17,7 @@
     // Masking on the basis of pixel value needs to be done before anything else, so the values are pristine.
     if (options->doMaskBuild) {
-        pmReadoutGenerateMask(input, options->satMask, options->badMask);
+        psImageMaskType satMask = options->doMaskSat ? options->satMask : 0;
+        psImageMaskType lowMask = options->doMaskLow ? options->lowMask : 0;
+        pmReadoutGenerateMask(input, satMask, lowMask);
     }
     // apply the externally supplied mask to the input->mask pixels
@@ -54,5 +56,5 @@
         if (!pmBiasSubtract(input, options->overscan, bias, oldDark, view)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to subtract bias.");
-	    psFree(detview);
+            psFree(detview);
             return false;
         }
@@ -66,7 +68,7 @@
 
     if (options->doDark && dark) {
-        if (!pmDarkApply(input, dark, options->maskValue)) {
+        if (!pmDarkApply(input, dark, options->darkMask)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to subtract dark.");
-	    psFree(detview);
+            psFree(detview);
             return false;
         }
@@ -74,8 +76,8 @@
 
     if (options->doRemnance) {
-        if (!pmRemnance(input, options->maskValue, options->badMask,
+        if (!pmRemnance(input, options->maskValue, options->lowMask,
                         options->remnanceSize, options->remnanceThresh)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to mask remnance.");
-	    psFree(detview);
+            psFree(detview);
             return false;
         }
@@ -86,5 +88,5 @@
         pmReadout *shutter = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.SHUTTER");
         if (!pmShutterCorrectionApply(input, shutter, pmConfigMaskGet("FLAT", config))) {
-	    psFree(detview);
+            psFree(detview);
             return false;
         }
@@ -95,5 +97,5 @@
         pmReadout *flat = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.FLAT");
         if (!pmFlatField(input, flat, options->flatMask)) {
-	    psFree(detview);
+            psFree(detview);
             return false;
         }
@@ -117,30 +119,35 @@
     psMetadata *normlist = psMetadataLookupMetadata(&mdok, config->arguments, "NORMALIZATION.TABLE");
     if (normlist) {
-	pmFPAfile *inputFile = psMetadataLookupPtr(&mdok, config->files, "PPIMAGE.INPUT");
+        pmFPAfile *inputFile = psMetadataLookupPtr(&mdok, config->files, "PPIMAGE.INPUT");
 
-	// get the menu of class IDs
-        psMetadata *menu = psMetadataLookupMetadata(&mdok, inputFile->camera, "CLASSID"); 
+        // get the menu of class IDs
+        psMetadata *menu = psMetadataLookupMetadata(&mdok, inputFile->camera, "CLASSID");
         if (!menu) {
             psError(PS_ERR_IO, false, "Unable to find CLASSID metadata in camera configuration");
-	    psFree(detview);
+            psFree(detview);
             return false;
         }
-	// get the rule for class_id for the desired class
-        const char *rule = psMetadataLookupStr(&mdok, menu, options->normClass); 
+        // get the rule for class_id for the desired class
+        const char *rule = psMetadataLookupStr(&mdok, menu, options->normClass);
         if (!rule) {
             psError(PS_ERR_IO, false, "Unable to find NORM.CLASS value %s in CLASSID in camera configuration", options->normClass);
-	    psFree(detview);
+            psFree(detview);
             return false;
         }
-	// get the class_id from the rule
+        // get the class_id from the rule
         char *classID = pmFPAfileNameFromRule(rule, inputFile, view);
         if (!classID) {
             psError(PS_ERR_IO, false, "error converting CLASSID rule %s to name\n", rule);
-	    psFree(detview);
+            psFree(detview);
             return false;
         }
 
-	// get normalization from the class_id
-	float norm = psMetadataLookupF32 (&mdok, normlist, classID);
+        // get normalization from the class_id
+        float norm = psMetadataLookupF32 (&mdok, normlist, classID);
+        if (!mdok) {
+            psError(PS_ERR_IO, false, "failed to find class ID %s in normalization table\n", classID);
+            psFree(detview);
+            return false;
+        }
 
         pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest
@@ -150,8 +157,8 @@
         psFree(comment);
 
-	// apply the normalization
+        // apply the normalization
         psBinaryOp(input->image, input->image, "*", psScalarAlloc(norm, PS_TYPE_F32));
 
-	psFree (classID);
+        psFree (classID);
     }
 # endif
@@ -160,5 +167,5 @@
         pmCell *fringe = pmFPAfileThisCell(config->files, detview, "PPIMAGE.FRINGE");
         if (!ppImageDetrendFringeMeasure(input, fringe, false, options)) {
-	    psFree(detview);
+            psFree(detview);
             return false;
         }
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageErrorCodes.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageErrorCodes.dat	(revision 24244)
@@ -1,6 +1,6 @@
 #
 # This file is used to generate ppImageErrorClasses.h
-#
-BASE = 500		First value we use; lower values belong to psLib
+
+BASE = 5000		First value we use; lower values belong to psLib
 # these errors correspond to standard exit conditions
 ARGUMENTS               Incorrect arguments
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageErrorCodes.h.in	(revision 24244)
@@ -9,5 +9,5 @@
  */
 typedef enum {
-    PPIMAGE_ERR_BASE = 512,
+    PPIMAGE_ERR_BASE = 5000,
     PPIMAGE_ERR_${ErrorCode},
     PPIMAGE_ERR_NERROR
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageLoop.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageLoop.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageLoop.c	(revision 24244)
@@ -13,7 +13,12 @@
 bool ppImageLoop(pmConfig *config, ppImageOptions *options)
 {
-    psMetadata *stats = options->doStats ? psMetadataAlloc() : NULL; // Statistics to output
+    psMetadata *stats = NULL;           // Statistics to output
     float timeDetrend = 0;              // Amount of time spent in detrend
     float timePhot = 0;                 // Amount of time spent in photometry
+
+    if (options->doStats) {
+        stats = psMetadataAlloc();
+        psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", 0, "No problems", 0);
+    }
 
     bool status;                        // Status of MD lookup
@@ -46,4 +51,18 @@
         }
 
+	// crosstalk measurement needs to be done on the entire chip at once, and before
+	// signal levels are modified by the detrending.  If crosstalk measurement is
+	// requested, the read-level for the images is set to CHIP.
+	if (!ppImageMeasureCrosstalk(config, options, view)) {
+	  ESCAPE("Unable to perform crosstalk correction");
+	}
+
+	// crosstalk correction needs to be done on the entire chip at once, and before
+	// signal levels are modified by the detrending.  If crosstalk correction is
+	// requested, the read-level for the images is set to CHIP.
+	if (!ppImageCorrectCrosstalk(config, options, view)) {
+	  ESCAPE("Unable to perform crosstalk correction");
+	}
+
         psTimerStart(TIMER_DETREND);
         pmCell *cell;                   // Cell from chip
@@ -80,4 +99,12 @@
                 }
 
+                // perform the detrend analysis
+                if (!ppImageParityFlip(config, options, view)) {
+                    ESCAPE("Unable to detrend readout");
+                }
+
+		// XXX TEST:
+		// psphotSaveImage (NULL, readout->image, "test.image.fits");
+
                 // XXX set the options->*Mask values here (after the mask images have been loaded
                 // and before any of the value are used)
@@ -134,5 +161,5 @@
         psTimerStart(TIMER_PHOT);
         if (options->doPhotom) {
-            if (!ppImagePhotom(config, view)) {
+            if (!ppImagePhotom(stats, config, view)) {
                 ESCAPE("error running photometry.");
             }
@@ -195,5 +222,5 @@
     // this also performs the psastro file IO
     if (options->doAstromChip || options->doAstromMosaic) {
-        if (!ppImageAstrom(config)) {
+        if (!ppImageAstrom(config, stats)) {
             ESCAPE("error running astrometry.");
         }
@@ -218,5 +245,5 @@
     psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
     if (dump_file) {
-        pmConfigDump(config, input->fpa, dump_file);
+        pmConfigDump(config, dump_file);
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageMeasureCrosstalk.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageMeasureCrosstalk.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageMeasureCrosstalk.c	(revision 24244)
@@ -0,0 +1,187 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "ppImage.h"
+
+#define ESCAPE(MESSAGE) {				\
+	psError(PS_ERR_UNKNOWN, false, MESSAGE);	\
+	psFree(view);					\
+	return false;					\
+    }
+
+// For the moment, this implementation is VERY GPC-specific
+
+bool ppImageMeasureCrosstalk(pmConfig *config, ppImageOptions *options, pmFPAview *view)
+{
+    bool status;
+    const float CUTOFF = 60000;
+    const int MIN_MATCHED_PIXELS = 500;  // require a significant area of high signal
+    char name[64];
+
+    if (!options->doCrosstalkMeasure) return true;
+
+    psTimerStart("crosstalk");
+
+    // confirm that this camera is GPC1
+
+    // find the currently selected chip
+    pmChip *chip = pmFPAfileThisChip(config->files, view, "PPIMAGE.INPUT");
+    pmFPA *fpa = chip->parent;
+    if (chip->cells->n != 64) return true;
+
+    // for each cell (xyNM), we want to find the pixels with values above a cutoff (40k, 60k, ?)
+    // for all of the other cells in the row (xyJM), we want to measure the (robust) median flux in the pixels
+    // corresponding to the high pixels in xyNM, and compare with the median flux for the cell
+
+    psArray *vectorSet = psArrayAlloc(8);
+    psArray *cellRows = psArrayAlloc(8);
+    psArray *imageRows = psArrayAlloc(8);
+    for (int i = 0; i < 8; i++) {
+	psArray *cellRow = psArrayAlloc(8);
+	cellRows->data[i] = cellRow;
+	psArray *imageRow = psArrayAlloc(8);
+	imageRows->data[i] = imageRow;
+	psVector *vector = psVectorAllocEmpty(100, PS_TYPE_F32);
+	vectorSet->data[i] = vector;
+    }
+
+    // assign the cells to arrays by row and column
+    for (int i = 0; i < chip->cells->n; i++) {
+
+	pmCell *cell = chip->cells->data[i];
+
+	// skip the video cells and empty cells (cell->readouts->n != 1)
+	if (cell->readouts->n != 1) {
+	    psWarning ("Skipping Empty and Video Cell for ppImageCrosstalk");
+	    continue;
+	}
+
+	// place the image from this cell in the 2D array:
+	const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME");	    
+	int row = cellName[3] - '0';
+	int col = cellName[2] - '0';
+	assert (row >= 0 && row < 8);
+	assert (col >= 0 && col < 8);
+
+	pmReadout *readout = cell->readouts->data[0];
+	psImage *image = readout->image;
+
+	psArray *cellRow = cellRows->data[row];
+	cellRow->data[col] = psMemIncrRefCounter(cell);
+
+	psArray *imageRow = imageRows->data[row];
+	imageRow->data[col] = psMemIncrRefCounter(image);
+    }
+
+    psVector *sample = NULL;
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
+    psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN);
+
+    // measure the background for each cell
+    for (int row = 0; row < cellRows->n; row++) {
+	psArray *cellRow = cellRows->data[row];
+	psArray *imageRow = imageRows->data[row];
+	for (int col = 0; col < cellRow->n; col++) {
+	    pmCell *cell = cellRow->data[col];
+	    if (!cell) continue;
+	    psImage *image = imageRow->data[col];
+	    if (!image) continue; // XXX assert on this?
+
+	    psStatsInit (stats);
+	    if (!psImageBackground (stats, &sample, image, NULL, 0, rng)) continue;
+	    psMetadataAddF32 (cell->analysis, PS_LIST_TAIL, "XTALK.REF", PS_META_REPLACE, "crosstalk measurement", stats->robustMedian);
+	    // fprintf (stderr, "xtalk.ref (xy%d%d) : %f\n", col, row, stats->robustMedian);
+	}
+    }
+
+    for (int row = 0; row < cellRows->n; row++) {
+	psArray *cellRow = cellRows->data[row];
+	psArray *imageRow = imageRows->data[row];
+	for (int col = 0; col < cellRow->n; col++) {
+	    psImage *image = imageRow->data[col];
+	    if (!image) continue;
+
+	    // initialize the storage vectors
+	    for (int i = 0; i < 8; i++) {
+		psVector *vector = vectorSet->data[i];
+		vector->n = 0;
+	    }
+
+	    int nBright = 0;
+
+	    // generate a vector for each of the other cells in this row
+	    // containing only the pixels for which this cell is > CUTOFF
+	    for (int iy = 0; iy < image->numRows; iy++) {
+		for (int ix = 0; ix < image->numCols; ix++) {
+
+		    if (image->data.F32[iy][ix] < CUTOFF) continue;
+		    nBright ++;
+
+		    // this is a pixel of interest in the target cell; extract the matched pixels
+			
+		    for (int i = 0; i < 8; i++) {
+			if (i == col) continue;
+
+			psImage *matchImage = imageRow->data[i];
+			if (!matchImage) continue;
+
+			psVector *vector = vectorSet->data[i];
+			psVectorAppend (vector, matchImage->data.F32[iy][ix]);
+		    }
+		}
+	    }
+	    // fprintf (stderr, "nBright for xy%d%d : %d\n", col, row, nBright);
+	    if (nBright < MIN_MATCHED_PIXELS) continue;
+
+	    // pmCell *refCell = cellRow->data[col];
+	    // float refBackground = psMetadataLookupF32 (&status, refCell->analysis, "XTALK.REF");
+	    // float swing = CUTOFF - refBackground;
+
+	    // now we need to measure the median for these vectors
+	    for (int i = 0; i < 8; i++) {
+		if (i == col) continue;
+		psVector *vector = vectorSet->data[i];
+		if (vector->n < MIN_MATCHED_PIXELS) continue;
+
+		pmCell *cell = cellRow->data[i];
+		if (!cell) continue; // XXX assert on this?
+
+		psStatsInit (stats);
+		if (!psVectorStats (stats, vector, NULL, NULL, 0)) {
+		    psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		    return false;
+		}
+		    
+		float background = psMetadataLookupF32 (&status, cell->analysis, "XTALK.REF");
+		float xtalk = (stats->robustMedian - background) / CUTOFF;
+
+		// Put version information into the header
+		pmHDU *hdu = pmHDUGetHighest(fpa, chip, cell);
+		if (hdu) {
+		    // add this to the PHU header 
+		    snprintf (name, 64, "XT_%d%d_%d%d", col, row, i, row);
+		    psMetadataAddF32 (hdu->header, PS_LIST_TAIL, name, PS_META_REPLACE, "crosstalk measurement", xtalk);
+		}
+
+		// keyword for resulting value:
+		snprintf (name, 64, "XTALK_%d%d", i, row);
+
+		psMetadataAddF32 (cell->analysis, PS_LIST_TAIL, name, PS_META_REPLACE, "crosstalk measurement", xtalk);
+		// fprintf (stderr, "xtalk (xy%d%d on xy%d%d) : %f -> delta is %f, slope if %e\n", col, row, i, row, stats->robustMedian, stats->robustMedian - background, xtalk);
+	    }
+	}
+    }
+    psLogMsg ("ppImage", 3, "crosstalk measurement: %f sec\n", psTimerMark ("crosstalk"));
+
+    psFree (vectorSet);
+    psFree (cellRows);
+    psFree (imageRows);
+    psFree (stats);
+    psFree (rng);
+    psFree (sample);
+
+    // need to free all sorts of things here....
+    return true;
+}
+
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageOptions.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageOptions.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageOptions.c	(revision 24244)
@@ -19,5 +19,7 @@
     // actions which ppImage should perform
     options->doMaskBuild     = false;   // Build internal mask
-    options->doVarianceBuild   = false;   // Build internal variance
+    options->doMaskSat       = false;   // mask saturated pixels
+    options->doMaskLow       = false;   // mask low pixels
+    options->doVarianceBuild = false;   // Build internal variance
     options->doMask          = false;   // Mask bad pixels
     options->doNonLin        = false;   // Non-linearity correction
@@ -33,4 +35,5 @@
     options->doAstromMosaic  = false;   // Astrometry (full-mosaic)
     options->doStats         = false;   // Measure and save image statistics
+    options->applyParity     = false;   // Apply Cell parities
 
     // output files requested
@@ -53,8 +56,13 @@
     options->maskValue       = 0x00;    // Default mask value (used to skip / ignore pixels)
     options->satMask         = 0x00;    // Saturated pixels (supplied to pmReadoutGenerateMask)
-    options->badMask         = 0x00;    // Bad (low) pixels (supplied to pmReadoutGenerateMask)
+    options->lowMask         = 0x00;    // out-of-bounds (low) pixels (supplied to pmReadoutGenerateMask)
     options->flatMask        = 0x00;    // Bad flat pixels (supplied to pmFlatField)
+    options->darkMask        = 0x00;    // Bad dark pixels (supplied to pmDarkApply)
     options->blankMask       = 0x00;    // Blank (no data, cell gap) pixels (supplied to pmChipMosaic, pmFPAMosaic)
     options->markValue       = 0x00;    // A safe bit for internal marking
+
+    // crosstalk options
+    options->doCrosstalkMeasure = false;   // measure crosstalk
+    options->doCrosstalkCorrect = false;   // correct crosstalk
 
     // Non-linearity default options
@@ -197,8 +205,14 @@
     // for these images, even if not required otherwise
     options->doMaskBuild = psMetadataLookupBool(NULL, recipe, "MASK.BUILD");
+    options->doMaskSat   = psMetadataLookupBool(NULL, recipe, "MASK.SATURATED");
+    options->doMaskLow   = psMetadataLookupBool(NULL, recipe, "MASK.LOW");
     options->doVarianceBuild = psMetadataLookupBool(NULL, recipe, "VARIANCE.BUILD");
 
     // Mask recipe options (note that mask bit values are set in ppImageSetMaskBits.c)
     options->doMask = psMetadataLookupBool(NULL, recipe, "MASK");
+
+    // Mask recipe options (note that mask bit values are set in ppImageSetMaskBits.c)
+    options->doCrosstalkMeasure = psMetadataLookupBool(NULL, recipe, "CROSSTALK.MEASURE");
+    options->doCrosstalkCorrect = psMetadataLookupBool(NULL, recipe, "CROSSTALK.CORRECT");
 
     options->doBias = psMetadataLookupBool(NULL, recipe, "BIAS");
@@ -214,4 +228,6 @@
         options->doStats = true;
     }
+
+    options->applyParity = psMetadataLookupBool(NULL, recipe, "APPLY.CELL.PARITY");
 
     // binned image options
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageParityFlip.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageParityFlip.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageParityFlip.c	(revision 24244)
@@ -0,0 +1,127 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "ppImage.h"
+
+// Update a concept to the assumed value
+#define FIX_CONCEPT(SOURCE, NAME, TYPE, VALUE) {        \
+psMetadataItem *item = psMetadataLookup(SOURCE, NAME); \
+item->data.TYPE = VALUE; }
+
+bool ppImageParityFlip (pmConfig *config, const ppImageOptions *options, const pmFPAview *view) {
+
+    bool status;
+
+    if (!options->applyParity) return true;
+
+    // find the currently selected readout
+    pmReadout *readout = pmFPAfileThisReadout(config->files, view, "PPIMAGE.INPUT");
+
+    pmCell *cell = readout->parent;
+
+    int xParity = psMetadataLookupS32(&status, cell->concepts, "CELL.XPARITY");
+    if (!status || (xParity != -1 && xParity != 1)) {
+        psWarning("CELL.XPARITY is not set for the input cell; assuming 1.\n");
+        FIX_CONCEPT(cell->concepts, "CELL.XPARITY", S32, 1);
+        xParity = 1;
+    }
+    int yParity = psMetadataLookupS32(&status, cell->concepts, "CELL.YPARITY");
+    if (!status || (yParity != -1 && yParity != 1)) {
+        psWarning("CELL.YPARITY is not set for the input cell; assuming 1.\n");
+        FIX_CONCEPT(cell->concepts, "CELL.YPARITY", S32, 1);
+        yParity = 1;
+    }
+
+    // for this case, nothing to be done:
+    if ((xParity == 1) && (yParity == 1)) return true;
+    
+    psImage *image = readout->image;
+    psImage *var   = readout->variance;
+    psImage *mask  = readout->mask;
+
+    if (var) psAssert (image->numCols == var->numCols,  "image and variance sizes are mismatched");
+    if (var) psAssert (image->numRows == var->numRows,  "image and variance sizes are mismatched");
+    if (mask) psAssert (image->numCols == mask->numCols, "image and mask sizes are mismatched");
+    if (mask) psAssert (image->numRows == mask->numRows, "image and mask sizes are mismatched");
+
+    int Nx = image->numCols;
+    int Ny = image->numRows;
+
+    // we need working buffers for the image, variance, and mask:
+    psF32 *imrow = (psF32 *) psAlloc (Nx*sizeof(psF32));
+    psF32 *wtrow = (psF32 *) psAlloc (Nx*sizeof(psF32));
+    psImageMaskType *mkrow = (psImageMaskType *) psAlloc (Nx*sizeof(psImageMaskType));
+
+    // the three cases (+1,-1), (-1,+1), (-1,-1) should be handled independently
+    if ((xParity == -1) && (yParity == +1)) {
+	for (int iy = 0; iy < Ny; iy++) {
+	    for (int ix = 0; ix < Nx; ix++) {
+		imrow[Nx-1-ix] = image->data.F32[iy][ix];
+	    }
+	    for (int ix = 0; (var != NULL) && (ix < Nx); ix++) {
+		wtrow[Nx-1-ix] = var->data.F32[iy][ix];
+	    }
+	    for (int ix = 0; (mask != NULL) && (ix < Nx); ix++) {
+		mkrow[Nx-1-ix] = mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix];
+	    }
+	    memcpy (image->data.F32[iy], imrow, Nx*sizeof(psF32));
+	    if (var) memcpy (var->data.F32[iy],   wtrow, Nx*sizeof(psF32));
+	    if (mask) memcpy (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy], mkrow, Nx*sizeof(psImageMaskType));
+	}
+    }
+    if ((xParity == +1) && (yParity == -1)) {
+	for (int iy = 0; iy < Ny/2; iy++) {
+	    memcpy (imrow, image->data.F32[iy], Nx*sizeof(psF32));
+	    memcpy (image->data.F32[iy], image->data.F32[Ny-1-iy], Nx*sizeof(psF32));
+	    memcpy (image->data.F32[Ny-1-iy], imrow, Nx*sizeof(psF32));
+
+	    if (var) {
+		memcpy (wtrow, var->data.F32[iy], Nx*sizeof(psF32));
+		memcpy (var->data.F32[iy], var->data.F32[Ny-1-iy], Nx*sizeof(psF32));
+		memcpy (var->data.F32[Ny-1-iy], wtrow, Nx*sizeof(psF32));
+	    }
+
+	    if (mask) {
+		memcpy (mkrow, mask->data.PS_TYPE_IMAGE_MASK_DATA[iy], Nx*sizeof(psImageMaskType));
+		memcpy (mask->data.PS_TYPE_IMAGE_MASK_DATA[iy], mask->data.PS_TYPE_IMAGE_MASK_DATA[Ny-1-iy], Nx*sizeof(psImageMaskType));
+		memcpy (mask->data.PS_TYPE_IMAGE_MASK_DATA[Ny-1-iy], mkrow, Nx*sizeof(psImageMaskType));
+	    }
+	}
+    }
+    if ((xParity == -1) && (yParity == -1)) {
+	for (int iy = 0; iy < Ny/2; iy++) {
+	    for (int ix = 0; ix < Nx; ix++) {
+		imrow[Nx-1-ix] = image->data.F32[iy][ix];
+	    }
+	    for (int ix = 0; var && (ix < Nx); ix++) {
+		wtrow[Nx-1-ix] = var->data.F32[iy][ix];
+	    }
+	    for (int ix = 0; mask && (ix < Nx); ix++) {
+		mkrow[Nx-1-ix] = mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix];
+	    }
+
+	    for (int ix = 0; ix < Nx; ix++) {
+		image->data.F32[iy][Nx-1-ix] = image->data.F32[Ny-1-iy][ix];
+	    }
+	    for (int ix = 0; var && (ix < Nx); ix++) {
+		var->data.F32[iy][Nx-1-ix] = var->data.F32[Ny-1-iy][ix];
+	    }
+	    for (int ix = 0; mask && (ix < Nx); ix++) {
+		mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][Nx-1-ix] = mask->data.PS_TYPE_IMAGE_MASK_DATA[Ny-1-iy][ix];
+	    }
+
+	    memcpy (image->data.F32[Ny-1-iy], imrow, Nx*sizeof(psF32));
+	    if (var) memcpy (var->data.F32[Ny-1-iy], wtrow, Nx*sizeof(psF32));
+	    if (mask) memcpy (mask->data.PS_TYPE_IMAGE_MASK_DATA[Ny-1-iy], mkrow, Nx*sizeof(psImageMaskType));
+	}
+    }
+
+    // FIX_CONCEPT(cell->concepts, "CELL.XPARITY", S32, 1);
+    // FIX_CONCEPT(cell->concepts, "CELL.YPARITY", S32, 1);
+
+    psFree (imrow);
+    psFree (wtrow);
+    psFree (mkrow);
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImagePhotom.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImagePhotom.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImagePhotom.c	(revision 24244)
@@ -6,5 +6,5 @@
 
 // In this function, we perform the psphot analysis routine for the chip-mosaicked images
-bool ppImagePhotom (pmConfig *config, pmFPAview *view) {
+bool ppImagePhotom(psMetadata *stats, pmConfig *config, pmFPAview *view) {
 
     bool status;
@@ -12,5 +12,5 @@
     pmReadout *readout;
 
-    psphotInit ();
+    psphotInit();
 
     // find or define a pmFPAfile PSPHOT.INPUT
@@ -37,14 +37,22 @@
             // run the actual photometry analysis
             if (!psphotReadout (config, view)) {
-                psError(psErrorCodeLast(), false, "failure in psphotReadout for chip %d, cell %d, readout %d\n", view->chip, view->cell, view->readout);
-                return false;
+                // This is likely a data quality issue
+                // XXX Split into multiple cases using error codes?
+                psErrorStackPrint(stderr, "Unable to perform photometry on image");
+                psWarning("Unable to perform photometry on image --- suspect bad data quality.");
+                if (stats && psMetadataLookupS32(NULL, stats, "QUALITY") == 0) {
+                    psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE,
+                                     "Unable to perform photometry on image", psErrorCodeLast());
+                }
+                psErrorClear();
+                psphotFilesActivate(config, false);
             }
 
-	    // we want to save the MASK as modified by psphot, but not the data or weight
-	    // free the old mask and replace with a memory copy of the new mask
-	    pmReadout *oldReadout = pmFPAviewThisReadout (view, input->src);
-	    pmReadout *newReadout = pmFPAviewThisReadout (view, input->fpa);
-	    psFree (oldReadout->mask);
-	    oldReadout->mask = psMemIncrRefCounter (newReadout->mask);
+            // we want to save the MASK as modified by psphot, but not the data or weight
+            // free the old mask and replace with a memory copy of the new mask
+            pmReadout *oldReadout = pmFPAviewThisReadout(view, input->src);
+            pmReadout *newReadout = pmFPAviewThisReadout(view, input->fpa);
+            psFree (oldReadout->mask);
+            oldReadout->mask = psMemIncrRefCounter(newReadout->mask);
         }
     }
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageReplaceBackground.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageReplaceBackground.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageReplaceBackground.c	(revision 24244)
@@ -147,5 +147,5 @@
             if (!isfinite(value)) {
                 image->data.F32[y][x] = NAN;
-                mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= options->badMask;
+                mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= options->lowMask;
             } else {
                 image->data.F32[y][x] -= value;
@@ -164,5 +164,5 @@
                 if (!isfinite(value)) {
                     image->data.F32[y][x] = NAN;
-                    mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= options->badMask;
+                    mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= options->lowMask;
                 } else {
                     image->data.F32[y][x] -= value;
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageSetMaskBits.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageSetMaskBits.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageSetMaskBits.c	(revision 24244)
@@ -14,7 +14,11 @@
     // at this point we know we have valid values for required entries SAT, BAD, FLAT, BLANK:
 
-    // mask for non-linear flat corrections
+    // mask for bad flat corrections
     options->flatMask = pmConfigMaskGet("FLAT", config);
     psAssert (options->flatMask, "flat mask not set");
+
+    // mask for bad dark corrections
+    options->darkMask = pmConfigMaskGet("DARK", config);
+    psAssert (options->darkMask, "dark mask not set");
 
     // mask for non-existent data  (default to DETECTOR if not defined)
@@ -27,6 +31,10 @@
 
     // mask for below-range data  (default to RANGE if not defined)
-    options->badMask = pmConfigMaskGet("BAD", config);
-    psAssert (options->badMask, "bad mask not set");
+    options->lowMask = pmConfigMaskGet("LOW", config);
+    if (!options->lowMask) {
+        // look up old name for backward compatability
+        options->lowMask = pmConfigMaskGet("BAD", config);
+    }
+    psAssert (options->lowMask, "low mask not set");
 
     // save MASK and MARK on the PSPHOT recipe
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageStatsOutput.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageStatsOutput.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageStatsOutput.c	(revision 24244)
@@ -31,4 +31,6 @@
     psFree(resolved);
 
+    pmConfigRunFilenameAddWrite(config, "STATS", statsName);
+
     return true;
 }
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageVersion.c	(revision 24244)
@@ -4,4 +4,5 @@
 
 #include "ppImage.h"
+#include "ppImageVersionDefinitions.h"
 
 #ifndef PPIMAGE_VERSION
@@ -15,11 +16,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString ppImageVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PPIMAGE_BRANCH), xstr(PPIMAGE_VERSION));
+    psStringAppend(&value, "%s@%s", PPIMAGE_BRANCH, PPIMAGE_VERSION);
     return value;
 }
@@ -27,5 +25,5 @@
 psString ppImageSource(void)
 {
-    return psStringCopy (xstr(PPIMAGE_SOURCE));
+    return psStringCopy(PPIMAGE_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PPIMAGE_VERSION_DEFINITIONS_H
+#define PPIMAGE_VERSION_DEFINITIONS_H
+
+#define PPIMAGE_VERSION @PPIMAGE_VERSION@ // SVN version
+#define PPIMAGE_BRANCH  @PPIMAGE_BRANCH@  // SVN branch
+#define PPIMAGE_SOURCE  @PPIMAGE_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/configure.ac	(revision 24244)
@@ -21,4 +21,6 @@
 PKG_CHECK_MODULES([PPSTATS], [ppStats >= 1.0.0]) 
 
+IPP_VERSION
+
 IPP_STDOPTS
 CFLAGS="${CFLAGS=} -Wall -Werror"
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/Makefile.am	(revision 24244)
@@ -1,14 +1,27 @@
 bin_PROGRAMS = ppMerge
 
-# PPMERGE_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-# PPMERGE_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-# PPMERGE_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
-# 
-# # Force recompilation of ppMergeVersion.c, since it gets the version information
-# ppMergeVersion.c: FORCE
-# 	touch ppMergeVersion.c
-# FORCE: ;
+if HAVE_SVNVERSION
+PPMERGE_VERSION=`$(SVNVERSION) ..`
+else
+PPMERGE_VERSION="UNKNOWN"
+endif
 
-ppMerge_CFLAGS = $(PPMERGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPPMERGE_VERSION=$(SVN_VERSION) -DPPMERGE_BRANCH=$(SVN_BRANCH) -DPPMERGE_SOURCE=$(SVN_SOURCE)
+if HAVE_SVN
+PPMERGE_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPMERGE_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PPMERGE_BRANCH="UNKNOWN"
+PPMERGE_SOURCE="UNKNOWN"
+endif
+
+# Force recompilation of ppMergeVersion.c, since it gets the version information
+ppMergeVersion.c: ppMergeVersionDefinitions.h
+ppMergeVersionDefinitions.h: ppMergeVersionDefinitions.h.in FORCE
+	-$(RM) ppMergeVersionDefinitions.h
+	$(SED) -e "s|@PPMERGE_VERSION@|\"$(PPMERGE_VERSION)\"|" -e "s|@PPMERGE_BRANCH@|\"$(PPMERGE_BRANCH)\"|" -e "s|@PPMERGE_SOURCE@|\"$(PPMERGE_SOURCE)\"|" ppMergeVersionDefinitions.h.in > ppMergeVersionDefinitions.h
+FORCE: ;
+
+
+ppMerge_CFLAGS = $(PPMERGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
 ppMerge_LDFLAGS = $(PPMERGE_LIBS) $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMerge.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMerge.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMerge.c	(revision 24244)
@@ -95,4 +95,5 @@
         psFree(statsOut);
         fclose(statsFile);
+        pmConfigRunFilenameAddWrite(config, "STATS", statsName);
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeArguments.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeArguments.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeArguments.c	(revision 24244)
@@ -156,4 +156,5 @@
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-nkeep",    0, "Minimum number of pixels in stack to keep", 0);
     psMetadataAddBool(arguments, PS_LIST_TAIL, "-variances", 0, "Use image variances in combination?", false);
+    psMetadataAddBool(arguments, PS_LIST_TAIL, "-use-masks", 0, "Use image masks in combination?", false);
 
     // XXX EAM : not clear this should be allowed on the command line.
@@ -179,5 +180,5 @@
 
     /** Mask construction parameters */
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-mask-suspect",  0, "Threshold for suspect pixels (sigma)", NAN);
+    psMetadataAddF32(arguments, PS_LIST_TAIL, "-mask-suspect-sigma",  0, "Threshold for suspect pixels (sigma)", NAN);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-mask-bad",      0, "Threshold for bad pixels (sigma)", NAN);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask-mode",     0, "Mode to identify bad pixels", NULL);
@@ -308,4 +309,5 @@
     VALUE_ARG_RECIPE_INT("-nkeep",      "NKEEP",    S32, 0);
     VALUE_ARG_RECIPE_BOOL("-variances", "VARIANCES");
+    VALUE_ARG_RECIPE_BOOL("-use-masks", "INPUTS.MASKS.USE");
 
     // XXX we do not supply this on the command line
@@ -329,5 +331,8 @@
 
     /** Mask construction parameters */
-    VALUE_ARG_RECIPE_FLOAT("-mask-suspect", "MASK.SUSPECT", F32);
+    VALUE_ARG_RECIPE_FLOAT("-mask-suspect-sigma", "MASK.SUSPECT.SIGMA", F32);
+    VALUE_ARG_RECIPE_FLOAT("-mask-suspect-min",   "MASK.SUSPECT.MIN", F32);
+    VALUE_ARG_RECIPE_FLOAT("-mask-suspect-max",   "MASK.SUSPECT.MAX", F32);
+    VALUE_ARG_RECIPE_STR("-mask-mode", "MASK.SUSPECT.MODE");
     VALUE_ARG_RECIPE_FLOAT("-mask-bad",     "MASK.BAD",     F32);
     VALUE_ARG_RECIPE_INT("-mask-grow",      "MASK.GROW",    S32, 0);
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeErrorCodes.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeErrorCodes.dat	(revision 24244)
@@ -2,5 +2,5 @@
 # This file is used to generate ppMergeErrorClasses.h
 #
-BASE = 700		First value we use; lower values belong to psLib
+BASE = 6000		First value we use; lower values belong to psLib
 # these errors correspond to standard exit conditions
 ARGUMENTS               Incorrect arguments
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeErrorCodes.h.in	(revision 24244)
@@ -21,5 +21,5 @@
  */
 typedef enum {
-    PPMERGE_ERR_BASE = 512,
+    PPMERGE_ERR_BASE = 6000,
     PPMERGE_ERR_${ErrorCode},
     PPMERGE_ERR_NERROR
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeLoop.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeLoop.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeLoop.c	(revision 24244)
@@ -36,4 +36,5 @@
     bool mdok;                          ///< Status of MD lookup
     bool haveMasks = psMetadataLookupBool(&mdok, arguments, "INPUTS.MASKS"); // Do we have masks?
+    bool useMasks = psMetadataLookupBool(&mdok, arguments, "INPUTS.MASKS.USE"); // Do we have masks?
     bool haveVariances = psMetadataLookupBool(&mdok, arguments, "INPUTS.VARIANCES"); // Do we have variances?
 
@@ -73,5 +74,5 @@
 
     pmCombineParams *combination = pmCombineParamsAlloc(combineStat); ///< Combination parameters
-    combination->maskVal = maskVal;
+    combination->maskVal = useMasks ? maskVal : 0;
     combination->blank = pmConfigMaskGet("BLANK", config);
     combination->nKeep = nKeep;
@@ -149,4 +150,8 @@
         }
         pmCell *outCell;                ///< Cell of interest
+
+	// XXX TEST : force a single loop
+        // outCell = pmFPAviewNextCell(view, outFPA, 1); {
+
         while ((outCell = pmFPAviewNextCell(view, outFPA, 1))) {
             if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
@@ -164,4 +169,7 @@
                 pmHDU *hdu = pmHDUGetHighest(outFPA, outChip, outCell); // HDU for file
                 if (hdu && hdu != lastHDU) {
+                    if (!hdu->header) {
+                        hdu->header = psMetadataAlloc();
+                    }
                     ppMergeVersionHeader(hdu->header);
                     lastHDU = hdu;
@@ -229,5 +237,5 @@
                 psAssert (fileGroups->n > 0, "no valid file groups defined");
                 fileGroup = fileGroups->data[0];
-                if (!pmShutterCorrectionGeneratePrepare(outRO, pattern, fileGroup->readouts, maskVal)) {
+                if (!pmShutterCorrectionGeneratePrepare(outRO, pattern, fileGroup->readouts, combination->maskVal)) {
                     goto ERROR;
                 }
@@ -411,26 +419,45 @@
             }
 
-	    // calculate CTEMASK after stats so stats reflect median image
+            // calculate CTEMASK after stats so stats reflect median image
             if (type == PPMERGE_TYPE_CTEMASK && outRO) {
-		// need to apply range cuts on the output image
-		psAssert (outRO->mask, "mask is not defined");
-		psAssert (outRO->mask->numCols == outRO->image->numCols, "mismatch between image and mask");
-		psAssert (outRO->mask->numRows == outRO->image->numRows, "mismatch between image and mask");
-
-		// CTEMASK parameters
-		float cteMin = psMetadataLookupF32(NULL, arguments, "CTE.MIN"); // Number of fringe points
-
-		char *cteMaskName = psMetadataLookupStr (&mdok, config->arguments, "MASK.SET.VALUE");
-		psImageMaskType cteMaskValue = pmConfigMaskGet(cteMaskName, config);
-
-		psF32 **outputImage = outRO->image->data.F32; 
-		psImageMaskType **outputMask = outRO->mask->data.PS_TYPE_IMAGE_MASK_DATA; 
-		for (int iy = 0; iy < outRO->image->numRows; iy++) {
-		    for (int ix = 0; ix < outRO->image->numCols; ix++) {
-			if (outputImage[iy][ix] < cteMin) {
-			    outputMask[iy][ix] |= cteMaskValue;
-			}
-		    }
+                // need to apply range cuts on the output image
+                psAssert (outRO->mask, "mask is not defined");
+                psAssert (outRO->mask->numCols == outRO->image->numCols, "mismatch between image and mask");
+                psAssert (outRO->mask->numRows == outRO->image->numRows, "mismatch between image and mask");
+
+                // CTEMASK parameters
+                float cteMin = psMetadataLookupF32(NULL, arguments, "CTE.MIN"); // Number of fringe points
+
+                char *cteMaskName = psMetadataLookupStr (&mdok, config->arguments, "MASK.SET.VALUE");
+                psImageMaskType cteMaskValue = pmConfigMaskGet(cteMaskName, config);
+
+		if (0) {
+		  psFits *fits = NULL;
+		  fits = psFitsOpen ("combine.fits", "w");
+		  psFitsWriteImage (fits, NULL, outRO->image, 0, NULL);
+		  psFitsClose (fits);
+
+		  fits = psFitsOpen ("inmask.fits", "w");
+		  psFitsWriteImage (fits, NULL, outRO->mask, 0, NULL);
+		  psFitsClose (fits);
 		}
+
+                psF32 **outputImage = outRO->image->data.F32;
+                psImageMaskType **outputMask = outRO->mask->data.PS_TYPE_IMAGE_MASK_DATA;
+                for (int iy = 0; iy < outRO->image->numRows; iy++) {
+                    for (int ix = 0; ix < outRO->image->numCols; ix++) {
+                        if (outputImage[iy][ix] < cteMin) {
+                            outputMask[iy][ix] |= cteMaskValue;
+                        }
+                    }
+                }
+
+		if (0) {
+		  psFits *fits = NULL;
+		  fits = psFitsOpen ("otmask.fits", "w");
+		  psFitsWriteImage (fits, NULL, outRO->mask, 0, NULL);
+		  psFitsClose (fits);
+		}
+
             }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeMask.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeMask.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeMask.c	(revision 24244)
@@ -31,5 +31,14 @@
     int sample = psMetadataLookupS32(NULL, config->arguments, "SAMPLE"); ///< Size of sample for statistics
     bool chipStats = psMetadataLookupBool(&mdok, config->arguments, "MASK.CHIPSTATS"); ///< Statistics on chip?
-    float maskSuspect = psMetadataLookupF32(NULL, config->arguments, "MASK.SUSPECT"); ///< Threshold for suspect pixels
+
+    char *maskSuspectMode  = psMetadataLookupStr(NULL, config->arguments, "MASK.SUSPECT.MODE"); ///< Threshold for suspect pixels
+    if (strcasecmp(maskSuspectMode, "SIGMA") && strcasecmp(maskSuspectMode, "VALUE")) {
+        psError (PS_ERR_UNKNOWN, true, "Invalid choice for MASK.SUSPECT.MODE: %s\n", maskSuspectMode);
+        return false;
+    }
+    float maskSuspectSigma = psMetadataLookupF32(NULL, config->arguments, "MASK.SUSPECT.SIGMA"); ///< Threshold for suspect pixels
+    float maskSuspectMin   = psMetadataLookupF32(NULL, config->arguments, "MASK.SUSPECT.MIN"); ///< Threshold for suspect pixels
+    float maskSuspectMax   = psMetadataLookupF32(NULL, config->arguments, "MASK.SUSPECT.MAX"); ///< Threshold for suspect pixels
+
     float maskBad = psMetadataLookupF32(NULL, config->arguments, "MASK.BAD"); ///< Threshold for bad pixels
     pmMaskIdentifyMode maskMode = psMetadataLookupS32(NULL, config->arguments, "MASK.MODE"); ///< Mode for identifying bad pixels
@@ -111,5 +120,5 @@
             }
 
-            if (!ppMergeFileOpenInput(config, view, i)) {
+            if (!ppMergeFileOpenInput(config, inView, i)) {
                 psError(PS_ERR_UNKNOWN, false, "Unable to open file %d", i);
                 psFree(inView);
@@ -154,43 +163,64 @@
                 valueIndex = 0;
             }
-            for (int i = 0; i < num; i++) {
-                int pixel = numPix * psRandomUniform(rng);
-                int x = pixel % numCols;
-                int y = pixel / numCols;
-                if (mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskValRaw)) continue;
-                if (outMask && (outMask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskValOut)) continue;
-                if (!isfinite(image->data.F32[y][x])) continue;
-
-                values->data.F32[valueIndex++] = image->data.F32[y][x];
-            }
-
-            if (!chipStats) {
-                values->n = valueIndex;
-                if (!psVectorStats(statistics, values, NULL, NULL, 0)) {
-                    psError(PS_ERR_UNKNOWN, false, "Unable to do statistics on readout.");
-                    psFree(inView);
-                    psFree(readout);
-                    goto MERGE_MASK_ERROR;
-                }
-
-                float mean = psStatsGetValue(statistics, meanStat);
-                float stdev = psStatsGetValue(statistics, stdevStat);
-
-                // this function increments the count for each suspect pixel in each input plane
-                // maskValRaw is used to test for valid input pixels
-                if (!pmMaskFlagSuspectPixels(outRO, readout, mean, stdev, maskSuspect, maskValRaw)) {
-                    psError(PS_ERR_UNKNOWN, false, "Unable to find suspect values in file %d", i);
-                    psFree(inView);
-                    psFree(readout);
-                    goto MERGE_MASK_ERROR;
-                }
-                pmCellFreeData(inCell);
-
-                if (!pmFPAfileIOChecks(config, inView, PM_FPA_AFTER)) {
-                    psFree(inView);
-                    psFree(readout);
-                    goto MERGE_MASK_ERROR;
-                }
-            }
+
+	    if (!strcasecmp(maskSuspectMode, "VALUE")) {
+		// this function increments the count for each suspect pixel in each input plane
+		// maskValRaw is used to test for valid input pixels
+		if (!pmMaskFlagSuspectPixelsByValue(outRO, readout, maskSuspectMin, maskSuspectMax, maskValRaw)) {
+		    psError(PS_ERR_UNKNOWN, false, "Unable to find suspect values in file %d", i);
+		    psFree(inView);
+		    psFree(readout);
+		    goto MERGE_MASK_ERROR;
+		}
+		pmCellFreeData(inCell);
+
+		if (!pmFPAfileIOChecks(config, inView, PM_FPA_AFTER)) {
+		    psFree(inView);
+		    psFree(readout);
+		    goto MERGE_MASK_ERROR;
+		}
+	    } else {
+		// extract a subset of pixels for stats measurement -- don't use pixels which are masked for this calculation
+		for (int i = 0; i < num; i++) {
+		    int pixel = numPix * psRandomUniform(rng);
+		    int x = pixel % numCols;
+		    int y = pixel / numCols;
+		    if (mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskValRaw)) continue;
+		    if (outMask && (outMask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskValOut)) continue;
+		    if (!isfinite(image->data.F32[y][x])) continue;
+
+		    values->data.F32[valueIndex] = image->data.F32[y][x];
+		    valueIndex++;
+		}
+
+		// for per-readout stats, measure the stats and find the suspect pixels
+		if (!chipStats) {
+		    values->n = valueIndex;
+		    if (!psVectorStats(statistics, values, NULL, NULL, 0)) {
+			psError(PS_ERR_UNKNOWN, false, "Unable to do statistics on readout.");
+			psFree(inView);
+			psFree(readout);
+			goto MERGE_MASK_ERROR;
+		    }
+		    float mean = psStatsGetValue(statistics, meanStat);
+		    float stdev = psStatsGetValue(statistics, stdevStat);
+
+		    // this function increments the count for each suspect pixel in each input plane
+		    // maskValRaw is used to test for valid input pixels
+		    if (!pmMaskFlagSuspectPixelsBySigma(outRO, readout, mean, stdev, maskSuspectSigma, maskValRaw)) {
+			psError(PS_ERR_UNKNOWN, false, "Unable to find suspect values in file %d", i);
+			psFree(inView);
+			psFree(readout);
+			goto MERGE_MASK_ERROR;
+		    }
+		    pmCellFreeData(inCell);
+
+		    if (!pmFPAfileIOChecks(config, inView, PM_FPA_AFTER)) {
+			psFree(inView);
+			psFree(readout);
+			goto MERGE_MASK_ERROR;
+		    }
+		}
+	    }
             psFree(readout);
             psFree(outRO);
@@ -198,5 +228,6 @@
 
         // Additional run through cells if we want chip-level statistics
-        if (chipStats && valueIndex > 0) {
+	// only used for MASK.SUSPECT.MODE == SIGMA
+        if (!strcasecmp(maskSuspectMode, "SIGMA") && chipStats && valueIndex > 0) {
             values->n = valueIndex;
             if (!psVectorStats(statistics, values, NULL, NULL, 0)) {
@@ -221,5 +252,5 @@
                 float stdev = psStatsGetValue(statistics, stdevStat);
 
-                if (!pmMaskFlagSuspectPixels(outRO, readout, mean, stdev, maskSuspect, maskValRaw)) {
+                if (!pmMaskFlagSuspectPixelsBySigma(outRO, readout, mean, stdev, maskSuspectSigma, maskValRaw)) {
                     psError(PS_ERR_UNKNOWN, false, "Unable to find suspect values in file %d", i);
                     goto MERGE_MASK_ERROR;
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeVersion.c	(revision 24244)
@@ -21,4 +21,5 @@
 
 #include "ppMergeVersion.h"
+#include "ppMergeVersionDefinitions.h"
 
 #ifndef PPMERGE_VERSION
@@ -32,11 +33,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString ppMergeVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PPMERGE_BRANCH), xstr(PPMERGE_VERSION));
+    psStringAppend(&value, "%s@%s", PPMERGE_BRANCH, PPMERGE_VERSION);
     return value;
 }
@@ -44,5 +42,5 @@
 psString ppMergeSource(void)
 {
-    return psStringCopy(xstr(PPMERGE_SOURCE));
+    return psStringCopy(PPMERGE_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PPMERGE_VERSION_DEFINITIONS_H
+#define PPMERGE_VERSION_DEFINITIONS_H
+
+#define PPMERGE_VERSION @PPMERGE_VERSION@ // SVN version
+#define PPMERGE_BRANCH  @PPMERGE_BRANCH@  // SVN branch
+#define PPMERGE_SOURCE  @PPMERGE_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/Makefile.am	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/Makefile.am	(revision 24244)
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES = *.pyc *~ core core.*
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/autogen.sh
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/autogen.sh	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/autogen.sh	(revision 24244)
@@ -0,0 +1,103 @@
+#!/bin/sh
+# Run this to generate all the initial makefiles, etc.
+
+srcdir=`dirname $0`
+test -z "$srcdir" && srcdir=.
+
+ORIGDIR=`pwd`
+cd $srcdir
+
+PROJECT=ppMops
+TEST_TYPE=-f
+# change this to be a unique filename in the top level dir
+FILE=autogen.sh
+
+DIE=0
+
+LIBTOOLIZE=libtoolize
+ACLOCAL="aclocal $ACLOCAL_FLAGS"
+AUTOHEADER=autoheader
+AUTOMAKE=automake
+AUTOCONF=autoconf
+
+($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $LIBTOOLIZE installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/"
+        DIE=1
+}
+
+($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $ACLOCAL installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/"
+        DIE=1
+}
+
+($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $AUTOHEADER installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/"
+        DIE=1
+}
+
+($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $AUTOMAKE installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/"
+        DIE=1
+}
+
+($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $AUTOCONF installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/"
+        DIE=1
+}
+
+if test "$DIE" -eq 1; then
+        exit 1
+fi
+
+test $TEST_TYPE $FILE || {
+        echo "You must run this script in the top-level $PROJECT directory"
+        exit 1
+}
+
+if test -z "$*"; then
+        echo "I am going to run ./configure with no arguments - if you wish "
+        echo "to pass any to it, please specify them on the $0 command line."
+fi
+
+$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed"
+$ACLOCAL || echo "$ACLOCAL failed"
+$AUTOHEADER || echo "$AUTOHEADER failed"
+$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE  failed"
+$AUTOCONF || echo "$AUTOCONF failed"
+
+cd $ORIGDIR
+
+run_configure=true
+for arg in $*; do
+    case $arg in
+        --no-configure)
+            run_configure=false
+            ;;
+        *)
+            ;;
+    esac
+done
+
+if $run_configure; then
+    $srcdir/configure --enable-maintainer-mode "$@"
+    echo
+    echo "Now type 'make' to compile $PROJECT."
+else
+    echo
+    echo "Now run 'configure' and 'make' to compile $PROJECT."
+fi
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/configure.ac	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/configure.ac	(revision 24244)
@@ -0,0 +1,41 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_PREREQ(2.61)
+
+AC_INIT([ppMops], [0.1.1], [ipp-support@ifa.hawaii.edu])
+AC_CONFIG_SRCDIR([src])
+
+AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2])
+AM_CONFIG_HEADER([src/config.h])
+AM_MAINTAINER_MODE
+
+IPP_STDCFLAGS
+
+AC_LANG(C)
+AC_GNU_SOURCE
+AC_PROG_CC_C99
+AC_PROG_INSTALL
+AC_PROG_LIBTOOL
+AC_SYS_LARGEFILE
+
+PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0])
+
+AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing])
+if test "$ERRORCODES" = "missing" ; then
+  AC_MSG_ERROR([psParseErrorCodes is required])
+fi
+
+dnl Set CFLAGS for build
+IPP_STDOPTS
+CFLAGS="${CFLAGS} -Wall -Werror"
+
+IPP_VERSION
+
+AC_SUBST([PPSUB_CFLAGS])
+AC_SUBST([PPSUB_LIBS])
+
+AC_CONFIG_FILES([
+  Makefile
+  src/Makefile
+])
+
+AC_OUTPUT
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/src/Makefile.am	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/src/Makefile.am	(revision 24244)
@@ -0,0 +1,41 @@
+bin_PROGRAMS = ppMops
+
+if HAVE_SVNVERSION
+PPMOPS_VERSION=`$(SVNVERSION) ..`
+else
+PPMOPS_VERSION="UNKNOWN"
+endif
+
+if HAVE_SVN
+PPMOPS_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPMOPS_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PPMOPS_BRANCH="UNKNOWN"
+PPMOPS_SOURCE="UNKNOWN"
+endif
+
+# Force recompilation of ppMopsVersion.c, since it gets the version information
+ppMopsVersion.c: ppMopsVersionDefinitions.h
+ppMopsVersionDefinitions.h: ppMopsVersionDefinitions.h.in FORCE
+	-$(RM) ppMopsVersionDefinitions.h
+	$(SED) -e "s|@PPMOPS_VERSION@|\"$(PPMOPS_VERSION)\"|" -e "s|@PPMOPS_BRANCH@|\"$(PPMOPS_BRANCH)\"|" -e "s|@PPMOPS_SOURCE@|\"$(PPMOPS_SOURCE)\"|" ppMopsVersionDefinitions.h.in > ppMopsVersionDefinitions.h
+FORCE: ;
+
+ppMops_CPPFLAGS = $(PSLIB_CFLAGS) $(PPMOPS_CFLAGS)
+ppMops_LDFLAGS  = $(PSLIB_LIBS)   $(PPMOPS_LIBS)
+
+ppMops_SOURCES =		\
+	ppMops.c		\
+	ppMopsVersion.c		\
+	ppMopsData.c			
+
+noinst_HEADERS = \
+	ppMops.h
+
+
+clean-local:
+	-rm -f TAGS
+
+# Tags for emacs
+tags:
+	etags `find . -name \*.[ch] -print`
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMops.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMops.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMops.c	(revision 24244)
@@ -0,0 +1,163 @@
+#include <stdio.h>
+#include <pslib.h>
+
+#include "ppMops.h"
+
+int main(int argc, char *argv[])
+{
+    if (argc != 4) {
+        fprintf(stderr, "Insufficient arguments.\n");
+        fprintf(stderr, "Usage: %s DETECTIONS ZP OUTPUT\n", argv[0]);
+        exit(PS_EXIT_CONFIG_ERROR);
+    }
+
+    ppMopsData *data = ppMopsDataAlloc(); // Configuration data
+    data->detections = psStringCopy(argv[1]);
+    data->zp = atof(argv[2]);
+    data->output = psStringCopy(argv[3]);
+
+    psArray *detections = NULL;         // Detections
+    psMetadata *header = NULL;          // Header for detections
+    {
+        psFits *fits = psFitsOpen(data->detections, "r"); // FITS file
+        header = psFitsReadHeader(NULL, fits);
+        if (!header) {
+            psErrorStackPrint(stderr, "Unable to read header");
+            psFitsClose(fits);
+            psFree(data);
+            exit(PS_EXIT_DATA_ERROR);
+        }
+        if (!psFitsMoveExtName(fits, IN_EXTNAME)) {
+            psErrorStackPrint(stderr, "Unable to move to extension %s", IN_EXTNAME);
+            psFitsClose(fits);
+            psFree(header);
+            psFree(data);
+            exit(PS_EXIT_DATA_ERROR);
+        }
+        detections = psFitsReadTable(fits);
+        psFitsClose(fits);
+        if (!detections) {
+            psErrorStackPrint(stderr, "Unable to read detections");
+            psFree(data);
+            psFree(header);
+            exit(PS_EXIT_DATA_ERROR);
+        }
+    }
+
+    // Translate the columns
+    int numRows = detections->n;        // Number of rows
+    psArray *output = psArrayAlloc(numRows); // Output table
+    double plateScale = 0.0;                 // Average plate scale
+    for (int i = 0; i < numRows; i++) {
+        psMetadata *inRow = detections->data[i]; // Input row
+        psMetadata *outRow = output->data[i] = psMetadataAlloc(); // Output row
+
+        double ra = psMetadataLookupF64(NULL, inRow, "RA_PSF");
+        double dec = psMetadataLookupF64(NULL, inRow, "DEC_PSF");
+        double mag = psMetadataLookupF64(NULL, inRow, "PSF_INST_MAG") + data->zp;
+        double magErr = psMetadataLookupF64(NULL, inRow, "PSF_INST_MAG_SIG");
+        double ext = psMetadataLookupF64(NULL, inRow, "EXT_NSIGMA");
+        double xErr = psMetadataLookupF64(NULL, inRow, "X_PSF_SIG");
+        double yErr = psMetadataLookupF64(NULL, inRow, "Y_PSF_SIG");
+        double scale = psMetadataLookupF64(NULL, inRow, "PLTSCALE");
+        double angle = psMetadataLookupF64(NULL, inRow, "POSANGLE");
+        psU32 flags = psMetadataLookupU32(NULL, inRow, "FLAGS");
+
+        plateScale += scale;
+
+        // XXX Not at all sure I've got the angles around the right way here...
+        double cosAngle = cos(angle), sinAngle = sin(angle);
+        double raErr = scale * sqrt(PS_SQR(cosAngle) * PS_SQR(xErr) + PS_SQR(sinAngle) * PS_SQR(yErr));
+        double decErr = scale * sqrt(PS_SQR(sinAngle) * PS_SQR(xErr) + PS_SQR(cosAngle) * PS_SQR(yErr));
+
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "RA_DEG", 0, "Right ascension (degrees)", ra);
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "DEC_DEG", 0, "Declination (degrees)", dec);
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "RA_SIG", 0, "Right ascension error (degrees)", raErr);
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "DEC_SIG", 0, "Declination error (degrees)", decErr);
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "MAG", 0, "Magnitude", mag);
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "MAG_SIG", 0, "Magnitude error", magErr);
+        psMetadataAddU32(outRow, PS_LIST_TAIL, "FLAGS", 0, "Detection bit flags", flags);
+
+        // The below need fixing
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "STARPSF", 0, "EXT_NSIGMA", ext);
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "ANG", 0, "Position angle of trail (degrees)", 0.0);
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "ANG_SIG", 0, "Position angle error (degrees)", 0.0);
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "LEN", 0, "Length of trail (arcsec)", 0.0);
+        psMetadataAddF64(outRow, PS_LIST_TAIL, "LEN_SIG", 0, "Length error (arcsec)", 0.0);
+    }
+    plateScale /= numRows;
+    psFree(detections);
+
+    // Translate the header
+    psMetadata *outHeader = psMetadataAlloc(); // Output header
+    ppMopsVersionHeader(outHeader);
+    {
+        double ra = psMetadataLookupF64(NULL, header, "FPA.RA");
+        double dec = psMetadataLookupF64(NULL, header, "FPA.DEC");
+        const char *filter = psMetadataLookupStr(NULL, header, "FPA.FILTER");
+        float exptime = psMetadataLookupF32(NULL, header, "EXPTIME");
+        double angle = psMetadataLookupF64(NULL, header, "FPA.POSANGLE");
+        double alt = psMetadataLookupF64(NULL, header, "FPA.ALT");
+        double az = psMetadataLookupF64(NULL, header, "FPA.AZ");
+        int imageid = psMetadataLookupS32(NULL, header, "IMAGEID");
+
+        float psf = plateScale * 0.5 * (psMetadataLookupF32(NULL, header, "FWHM_MAJ") +
+                                        psMetadataLookupF32(NULL, header, "FWHM_MIN"));
+
+        // XXX This is wrong
+        int fpaid = psMetadataLookupS32(NULL, header, "IMAGEID");
+
+
+        psMetadataAddF64(outHeader, PS_LIST_TAIL, "RA", 0, "Right ascension of boresight", ra);
+        psMetadataAddF64(outHeader, PS_LIST_TAIL, "DEC", 0, "Declination of boresight", dec);
+        psMetadataAddStr(outHeader, PS_LIST_TAIL, "FILTER", 0, "Filter name", filter);
+        psMetadataAddF64(outHeader, PS_LIST_TAIL, "EXPTIME", 0, "Exposure time (sec)", exptime);
+        psMetadataAddF64(outHeader, PS_LIST_TAIL, "ROTANGLE", 0, "Rotator position angle", angle);
+        psMetadataAddF64(outHeader, PS_LIST_TAIL, "TEL_ALT", 0, "Telescope altitude", alt);
+        psMetadataAddF64(outHeader, PS_LIST_TAIL, "TEL_AZ", 0, "Telescope azimuth", az);
+        psMetadataAddF64(outHeader, PS_LIST_TAIL, "DIFFIMID", 0, "Difference image identifier", imageid);
+        psMetadataAddF64(outHeader, PS_LIST_TAIL, "FPA_ID", 0, "Exposure identifier", fpaid);
+        psMetadataAddS32(outHeader, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", OBSERVATORY_CODE);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "STARPSF", 0, "Stellar PSF (arcsec)", psf);
+
+        // These are completely fake
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "LIMITMAG", 0, "Limiting magnitude", 25.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE1", 0, "Detection efficiency (FAKE)", 0.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE2", 0, "Detection efficiency (FAKE)", 0.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE3", 0, "Detection efficiency (FAKE)", 0.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE4", 0, "Detection efficiency (FAKE)", 0.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE5", 0, "Detection efficiency (FAKE)", 0.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE6", 0, "Detection efficiency (FAKE)", 0.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE7", 0, "Detection efficiency (FAKE)", 0.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE8", 0, "Detection efficiency (FAKE)", 0.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE9", 0, "Detection efficiency (FAKE)", 0.0);
+        psMetadataAddF32(outHeader, PS_LIST_TAIL, "DE10", 0, "Detection efficiency (FAKE)", 0.0);
+    }
+    psFree(header);
+
+    // Write the new table
+    {
+        psFits *fits = psFitsOpen(data->output, "w"); // FITS file
+        if (!fits) {
+            psErrorStackPrint(stderr, "Unable to open %s for writing", data->output);
+            psFree(outHeader);
+            psFree(output);
+            psFree(data);
+            exit(PS_EXIT_SYS_ERROR);
+        }
+        if (!psFitsWriteTable(fits, outHeader, output, OUT_EXTNAME)) {
+            psErrorStackPrint(stderr, "Unable to write table.");
+            psFree(outHeader);
+            psFree(output);
+            psFree(data);
+            exit(PS_EXIT_SYS_ERROR);
+        }
+        psFitsClose(fits);
+    }
+
+    psFree(outHeader);
+    psFree(output);
+    psFree(data);
+
+    return PS_EXIT_SUCCESS;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMops.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMops.h	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMops.h	(revision 24244)
@@ -0,0 +1,37 @@
+#ifndef PP_MOPS_H
+#define PP_MOPS_H
+
+#include <pslib.h>
+
+#define IN_EXTNAME "SkyChip.psf"                // Extension name for data in input
+#define OBSERVATORY_CODE 566                    // IAU Observatory Code
+#define OUT_EXTNAME "MOPS_TRANSIENT_DETECTIONS" // Extension name for data in output
+
+// Configuration data
+typedef struct {
+    psString detections;                // Detections filename
+    float zp;                           // Magnitude zero point
+    psString output;                    // Output filename
+} ppMopsData;
+
+// Allocator
+ppMopsData *ppMopsDataAlloc(void);
+
+
+
+/// Return version
+psString ppMopsVersion(void);
+
+/// Return source
+psString ppMopsSource(void);
+
+/// Return detailed version information
+psString ppMopsVersionLong(void);
+
+/// Put version into header
+bool ppMopsVersionHeader(psMetadata *header);
+
+/// Print version information
+void ppMopsVersionPrint(void);
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMopsData.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMopsData.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMopsData.c	(revision 24244)
@@ -0,0 +1,29 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+
+#include "ppMops.h"
+
+static void mopsDataFree(ppMopsData *data)
+{
+    psFree(data->detections);
+    psFree(data->output);
+    return;
+}
+
+ppMopsData *ppMopsDataAlloc(void)
+{
+    ppMopsData *data = psAlloc(sizeof(ppMopsData)); // Data to return
+    psMemSetDeallocator(data, (psFreeFunc)mopsDataFree);
+
+    data->detections = NULL;
+    data->zp = NAN;
+    data->output = NULL;
+
+    return data;
+}
+
+
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMopsVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMopsVersion.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMopsVersion.c	(revision 24244)
@@ -0,0 +1,100 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+
+#include "ppMops.h"
+#include "ppMopsVersionDefinitions.h"
+
+#ifndef PPMOPS_VERSION
+#error "PPMOPS_VERSION is not set"
+#endif
+#ifndef PPMOPS_BRANCH
+#error "PPMOPS_BRANCH is not set"
+#endif
+#ifndef PPMOPS_SOURCE
+#error "PPMOPS_SOURCE is not set"
+#endif
+
+psString ppMopsVersion(void)
+{
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", PPMOPS_BRANCH, PPMOPS_VERSION);
+    return value;
+}
+
+psString ppMopsSource(void)
+{
+    return psStringCopy(PPMOPS_SOURCE);
+}
+
+psString ppMopsVersionLong(void)
+{
+    psString version = ppMopsVersion();  // Version, to return
+    psString source = ppMopsSource();    // Source
+
+    psStringPrepend(&version, "ppMops ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
+
+    return version;
+};
+
+
+bool ppMopsVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "ppMops at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+
+    psLibVersionHeader(header);
+
+    psString version = ppMopsVersion(); // Software version
+    psString source  = ppMopsSource();  // Software source
+
+    psStringPrepend(&version, "ppMops version: ");
+    psStringPrepend(&source, "ppMops source: ");
+
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+
+    psFree(version);
+    psFree(source);
+
+    return true;
+}
+
+void ppMopsVersionPrint(void)
+{
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("ppMops", PS_LOG_INFO, "ppMops at %s", timeString);
+    psFree(timeString);
+
+    psString pslib = psLibVersionLong();// psLib version
+    psString ppMops = ppMopsVersionLong(); // ppMops version
+
+    psLogMsg("ppMops", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("ppMops", PS_LOG_INFO, "%s", ppMops);
+
+    psFree(pslib);
+    psFree(ppMops);
+
+    return;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMopsVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMopsVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppMopsVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PPMOPS_VERSION_DEFINITIONS_H
+#define PPMOPS_VERSION_DEFINITIONS_H
+
+#define PPMOPS_VERSION @PPMOPS_VERSION@ // SVN version
+#define PPMOPS_BRANCH  @PPMOPS_BRANCH@  // SVN branch
+#define PPMOPS_SOURCE  @PPMOPS_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppSubVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppSubVersion.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppMops/src/ppSubVersion.c	(revision 24244)
@@ -0,0 +1,127 @@
+/** @file ppSubVersion.c
+ *
+ *  @brief
+ *
+ *  @ingroup ppSub
+ *
+ *  @author IfA
+ *  @version $Revision: 1.4 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-18 00:31:20 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+#include <ppStats.h>
+#include <psphot.h>
+
+#include "ppSub.h"
+#include "ppSubVersionDefinitions.h"
+
+#ifndef PPSUB_VERSION
+#error "PPSUB_VERSION is not set"
+#endif
+#ifndef PPSUB_BRANCH
+#error "PPSUB_BRANCH is not set"
+#endif
+#ifndef PPSUB_SOURCE
+#error "PPSUB_SOURCE is not set"
+#endif
+
+psString ppSubVersion(void)
+{
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", PPSUB_BRANCH, PPSUB_VERSION);
+    return value;
+}
+
+psString ppSubSource(void)
+{
+    return psStringCopy(PPSUB_SOURCE);
+}
+
+psString ppSubVersionLong(void)
+{
+    psString version = ppSubVersion();  // Version, to return
+    psString source = ppSubSource();    // Source
+
+    psStringPrepend(&version, "ppSub ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
+
+    return version;
+};
+
+
+bool ppSubVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "ppSub at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+    psphotVersionHeader(header);
+    ppStatsVersionHeader(header);
+
+    psString version = ppSubVersion(); // Software version
+    psString source  = ppSubSource();  // Software source
+
+    psStringPrepend(&version, "ppSub version: ");
+    psStringPrepend(&source, "ppSub source: ");
+
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+
+    psFree(version);
+    psFree(source);
+
+    return true;
+}
+
+void ppSubVersionPrint(void)
+{
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("ppSub", PS_LOG_INFO, "ppSub at %s", timeString);
+    psFree(timeString);
+
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString psphot = psphotVersionLong(); // psphot version
+    psString ppStats = ppStatsVersionLong(); // ppStats version
+    psString ppSub = ppSubVersionLong(); // ppSub version
+
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", psphot);
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", ppStats);
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", ppSub);
+
+    psFree(pslib);
+    psFree(psmodules);
+    psFree(psphot);
+    psFree(ppStats);
+    psFree(ppSub);
+
+    return;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppNorm/src/ppNormErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppNorm/src/ppNormErrorCodes.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppNorm/src/ppNormErrorCodes.dat	(revision 24244)
@@ -2,5 +2,5 @@
 # This file is used to generate ppNormErrorClasses.h
 #
-BASE = 600		First value we use; lower values belong to psLib
+BASE = 7000		First value we use; lower values belong to psLib
 # these errors correspond to standard exit conditions
 ARGUMENTS               Incorrect arguments
Index: /branches/cnb_branches/cnb_branch_20090301/ppNorm/src/ppNormErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppNorm/src/ppNormErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppNorm/src/ppNormErrorCodes.h.in	(revision 24244)
@@ -9,5 +9,5 @@
  */
 typedef enum {
-    PPNORM_ERR_BASE = 512,
+    PPNORM_ERR_BASE = 7000,
     PPNORM_ERR_${ErrorCode},
     PPNORM_ERR_NERROR
Index: /branches/cnb_branches/cnb_branch_20090301/ppSim/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSim/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSim/configure.ac	(revision 24244)
@@ -22,4 +22,6 @@
 PKG_CHECK_MODULES([PSASTRO],  [psastro >= 0.9.0])
 
+IPP_VERSION
+
 IPP_STDOPTS
 CFLAGS="${CFLAGS=} -Wall -Werror"
Index: /branches/cnb_branches/cnb_branch_20090301/ppSim/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSim/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSim/src/Makefile.am	(revision 24244)
@@ -1,14 +1,26 @@
 bin_PROGRAMS = ppSim ppSimSequence
 
-# PPSIM_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-# PPSIM_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-# PPSIM_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
-# 
-# # Force recompilation of ppSimVersion.c, since it gets the version information
-# ppSimVersion.c: FORCE
-# 	touch ppSimVersion.c
-# FORCE: ;
+if HAVE_SVNVERSION
+PPSIM_VERSION=`$(SVNVERSION) ..`
+else
+PPSIM_VERSION="UNKNOWN"
+endif
 
-ppSim_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PSASTRO_CFLAGS) $(ppSim_CFLAGS) -DPPSIM_VERSION=$(SVN_VERSION) -DPPSIM_BRANCH=$(SVN_BRANCH) -DPPSIM_SOURCE=$(SVN_SOURCE)
+if HAVE_SVN
+PPSIM_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPSIM_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PPSIM_BRANCH="UNKNOWN"
+PPSIM_SOURCE="UNKNOWN"
+endif
+
+# Force recompilation of ppSimVersion.c, since it gets the version information
+ppSimVersion.c: ppSimVersionDefinitions.h
+ppSimVersionDefinitions.h: ppSimVersionDefinitions.h.in FORCE
+	-$(RM) ppSimVersionDefinitions.h
+	$(SED) -e "s|@PPSIM_VERSION@|\"$(PPSIM_VERSION)\"|" -e "s|@PPSIM_BRANCH@|\"$(PPSIM_BRANCH)\"|" -e "s|@PPSIM_SOURCE@|\"$(PPSIM_SOURCE)\"|" ppSimVersionDefinitions.h.in > ppSimVersionDefinitions.h
+FORCE: ;
+
+ppSim_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PSASTRO_CFLAGS) $(ppSim_CFLAGS)
 ppSim_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PSPHOT_LIBS) $(PSASTRO_LIBS)
 ppSim_SOURCES = \
Index: /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSim.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSim.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSim.h	(revision 24244)
@@ -183,5 +183,5 @@
 char *ppSimTypeToString (ppSimType type);
 
-float ppSimGetZeroPoint (psMetadata *recipe, char *filter);
+float ppSimGetZeroPoint(psMetadata *recipe, const char *filter);
 
 bool ppSimMergeReadouts (pmConfig *config, pmFPAview *view);
Index: /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimArguments.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimArguments.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimArguments.c	(revision 24244)
@@ -31,4 +31,5 @@
     psMetadataAddStr(arguments,  PS_LIST_TAIL, "-format", 0, "Camera format name", NULL);
     psMetadataAddStr(arguments,  PS_LIST_TAIL, "-type", 0, "Exposure type (BIAS|DARK|FLAT|OBJECT)", NULL);
+    psMetadataAddStr(arguments,  PS_LIST_TAIL, "-obs_mode", 0, "observation mode", NULL);
     psMetadataAddStr(arguments,  PS_LIST_TAIL, "-filter", 0, "Filter name", NULL);
     psMetadataAddF32(arguments,  PS_LIST_TAIL, "-exptime", 0, "Exposure time (s)", NAN);
@@ -108,4 +109,9 @@
     ppSimArgToRecipeF32(&status, options, "STARS.DENSITY", arguments, "-starsdensity");
     ppSimArgToRecipeBool(&status, options, "PHOTOM",        arguments, "+photom");
+    psString obs_mode = psMetadataLookupStr(&status, arguments, "-obs_mode");
+    if (obs_mode) {
+        psMetadataAddStr(options, PS_LIST_TAIL, "OBS_MODE", 0, "observation mode", obs_mode);
+    }
+
 
     // if we are loading the input image (not creating it), then we can skip the remaining arguments
Index: /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimCreate.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimCreate.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimCreate.c	(revision 24244)
@@ -12,16 +12,16 @@
     pmFPAfile *input = pmFPAfileDefineFromArgs (&status, config, "PPSIM.INPUT", "INPUT");
     if (!input) {
-	// if we have not specified the camera already, we need to interpolate the recipes associated with this camera, and read other command-line recipes
-	if (!pmConfigReadRecipes(config, PM_RECIPE_SOURCE_CL)) {
-	    psError(PS_ERR_IO, false, "Error merging recipes from camera config for %s", config->cameraName);
-	    return NULL;
-	}
+        // if we have not specified the camera already, we need to interpolate the recipes associated with this camera, and read other command-line recipes
+        if (!pmConfigReadRecipes(config, PM_RECIPE_SOURCE_CL)) {
+            psError(PS_ERR_IO, false, "Error merging recipes from camera config for %s", config->cameraName);
+            return NULL;
+        }
     } else {
-	// If an image is supplied, we still generate a fake image and merge them together downstream
-	// (otherwise, we get the variance wrong).
-	if (input->type != PM_FPA_FILE_IMAGE) {
-	    psError(PS_ERR_IO, true, "PPSIM.INPUT is not of type IMAGE");
-	    return NULL;
-	}
+        // If an image is supplied, we still generate a fake image and merge them together downstream
+        // (otherwise, we get the variance wrong).
+        if (input->type != PM_FPA_FILE_IMAGE) {
+            psError(PS_ERR_IO, true, "PPSIM.INPUT is not of type IMAGE");
+            return NULL;
+        }
     }
 
@@ -30,6 +30,6 @@
     fpa = pmFPAConstruct(config->camera, config->cameraName); // FPA to contain the observation
     if (!fpa) {
-	psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to construct an FPA from camera configuration.");
-	return NULL;
+        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to construct an FPA from camera configuration.");
+        return NULL;
     }
 
@@ -37,11 +37,11 @@
     pmFPAfile *output = pmFPAfileDefineOutput(config, fpa, "PPSIM.OUTPUT");
     if (!output) {
-	psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to create output file from PPSIM.OUTPUT. Did you forget to specify the format?");
-	return NULL;
+        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to create output file from PPSIM.OUTPUT. Did you forget to specify the format?");
+        return NULL;
     }
     if (output->type != PM_FPA_FILE_IMAGE) {
-	psError(PS_ERR_BAD_PARAMETER_TYPE, true, "PPSIM.OUTPUT type is not IMAGE");
-	psFree(fpa);
-	return NULL;
+        psError(PS_ERR_BAD_PARAMETER_TYPE, true, "PPSIM.OUTPUT type is not IMAGE");
+        psFree(fpa);
+        return NULL;
     }
     // XXX we should not require the output image to be written
@@ -58,42 +58,42 @@
 
     if (type == PPSIM_TYPE_OBJECT) {
-	// adjust the seeing by the scale
-	float seeing = psMetadataLookupF32(&status, recipe, "SEEING");
-	if (isnan(seeing)) {
-	    psError(PS_ERR_BAD_PARAMETER_TYPE, true, "seeing is not defined");
-	    psFree(fpa);
-	    return NULL;
-	}
-	float scale = psMetadataLookupF32(&status, recipe, "PIXEL.SCALE");
-	if (isnan(scale)) {
-	    psError(PS_ERR_BAD_PARAMETER_TYPE, true, "pixel scale is not defined");
-	    psFree(fpa);
-	    return NULL;
-	}
-	psMetadataAddF32(recipe, PS_LIST_TAIL, "SEEING", PS_META_REPLACE, "Seeing SIGMA (pixels)", seeing / 2.0 / sqrt(2.0 * log(2.0)) / scale);
-
-	// if we have been supplied an input image, but no ra & dec, use the input image values
-	if (input) {
-	    float ra = psMetadataLookupF32(&status, recipe, "RA");
-	    if (isnan(ra)) {
-		ra = psMetadataLookupF64(&status, input->fpa->concepts, "FPA.RA");
-		psMetadataAddF32(recipe, PS_LIST_TAIL, "RA", PS_META_REPLACE, "ra (radians)", ra);
-	    }
-	    float dec = psMetadataLookupF32(&status, recipe, "DEC");
-	    if (isnan(dec)) {
-		dec = psMetadataLookupF64(&status, input->fpa->concepts, "FPA.DEC");
-		psMetadataAddF32(recipe, PS_LIST_TAIL, "DEC", PS_META_REPLACE, "dec (radians)", dec);
-	    }
-	}
+        // adjust the seeing by the scale
+        float seeing = psMetadataLookupF32(&status, recipe, "SEEING");
+        if (isnan(seeing)) {
+            psError(PS_ERR_BAD_PARAMETER_TYPE, true, "seeing is not defined");
+            psFree(fpa);
+            return NULL;
+        }
+        float scale = psMetadataLookupF32(&status, recipe, "PIXEL.SCALE");
+        if (isnan(scale)) {
+            psError(PS_ERR_BAD_PARAMETER_TYPE, true, "pixel scale is not defined");
+            psFree(fpa);
+            return NULL;
+        }
+        psMetadataAddF32(recipe, PS_LIST_TAIL, "SEEING", PS_META_REPLACE, "Seeing SIGMA (pixels)", seeing / 2.0 / sqrt(2.0 * log(2.0)) / scale);
+
+        // if we have been supplied an input image, but no ra & dec, use the input image values
+        if (input) {
+            float ra = psMetadataLookupF32(&status, recipe, "RA");
+            if (isnan(ra)) {
+                ra = psMetadataLookupF64(&status, input->fpa->concepts, "FPA.RA");
+                psMetadataAddF32(recipe, PS_LIST_TAIL, "RA", PS_META_REPLACE, "ra (radians)", ra);
+            }
+            float dec = psMetadataLookupF32(&status, recipe, "DEC");
+            if (isnan(dec)) {
+                dec = psMetadataLookupF64(&status, input->fpa->concepts, "FPA.DEC");
+                psMetadataAddF32(recipe, PS_LIST_TAIL, "DEC", PS_META_REPLACE, "dec (radians)", dec);
+            }
+        }
     }
 
     if ((type == PPSIM_TYPE_OBJECT) || (type == PPSIM_TYPE_FLAT)) {
-	// determine the zeropoint from the filter
-	float zp = psMetadataLookupF32(&status, recipe, "ZEROPOINT");
-	if (isnan(zp)) {
-	    char *filter = psMetadataLookupStr(&status, recipe, "FILTER");
-	    float zp = ppSimGetZeroPoint (recipe, filter);
-	    psMetadataAddF32(recipe, PS_LIST_TAIL, "ZEROPOINT", PS_META_REPLACE, "Photometric zeropoint", zp);
-	}
+        // determine the zeropoint from the filter
+        float zp = psMetadataLookupF32(&status, recipe, "ZEROPOINT");
+        if (isnan(zp)) {
+            char *filter = psMetadataLookupStr(&status, recipe, "FILTER");
+            float zp = ppSimGetZeroPoint (recipe, filter);
+            psMetadataAddF32(recipe, PS_LIST_TAIL, "ZEROPOINT", PS_META_REPLACE, "Photometric zeropoint", zp);
+        }
     }
 
@@ -103,64 +103,64 @@
     if (doPhotom) {
 
-	// XXX at the moment, we can perform photometry on the fake sources and the forced
-	// photometry positions, but not one or the other.  Also, we only support photometry in
-	// the context of an image previously analysed by psphot.  Add support for:
-	// * photometry of a pure fake image (requires peak detection and psf creation)
-	// * photometry of forced source positions without fake image (this might work, it just
-	// requires not generating any fake features).
-
-	if (!input) {
-	    psError(PS_ERR_UNKNOWN, false, "input image not found; currently required for photometry");
-	    return NULL;
-	}
-
-	// we need a chip image if we perform photometry (is it necessary to build it if we don't use it?)
-	pmFPAfile *fakeImage = pmFPAfileDefineChipMosaic(config, output->fpa, "PPSIM.FAKE.CHIP");
-	if (!fakeImage) {
-	    psError(PS_ERR_IO, false, _("Unable to generate new file from PPSIM.FAKE.CHIP"));
-	    psFree(fpa);
-	    return NULL;
-	}
-	if (fakeImage->type != PM_FPA_FILE_IMAGE) {
-	    psError(PS_ERR_IO, true, "PPSIM.FAKE.CHIP is not of type IMAGE");
-	    psFree(fpa);
-	    return NULL;
-	}
-
-	// we need a chip image if we perform photometry (is it necessary to build it if we don't use it?)
-	pmFPAfile *forceImage = NULL;
-	if (input) {
-	    forceImage = pmFPAfileDefineChipMosaic(config, input->fpa, "PPSIM.FORCE.CHIP");
-	    if (!forceImage) {
-		psError(PS_ERR_IO, false, _("Unable to generate new file from PPSIM.FORCE.CHIP"));
-		psFree(fpa);
-		return NULL;
-	    }
-	    if (forceImage->type != PM_FPA_FILE_IMAGE) {
-		psError(PS_ERR_IO, true, "PPSIM.FORCE.CHIP is not of type IMAGE");
-		psFree(fpa);
-		return NULL;
-	    }
-	}
-
-	// define associated psphot input/output files
-	if (!ppSimPhotomFiles (config, fakeImage, forceImage)) {
-	    psError(PSPHOT_ERR_CONFIG, false, "Trouble defining the additional input/output files for psphot");
-	    psFree(fpa);
-	    return NULL;
-	}
+        // XXX at the moment, we can perform photometry on the fake sources and the forced
+        // photometry positions, but not one or the other.  Also, we only support photometry in
+        // the context of an image previously analysed by psphot.  Add support for:
+        // * photometry of a pure fake image (requires peak detection and psf creation)
+        // * photometry of forced source positions without fake image (this might work, it just
+        // requires not generating any fake features).
+
+        if (!input) {
+            psError(PS_ERR_UNKNOWN, false, "input image not found; currently required for photometry");
+            return NULL;
+        }
+
+        // we need a chip image if we perform photometry (is it necessary to build it if we don't use it?)
+        pmFPAfile *fakeImage = pmFPAfileDefineChipMosaic(config, output->fpa, "PPSIM.FAKE.CHIP");
+        if (!fakeImage) {
+            psError(PS_ERR_IO, false, _("Unable to generate new file from PPSIM.FAKE.CHIP"));
+            psFree(fpa);
+            return NULL;
+        }
+        if (fakeImage->type != PM_FPA_FILE_IMAGE) {
+            psError(PS_ERR_IO, true, "PPSIM.FAKE.CHIP is not of type IMAGE");
+            psFree(fpa);
+            return NULL;
+        }
+
+        // we need a chip image if we perform photometry (is it necessary to build it if we don't use it?)
+        pmFPAfile *forceImage = NULL;
+        if (input) {
+            forceImage = pmFPAfileDefineChipMosaic(config, input->fpa, "PPSIM.FORCE.CHIP");
+            if (!forceImage) {
+                psError(PS_ERR_IO, false, _("Unable to generate new file from PPSIM.FORCE.CHIP"));
+                psFree(fpa);
+                return NULL;
+            }
+            if (forceImage->type != PM_FPA_FILE_IMAGE) {
+                psError(PS_ERR_IO, true, "PPSIM.FORCE.CHIP is not of type IMAGE");
+                psFree(fpa);
+                return NULL;
+            }
+        }
+
+        // define associated psphot input/output files
+        if (!ppSimPhotomFiles (config, fakeImage, forceImage)) {
+            psError(PSPHOT_ERR_CONFIG, false, "Trouble defining the additional input/output files for psphot");
+            psFree(fpa);
+            return NULL;
+        }
     } else {
-	// have we supplied a psf model?  this happens in ppSimPhotomFiles if we request a photometry
-	// analysis.  however, even if we do not, a psf model may be used to generate the fake
-	// sources.
-	if (psMetadataLookupPtr(NULL, config->arguments, "PSPHOT.PSF")) {
-	    // tie the psf file to the chipMosaic 
-	    pmFPAfileBindFromArgs(&status, output, config, "PSPHOT.PSF.LOAD", "PSPHOT.PSF");
-	    if (!status) {
-		psError(PS_ERR_UNKNOWN, false, "Failed to find/build PSPHOT.PSF.LOAD");
-		psFree(fpa);
-		return NULL;
-	    }
-	}
+        // have we supplied a psf model?  this happens in ppSimPhotomFiles if we request a photometry
+        // analysis.  however, even if we do not, a psf model may be used to generate the fake
+        // sources.
+        if (psMetadataLookupPtr(NULL, config->arguments, "PSPHOT.PSF")) {
+            // tie the psf file to the chipMosaic
+            pmFPAfileBindFromArgs(&status, output, config, "PSPHOT.PSF.LOAD", "PSPHOT.PSF");
+            if (!status) {
+                psError(PS_ERR_UNKNOWN, false, "Failed to find/build PSPHOT.PSF.LOAD");
+                psFree(fpa);
+                return NULL;
+            }
+        }
     }
 
@@ -169,6 +169,6 @@
     pmFPAfile *simSources = pmFPAfileDefineOutput (config, output->fpa, "PPSIM.SOURCES");
     if (!simSources) {
-	psError(PS_ERR_UNKNOWN, false, "Cannot find a rule for PPSIM.SOURCES");
-	return false;
+        psError(PS_ERR_UNKNOWN, false, "Cannot find a rule for PPSIM.SOURCES");
+        return false;
     }
     simSources->save = true;
@@ -176,14 +176,19 @@
     // if we have loaded an input image, we derive certain values from the image, if possible
     if (input) {
-	// we need to extract certain metadata from the image and populate the recipe.
-	// or else we need to set the fpa concepts based on the recipe options...
-
-	psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, PPSIM_RECIPE); // Recipe
-
-	ppSimArgToRecipeF32(&status, recipe, "EXPTIME", input->fpa->concepts, "FPA.EXPOSURE");
-	char *filter = ppSimArgToRecipeStr(&status, recipe, "FILTER", input->fpa->concepts, "FPA.FILTER");
-
-	float zp = ppSimGetZeroPoint (recipe, filter);
-	psMetadataAddF32(recipe, PS_LIST_TAIL, "ZEROPOINT", PS_META_REPLACE, "Photometric zeropoint", zp);
+        // we need to extract certain metadata from the image and populate the recipe.
+        // or else we need to set the fpa concepts based on the recipe options...
+
+        psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, PPSIM_RECIPE); // Recipe
+
+        ppSimArgToRecipeF32(&status, recipe, "EXPTIME", input->fpa->concepts, "FPA.EXPOSURE");
+        char *filter = ppSimArgToRecipeStr(&status, recipe, "FILTER", input->fpa->concepts, "FPA.FILTERID");
+
+        float zp = ppSimGetZeroPoint(recipe, filter);
+        if (!isfinite(zp)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to find zero point for filter %s", filter);
+            psFree(fpa);
+            return NULL;
+        }
+        psMetadataAddF32(recipe, PS_LIST_TAIL, "ZEROPOINT", PS_META_REPLACE, "Photometric zeropoint", zp);
     }
 
@@ -193,34 +198,34 @@
 
     if (phuLevel == PM_FPA_LEVEL_FPA) {
-	if (!pmFPAAddSourceFromView(fpa, "Simulation", view, output->format)) {
-	    psError(PS_ERR_UNKNOWN, false, "Unable to add PHU to FPA.");
-	    psFree(fpa);
-	    psFree(view);
-	    return NULL;
-	}
+        if (!pmFPAAddSourceFromView(fpa, "Simulation", view, output->format)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to add PHU to FPA.");
+            psFree(fpa);
+            psFree(view);
+            return NULL;
+        }
     }
 
     pmChip *chip;                       // Chip from FPA
     while ((chip = pmFPAviewNextChip(view, fpa, 1))) {
-	if (phuLevel == PM_FPA_LEVEL_CHIP) {
-	    if (!pmFPAAddSourceFromView(fpa, "Simulation", view, output->format)) {
-		psError(PS_ERR_UNKNOWN, false, "Unable to add PHU to FPA.");
-		psFree(fpa);
-		psFree(view);
-		return NULL;
-	    }
-	}
-
-	pmCell *cell;                   // Cell from chip
-	while ((cell = pmFPAviewNextCell(view, fpa, 1))) {
-	    if (phuLevel == PM_FPA_LEVEL_CELL) {
-		if (!pmFPAAddSourceFromView(fpa, "Simulation", view, output->format)) {
-		    psError(PS_ERR_UNKNOWN, false, "Unable to add PHU to FPA.");
-		    psFree(fpa);
-		    psFree(view);
-		    return NULL;
-		}
-	    }
-	}
+        if (phuLevel == PM_FPA_LEVEL_CHIP) {
+            if (!pmFPAAddSourceFromView(fpa, "Simulation", view, output->format)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to add PHU to FPA.");
+                psFree(fpa);
+                psFree(view);
+                return NULL;
+            }
+        }
+
+        pmCell *cell;                   // Cell from chip
+        while ((cell = pmFPAviewNextCell(view, fpa, 1))) {
+            if (phuLevel == PM_FPA_LEVEL_CELL) {
+                if (!pmFPAAddSourceFromView(fpa, "Simulation", view, output->format)) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to add PHU to FPA.");
+                    psFree(fpa);
+                    psFree(view);
+                    return NULL;
+                }
+            }
+        }
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimSequenceObject.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimSequenceObject.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimSequenceObject.c	(revision 24244)
@@ -95,4 +95,5 @@
 			    psStringAppend (&command, " -dec %f", dec);
 			    psStringAppend (&command, " -pa %f", pos);
+                            psStringAppend (&command, " -obs_mode OBJECT.%s", (char *) filters->data[i]);
 
 			    double frnd = psRandomUniform(rng);
Index: /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimUtils.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimUtils.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimUtils.c	(revision 24244)
@@ -112,7 +112,7 @@
 ppSimType ppSimTypeFromString (char *typeStr) {
 
-    if (!strcasecmp (typeStr, "BIAS")) 	 return PPSIM_TYPE_BIAS;
-    if (!strcasecmp (typeStr, "DARK")) 	 return PPSIM_TYPE_DARK;
-    if (!strcasecmp (typeStr, "FLAT")) 	 return PPSIM_TYPE_FLAT;
+    if (!strcasecmp (typeStr, "BIAS"))   return PPSIM_TYPE_BIAS;
+    if (!strcasecmp (typeStr, "DARK"))   return PPSIM_TYPE_DARK;
+    if (!strcasecmp (typeStr, "FLAT"))   return PPSIM_TYPE_FLAT;
     if (!strcasecmp (typeStr, "OBJECT")) return PPSIM_TYPE_OBJECT;
     psAbort("Should never get here.");
@@ -133,7 +133,9 @@
 
     char *typeStr = psMetadataLookupStr(NULL, recipe, "IMAGE.TYPE"); // Type of image to simulate
+    char *obs_mode = psMetadataLookupStr(NULL, recipe, "OBS_MODE");
 
     psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.OBSTYPE", PS_META_REPLACE, "Observation type", typeStr);
     psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.OBJECT", PS_META_REPLACE, "Observation name", typeStr);
+    psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.OBS.MODE", PS_META_REPLACE, "Observation mode", obs_mode);
     psMetadataAddF32(fpa->concepts, PS_LIST_TAIL, "FPA.EXPOSURE", PS_META_REPLACE, "Exposure time (sec)", expTime);
     psMetadataAddStr(fpa->concepts, PS_LIST_TAIL, "FPA.FILTERID", PS_META_REPLACE, "Filter name", filter);
@@ -174,5 +176,5 @@
     if (binning <= 0) {
         psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Binning (%d) is non-positive.", binning);
-	exit(PS_EXIT_CONFIG_ERROR);
+        exit(PS_EXIT_CONFIG_ERROR);
     }
     return true;
@@ -181,8 +183,8 @@
 // Get a value from the command-line arguments and add it to recipe options
 float ppSimArgToRecipeF32(bool *status,
-			  psMetadata *options,    // Target to which to add value
-			  const char *recipeName, // Name for value in the recipe
-			  psMetadata *arguments,  // Command-line arguments
-			  const char *argName	    // Argument name in the command-line arguments
+                          psMetadata *options,    // Target to which to add value
+                          const char *recipeName, // Name for value in the recipe
+                          psMetadata *arguments,  // Command-line arguments
+                          const char *argName       // Argument name in the command-line arguments
     )
 {
@@ -198,8 +200,8 @@
 // Get a value from the command-line arguments and add it to recipe options
 int ppSimArgToRecipeS32(bool *status,
-			psMetadata *options,    // Target to which to add value
-			const char *recipeName, // Name for value in the recipe
-			psMetadata *arguments,  // Command-line arguments
-			const char *argName	    // Argument name in the command-line arguments
+                        psMetadata *options,    // Target to which to add value
+                        const char *recipeName, // Name for value in the recipe
+                        psMetadata *arguments,  // Command-line arguments
+                        const char *argName         // Argument name in the command-line arguments
     )
 {
@@ -214,8 +216,8 @@
 // Get a value from the command-line arguments and add it to recipe options
 bool ppSimArgToRecipeBool(bool *status,
-			  psMetadata *options,    // Target to which to add value
-			  const char *recipeName, // Name for value in the recipe
-			  psMetadata *arguments,  // Command-line arguments
-			  const char *argName	    // Argument name in the command-line arguments
+                          psMetadata *options,    // Target to which to add value
+                          const char *recipeName, // Name for value in the recipe
+                          psMetadata *arguments,  // Command-line arguments
+                          const char *argName       // Argument name in the command-line arguments
     )
 {
@@ -231,8 +233,8 @@
 // if it is not specified, do not override the existing recipe value
 char *ppSimArgToRecipeStr(bool *status,
-			  psMetadata *options,    // Target to which to add value
-			  const char *recipeName, // Name for value in the recipe
-			  psMetadata *arguments,  // Command-line arguments
-			  const char *argName	    // Argument name in the command-line arguments
+                          psMetadata *options,    // Target to which to add value
+                          const char *recipeName, // Name for value in the recipe
+                          psMetadata *arguments,  // Command-line arguments
+                          const char *argName       // Argument name in the command-line arguments
     )
 {
@@ -241,5 +243,5 @@
     char *value = psMetadataLookupStr(&myStatus, arguments, argName); // Value of interest
     if (status) {
-	*status = myStatus;
+        *status = myStatus;
     }
     if (!value) return NULL;
@@ -248,32 +250,17 @@
 }
 
-float ppSimGetZeroPoint (psMetadata *recipe, char *filter) {
-
-    bool mdok;
-    float zp;
+float ppSimGetZeroPoint(psMetadata *recipe, const char *filter)
+{
+    PS_ASSERT_METADATA_NON_NULL(recipe, NAN);
+    PS_ASSERT_STRING_NON_EMPTY(filter, NAN);
 
     // use the filter to get the zeropoint from the recipe
-    psMetadataItem *zpItem = psMetadataLookup (recipe, "ZEROPTS");
-    // check that item is multi...
-	    
-    psArray *entries = psListToArray (zpItem->data.list);
-	  
-    // search for matching filter
-    for (int i = 0; i < entries->n; i++) {
-	psMetadataItem *item = entries->data[i];
-	psMetadata *entry = item->data.V;
-
-	char *filterName = psMetadataLookupStr (&mdok, entry, "FILTER");
-	assert (filterName);
-
-	if (strcmp(filterName, filter)) continue;
-
-	zp = psMetadataLookupF32 (&mdok, entry, "ZERO_PT");
-	assert (mdok);
-	psFree (entries);
-	return zp;
-    }
-    psFree (entries);
-    return NAN;
+    psMetadata *zeropoints = psMetadataLookupMetadata(NULL, recipe, "ZEROPTS");
+    if (!zeropoints) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to find ZEROPTS in recipe");
+        return NAN;
+    }
+
+    return psMetadataLookupF32(NULL, zeropoints, filter);
 }
 
@@ -295,5 +282,5 @@
 
     for (int i = 0; i < sources->n; i++) {
-	pmSource *source = sources->data[i];
+        pmSource *source = sources->data[i];
 
         // allocate image, weight, mask arrays for each peak (square of radius OUTER)
Index: /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimVersion.c	(revision 24244)
@@ -1,3 +1,4 @@
 #include "ppSim.h"
+#include "ppSimVersionDefinitions.h"
 
 #ifndef PPSIM_VERSION
@@ -11,11 +12,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString ppSimVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PPSIM_BRANCH), xstr(PPSIM_VERSION));
+    psStringAppend(&value, "%s@%s", PPSIM_BRANCH, PPSIM_VERSION);
     return value;
 }
@@ -23,5 +21,5 @@
 psString ppSimSource(void)
 {
-    return psStringCopy(xstr(PPSIM_SOURCE));
+    return psStringCopy(PPSIM_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PPSIM_VERSION_DEFINITIONS_H
+#define PPSIM_VERSION_DEFINITIONS_H
+
+#define PPSIM_VERSION @PPSIM_VERSION@ // SVN version
+#define PPSIM_BRANCH  @PPSIM_BRANCH@  // SVN branch
+#define PPSIM_SOURCE  @PPSIM_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/Doxyfile.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/Doxyfile.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/Doxyfile.in	(revision 24244)
@@ -0,0 +1,1310 @@
+# Doxyfile 1.5.4
+
+# This file describes the settings to be used by the documentation system
+# doxygen (www.doxygen.org) for a project
+#
+# All text after a hash (#) is considered a comment and will be ignored
+# The format is:
+#       TAG = value [value, ...]
+# For lists items can also be appended using:
+#       TAG += value [value, ...]
+# Values that contain spaces should be placed between quotes (" ")
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+
+# This tag specifies the encoding used for all characters in the config file that 
+# follow. The default is UTF-8 which is also the encoding used for all text before 
+# the first occurrence of this tag. Doxygen uses libiconv (or the iconv built into 
+# libc) for the transcoding. See http://www.gnu.org/software/libiconv for the list of 
+# possible encodings.
+
+DOXYFILE_ENCODING      = UTF-8
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded 
+# by quotes) that should identify the project.
+
+PROJECT_NAME           = ppArith
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. 
+# This could be handy for archiving the generated documentation or 
+# if some version control system is used.
+
+PROJECT_NUMBER         = ipp-2.7.dev
+
+# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) 
+# base path where the generated documentation will be put. 
+# If a relative path is entered, it will be relative to the location 
+# where doxygen was started. If left blank the current directory will be used.
+
+OUTPUT_DIRECTORY       = ./docs
+
+# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create 
+# 4096 sub-directories (in 2 levels) under the output directory of each output 
+# format and will distribute the generated files over these directories. 
+# Enabling this option can be useful when feeding doxygen a huge amount of 
+# source files, where putting all generated files in the same directory would 
+# otherwise cause performance problems for the file system.
+
+CREATE_SUBDIRS         = NO
+
+# The OUTPUT_LANGUAGE tag is used to specify the language in which all 
+# documentation generated by doxygen is written. Doxygen will use this 
+# information to generate all constant output in the proper language. 
+# The default language is English, other supported languages are: 
+# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, 
+# Croatian, Czech, Danish, Dutch, Finnish, French, German, Greek, Hungarian, 
+# Italian, Japanese, Japanese-en (Japanese with English messages), Korean, 
+# Korean-en, Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, 
+# Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian.
+
+OUTPUT_LANGUAGE        = English
+
+# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will 
+# include brief member descriptions after the members that are listed in 
+# the file and class documentation (similar to JavaDoc). 
+# Set to NO to disable this.
+
+BRIEF_MEMBER_DESC      = YES
+
+# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend 
+# the brief description of a member or function before the detailed description. 
+# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the 
+# brief descriptions will be completely suppressed.
+
+REPEAT_BRIEF           = YES
+
+# This tag implements a quasi-intelligent brief description abbreviator 
+# that is used to form the text in various listings. Each string 
+# in this list, if found as the leading text of the brief description, will be 
+# stripped from the text and the result after processing the whole list, is 
+# used as the annotated text. Otherwise, the brief description is used as-is. 
+# If left blank, the following values are used ("$name" is automatically 
+# replaced with the name of the entity): "The $name class" "The $name widget" 
+# "The $name file" "is" "provides" "specifies" "contains" 
+# "represents" "a" "an" "the"
+
+ABBREVIATE_BRIEF       = 
+
+# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then 
+# Doxygen will generate a detailed section even if there is only a brief 
+# description.
+
+ALWAYS_DETAILED_SEC    = NO
+
+# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all 
+# inherited members of a class in the documentation of that class as if those 
+# members were ordinary class members. Constructors, destructors and assignment 
+# operators of the base classes will not be shown.
+
+INLINE_INHERITED_MEMB  = NO
+
+# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full 
+# path before files name in the file list and in the header files. If set 
+# to NO the shortest path that makes the file name unique will be used.
+
+FULL_PATH_NAMES        = NO
+
+# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag 
+# can be used to strip a user-defined part of the path. Stripping is 
+# only done if one of the specified strings matches the left-hand part of 
+# the path. The tag can be used to show relative paths in the file list. 
+# If left blank the directory from which doxygen is run is used as the 
+# path to strip.
+
+STRIP_FROM_PATH        = 
+
+# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of 
+# the path mentioned in the documentation of a class, which tells 
+# the reader which header file to include in order to use a class. 
+# If left blank only the name of the header file containing the class 
+# definition is used. Otherwise one should specify the include paths that 
+# are normally passed to the compiler using the -I flag.
+
+STRIP_FROM_INC_PATH    = 
+
+# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter 
+# (but less readable) file names. This can be useful is your file systems 
+# doesn't support long names like on DOS, Mac, or CD-ROM.
+
+SHORT_NAMES            = NO
+
+# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen 
+# will interpret the first line (until the first dot) of a JavaDoc-style 
+# comment as the brief description. If set to NO, the JavaDoc 
+# comments will behave just like regular Qt-style comments 
+# (thus requiring an explicit @brief command for a brief description.)
+
+JAVADOC_AUTOBRIEF      = NO
+
+# If the QT_AUTOBRIEF tag is set to YES then Doxygen will 
+# interpret the first line (until the first dot) of a Qt-style 
+# comment as the brief description. If set to NO, the comments 
+# will behave just like regular Qt-style comments (thus requiring 
+# an explicit \brief command for a brief description.)
+
+QT_AUTOBRIEF           = NO
+
+# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen 
+# treat a multi-line C++ special comment block (i.e. a block of //! or /// 
+# comments) as a brief description. This used to be the default behaviour. 
+# The new default is to treat a multi-line C++ comment block as a detailed 
+# description. Set this tag to YES if you prefer the old behaviour instead.
+
+MULTILINE_CPP_IS_BRIEF = NO
+
+# If the DETAILS_AT_TOP tag is set to YES then Doxygen 
+# will output the detailed description near the top, like JavaDoc.
+# If set to NO, the detailed description appears after the member 
+# documentation.
+
+DETAILS_AT_TOP         = NO
+
+# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented 
+# member inherits the documentation from any documented member that it 
+# re-implements.
+
+INHERIT_DOCS           = YES
+
+# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce 
+# a new page for each member. If set to NO, the documentation of a member will 
+# be part of the file/class/namespace that contains it.
+
+SEPARATE_MEMBER_PAGES  = NO
+
+# The TAB_SIZE tag can be used to set the number of spaces in a tab. 
+# Doxygen uses this value to replace tabs by spaces in code fragments.
+
+TAB_SIZE               = 4
+
+# This tag can be used to specify a number of aliases that acts 
+# as commands in the documentation. An alias has the form "name=value". 
+# For example adding "sideeffect=\par Side Effects:\n" will allow you to 
+# put the command \sideeffect (or @sideeffect) in the documentation, which 
+# will result in a user-defined paragraph with heading "Side Effects:". 
+# You can put \n's in the value part of an alias to insert newlines.
+
+ALIASES                = 
+
+# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C 
+# sources only. Doxygen will then generate output that is more tailored for C. 
+# For instance, some of the names that are used will be different. The list 
+# of all members will be omitted, etc.
+
+OPTIMIZE_OUTPUT_FOR_C  = NO
+
+# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java 
+# sources only. Doxygen will then generate output that is more tailored for Java. 
+# For instance, namespaces will be presented as packages, qualified scopes 
+# will look different, etc.
+
+OPTIMIZE_OUTPUT_JAVA   = NO
+
+# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want to 
+# include (a tag file for) the STL sources as input, then you should 
+# set this tag to YES in order to let doxygen match functions declarations and 
+# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. 
+# func(std::string) {}). This also make the inheritance and collaboration 
+# diagrams that involve STL classes more complete and accurate.
+
+BUILTIN_STL_SUPPORT    = NO
+
+# If you use Microsoft's C++/CLI language, you should set this option to YES to
+# enable parsing support.
+
+CPP_CLI_SUPPORT        = NO
+
+# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. 
+# Doxygen will parse them like normal C++ but will assume all classes use public 
+# instead of private inheritance when no explicit protection keyword is present.
+
+SIP_SUPPORT            = NO
+
+# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC 
+# tag is set to YES, then doxygen will reuse the documentation of the first 
+# member in the group (if any) for the other members of the group. By default 
+# all members of a group must be documented explicitly.
+
+DISTRIBUTE_GROUP_DOC   = NO
+
+# Set the SUBGROUPING tag to YES (the default) to allow class member groups of 
+# the same type (for instance a group of public functions) to be put as a 
+# subgroup of that type (e.g. under the Public Functions section). Set it to 
+# NO to prevent subgrouping. Alternatively, this can be done per class using 
+# the \nosubgrouping command.
+
+SUBGROUPING            = YES
+
+# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct (or union) is 
+# documented as struct with the name of the typedef. So 
+# typedef struct TypeS {} TypeT, will appear in the documentation as a struct 
+# with name TypeT. When disabled the typedef will appear as a member of a file, 
+# namespace, or class. And the struct will be named TypeS. This can typically 
+# be useful for C code where the coding convention is that all structs are 
+# typedef'ed and only the typedef is referenced never the struct's name.
+
+TYPEDEF_HIDES_STRUCT   = NO
+
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+
+# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in 
+# documentation are documented, even if no documentation was available. 
+# Private class members and static file members will be hidden unless 
+# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES
+
+EXTRACT_ALL            = NO
+
+# If the EXTRACT_PRIVATE tag is set to YES all private members of a class 
+# will be included in the documentation.
+
+EXTRACT_PRIVATE        = NO
+
+# If the EXTRACT_STATIC tag is set to YES all static members of a file 
+# will be included in the documentation.
+
+EXTRACT_STATIC         = NO
+
+# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) 
+# defined locally in source files will be included in the documentation. 
+# If set to NO only classes defined in header files are included.
+
+EXTRACT_LOCAL_CLASSES  = YES
+
+# This flag is only useful for Objective-C code. When set to YES local 
+# methods, which are defined in the implementation section but not in 
+# the interface are included in the documentation. 
+# If set to NO (the default) only methods in the interface are included.
+
+EXTRACT_LOCAL_METHODS  = NO
+
+# If this flag is set to YES, the members of anonymous namespaces will be extracted 
+# and appear in the documentation as a namespace called 'anonymous_namespace{file}', 
+# where file will be replaced with the base name of the file that contains the anonymous 
+# namespace. By default anonymous namespace are hidden.
+
+EXTRACT_ANON_NSPACES   = NO
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all 
+# undocumented members of documented classes, files or namespaces. 
+# If set to NO (the default) these members will be included in the 
+# various overviews, but no documentation section is generated. 
+# This option has no effect if EXTRACT_ALL is enabled.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all 
+# undocumented classes that are normally visible in the class hierarchy. 
+# If set to NO (the default) these classes will be included in the various 
+# overviews. This option has no effect if EXTRACT_ALL is enabled.
+
+HIDE_UNDOC_CLASSES     = NO
+
+# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all 
+# friend (class|struct|union) declarations. 
+# If set to NO (the default) these declarations will be included in the 
+# documentation.
+
+HIDE_FRIEND_COMPOUNDS  = NO
+
+# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any 
+# documentation blocks found inside the body of a function. 
+# If set to NO (the default) these blocks will be appended to the 
+# function's detailed documentation block.
+
+HIDE_IN_BODY_DOCS      = NO
+
+# The INTERNAL_DOCS tag determines if documentation 
+# that is typed after a \internal command is included. If the tag is set 
+# to NO (the default) then the documentation will be excluded. 
+# Set it to YES to include the internal documentation.
+
+INTERNAL_DOCS          = NO
+
+# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate 
+# file names in lower-case letters. If set to YES upper-case letters are also 
+# allowed. This is useful if you have classes or files whose names only differ 
+# in case and if your file system supports case sensitive file names. Windows 
+# and Mac users are advised to set this option to NO.
+
+CASE_SENSE_NAMES       = YES
+
+# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen 
+# will show members with their full class and namespace scopes in the 
+# documentation. If set to YES the scope will be hidden.
+
+HIDE_SCOPE_NAMES       = NO
+
+# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen 
+# will put a list of the files that are included by a file in the documentation 
+# of that file.
+
+SHOW_INCLUDE_FILES     = YES
+
+# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] 
+# is inserted in the documentation for inline members.
+
+INLINE_INFO            = YES
+
+# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen 
+# will sort the (detailed) documentation of file and class members 
+# alphabetically by member name. If set to NO the members will appear in 
+# declaration order.
+
+SORT_MEMBER_DOCS       = YES
+
+# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the 
+# brief documentation of file, namespace and class members alphabetically 
+# by member name. If set to NO (the default) the members will appear in 
+# declaration order.
+
+SORT_BRIEF_DOCS        = NO
+
+# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be 
+# sorted by fully-qualified names, including namespaces. If set to 
+# NO (the default), the class list will be sorted only by class name, 
+# not including the namespace part. 
+# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES.
+# Note: This option applies only to the class list, not to the 
+# alphabetical list.
+
+SORT_BY_SCOPE_NAME     = NO
+
+# The GENERATE_TODOLIST tag can be used to enable (YES) or 
+# disable (NO) the todo list. This list is created by putting \todo 
+# commands in the documentation.
+
+GENERATE_TODOLIST      = YES
+
+# The GENERATE_TESTLIST tag can be used to enable (YES) or 
+# disable (NO) the test list. This list is created by putting \test 
+# commands in the documentation.
+
+GENERATE_TESTLIST      = YES
+
+# The GENERATE_BUGLIST tag can be used to enable (YES) or 
+# disable (NO) the bug list. This list is created by putting \bug 
+# commands in the documentation.
+
+GENERATE_BUGLIST       = YES
+
+# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or 
+# disable (NO) the deprecated list. This list is created by putting 
+# \deprecated commands in the documentation.
+
+GENERATE_DEPRECATEDLIST= YES
+
+# The ENABLED_SECTIONS tag can be used to enable conditional 
+# documentation sections, marked by \if sectionname ... \endif.
+
+ENABLED_SECTIONS       = 
+
+# The MAX_INITIALIZER_LINES tag determines the maximum number of lines 
+# the initial value of a variable or define consists of for it to appear in 
+# the documentation. If the initializer consists of more lines than specified 
+# here it will be hidden. Use a value of 0 to hide initializers completely. 
+# The appearance of the initializer of individual variables and defines in the 
+# documentation can be controlled using \showinitializer or \hideinitializer 
+# command in the documentation regardless of this setting.
+
+MAX_INITIALIZER_LINES  = 30
+
+# Set the SHOW_USED_FILES tag to NO to disable the list of files generated 
+# at the bottom of the documentation of classes and structs. If set to YES the 
+# list will mention the files that were used to generate the documentation.
+
+SHOW_USED_FILES        = YES
+
+# If the sources in your project are distributed over multiple directories 
+# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy 
+# in the documentation. The default is NO.
+
+SHOW_DIRECTORIES       = YES
+
+# The FILE_VERSION_FILTER tag can be used to specify a program or script that 
+# doxygen should invoke to get the current version for each file (typically from the 
+# version control system). Doxygen will invoke the program by executing (via 
+# popen()) the command <command> <input-file>, where <command> is the value of 
+# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file 
+# provided by doxygen. Whatever the program writes to standard output 
+# is used as the file version. See the manual for examples.
+
+FILE_VERSION_FILTER    = 
+
+#---------------------------------------------------------------------------
+# configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+
+# The QUIET tag can be used to turn on/off the messages that are generated 
+# by doxygen. Possible values are YES and NO. If left blank NO is used.
+
+QUIET                  = NO
+
+# The WARNINGS tag can be used to turn on/off the warning messages that are 
+# generated by doxygen. Possible values are YES and NO. If left blank 
+# NO is used.
+
+WARNINGS               = YES
+
+# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings 
+# for undocumented members. If EXTRACT_ALL is set to YES then this flag will 
+# automatically be disabled.
+
+WARN_IF_UNDOCUMENTED   = YES
+
+# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for 
+# potential errors in the documentation, such as not documenting some 
+# parameters in a documented function, or documenting parameters that 
+# don't exist or using markup commands wrongly.
+
+WARN_IF_DOC_ERROR      = YES
+
+# This WARN_NO_PARAMDOC option can be abled to get warnings for 
+# functions that are documented, but have no documentation for their parameters 
+# or return value. If set to NO (the default) doxygen will only warn about 
+# wrong or incomplete parameter documentation, but not about the absence of 
+# documentation.
+
+WARN_NO_PARAMDOC       = NO
+
+# The WARN_FORMAT tag determines the format of the warning messages that 
+# doxygen can produce. The string should contain the $file, $line, and $text 
+# tags, which will be replaced by the file and line number from which the 
+# warning originated and the warning text. Optionally the format may contain 
+# $version, which will be replaced by the version of the file (if it could 
+# be obtained via FILE_VERSION_FILTER)
+
+WARN_FORMAT            = "$file:$line: $text "
+
+# The WARN_LOGFILE tag can be used to specify a file to which warning 
+# and error messages should be written. If left blank the output is written 
+# to stderr.
+
+WARN_LOGFILE           = 
+
+#---------------------------------------------------------------------------
+# configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag can be used to specify the files and/or directories that contain 
+# documented source files. You may enter file names like "myfile.cpp" or 
+# directories like "/usr/src/myproject". Separate the files or directories 
+# with spaces.
+
+INPUT                  = ./src
+
+# This tag can be used to specify the character encoding of the source files that 
+# doxygen parses. Internally doxygen uses the UTF-8 encoding, which is also the default 
+# input encoding. Doxygen uses libiconv (or the iconv built into libc) for the transcoding. 
+# See http://www.gnu.org/software/libiconv for the list of possible encodings.
+
+INPUT_ENCODING         = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the 
+# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp 
+# and *.h) to filter out the source-files in the directories. If left 
+# blank the following patterns are tested: 
+# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx 
+# *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.py *.f90
+
+FILE_PATTERNS          = *.h
+
+# The RECURSIVE tag can be used to turn specify whether or not subdirectories 
+# should be searched for input files as well. Possible values are YES and NO. 
+# If left blank NO is used.
+
+RECURSIVE              = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should 
+# excluded from the INPUT source files. This way you can easily exclude a 
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+
+EXCLUDE                = 
+
+# The EXCLUDE_SYMLINKS tag can be used select whether or not files or 
+# directories that are symbolic links (a Unix filesystem feature) are excluded 
+# from the input.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the 
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude 
+# certain files from those directories. Note that the wildcards are matched 
+# against the file with absolute path, so to exclude all test directories 
+# for example use the pattern */test/*
+
+EXCLUDE_PATTERNS       = 
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names 
+# (namespaces, classes, functions, etc.) that should be excluded from the output. 
+# The symbol name can be a fully qualified name, a word, or if the wildcard * is used, 
+# a substring. Examples: ANamespace, AClass, AClass::ANamespace, ANamespace::*Test
+
+EXCLUDE_SYMBOLS        = 
+
+# The EXAMPLE_PATH tag can be used to specify one or more files or 
+# directories that contain example code fragments that are included (see 
+# the \include command).
+
+EXAMPLE_PATH           = 
+
+# If the value of the EXAMPLE_PATH tag contains directories, you can use the 
+# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp 
+# and *.h) to filter out the source-files in the directories. If left 
+# blank all files are included.
+
+EXAMPLE_PATTERNS       = 
+
+# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be 
+# searched for input files to be used with the \include or \dontinclude 
+# commands irrespective of the value of the RECURSIVE tag. 
+# Possible values are YES and NO. If left blank NO is used.
+
+EXAMPLE_RECURSIVE      = NO
+
+# The IMAGE_PATH tag can be used to specify one or more files or 
+# directories that contain image that are included in the documentation (see 
+# the \image command).
+
+IMAGE_PATH             = 
+
+# The INPUT_FILTER tag can be used to specify a program that doxygen should 
+# invoke to filter for each input file. Doxygen will invoke the filter program 
+# by executing (via popen()) the command <filter> <input-file>, where <filter> 
+# is the value of the INPUT_FILTER tag, and <input-file> is the name of an 
+# input file. Doxygen will then use the output that the filter program writes 
+# to standard output.  If FILTER_PATTERNS is specified, this tag will be 
+# ignored.
+
+INPUT_FILTER           = 
+
+# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern 
+# basis.  Doxygen will compare the file name with each pattern and apply the 
+# filter if there is a match.  The filters are a list of the form: 
+# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further 
+# info on how filters are used. If FILTER_PATTERNS is empty, INPUT_FILTER 
+# is applied to all files.
+
+FILTER_PATTERNS        = 
+
+# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using 
+# INPUT_FILTER) will be used to filter the input files when producing source 
+# files to browse (i.e. when SOURCE_BROWSER is set to YES).
+
+FILTER_SOURCE_FILES    = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to source browsing
+#---------------------------------------------------------------------------
+
+# If the SOURCE_BROWSER tag is set to YES then a list of source files will 
+# be generated. Documented entities will be cross-referenced with these sources. 
+# Note: To get rid of all source code in the generated output, make sure also 
+# VERBATIM_HEADERS is set to NO. If you have enabled CALL_GRAPH or CALLER_GRAPH 
+# then you must also enable this option. If you don't then doxygen will produce 
+# a warning and turn it on anyway
+
+SOURCE_BROWSER         = NO
+
+# Setting the INLINE_SOURCES tag to YES will include the body 
+# of functions and classes directly in the documentation.
+
+INLINE_SOURCES         = NO
+
+# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct 
+# doxygen to hide any special comment blocks from generated source code 
+# fragments. Normal C and C++ comments will always remain visible.
+
+STRIP_CODE_COMMENTS    = YES
+
+# If the REFERENCED_BY_RELATION tag is set to YES (the default) 
+# then for each documented function all documented 
+# functions referencing it will be listed.
+
+REFERENCED_BY_RELATION = YES
+
+# If the REFERENCES_RELATION tag is set to YES (the default) 
+# then for each documented function all documented entities 
+# called/used by that function will be listed.
+
+REFERENCES_RELATION    = YES
+
+# If the REFERENCES_LINK_SOURCE tag is set to YES (the default)
+# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from
+# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will
+# link to the source code.  Otherwise they will link to the documentstion.
+
+REFERENCES_LINK_SOURCE = YES
+
+# If the USE_HTAGS tag is set to YES then the references to source code 
+# will point to the HTML generated by the htags(1) tool instead of doxygen 
+# built-in source browser. The htags tool is part of GNU's global source 
+# tagging system (see http://www.gnu.org/software/global/global.html). You 
+# will need version 4.8.6 or higher.
+
+USE_HTAGS              = NO
+
+# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen 
+# will generate a verbatim copy of the header file for each class for 
+# which an include is specified. Set to NO to disable this.
+
+VERBATIM_HEADERS       = YES
+
+#---------------------------------------------------------------------------
+# configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+
+# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index 
+# of all compounds will be generated. Enable this if the project 
+# contains a lot of classes, structs, unions or interfaces.
+
+ALPHABETICAL_INDEX     = NO
+
+# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then 
+# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns 
+# in which this list will be split (can be a number in the range [1..20])
+
+COLS_IN_ALPHA_INDEX    = 5
+
+# In case all classes in a project start with a common prefix, all 
+# classes will be put under the same header in the alphabetical index. 
+# The IGNORE_PREFIX tag can be used to specify one or more prefixes that 
+# should be ignored while generating the index headers.
+
+IGNORE_PREFIX          = 
+
+#---------------------------------------------------------------------------
+# configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES (the default) Doxygen will 
+# generate HTML output.
+
+GENERATE_HTML          = YES
+
+# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. 
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# put in front of it. If left blank `html' will be used as the default path.
+
+HTML_OUTPUT            = html
+
+# The HTML_FILE_EXTENSION tag can be used to specify the file extension for 
+# each generated HTML page (for example: .htm,.php,.asp). If it is left blank 
+# doxygen will generate files with .html extension.
+
+HTML_FILE_EXTENSION    = .html
+
+# The HTML_HEADER tag can be used to specify a personal HTML header for 
+# each generated HTML page. If it is left blank doxygen will generate a 
+# standard header.
+
+HTML_HEADER            = 
+
+# The HTML_FOOTER tag can be used to specify a personal HTML footer for 
+# each generated HTML page. If it is left blank doxygen will generate a 
+# standard footer.
+
+HTML_FOOTER            = 
+
+# The HTML_STYLESHEET tag can be used to specify a user-defined cascading 
+# style sheet that is used by each HTML page. It can be used to 
+# fine-tune the look of the HTML output. If the tag is left blank doxygen 
+# will generate a default style sheet. Note that doxygen will try to copy 
+# the style sheet file to the HTML output directory, so don't put your own 
+# stylesheet in the HTML output directory as well, or it will be erased!
+
+HTML_STYLESHEET        = 
+
+# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, 
+# files or namespaces will be aligned in HTML using tables. If set to 
+# NO a bullet list will be used.
+
+HTML_ALIGN_MEMBERS     = YES
+
+# If the GENERATE_HTMLHELP tag is set to YES, additional index files 
+# will be generated that can be used as input for tools like the 
+# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) 
+# of the generated HTML documentation.
+
+GENERATE_HTMLHELP      = NO
+
+# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML 
+# documentation will contain sections that can be hidden and shown after the 
+# page has loaded. For this to work a browser that supports 
+# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox 
+# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari).
+
+HTML_DYNAMIC_SECTIONS  = NO
+
+# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can 
+# be used to specify the file name of the resulting .chm file. You 
+# can add a path in front of the file if the result should not be 
+# written to the html output directory.
+
+CHM_FILE               = 
+
+# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can 
+# be used to specify the location (absolute path including file name) of 
+# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run 
+# the HTML help compiler on the generated index.hhp.
+
+HHC_LOCATION           = 
+
+# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag 
+# controls if a separate .chi index file is generated (YES) or that 
+# it should be included in the master .chm file (NO).
+
+GENERATE_CHI           = NO
+
+# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag 
+# controls whether a binary table of contents is generated (YES) or a 
+# normal table of contents (NO) in the .chm file.
+
+BINARY_TOC             = NO
+
+# The TOC_EXPAND flag can be set to YES to add extra items for group members 
+# to the contents of the HTML help documentation and to the tree view.
+
+TOC_EXPAND             = NO
+
+# The DISABLE_INDEX tag can be used to turn on/off the condensed index at 
+# top of each HTML page. The value NO (the default) enables the index and 
+# the value YES disables it.
+
+DISABLE_INDEX          = NO
+
+# This tag can be used to set the number of enum values (range [1..20]) 
+# that doxygen will group on one line in the generated HTML documentation.
+
+ENUM_VALUES_PER_LINE   = 4
+
+# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be
+# generated containing a tree-like index structure (just like the one that 
+# is generated for HTML Help). For this to work a browser that supports 
+# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, 
+# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are 
+# probably better off using the HTML help feature.
+
+GENERATE_TREEVIEW      = NO
+
+# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be 
+# used to set the initial width (in pixels) of the frame in which the tree 
+# is shown.
+
+TREEVIEW_WIDTH         = 250
+
+#---------------------------------------------------------------------------
+# configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will 
+# generate Latex output.
+
+GENERATE_LATEX         = NO
+
+# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. 
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# put in front of it. If left blank `latex' will be used as the default path.
+
+LATEX_OUTPUT           = latex
+
+# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be 
+# invoked. If left blank `latex' will be used as the default command name.
+
+LATEX_CMD_NAME         = latex
+
+# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to 
+# generate index for LaTeX. If left blank `makeindex' will be used as the 
+# default command name.
+
+MAKEINDEX_CMD_NAME     = makeindex
+
+# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact 
+# LaTeX documents. This may be useful for small projects and may help to 
+# save some trees in general.
+
+COMPACT_LATEX          = NO
+
+# The PAPER_TYPE tag can be used to set the paper type that is used 
+# by the printer. Possible values are: a4, a4wide, letter, legal and 
+# executive. If left blank a4wide will be used.
+
+PAPER_TYPE             = a4wide
+
+# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX 
+# packages that should be included in the LaTeX output.
+
+EXTRA_PACKAGES         = 
+
+# The LATEX_HEADER tag can be used to specify a personal LaTeX header for 
+# the generated latex document. The header should contain everything until 
+# the first chapter. If it is left blank doxygen will generate a 
+# standard header. Notice: only use this tag if you know what you are doing!
+
+LATEX_HEADER           = 
+
+# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated 
+# is prepared for conversion to pdf (using ps2pdf). The pdf file will 
+# contain links (just like the HTML output) instead of page references 
+# This makes the output suitable for online browsing using a pdf viewer.
+
+PDF_HYPERLINKS         = NO
+
+# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of 
+# plain latex in the generated Makefile. Set this option to YES to get a 
+# higher quality PDF documentation.
+
+USE_PDFLATEX           = NO
+
+# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. 
+# command to the generated LaTeX files. This will instruct LaTeX to keep 
+# running if errors occur, instead of asking the user for help. 
+# This option is also used when generating formulas in HTML.
+
+LATEX_BATCHMODE        = NO
+
+# If LATEX_HIDE_INDICES is set to YES then doxygen will not 
+# include the index chapters (such as File Index, Compound Index, etc.) 
+# in the output.
+
+LATEX_HIDE_INDICES     = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to the RTF output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output 
+# The RTF output is optimized for Word 97 and may not look very pretty with 
+# other RTF readers or editors.
+
+GENERATE_RTF           = NO
+
+# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. 
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# put in front of it. If left blank `rtf' will be used as the default path.
+
+RTF_OUTPUT             = rtf
+
+# If the COMPACT_RTF tag is set to YES Doxygen generates more compact 
+# RTF documents. This may be useful for small projects and may help to 
+# save some trees in general.
+
+COMPACT_RTF            = NO
+
+# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated 
+# will contain hyperlink fields. The RTF file will 
+# contain links (just like the HTML output) instead of page references. 
+# This makes the output suitable for online browsing using WORD or other 
+# programs which support those fields. 
+# Note: wordpad (write) and others do not support links.
+
+RTF_HYPERLINKS         = NO
+
+# Load stylesheet definitions from file. Syntax is similar to doxygen's 
+# config file, i.e. a series of assignments. You only have to provide 
+# replacements, missing definitions are set to their default value.
+
+RTF_STYLESHEET_FILE    = 
+
+# Set optional variables used in the generation of an rtf document. 
+# Syntax is similar to doxygen's config file.
+
+RTF_EXTENSIONS_FILE    = 
+
+#---------------------------------------------------------------------------
+# configuration options related to the man page output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_MAN tag is set to YES (the default) Doxygen will 
+# generate man pages
+
+GENERATE_MAN           = YES
+
+# The MAN_OUTPUT tag is used to specify where the man pages will be put. 
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# put in front of it. If left blank `man' will be used as the default path.
+
+MAN_OUTPUT             = man
+
+# The MAN_EXTENSION tag determines the extension that is added to 
+# the generated man pages (default is the subroutine's section .3)
+
+MAN_EXTENSION          = .3
+
+# If the MAN_LINKS tag is set to YES and Doxygen generates man output, 
+# then it will generate one additional man file for each entity 
+# documented in the real man page(s). These additional files 
+# only source the real man page, but without them the man command 
+# would be unable to find the correct page. The default is NO.
+
+MAN_LINKS              = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to the XML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_XML tag is set to YES Doxygen will 
+# generate an XML file that captures the structure of 
+# the code including all documentation.
+
+GENERATE_XML           = NO
+
+# The XML_OUTPUT tag is used to specify where the XML pages will be put. 
+# If a relative path is entered the value of OUTPUT_DIRECTORY will be 
+# put in front of it. If left blank `xml' will be used as the default path.
+
+XML_OUTPUT             = xml
+
+# The XML_SCHEMA tag can be used to specify an XML schema, 
+# which can be used by a validating XML parser to check the 
+# syntax of the XML files.
+
+XML_SCHEMA             = 
+
+# The XML_DTD tag can be used to specify an XML DTD, 
+# which can be used by a validating XML parser to check the 
+# syntax of the XML files.
+
+XML_DTD                = 
+
+# If the XML_PROGRAMLISTING tag is set to YES Doxygen will 
+# dump the program listings (including syntax highlighting 
+# and cross-referencing information) to the XML output. Note that 
+# enabling this will significantly increase the size of the XML output.
+
+XML_PROGRAMLISTING     = YES
+
+#---------------------------------------------------------------------------
+# configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will 
+# generate an AutoGen Definitions (see autogen.sf.net) file 
+# that captures the structure of the code including all 
+# documentation. Note that this feature is still experimental 
+# and incomplete at the moment.
+
+GENERATE_AUTOGEN_DEF   = NO
+
+#---------------------------------------------------------------------------
+# configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_PERLMOD tag is set to YES Doxygen will 
+# generate a Perl module file that captures the structure of 
+# the code including all documentation. Note that this 
+# feature is still experimental and incomplete at the 
+# moment.
+
+GENERATE_PERLMOD       = NO
+
+# If the PERLMOD_LATEX tag is set to YES Doxygen will generate 
+# the necessary Makefile rules, Perl scripts and LaTeX code to be able 
+# to generate PDF and DVI output from the Perl module output.
+
+PERLMOD_LATEX          = NO
+
+# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be 
+# nicely formatted so it can be parsed by a human reader.  This is useful 
+# if you want to understand what is going on.  On the other hand, if this 
+# tag is set to NO the size of the Perl module output will be much smaller 
+# and Perl will parse it just the same.
+
+PERLMOD_PRETTY         = YES
+
+# The names of the make variables in the generated doxyrules.make file 
+# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. 
+# This is useful so different doxyrules.make files included by the same 
+# Makefile don't overwrite each other's variables.
+
+PERLMOD_MAKEVAR_PREFIX = 
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor   
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will 
+# evaluate all C-preprocessor directives found in the sources and include 
+# files.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro 
+# names in the source code. If set to NO (the default) only conditional 
+# compilation will be performed. Macro expansion can be done in a controlled 
+# way by setting EXPAND_ONLY_PREDEF to YES.
+
+MACRO_EXPANSION        = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES 
+# then the macro expansion is limited to the macros specified with the 
+# PREDEFINED and EXPAND_AS_DEFINED tags.
+
+EXPAND_ONLY_PREDEF     = NO
+
+# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files 
+# in the INCLUDE_PATH (see below) will be search if a #include is found.
+
+SEARCH_INCLUDES        = YES
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that 
+# contain include files that are not input files but should be processed by 
+# the preprocessor.
+
+INCLUDE_PATH           = 
+
+# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard 
+# patterns (like *.h and *.hpp) to filter out the header-files in the 
+# directories. If left blank, the patterns specified with FILE_PATTERNS will 
+# be used.
+
+INCLUDE_FILE_PATTERNS  = 
+
+# The PREDEFINED tag can be used to specify one or more macro names that 
+# are defined before the preprocessor is started (similar to the -D option of 
+# gcc). The argument of the tag is a list of macros of the form: name 
+# or name=definition (no spaces). If the definition and the = are 
+# omitted =1 is assumed. To prevent a macro definition from being 
+# undefined via #undef or recursively expanded use the := operator 
+# instead of the = operator.
+
+PREDEFINED             = 
+
+# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then 
+# this tag can be used to specify a list of macro names that should be expanded. 
+# The macro definition that is found in the sources will be used. 
+# Use the PREDEFINED tag if you want to use a different macro definition.
+
+EXPAND_AS_DEFINED      = 
+
+# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then 
+# doxygen's preprocessor will remove all function-like macros that are alone 
+# on a line, have an all uppercase name, and do not end with a semicolon. Such 
+# function macros are typically used for boiler-plate code, and will confuse 
+# the parser if not removed.
+
+SKIP_FUNCTION_MACROS   = YES
+
+#---------------------------------------------------------------------------
+# Configuration::additions related to external references   
+#---------------------------------------------------------------------------
+
+# The TAGFILES option can be used to specify one or more tagfiles. 
+# Optionally an initial location of the external documentation 
+# can be added for each tagfile. The format of a tag file without 
+# this location is as follows: 
+#   TAGFILES = file1 file2 ... 
+# Adding location for the tag files is done as follows: 
+#   TAGFILES = file1=loc1 "file2 = loc2" ... 
+# where "loc1" and "loc2" can be relative or absolute paths or 
+# URLs. If a location is present for each tag, the installdox tool 
+# does not have to be run to correct the links.
+# Note that each tag file must have a unique name
+# (where the name does NOT include the path)
+# If a tag file is not located in the directory in which doxygen 
+# is run, you must also specify the path to the tagfile here.
+
+TAGFILES               = 
+
+# When a file name is specified after GENERATE_TAGFILE, doxygen will create 
+# a tag file that is based on the input files it reads.
+
+GENERATE_TAGFILE       = 
+
+# If the ALLEXTERNALS tag is set to YES all external classes will be listed 
+# in the class index. If set to NO only the inherited external classes 
+# will be listed.
+
+ALLEXTERNALS           = NO
+
+# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed 
+# in the modules index. If set to NO, only the current project's groups will 
+# be listed.
+
+EXTERNAL_GROUPS        = YES
+
+# The PERL_PATH should be the absolute path and name of the perl script 
+# interpreter (i.e. the result of `which perl').
+
+PERL_PATH              = /usr/bin/perl
+
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool   
+#---------------------------------------------------------------------------
+
+# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will 
+# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base 
+# or super classes. Setting the tag to NO turns the diagrams off. Note that 
+# this option is superseded by the HAVE_DOT option below. This is only a 
+# fallback. It is recommended to install and use dot, since it yields more 
+# powerful graphs.
+
+CLASS_DIAGRAMS         = YES
+
+# You can define message sequence charts within doxygen comments using the \msc 
+# command. Doxygen will then run the mscgen tool (see http://www.mcternan.me.uk/mscgen/) to 
+# produce the chart and insert it in the documentation. The MSCGEN_PATH tag allows you to 
+# specify the directory where the mscgen tool resides. If left empty the tool is assumed to 
+# be found in the default search path.
+
+MSCGEN_PATH            = 
+
+# If set to YES, the inheritance and collaboration graphs will hide 
+# inheritance and usage relations if the target is undocumented 
+# or is not a class.
+
+HIDE_UNDOC_RELATIONS   = YES
+
+# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is 
+# available from the path. This tool is part of Graphviz, a graph visualization 
+# toolkit from AT&T and Lucent Bell Labs. The other options in this section 
+# have no effect if this option is set to NO (the default)
+
+HAVE_DOT               = NO
+
+# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen 
+# will generate a graph for each documented class showing the direct and 
+# indirect inheritance relations. Setting this tag to YES will force the 
+# the CLASS_DIAGRAMS tag to NO.
+
+CLASS_GRAPH            = YES
+
+# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen 
+# will generate a graph for each documented class showing the direct and 
+# indirect implementation dependencies (inheritance, containment, and 
+# class references variables) of the class with other documented classes.
+
+COLLABORATION_GRAPH    = YES
+
+# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen 
+# will generate a graph for groups, showing the direct groups dependencies
+
+GROUP_GRAPHS           = YES
+
+# If the UML_LOOK tag is set to YES doxygen will generate inheritance and 
+# collaboration diagrams in a style similar to the OMG's Unified Modeling 
+# Language.
+
+UML_LOOK               = NO
+
+# If set to YES, the inheritance and collaboration graphs will show the 
+# relations between templates and their instances.
+
+TEMPLATE_RELATIONS     = NO
+
+# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT 
+# tags are set to YES then doxygen will generate a graph for each documented 
+# file showing the direct and indirect include dependencies of the file with 
+# other documented files.
+
+INCLUDE_GRAPH          = YES
+
+# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and 
+# HAVE_DOT tags are set to YES then doxygen will generate a graph for each 
+# documented header file showing the documented files that directly or 
+# indirectly include this file.
+
+INCLUDED_BY_GRAPH      = YES
+
+# If the CALL_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will 
+# generate a call dependency graph for every global function or class method. 
+# Note that enabling this option will significantly increase the time of a run. 
+# So in most cases it will be better to enable call graphs for selected 
+# functions only using the \callgraph command.
+
+CALL_GRAPH             = NO
+
+# If the CALLER_GRAPH, SOURCE_BROWSER and HAVE_DOT tags are set to YES then doxygen will 
+# generate a caller dependency graph for every global function or class method. 
+# Note that enabling this option will significantly increase the time of a run. 
+# So in most cases it will be better to enable caller graphs for selected 
+# functions only using the \callergraph command.
+
+CALLER_GRAPH           = NO
+
+# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen 
+# will graphical hierarchy of all classes instead of a textual one.
+
+GRAPHICAL_HIERARCHY    = YES
+
+# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES 
+# then doxygen will show the dependencies a directory has on other directories 
+# in a graphical way. The dependency relations are determined by the #include
+# relations between the files in the directories.
+
+DIRECTORY_GRAPH        = YES
+
+# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images 
+# generated by dot. Possible values are png, jpg, or gif
+# If left blank png will be used.
+
+DOT_IMAGE_FORMAT       = png
+
+# The tag DOT_PATH can be used to specify the path where the dot tool can be 
+# found. If left blank, it is assumed the dot tool can be found in the path.
+
+DOT_PATH               = 
+
+# The DOTFILE_DIRS tag can be used to specify one or more directories that 
+# contain dot files that are included in the documentation (see the 
+# \dotfile command).
+
+DOTFILE_DIRS           = 
+
+# The MAX_DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of 
+# nodes that will be shown in the graph. If the number of nodes in a graph 
+# becomes larger than this value, doxygen will truncate the graph, which is 
+# visualized by representing a node as a red box. Note that doxygen if the number 
+# of direct children of the root node in a graph is already larger than 
+# MAX_DOT_GRAPH_NOTES then the graph will not be shown at all. Also note 
+# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH.
+
+DOT_GRAPH_MAX_NODES    = 50
+
+# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the 
+# graphs generated by dot. A depth value of 3 means that only nodes reachable 
+# from the root by following a path via at most 3 edges will be shown. Nodes 
+# that lay further from the root node will be omitted. Note that setting this 
+# option to 1 or 2 may greatly reduce the computation time needed for large 
+# code bases. Also note that the size of a graph can be further restricted by 
+# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction.
+
+MAX_DOT_GRAPH_DEPTH    = 0
+
+# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent 
+# background. This is disabled by default, which results in a white background. 
+# Warning: Depending on the platform used, enabling this option may lead to 
+# badly anti-aliased labels on the edges of a graph (i.e. they become hard to 
+# read).
+
+DOT_TRANSPARENT        = NO
+
+# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output 
+# files in one run (i.e. multiple -o and -T options on the command line). This 
+# makes dot run faster, but since only newer versions of dot (>1.8.10) 
+# support this, this feature is disabled by default.
+
+DOT_MULTI_TARGETS      = NO
+
+# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will 
+# generate a legend page explaining the meaning of the various boxes and 
+# arrows in the dot generated graphs.
+
+GENERATE_LEGEND        = YES
+
+# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will 
+# remove the intermediate dot files that are used to generate 
+# the various graphs.
+
+DOT_CLEANUP            = YES
+
+#---------------------------------------------------------------------------
+# Configuration::additions related to the search engine   
+#---------------------------------------------------------------------------
+
+# The SEARCHENGINE tag specifies whether or not a search engine should be 
+# used. If set to NO the values of all tags below this one will be ignored.
+
+SEARCHENGINE           = NO
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/Makefile.am	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/Makefile.am	(revision 24244)
@@ -0,0 +1,3 @@
+SUBDIRS = src
+
+CLEANFILES = *.pyc *~ core core.*
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/autogen.sh
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/autogen.sh	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/autogen.sh	(revision 24244)
@@ -0,0 +1,103 @@
+#!/bin/sh
+# Run this to generate all the initial makefiles, etc.
+
+srcdir=`dirname $0`
+test -z "$srcdir" && srcdir=.
+
+ORIGDIR=`pwd`
+cd $srcdir
+
+PROJECT=ppSkycell
+TEST_TYPE=-f
+# change this to be a unique filename in the top level dir
+FILE=autogen.sh
+
+DIE=0
+
+LIBTOOLIZE=libtoolize
+ACLOCAL="aclocal $ACLOCAL_FLAGS"
+AUTOHEADER=autoheader
+AUTOMAKE=automake
+AUTOCONF=autoconf
+
+($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $LIBTOOLIZE installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/"
+        DIE=1
+}
+
+($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $ACLOCAL installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/"
+        DIE=1
+}
+
+($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $AUTOHEADER installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/"
+        DIE=1
+}
+
+($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $AUTOMAKE installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/"
+        DIE=1
+}
+
+($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || {
+        echo
+        echo "You must have $AUTOCONF installed to compile $PROJECT."
+        echo "Download the appropriate package for your distribution,"
+        echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/"
+        DIE=1
+}
+
+if test "$DIE" -eq 1; then
+        exit 1
+fi
+
+test $TEST_TYPE $FILE || {
+        echo "You must run this script in the top-level $PROJECT directory"
+        exit 1
+}
+
+if test -z "$*"; then
+        echo "I am going to run ./configure with no arguments - if you wish "
+        echo "to pass any to it, please specify them on the $0 command line."
+fi
+
+$LIBTOOLIZE --copy --force || echo "$LIBTOOLIZE failed"
+$ACLOCAL || echo "$ACLOCAL failed"
+$AUTOHEADER || echo "$AUTOHEADER failed"
+$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE  failed"
+$AUTOCONF || echo "$AUTOCONF failed"
+
+cd $ORIGDIR
+
+run_configure=true
+for arg in $*; do
+    case $arg in
+        --no-configure)
+            run_configure=false
+            ;;
+        *)
+            ;;
+    esac
+done
+
+if $run_configure; then
+    $srcdir/configure --enable-maintainer-mode "$@"
+    echo
+    echo "Now type 'make' to compile $PROJECT."
+else
+    echo
+    echo "Now run 'configure' and 'make' to compile $PROJECT."
+fi
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/configure.ac	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/configure.ac	(revision 24244)
@@ -0,0 +1,38 @@
+dnl Process this file with autoconf to produce a configure script.
+AC_PREREQ(2.61)
+
+AC_INIT([ppSkycell], [0.1.1], [ipp-support@ifa.hawaii.edu])
+AC_CONFIG_SRCDIR([src])
+
+AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2])
+AM_CONFIG_HEADER([src/config.h])
+AM_MAINTAINER_MODE
+
+IPP_STDCFLAGS
+
+AC_LANG(C)
+AC_GNU_SOURCE
+AC_PROG_CC_C99
+AC_PROG_INSTALL
+AC_PROG_LIBTOOL
+AC_SYS_LARGEFILE
+
+PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0])
+PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0])
+
+dnl Set CFLAGS for build
+IPP_STDOPTS
+CFLAGS="${CFLAGS} -Wall -Werror"
+
+IPP_VERSION
+
+AC_SUBST([PPSKYCELL_CFLAGS])
+AC_SUBST([PPSKYCELL_LIBS])
+
+AC_CONFIG_FILES([
+  Makefile
+  src/Makefile
+  Doxyfile
+])
+
+AC_OUTPUT
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/Makefile.am	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/Makefile.am	(revision 24244)
@@ -0,0 +1,47 @@
+bin_PROGRAMS = ppSkycell
+
+if HAVE_SVNVERSION
+PPSKYCELL_VERSION=`$(SVNVERSION) ..`
+else
+PPSKYCELL_VERSION="UNKNOWN"
+endif
+
+if HAVE_SVN
+PPSKYCELL_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPSKYCELL_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PPSKYCELL_BRANCH="UNKNOWN"
+PPSKYCELL_SOURCE="UNKNOWN"
+endif
+
+# Force recompilation of ppSkycellVersion.c, since it gets the version information
+ppSkycellVersion.c: ppSkycellVersionDefinitions.h
+ppSkycellVersionDefinitions.h: ppSkycellVersionDefinitions.h.in FORCE
+	-$(RM) ppSkycellVersionDefinitions.h
+	$(SED) -e "s|@PPSKYCELL_VERSION@|\"$(PPSKYCELL_VERSION)\"|" -e "s|@PPSKYCELL_BRANCH@|\"$(PPSKYCELL_BRANCH)\"|" -e "s|@PPSKYCELL_SOURCE@|\"$(PPSKYCELL_SOURCE)\"|" ppSkycellVersionDefinitions.h.in > ppSkycellVersionDefinitions.h
+FORCE: ;
+
+BUILT_SOURCES = ppSkycellVersionDefinitions.h
+
+
+ppSkycell_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSKYCELL_CFLAGS)
+ppSkycell_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PPSKYCELL_LIBS)
+
+ppSkycell_SOURCES =		\
+	ppSkycell.c		\
+	ppSkycellArguments.c	\
+	ppSkycellCamera.c	\
+	ppSkycellData.c		\
+	ppSkycellLoop.c		\
+	ppSkycellVersion.c            
+
+noinst_HEADERS = \
+	ppSkycell.h
+
+clean-local:
+	-rm -f TAGS
+
+# Tags for emacs
+tags:
+	etags `find . -name \*.[ch] -print`
+
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycell.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycell.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycell.c	(revision 24244)
@@ -0,0 +1,41 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+
+#include "ppSkycell.h"
+
+int main(int argc, char *argv[])
+{
+    ppSkycellData *data = ppSkycellDataInit(&argc, argv);
+    if (!data) {
+        psErrorStackPrint(stderr, "Unable to initialise.");
+        return PS_EXIT_CONFIG_ERROR;
+    }
+
+    if (!ppSkycellArguments(data, argc, argv)) {
+        psErrorStackPrint(stderr, "Unable to parse arguments.");
+        psFree(data);
+        return PS_EXIT_CONFIG_ERROR;
+    }
+
+    if (!ppSkycellCamera(data)) {
+        psErrorStackPrint(stderr, "Unable to parse camera configuration.");
+        psFree(data);
+        return PS_EXIT_CONFIG_ERROR;
+    }
+
+    if (!ppSkycellLoop(data)) {
+        psErrorStackPrint(stderr, "Unable to process data.");
+        psFree(data);
+        return PS_EXIT_DATA_ERROR;
+    }
+
+    psFree(data);
+
+    return PS_EXIT_SUCCESS;
+}
+
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycell.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycell.h	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycell.h	(revision 24244)
@@ -0,0 +1,38 @@
+#ifndef PP_SKYCELL_H
+#define PP_SKYCELL_H
+
+#include <pslib.h>
+#include <psmodules.h>
+
+#define PPSKYCELL_RECIPE "PPSKYCELL"    // Recipe name
+
+// Data for processing
+typedef struct {
+    psString imagesName;                // Filename with images
+    psString masksName;                 // Filename with masks
+    psString outRoot;                   // Output root name
+    int numInputs;                      // Number of inputs
+    psImageMaskType maskVal;            // Value to mask
+    int bin1, bin2;                     // Binning factors
+    pmConfig *config;                   // Configuration
+} ppSkycellData;
+
+/// Initialise data for processing
+ppSkycellData *ppSkycellDataInit(int *argc, char *argv[] // Command-line arguments
+    );
+
+/// Parse command-line arguments
+bool ppSkycellArguments(ppSkycellData *data, // Data for processing
+                        int argc, char *argv[] // Command-line arguments
+    );
+
+/// Parse camera configurations
+bool ppSkycellCamera(ppSkycellData *data // Data for processing
+    );
+
+/// Loop over input data, processing
+bool ppSkycellLoop(ppSkycellData *data // Data for processing
+    );
+
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellArguments.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellArguments.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellArguments.c	(revision 24244)
@@ -0,0 +1,70 @@
+/** @file ppSkycellArguments.c
+ *
+ *  @brief
+ *
+ *  @ingroup ppSkycell
+ *
+ *  @author IfA
+ *  @version $Revision: 1.6 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-06 19:45:30 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <pslib.h>
+#include <psmodules.h>
+
+#include "ppSkycell.h"
+
+/// Print usage information and die
+static void usage(const char *program,  // Name of the program
+                  psMetadata *arguments, // Command-line arguments
+                  ppSkycellData *data   // Run-time data
+    )
+{
+    fprintf(stderr, "\nPan-STARRS skycell JPEGifier\n\n");
+    fprintf(stderr, "Usage: %s -images INPUT.list [-masks MASK.list] OUTPUT_ROOT\n\n",
+            program);
+    fprintf(stderr, "\n");
+    psArgumentHelp(arguments);
+    psFree(arguments);
+    psFree(data);
+
+    pmConfigDone();
+    psLibFinalize();
+
+    exit(PS_EXIT_CONFIG_ERROR);
+}
+
+
+bool ppSkycellArguments(ppSkycellData *data, int argc, char *argv[])
+{
+    assert(data);
+    assert(data->config);
+
+    psMetadata *arguments = psMetadataAlloc(); // Command-line arguments
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-images", 0, "Filename with input images", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-masks", 0, "Filename with input masks", NULL);
+    if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 2) {
+        usage(argv[0], arguments, data);
+    }
+
+    data->imagesName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-images"));
+    data->masksName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-masks"));
+    data->outRoot = psStringCopy(argv[1]);
+
+    psTrace("ppSkycell", 1, "Done reading command-line arguments\n");
+    psFree(arguments);
+
+    PS_ASSERT_STRING_NON_EMPTY(data->imagesName, false);
+    PS_ASSERT_STRING_NON_EMPTY(data->outRoot, false);
+
+    return true;
+}
+
+
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellCamera.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellCamera.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellCamera.c	(revision 24244)
@@ -0,0 +1,124 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+
+#include "ppSkycell.h"
+
+/// Read list of images
+static psArray *fileList(const char *filename // Filename
+    )
+{
+    psString input = psSlurpFilename(filename);
+    if (!input) {
+        psError(PS_ERR_IO, false, "Unable to read %s", filename);
+        return false;
+    }
+    psArray *inputs = psStringSplitArray(input, "\n", false); // Input filenames
+    psFree(input);
+    if (!inputs || inputs->n == 0) {
+        psError(PS_ERR_IO, false, "Unable to read filenames from %s", filename);
+        psFree(inputs);
+        return NULL;
+    }
+    return inputs;
+}
+
+/// Add a single filename to the arguments as an array, so that it can be used with pmFPAfileBindFromArgs, etc
+static void fileArguments(const char *file, // The symbolic name for the file
+                          const char *name, // The name of the file
+                          const char *comment, // Description of the file
+                          pmConfig *config // Configuration
+    )
+{
+    psArray *files = psArrayAlloc(1); // Array with file names
+    files->data[0] = psStringCopy(name);
+    if (psMetadataLookup(config->arguments, file)) {
+        psMetadataRemoveKey(config->arguments, file);
+    }
+    psMetadataAddArray(config->arguments, PS_LIST_TAIL, file, 0, comment, files);
+    psFree(files);
+    return;
+}
+
+
+bool ppSkycellCamera(ppSkycellData *data // Run-time data
+    )
+{
+    psArray *images = fileList(data->imagesName); // Image names
+    if (!images) {
+        psError(psErrorCodeLast(), false, "No images provided.");
+        return false;
+    }
+    data->numInputs = images->n;
+
+    psArray *masks = NULL;              // Mask names
+    if (data->masksName) {
+        masks = fileList(data->masksName);
+        if (!masks) {
+            psError(psErrorCodeLast(), false, "No masks provided.");
+            psFree(images);
+            return false;
+        }
+        if (masks->n != data->numInputs) {
+            psError(PS_ERR_BAD_PARAMETER_SIZE, true, "Number of images (%ld) and masks (%ld) do not match",
+                    images->n, masks->n);
+            psFree(images);
+            psFree(masks);
+            return false;
+        }
+    }
+
+    psMetadataAddStr(data->config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Output root", data->outRoot);
+
+    for (int i = 0; i < data->numInputs; i++) {
+        bool status = false;             // Status of file definition
+        fileArguments("IMAGE", images->data[i], "Name of the image", data->config);
+        pmFPAfile *image = pmFPAfileDefineFromArgs(&status, data->config, "PPSKYCELL.IMAGE", "IMAGE"); // File
+        if (!status || !image) {
+            psError(PS_ERR_IO, false, "Failed to build file from PPSKYCELL.IMAGE");
+            // XXX Cleanup
+            return false;
+        }
+
+        if (data->masksName) {
+            fileArguments("MASK", masks->data[i], "Name of the mask", data->config);
+            if (!pmFPAfileBindFromArgs(&status, image, data->config, "PPSKYCELL.MASK", "MASK") || !status) {
+                psError(PS_ERR_IO, false, "Failed to build file from PPSKYCELL.MASK");
+                // XXX Cleanup
+                return false;
+            }
+        }
+    }
+
+    if (!pmFPAfileDefineOutput(data->config, NULL, "PPSKYCELL.JPEG1")) {
+        psError(psErrorCodeLast(), false, "Unable to define output.");
+        return false;
+    }
+    if (!pmFPAfileDefineOutput(data->config, NULL, "PPSKYCELL.JPEG2")) {
+        psError(psErrorCodeLast(), false, "Unable to define output.");
+        return false;
+    }
+
+    // Now the camera has been determined, we can read the recipe
+    psMetadata *recipe = psMetadataLookupMetadata(NULL, data->config->recipes, PPSKYCELL_RECIPE); // Recipe
+    if (!recipe) {
+        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSKYCELL_RECIPE);
+        return false;
+    }
+
+    psString maskString = psMetadataLookupStr(NULL, recipe, "MASKVAL"); // Mask values
+    data->maskVal = pmConfigMaskGet(maskString, data->config);
+    data->bin1 = psMetadataLookupS32(NULL, recipe, "BIN1");
+    data->bin2 = psMetadataLookupS32(NULL, recipe, "BIN2");
+
+    if (data->bin1 <= 0 || data->bin2 <= 0) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find legitimate values for BIN1 and BIN2");
+        return false;
+    }
+
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellData.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellData.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellData.c	(revision 24244)
@@ -0,0 +1,49 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+
+#include "ppSkycell.h"
+
+// Destructor
+static void skycellDataFree(ppSkycellData *data // Data to free
+    )
+{
+    psFree(data->imagesName);
+    psFree(data->masksName);
+    psFree(data->outRoot);
+    psFree(data->config);
+    return;
+}
+
+
+ppSkycellData *ppSkycellDataAlloc(void)
+{
+    ppSkycellData *data = psAlloc(sizeof(ppSkycellData)); // Processing data, to return
+    psMemSetDeallocator(data, (psFreeFunc)skycellDataFree);
+
+    data->imagesName = NULL;
+    data->masksName = NULL;
+    data->outRoot = NULL;
+    data->numInputs = 0;
+    data->maskVal = 0;
+    data->bin1 = 0;
+    data->bin2 = 0;
+    data->config = NULL;
+
+    return data;
+}
+
+
+ppSkycellData *ppSkycellDataInit(int *argc, char **argv)
+{
+    PS_ASSERT_PTR_NON_NULL(argc, NULL);
+    PS_ASSERT_PTR_NON_NULL(argv, NULL);
+
+    ppSkycellData *data = ppSkycellDataAlloc(); // Processing data, to return
+    data->config = pmConfigRead(argc, argv, PPSKYCELL_RECIPE);
+    return data;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellLoop.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellLoop.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellLoop.c	(revision 24244)
@@ -0,0 +1,332 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+
+#include "ppSkycell.h"
+
+#define BUFFER 16                       // Size of buffer for projections
+
+static void regionMinMax(psRegion *base,// Base region; modified
+                         const psRegion *compare // Comparison region
+                         )
+{
+    base->x0 = PS_MIN(base->x0, compare->x0);
+    base->x1 = PS_MAX(base->x1, compare->x1);
+    base->y0 = PS_MIN(base->y0, compare->y0);
+    base->y1 = PS_MAX(base->y1, compare->y1);
+    return;
+}
+
+static psRegion *skycellRegion(pmAstromWCS *wcs, // World Coordinate System
+                               int numCols, int numRows // Size of image
+    )
+{
+    psPlane *fromCoords = psPlaneAlloc(), *toCoords = psPlaneAlloc(); // Coordinates for transforms
+    psRegion *region = psRegionAlloc(INFINITY, -INFINITY, INFINITY, -INFINITY); // Region for skycell
+
+// Limit the region using the nominated point (X,Y)
+#define REGION_RANGE(X,Y) \
+    fromCoords->x = (X); \
+    fromCoords->y = (Y); \
+    psPlaneTransformApply(toCoords, wcs->trans, fromCoords); \
+    toCoords->x /= wcs->cdelt1; \
+    toCoords->y /= wcs->cdelt2; \
+    toCoords->x += wcs->crpix1; \
+    toCoords->y += wcs->crpix2; \
+    region->x0 = PS_MIN(region->x0, toCoords->x); \
+    region->x1 = PS_MAX(region->x1, toCoords->x); \
+    region->y0 = PS_MIN(region->y0, toCoords->y); \
+    region->y1 = PS_MAX(region->y1, toCoords->y);
+
+    REGION_RANGE(0, 0);                 // Lower left
+    psTrace("ppSkycell", 9, "Start: %.0f %.0f", toCoords->x, toCoords->y);
+    REGION_RANGE(0, numRows);           // Upper left
+    REGION_RANGE(numCols, 0);           // Lower right
+    REGION_RANGE(numCols, numRows);     // Upper right
+    psTrace("ppSkycell", 9, "Stop: %.0f %.0f\n", toCoords->x, toCoords->y);
+
+    return region;
+}
+
+// Activate/deactivate a single element for a list
+void fileActivationSingle(pmConfig *config, // Configuration
+                          const char **files, // Files to turn on/off
+                          bool state,   // Activation state
+                          int num // Number of file in sequence
+                          )
+{
+    assert(config);
+    for (int i = 0; files[i] != NULL; i++) {
+        pmFPAfileActivateSingle(config->files, state, files[i], num); // Activated file
+    }
+    return;
+}
+
+// Iterate down the hierarchy, loading files; we can get away with this because we're working on skycells
+static pmFPAview *filesIterateDown(pmConfig *config // Configuration
+                                   )
+{
+    assert(config);
+
+    pmFPAview *view = pmFPAviewAlloc(0); // Pointer into FPA hierarchy
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return NULL;
+    }
+    view->chip = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return NULL;
+    }
+    view->cell = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return NULL;
+    }
+    view->readout = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return NULL;
+    }
+    return view;
+}
+
+// Iterate up the hierarchy, writing files; we can get away with this because we're working on skycells
+static bool filesIterateUp(pmConfig *config // Configuration
+                           )
+{
+    assert(config);
+
+    pmFPAview *view = pmFPAviewAlloc(0); // Pointer into FPA hierarchy
+    view->chip = view->cell = view->readout = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+    view->readout = -1;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+    view->cell = -1;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+    view->chip = -1;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+    psFree(view);
+    return true;
+}
+
+
+bool ppSkycellLoop(ppSkycellData *data // Run-time data
+    )
+{
+    psVector *crval1 = psVectorAllocEmpty(BUFFER, PS_TYPE_F64); // CRVAL1 values
+    psVector *crval2 = psVectorAllocEmpty(BUFFER, PS_TYPE_F64); // CRVAL2 values
+    psVector *cdelt1 = psVectorAllocEmpty(BUFFER, PS_TYPE_F64); // CDELT1 values
+    psVector *cdelt2 = psVectorAllocEmpty(BUFFER, PS_TYPE_F64); // CDELT2 values
+    psArray *projRegions = psArrayAllocEmpty(BUFFER); // Region for projection
+    int numProj = 0;                    // Number of projections
+
+    psVector *target = psVectorAlloc(data->numInputs, PS_TYPE_S32); // Target for each input
+    psArray *imageRegions = psArrayAlloc(data->numInputs); // Region for image
+
+    for (int i = 0; i < data->numInputs; i++) {
+        pmFPAfile *file = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.IMAGE", i); // File to examine
+        // Header in the FPA should have been read as a part of defining the file...
+#if 0
+        bool pmFPAReadHeaderSet(pmFPA *fpa,     // FPA to read into
+                                psFits *fits,   // FITS file from which to read
+                                pmConfig *config // Configuration
+            );
+#endif
+        pmHDU *hdu = file->fpa->hdu;    // Header of interest
+
+        int numCols = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); // Number of columns
+        int numRows = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); // Number of rows
+
+        pmAstromWCS *wcs = pmAstromWCSfromHeader(hdu->header); // World Coordinate System
+        if (!wcs) {
+            psError(psErrorCodeLast(), false, "Unable to read WCS for image %d", i);
+            return false;
+        }
+
+        psRegion *region = imageRegions->data[i] = skycellRegion(wcs, numCols, numRows); // Region of image
+        psTrace("ppSkycell", 5, "Image region %d is: [%.0f:%.0f,%.0f:%.0f]\n",
+                i, region->x0, region->x1, region->y0, region->y1);
+
+        bool found = false;             // Found a projection?
+        for (int j = 0; j < numProj && !found; j++) {
+            if (wcs->crval1 == crval1->data.F64[j] && wcs->crval2 == crval2->data.F64[j] &&
+                wcs->cdelt1 == cdelt1->data.F64[j] && wcs->cdelt2 == cdelt1->data.F64[j]) {
+                regionMinMax(projRegions->data[j], region);
+                target->data.S32[i] = j;
+                found = true;
+                psTrace("ppSkycell", 3, "Image %d uses projection %d\n", i, j);
+            }
+        }
+
+        if (!found) {
+            psVectorAppend(crval1, wcs->crval1);
+            psVectorAppend(crval2, wcs->crval2);
+            psVectorAppend(cdelt1, wcs->cdelt1);
+            psVectorAppend(cdelt2, wcs->cdelt2);
+            psRegion *projRegion = psRegionAlloc(region->x0, region->x1, region->y0, region->y1);
+            psArrayAdd(projRegions, projRegions->n, projRegion);
+            psFree(projRegion);
+            target->data.S32[i] = numProj;
+            psTrace("ppSkycell", 3, "Image %d uses new projection\n", i);
+
+            numProj++;
+        }
+    }
+
+    pmFPAfileActivate(data->config->files, false, NULL);
+
+    for (int i = 0; i < numProj; i++) {
+        psRegion *projRegion = projRegions->data[i]; // Region for skycell projection
+        psTrace("ppSkycell", 2, "Projection %d: [%.0f:%.0f,%.0f:%.0f]\n",
+                i, projRegion->x0, projRegion->x1, projRegion->y0, projRegion->y1);
+        int xSize = projRegion->x1 - projRegion->x0 + 1; // Size of unbinned image
+        int ySize = projRegion->y1 - projRegion->y0 + 1; // Size of unbinned image
+        // Size of binned image 1
+        int numCols1 = xSize / (float)data->bin1 + 1.5, numRows1 = ySize / (float)data->bin1 + 1.5;
+        // Size of binned image 2
+        int numCols2 = numCols1 / (float)data->bin2 + 1.5, numRows2 = numRows1 / (float)data->bin2 + 1.5;
+
+        psImage *image1 = psImageAlloc(numCols1, numRows1, PS_TYPE_F32); // Binned image
+        psImage *image2 = psImageAlloc(numCols2, numRows2, PS_TYPE_F32); // Binned image
+        psImageInit(image1, 0);
+        psImageInit(image2, 0);
+
+        psImage *mask1 = NULL, *mask2 = NULL; // Binned masks
+        if (data->masksName) {
+            mask1 = psImageAlloc(numCols1, numRows1, PS_TYPE_IMAGE_MASK);
+            mask2 = psImageAlloc(numCols2, numRows2, PS_TYPE_IMAGE_MASK);
+            psImageInit(mask1, 0xFF);
+            psImageInit(mask2, 0xFF);
+        }
+
+        for (int j = 0; j < data->numInputs; j++) {
+            if (target->data.S32[j] != i) {
+                continue;
+            }
+
+            pmFPAfileActivateSingle(data->config->files, true, "PPSKYCELL.IMAGE", j);
+            if (data->masksName) {
+                pmFPAfileActivateSingle(data->config->files, true, "PPSKYCELL.MASK", j);
+            }
+
+            pmFPAview *view = filesIterateDown(data->config); // View to readout
+            if (!view) {
+                psError(psErrorCodeLast(), false, "Unable to iterate down.");
+                // XXX Cleanup
+                return false;
+            }
+
+            pmFPAfile *file = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.IMAGE", j);
+            pmReadout *inRO = pmFPAviewThisReadout(view, file->fpa); // Readout with input
+            psFree(view);
+
+            // Flip images; no idea why this has to be done, but apparently it does
+            {
+                psImage *rot = psImageRotate(NULL, inRO->image, M_PI, NAN, PS_INTERPOLATE_BILINEAR);
+                psFree(inRO->image);
+                inRO->image = rot;
+            }
+            if (inRO->mask) {
+                psImage *rot = psImageRotate(NULL, inRO->mask, M_PI, 0, PS_INTERPOLATE_FLAT);
+                psFree(inRO->mask);
+                inRO->mask = rot;
+            }
+
+            pmReadout *bin1RO = pmReadoutAlloc(NULL), *bin2RO = pmReadoutAlloc(NULL); // Binned readouts
+            if (!pmReadoutRebin(bin1RO, inRO, data->maskVal, data->bin1, data->bin1)) {
+                psError(psErrorCodeLast(), false, "Unable to rebin image");
+                // XXX Cleanup
+                return false;
+            }
+            if (!pmReadoutRebin(bin2RO, bin1RO, data->maskVal, data->bin2, data->bin2)) {
+                psError(psErrorCodeLast(), false, "Unable to rebin image");
+                // XXX Cleanup
+                return false;
+            }
+
+            psRegion *imageRegion = imageRegions->data[j]; // Region for image
+            // Offsets for image on skycell
+            int xOffset1 = (imageRegion->x0 - projRegion->x0) / (float)data->bin1;
+            int yOffset1 = (imageRegion->y0 - projRegion->y0) / (float)data->bin1;
+            int xOffset2 = xOffset1 / (float)data->bin2, yOffset2 = yOffset1 / (float)data->bin2;
+
+            // XXX Completely neglecting rotations
+            // The skycells are divided up neatly with them all having the same orientation
+            psImageOverlaySection(image1, bin1RO->image, xOffset1, yOffset1, "=");
+            psImageOverlaySection(image2, bin2RO->image, xOffset2, yOffset2, "=");
+            if (data->masksName) {
+                psImageOverlaySection(mask1, bin1RO->mask, xOffset1, yOffset1, "=");
+                psImageOverlaySection(mask2, bin2RO->mask, xOffset2, yOffset2, "=");
+            }
+
+            psFree(bin1RO);
+            psFree(bin2RO);
+            filesIterateUp(data->config);
+            psFree(file->fpa);
+            file->fpa = NULL;
+            if (data->masksName) {
+                pmFPAfile *file = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.MASK", j);
+                psFree(file->fpa);
+                file->fpa = NULL;
+            }
+            pmFPAfileActivate(data->config->files, false, NULL);
+        }
+
+        pmFPAfileActivate(data->config->files, true, "PPSKYCELL.JPEG1");
+        pmFPAfileActivate(data->config->files, true, "PPSKYCELL.JPEG2");
+        pmFPAview *view = filesIterateDown(data->config); // View to readout
+
+        pmCell *cell1 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.JPEG1"); // Rebinned cell 1
+        pmCell *cell2 = pmFPAfileThisCell(data->config->files, view, "PPSKYCELL.JPEG2"); // Rebinned cell 2
+        psFree(view);
+        pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts
+
+        ro1->image = image1;
+        ro2->image = image2;
+        ro1->mask = mask1;
+        ro2->mask = mask2;
+
+        ro1->data_exists = cell1->data_exists = cell1->parent->data_exists = true;
+        ro2->data_exists = cell2->data_exists = cell2->parent->data_exists = true;
+
+        pmFPAfile *file1 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.JPEG1", 0);
+        file1->save = true;
+        file1->index = i;
+        pmFPAfile *file2 = pmFPAfileSelectSingle(data->config->files, "PPSKYCELL.JPEG2", 0);
+        file2->save = true;
+        file2->index = i;
+#if 0
+        {
+            psString filename = NULL;   // Filename for image
+            psStringAppend(&filename, "skycell_%d_1.fits", i);
+            psFits *fits = psFitsOpen(filename, "w");
+            psFree(filename);
+            psFitsWriteImage(fits, NULL, image1, 0, NULL);
+            psFitsClose(fits);
+        }
+
+        {
+            psString filename = NULL;   // Filename for image
+            psStringAppend(&filename, "skycell_%d_2.fits", i);
+            psFits *fits = psFitsOpen(filename, "w");
+            psFree(filename);
+            psFitsWriteImage(fits, NULL, image2, 0, NULL);
+            psFitsClose(fits);
+        }
+#endif
+
+        filesIterateUp(data->config);
+    }
+
+    // XXX Cleanup
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellVersion.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellVersion.c	(revision 24244)
@@ -0,0 +1,116 @@
+/** @file ppSkycellVersion.c
+ *
+ *  @brief
+ *
+ *  @ingroup ppSkycell
+ *
+ *  @author IfA
+ *  @version $Revision: 1.3 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-06 19:45:30 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+
+#include "ppSkycell.h"
+#include "ppSkycellVersionDefinitions.h"
+
+#ifndef PPSKYCELL_VERSION
+#error "PPSKYCELL_VERSION is not set"
+#endif
+#ifndef PPSKYCELL_BRANCH
+#error "PPSKYCELL_BRANCH is not set"
+#endif
+#ifndef PPSKYCELL_SOURCE
+#error "PPSKYCELL_SOURCE is not set"
+#endif
+
+psString ppSkycellVersion(void)
+{
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", PPSKYCELL_BRANCH, PPSKYCELL_VERSION);
+    return value;
+}
+
+psString ppSkycellSource(void)
+{
+    return psStringCopy(PPSKYCELL_SOURCE);
+}
+
+psString ppSkycellVersionLong(void)
+{
+    psString version = ppSkycellVersion();  // Version, to return
+    psString source = ppSkycellSource();    // Source
+
+    psStringPrepend(&version, "ppSkycell ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
+
+    return version;
+};
+
+bool ppSkycellVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "ppSkycell at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+
+    psString version = ppSkycellVersion(); // Software version
+    psString source  = ppSkycellSource();  // Software source
+
+    psStringPrepend(&version, "ppSkycell version: ");
+    psStringPrepend(&version, "ppSkycell source: ");
+
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+
+    psFree(version);
+    psFree(source);
+
+    return true;
+}
+
+void ppSkycellVersionPrint(void)
+{
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("ppSkycell", PS_LOG_INFO, "ppSkycell at %s", timeString);
+    psFree(timeString);
+
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString ppSkycell = ppSkycellVersionLong(); // ppSkycell version
+
+    psLogMsg("ppSkycell", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("ppSkycell", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("ppSkycell", PS_LOG_INFO, "%s", ppSkycell);
+
+    psFree(pslib);
+    psFree(psmodules);
+    psFree(ppSkycell);
+
+    return;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSkycell/src/ppSkycellVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PPSKYCELL_VERSION_DEFINITIONS_H
+#define PPSKYCELL_VERSION_DEFINITIONS_H
+
+#define PPSKYCELL_VERSION @PPSKYCELL_VERSION@ // SVN version
+#define PPSKYCELL_BRANCH  @PPSKYCELL_BRANCH@  // SVN branch
+#define PPSKYCELL_SOURCE  @PPSKYCELL_SOURCE@  // SVN source
+
+#endif
Index: anches/cnb_branches/cnb_branch_20090301/ppStack/README.txt
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/README.txt	(revision 24243)
+++ 	(revision )
@@ -1,25 +1,0 @@
-ppStack needs to be optimised, principally to reduce the memory
-footprint (10 or so images will get it to over 4 GB).  The proposed
-strategy follows, more or less, the same as for ppMerge.
-
-* Convolution steps:
-  - For each file:
-    + read entire image
-    + convolve to reference PSF
-    + write convolved image
-
-* Combination step:
-  - For each chunk:
-    + For each file, read chunk
-    + combine chunk
-  - Probably have to repeat, to do the multi-pass rejection
-
-
-To do:
-* Need file PPSTACK.OUTPUT.CONV (image, mask, weight versions)
-* Need file PPSTACK.INPUT.CONV (image, mask, weight versions)
-* Check: does pmFPAfileChecks do piece-by-piece read?
-* Can we reset a file, so that we can read it twice through?
-* pmStack functions probably need work so that they operate on a piece
-  of the image at a time.  Will there be issues with the masks in the
-  second pass?
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/configure.ac	(revision 24244)
@@ -25,4 +25,6 @@
 CFLAGS="${CFLAGS=} -Wall -Werror"
 
+IPP_VERSION
+
 AC_SUBST([PPSTACK_CFLAGS])
 AC_SUBST([PPSTACK_LIBS])
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/Makefile.am	(revision 24244)
@@ -1,14 +1,26 @@
 bin_PROGRAMS = ppStack
 
-# PPSTACK_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-# PPSTACK_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-# PPSTACK_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
-# 
-# # Force recompilation of ppStackVersion.c, since it gets the version information
-# ppStackVersion.c: FORCE
-# 	touch ppStackVersion.c
-# FORCE: ;
+if HAVE_SVNVERSION
+PPSTACK_VERSION=`$(SVNVERSION) ..`
+else
+PPSTACK_VERSION="UNKNOWN"
+endif
 
-ppStack_CFLAGS 	= $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PPSTATS_CFLAGS) $(PPSTACK_CFLAGS) -DPPSTACK_VERSION=$(SVN_VERSION) -DPPSTACK_BRANCH=$(SVN_BRANCH) -DPPSTACK_SOURCE=$(SVN_SOURCE)
+if HAVE_SVN
+PPSTACK_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPSTACK_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PPSTACK_BRANCH="UNKNOWN"
+PPSTACK_SOURCE="UNKNOWN"
+endif
+
+# Force recompilation of ppStackVersion.c, since it gets the version information
+ppStackVersion.c: ppStackVersionDefinitions.h
+ppStackVersionDefinitions.h: ppStackVersionDefinitions.h.in FORCE
+	-$(RM) ppStackVersionDefinitions.h
+	$(SED) -e "s|@PPSTACK_VERSION@|\"$(PPSTACK_VERSION)\"|" -e "s|@PPSTACK_BRANCH@|\"$(PPSTACK_BRANCH)\"|" -e "s|@PPSTACK_SOURCE@|\"$(PPSTACK_SOURCE)\"|" ppStackVersionDefinitions.h.in > ppStackVersionDefinitions.h
+FORCE: ;
+
+ppStack_CFLAGS 	= $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PPSTATS_CFLAGS) $(PPSTACK_CFLAGS)
 ppStack_LDFLAGS = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PSPHOT_LIBS)   $(PPSTATS_LIBS)   $(PPSTACK_LIBS)
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackArguments.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackArguments.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackArguments.c	(revision 24244)
@@ -119,4 +119,12 @@
 {
     assert(config);
+
+    {
+        int argNum = psArgumentGet(argc, argv, "-debug"); // Debugging argument number
+        if (argNum) {
+            psArgumentRemove(argNum, &argc, argv);
+            pmSubtractionRegions(true);
+        }
+    }
 
     // This capability makes things much faster when debugging
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackCombineFinal.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackCombineFinal.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackCombineFinal.c	(revision 24244)
@@ -47,4 +47,24 @@
     }
 
+    // Sum covariance matrices
+    double sumWeights = 0.0;            // Sum of weights
+    for (int i = 0; i < options->num; i++) {
+        if (options->inputMask->data.U8[i]) {
+            psFree(options->covariances->data[i]);
+            options->covariances->data[i] = NULL;
+            continue;
+        }
+        psKernel *covar = options->covariances->data[i]; // Covariance matrix
+        float weight = options->weightings->data.F32[i]; // Weight to apply
+        psBinaryOp(covar->image, covar->image, "*", psScalarAlloc(weight, PS_TYPE_F32));
+        sumWeights += weight;
+    }
+    pmReadout *outRO = options->outRO;  // Output readout
+    outRO->covariance = psImageCovarianceSum(options->covariances);
+    psBinaryOp(outRO->covariance->image, outRO->covariance->image, "/",
+               psScalarAlloc(sumWeights, PS_TYPE_F32));
+    psFree(options->covariances); options->covariances = NULL;
+    psImageCovarianceTransfer(outRO->variance, outRO->covariance);
+
 #ifdef TESTING
     pmStackVisualPlotTestImage(outRO->image, "combined_initial.fits");
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackCombineInitial.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackCombineInitial.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackCombineInitial.c	(revision 24244)
@@ -87,13 +87,4 @@
 #endif
 
-    // Sum covariance matrices
-    for (int i = 0; i < options->num; i++) {
-        if (options->inputMask->data.U8[i]) {
-            psFree(options->covariances->data[i]);
-            options->covariances->data[i] = NULL;
-        }
-    }
-    options->outRO->covariance = psImageCovarianceSum(options->covariances);
-    psFree(options->covariances); options->covariances = NULL;
     psFree(options->matchChi2); options->matchChi2 = NULL;
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackConvolve.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackConvolve.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackConvolve.c	(revision 24244)
@@ -112,24 +112,22 @@
 
         // Write the temporary convolved files
-        if (options->convolve) {
-            pmHDU *hdu = readout->parent->parent->parent->hdu; // HDU for convolved image
-            assert(hdu);
-            ppStackWriteImage(options->imageNames->data[i], hdu->header, readout->image, config);
-            psMetadata *maskHeader = psMetadataCopy(NULL, hdu->header); // Copy of header, for mask
-            pmConfigMaskWriteHeader(config, maskHeader);
-            ppStackWriteImage(options->maskNames->data[i], maskHeader, readout->mask, config);
-            psFree(maskHeader);
-            psImageCovarianceTransfer(readout->variance, readout->covariance);
-            ppStackWriteImage(options->varianceNames->data[i], hdu->header, readout->variance, config);
+        pmHDU *hdu = readout->parent->parent->parent->hdu; // HDU for convolved image
+        assert(hdu);
+        ppStackWriteImage(options->imageNames->data[i], hdu->header, readout->image, config);
+        psMetadata *maskHeader = psMetadataCopy(NULL, hdu->header); // Copy of header, for mask
+        pmConfigMaskWriteHeader(config, maskHeader);
+        ppStackWriteImage(options->maskNames->data[i], maskHeader, readout->mask, config);
+        psFree(maskHeader);
+        psImageCovarianceTransfer(readout->variance, readout->covariance);
+        ppStackWriteImage(options->varianceNames->data[i], hdu->header, readout->variance, config);
 #ifdef TESTING
-            {
-                psString name = NULL;
-                psStringAppend(&name, "covariance_%d.fits", i);
-                ppStackWriteImage(name, hdu->header, readout->covariance->image, config);
-                pmStackVisualPlotTestImage(readout->covariance->image, name);
-                psFree(name);
-            }
+        {
+            psString name = NULL;
+            psStringAppend(&name, "covariance_%d.fits", i);
+            ppStackWriteImage(name, hdu->header, readout->covariance->image, config);
+            pmStackVisualPlotTestImage(readout->covariance->image, name);
+            psFree(name);
+        }
 #endif
-        }
 
         pmCell *inCell = readout->parent; // Input cell
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackFinish.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackFinish.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackFinish.c	(revision 24244)
@@ -58,4 +58,5 @@
         psFree((void *)statsMDC);
         fclose(options->statsFile); options->statsFile = NULL;
+        pmConfigRunFilenameAddWrite(config, "STATS", psMetadataLookupStr(NULL, config->arguments, "STATS"));
     }
 
@@ -65,6 +66,5 @@
     psString dump = psMetadataLookupStr(&mdok, config->arguments, "DUMP_CONFIG"); // File for config
     if (dump) {
-        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSTACK.INPUT"); // Input file
-        pmConfigDump(config, input->fpa, dump);
+        pmConfigDump(config, dump);
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackMatch.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackMatch.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackMatch.c	(revision 24244)
@@ -209,5 +209,5 @@
             pmFPAview *view = pmFPAviewAlloc(0); // View to readout of interest
             view->chip = view->cell = view->readout = 0;
-            psString filename = pmFPAfileNameFromRule(filerule->rule, file, view); // Filename of interest
+            psString filename = pmFPAfileNameFromRule(file->filerule, file, view); // Filename of interest
 
             // Read convolution kernel
@@ -241,12 +241,6 @@
                 !readImage(&readout->variance, options->varianceNames->data[index], config)) {
                 psError(PS_ERR_IO, false, "Unable to read previously produced image.");
-                psFree(imageName);
-                psFree(maskName);
-                psFree(varianceName);
                 return false;
             }
-            psFree(imageName);
-            psFree(maskName);
-            psFree(varianceName);
 
             psRegion *region = psMetadataLookupPtr(NULL, conv->analysis,
@@ -317,4 +311,6 @@
             }
 
+            fake->mask = psImageCopy(NULL, readout->mask, PS_TYPE_IMAGE_MASK);
+
             // Add the background into the target image
             psImage *bgImage = stackBackgroundModel(readout, config); // Image of background
@@ -361,4 +357,7 @@
                     psFree(stampSources);
                     psFree(conv);
+                    if (threads > 0) {
+                        pmSubtractionThreadsFinalize(readout, fake);
+                    }
                     return false;
                 }
@@ -375,4 +374,7 @@
                     psFree(stampSources);
                     psFree(conv);
+                    if (threads > 0) {
+                        pmSubtractionThreadsFinalize(readout, fake);
+                    }
                     return false;
                 }
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackPhotometry.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackPhotometry.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackPhotometry.c	(revision 24244)
@@ -69,8 +69,14 @@
 
     if (!psphotReadoutKnownSources(config, photView, inSources)) {
-        // Clear the error, so that the output files are written.
-        psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry on stacked image.");
-        psFree(photView);
-        return false;
+        // This is likely a data quality issue
+        // XXX Split into multiple cases using error codes?
+        psErrorStackPrint(stderr, "Unable to perform photometry on image");
+        psWarning("Unable to perform photometry on image --- suspect bad data quality.");
+        if (options->stats && psMetadataLookupS32(NULL, options->stats, "QUALITY") == 0) {
+            psMetadataAddS32(options->stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE,
+                             "Unable to perform photometry on image", psErrorCodeLast());
+        }
+        psErrorClear();
+        psphotFilesActivate(config, false);
     }
 
@@ -88,5 +94,5 @@
         psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources
         psMetadataAddS32(options->stats, PS_LIST_TAIL, "NUM_SOURCES", 0,
-                         "Number of sources detected", sources->n);
+                         "Number of sources detected", sources ? sources->n : 0);
         psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_PHOT", 0,
                          "Time to do photometry", psTimerMark("PPSTACK_PHOT"));
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackSetup.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackSetup.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackSetup.c	(revision 24244)
@@ -13,29 +13,4 @@
 #define BUFFER 16                       // Buffer for name array
 
-// Generate an array of input filenames
-static psArray *stackNameArray(const ppStackOptions *options, // Stack options
-                               pmConfig *config, // Configuration
-                               const char *name // Name of file
-    )
-{
-    psAssert(config, "Require configuration");
-    psAssert(config, "Require file name");
-
-    psArray *array = psArrayAllocEmpty(options->num); // Array with filenames
-    pmFPAview *view = pmFPAviewAlloc(0);// View to readout
-    view->chip = view->cell = view->readout = -1;
-
-    for (int i = 0; i < options->num; i++) {
-        pmFPAfile *file = pmFPAfileSelectSingle(config->files, name, i); // File of interest
-
-        psString filename = pmFPAfileName(file, view, config); // Filename of interest
-        psAssert(filename, "Can't determine filename");
-        psArrayAdd(array, array->n, filename);
-        psFree(filename);               // Drop reference
-    }
-
-    return array;
-}
-
 
 bool ppStackSetup(ppStackOptions *options, pmConfig *config)
@@ -47,5 +22,5 @@
     psAssert(recipe, "We've thrown an error on this before.");
 
-    options->convolve = psMetadataLookupBool(NULL, config->arguments, "CONVOLVE"); // Convolve images?
+    options->convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images?
     if (!psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) {
         psWarning("No PSFs provided --- unable to convolve to common PSF.");
@@ -68,52 +43,47 @@
         psFree(resolved);
         options->stats = psMetadataAlloc();
+        psMetadataAddS32(options->stats, PS_LIST_TAIL, "QUALITY", 0, "No problems", 0);
     }
 
     // Generate temporary names for convolved images
-    if (options->convolve) {
-        const char *tempDir = psMetadataLookupStr(NULL, recipe, "TEMP.DIR"); // Directory for temporary images
-        if (!tempDir) {
-            psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find TEMP.DIR in recipe");
-            return false;
-        }
+    const char *tempDir = psMetadataLookupStr(NULL, recipe, "TEMP.DIR"); // Directory for temporary images
+    if (!tempDir) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find TEMP.DIR in recipe");
+        return false;
+    }
 
-        psString outputName = psStringCopy(psMetadataLookupStr(NULL, config->arguments,
-                                                               "OUTPUT")); // Name for temporary files
-        const char *tempName = basename(outputName);
-        if (!tempName) {
-            psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to construct basename for temporary files.");
-            psFree(outputName);
-            return false;
-        }
+    psString outputName = psStringCopy(psMetadataLookupStr(NULL, config->arguments,
+                                                           "OUTPUT")); // Name for temporary files
+    const char *tempName = basename(outputName);
+    if (!tempName) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to construct basename for temporary files.");
+        psFree(outputName);
+        return false;
+    }
 
-        const char *tempImage = psMetadataLookupStr(NULL, recipe, "TEMP.IMAGE"); // Suffix for images
-        const char *tempMask = psMetadataLookupStr(NULL, recipe, "TEMP.MASK"); // Suffix for masks
-        const char *tempVariance = psMetadataLookupStr(NULL, recipe, "TEMP.VARIANCE"); // Suffix for var maps
-        if (!tempImage || !tempMask || !tempVariance) {
-            psError(PS_ERR_BAD_PARAMETER_VALUE, false,
-                    "Unable to find TEMP.IMAGE, TEMP.MASK and TEMP.VARIANCE in recipe");
-            psFree(outputName);
-            return false;
-        }
+    const char *tempImage = psMetadataLookupStr(NULL, recipe, "TEMP.IMAGE"); // Suffix for images
+    const char *tempMask = psMetadataLookupStr(NULL, recipe, "TEMP.MASK"); // Suffix for masks
+    const char *tempVariance = psMetadataLookupStr(NULL, recipe, "TEMP.VARIANCE"); // Suffix for var maps
+    if (!tempImage || !tempMask || !tempVariance) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, false,
+                "Unable to find TEMP.IMAGE, TEMP.MASK and TEMP.VARIANCE in recipe");
+        psFree(outputName);
+        return false;
+    }
 
-        options->imageNames = psArrayAlloc(num);
-        options->maskNames = psArrayAlloc(num);
-        options->varianceNames = psArrayAlloc(num);
-        for (int i = 0; i < num; i++) {
-            psString imageName = NULL, maskName = NULL, varianceName = NULL; // Names for convolved images
-            psStringAppend(&imageName, "%s/%s.%d.%s", tempDir, tempName, i, tempImage);
-            psStringAppend(&maskName, "%s/%s.%d.%s", tempDir, tempName, i, tempMask);
-            psStringAppend(&varianceName, "%s/%s.%d.%s", tempDir, tempName, i, tempVariance);
-            psTrace("ppStack", 5, "Temporary files: %s %s %s\n", imageName, maskName, varianceName);
-            options->imageNames->data[i] = imageName;
-            options->maskNames->data[i] = maskName;
-            options->varianceNames->data[i] = varianceName;
-        }
-        psFree(outputName);
-    } else {
-        options->imageNames = stackNameArray(options, config, "PPSTACK.INPUT");
-        options->maskNames = stackNameArray(options, config, "PPSTACK.INPUT.MASK");
-        options->varianceNames = stackNameArray(options, config, "PPSTACK.INPUT.VARIANCE");
+    options->imageNames = psArrayAlloc(num);
+    options->maskNames = psArrayAlloc(num);
+    options->varianceNames = psArrayAlloc(num);
+    for (int i = 0; i < num; i++) {
+        psString imageName = NULL, maskName = NULL, varianceName = NULL; // Names for convolved images
+        psStringAppend(&imageName, "%s/%s.%d.%s", tempDir, tempName, i, tempImage);
+        psStringAppend(&maskName, "%s/%s.%d.%s", tempDir, tempName, i, tempMask);
+        psStringAppend(&varianceName, "%s/%s.%d.%s", tempDir, tempName, i, tempVariance);
+        psTrace("ppStack", 5, "Temporary files: %s %s %s\n", imageName, maskName, varianceName);
+        options->imageNames->data[i] = imageName;
+        options->maskNames->data[i] = maskName;
+        options->varianceNames->data[i] = varianceName;
     }
+    psFree(outputName);
 
     if (!pmConfigMaskSetBits(NULL, NULL, config)) {
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackVersion.c	(revision 24244)
@@ -10,4 +10,5 @@
 
 #include "ppStack.h"
+#include "ppStackVersionDefinitions.h"
 
 #ifndef PPSTACK_VERSION
@@ -21,11 +22,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString ppStackVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PPSTACK_BRANCH), xstr(PPSTACK_VERSION));
+    psStringAppend(&value, "%s@%s", PPSTACK_BRANCH, PPSTACK_VERSION);
     return value;
 }
@@ -33,5 +31,5 @@
 psString ppStackSource(void)
 {
-    return psStringCopy(xstr(PPSTACK_SOURCE));
+    return psStringCopy(PPSTACK_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PPSTACK_VERSION_DEFINITIONS_H
+#define PPSTACK_VERSION_DEFINITIONS_H
+
+#define PPSTACK_VERSION @PPSTACK_VERSION@ // SVN version
+#define PPSTACK_BRANCH  @PPSTACK_BRANCH@  // SVN branch
+#define PPSTACK_SOURCE  @PPSTACK_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppStats/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStats/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStats/configure.ac	(revision 24244)
@@ -23,4 +23,6 @@
 CFLAGS="${CFLAGS=} -Wall -Werror -std=c99"
 
+IPP_VERSION
+
 AC_SUBST([PPSTATS_CFLAGS])
 AC_SUBST([PPSTATS_LIBS])
Index: /branches/cnb_branches/cnb_branch_20090301/ppStats/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStats/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStats/src/Makefile.am	(revision 24244)
@@ -1,14 +1,26 @@
 lib_LTLIBRARIES = libppStats.la
 
-# PPSTATS_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-# PPSTATS_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-# PPSTATS_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
-# 
-# # Force recompilation of ppStatsVersion.c, since it gets the version information
-# ppStatsVersion.c: FORCE
-# 	touch ppStatsVersion.c
-# FORCE: ;
+if HAVE_SVNVERSION
+PPSTATS_VERSION=`$(SVNVERSION) ..`
+else
+PPSTATS_VERSION="UNKNOWN"
+endif
 
-libppStats_la_CFLAGS = $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPPSTATS_VERSION=$(SVN_VERSION) -DPPSTATS_BRANCH=$(SVN_BRANCH) -DPPSTATS_SOURCE=$(SVN_SOURCE)
+if HAVE_SVN
+PPSTATS_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPSTATS_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PPSTATS_BRANCH="UNKNOWN"
+PPSTATS_SOURCE="UNKNOWN"
+endif
+
+# Force recompilation of ppStatsVersion.c, since it gets the version information
+ppStatsVersion.c: ppStatsVersionDefinitions.h
+ppStatsVersionDefinitions.h: ppStatsVersionDefinitions.h.in FORCE
+	-$(RM) ppStatsVersionDefinitions.h
+	$(SED) -e "s|@PPSTATS_VERSION@|\"$(PPSTATS_VERSION)\"|" -e "s|@PPSTATS_BRANCH@|\"$(PPSTATS_BRANCH)\"|" -e "s|@PPSTATS_SOURCE@|\"$(PPSTATS_SOURCE)\"|" ppStatsVersionDefinitions.h.in > ppStatsVersionDefinitions.h
+FORCE: ;
+
+libppStats_la_CFLAGS = $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
 libppStats_la_LDFLAGS = $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsErrorCodes.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsErrorCodes.dat	(revision 24244)
@@ -2,5 +2,5 @@
 # This file is used to generate ppStatsErrorClasses.h
 #
-BASE = 400		First value we use; lower values belong to psLib
+BASE = 2000		First value we use; lower values belong to psLib
 # these errors correspond to standard exit conditions
 ARGUMENTS               Incorrect arguments
Index: /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsErrorCodes.h.in	(revision 24244)
@@ -9,5 +9,5 @@
  */
 typedef enum {
-    PPSTATS_ERR_BASE = 512,
+    PPSTATS_ERR_BASE = 2000,
     PPSTATS_ERR_${ErrorCode},
     PPSTATS_ERR_NERROR
Index: /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsFromMetadataStats.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsFromMetadataStats.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsFromMetadataStats.c	(revision 24244)
@@ -38,5 +38,8 @@
 
         psStats *stats = psStatsAlloc(option);
-        psVectorStats(stats, entry->vector, NULL, NULL, 0);
+        if (!psVectorStats(stats, entry->vector, NULL, NULL, 0)) {
+	    psError(PS_ERR_UNKNOWN, false, "failure to measure stats for %s", entry->statistic);
+	    continue;
+	}
 
         double value;
Index: /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsReadout.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsReadout.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsReadout.c	(revision 24244)
@@ -92,7 +92,7 @@
         }
         if (!psVectorStats(data->stats, sampleValues, NULL, sampleMask, 1)) {
-            psWarning("Unable to perform statistics on readout %s.\n", readoutName);
-            psErrorClear();
-        }
+	    psError(PS_ERR_UNKNOWN, false, "failure to measure stats for readout %s", readoutName);
+	}
+	// XXX raise if we get a NAN? psWarning("Unable to perform statistics on readout %s.\n", readoutName);
         psFree(sampleValues);
         psFree(sampleMask);
Index: /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsVersion.c	(revision 24244)
@@ -1,3 +1,4 @@
 #include "ppStatsInternal.h"
+#include "ppStatsVersionDefinitions.h"
 
 #ifndef PPSTATS_VERSION
@@ -11,11 +12,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString ppStatsVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PPSTATS_BRANCH), xstr(PPSTATS_VERSION));
+    psStringAppend(&value, "%s@%s", PPSTATS_BRANCH, PPSTATS_VERSION);
     return value;
 }
@@ -23,5 +21,5 @@
 psString ppStatsSource(void)
 {
-    return psStringCopy(xstr(PPSTATS_SOURCE));
+    return psStringCopy(PPSTATS_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PPSTATS_VERSION_DEFINITIONS_H
+#define PPSTATS_VERSION_DEFINITIONS_H
+
+#define PPSTATS_VERSION @PPSTATS_VERSION@ // SVN version
+#define PPSTATS_BRANCH  @PPSTATS_BRANCH@  // SVN branch
+#define PPSTATS_SOURCE  @PPSTATS_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/configure.ac	(revision 24244)
@@ -23,7 +23,14 @@
 PKG_CHECK_MODULES([PSPHOT], [psphot >= 0.9.0]) 
 
+AC_PATH_PROG([ERRORCODES], [psParseErrorCodes], [missing])
+if test "$ERRORCODES" = "missing" ; then
+  AC_MSG_ERROR([psParseErrorCodes is required])
+fi
+
 dnl Set CFLAGS for build
 IPP_STDOPTS
 CFLAGS="${CFLAGS} -Wall -Werror"
+
+IPP_VERSION
 
 AC_SUBST([PPSUB_CFLAGS])
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/Makefile.am	(revision 24244)
@@ -1,34 +1,48 @@
 bin_PROGRAMS = ppSub ppSubKernel
 
-# PPSUB_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-# PPSUB_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-# PPSUB_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
-# 
-# # Force recompilation of ppSubVersion.c, since it gets the version information
-# ppSubVersion.c: FORCE
-# 	touch ppSubVersion.c
-# FORCE: ;
+if HAVE_SVNVERSION
+PPSUB_VERSION=`$(SVNVERSION) ..`
+else
+PPSUB_VERSION="UNKNOWN"
+endif
 
-ppSub_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PPSUB_CFLAGS) -DPPSUB_VERSION=$(SVN_VERSION) -DPPSUB_BRANCH=$(SVN_BRANCH) -DPPSUB_SOURCE=$(SVN_SOURCE)
+if HAVE_SVN
+PPSUB_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PPSUB_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PPSUB_BRANCH="UNKNOWN"
+PPSUB_SOURCE="UNKNOWN"
+endif
+
+# Force recompilation of ppSubVersion.c, since it gets the version information
+ppSubVersion.c: ppSubVersionDefinitions.h
+ppSubVersionDefinitions.h: ppSubVersionDefinitions.h.in FORCE
+	-$(RM) ppSubVersionDefinitions.h
+	$(SED) -e "s|@PPSUB_VERSION@|\"$(PPSUB_VERSION)\"|" -e "s|@PPSUB_BRANCH@|\"$(PPSUB_BRANCH)\"|" -e "s|@PPSUB_SOURCE@|\"$(PPSUB_SOURCE)\"|" ppSubVersionDefinitions.h.in > ppSubVersionDefinitions.h
+FORCE: ;
+
+ppSub_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PPSUB_CFLAGS)
 ppSub_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PPSTATS_LIBS)   $(PSPHOT_LIBS)   $(PPSUB_LIBS)
 
-ppSub_SOURCES =			 \
-	ppSub.c			 \
-	ppSubArguments.c	 \
-	ppSubVersion.c	         \
-	ppSubBackground.c	 \
-	ppSubCamera.c		 \
-	ppSubLoop.c		 \
-	ppSubReadout.c		 \
-	ppSubDefineOutput.c      \
-	ppSubExtras.c            \
-	ppSubMakePSF.c           \
-	ppSubMatchPSFs.c         \
-	ppSubReadoutPhotometry.c \
-	ppSubReadoutSubtract.c   \
-	ppSubReadoutUpdate.c     \
-	ppSubSetMasks.c          \
-	ppSubReadoutRenorm.c     \
-	ppSubVarianceFactors.c
+ppSub_SOURCES =				\
+	ppSub.c				\
+	ppSubArguments.c		\
+	ppSubVersion.c			\
+	ppSubBackground.c		\
+	ppSubCamera.c			\
+	ppSubData.c			\
+	ppSubErrorCodes.c		\
+	ppSubFiles.c			\
+	ppSubLoop.c			\
+	ppSubDefineOutput.c		\
+	ppSubExtras.c			\
+	ppSubMakePSF.c			\
+	ppSubMatchPSFs.c		\
+	ppSubReadoutInverse.c		\
+	ppSubReadoutJpeg.c		\
+	ppSubReadoutPhotometry.c	\
+	ppSubReadoutStats.c		\
+	ppSubReadoutSubtract.c		\
+	ppSubSetMasks.c
 
 ppSubKernel_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSUB_CFLAGS)
@@ -41,4 +55,16 @@
 	ppSub.h
 
+
+### Error codes.
+BUILT_SOURCES = ppSubErrorCodes.h ppSubErrorCodes.c
+CLEANFILES = ppSubErrorCodes.h ppSubErrorCodes.c
+
+ppSubErrorCodes.h : ppSubErrorCodes.dat ppSubErrorCodes.h.in
+	$(ERRORCODES) --data=ppSubErrorCodes.dat --outdir=. ppSubErrorCodes.h
+
+ppSubErrorCodes.c : ppSubErrorCodes.dat ppSubErrorCodes.c.in ppSubErrorCodes.h
+	$(ERRORCODES) --data=ppSubErrorCodes.dat --outdir=. ppSubErrorCodes.c
+
+
 clean-local:
 	-rm -f TAGS
@@ -47,3 +73,2 @@
 tags:
 	etags `find . -name \*.[ch] -print`
-
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSub.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSub.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSub.c	(revision 24244)
@@ -28,4 +28,5 @@
     psLibInit(NULL);
 
+    ppSubData *data = NULL;             // Processing data
     pmConfig *config = pmConfigRead(&argc, argv, PPSUB_RECIPE); // Configuration
     if (!config) {
@@ -40,4 +41,5 @@
         psErrorStackPrint(stderr, "Error initialising model classes.\n");
         exitValue = PS_EXIT_PROG_ERROR;
+        psFree(config);
         goto die;
     }
@@ -46,8 +48,11 @@
         psErrorStackPrint(stderr, "Error initialising psphot.\n");
         exitValue = PS_EXIT_PROG_ERROR;
+        psFree(config);
         goto die;
     }
 
-    if (!ppSubArgumentsSetup(argc, argv, config)) {
+    data = ppSubDataAlloc(config);
+
+    if (!ppSubArguments(argc, argv, data)) {
         psErrorStackPrint(stderr, "Error reading arguments.\n");
         exitValue = PS_EXIT_CONFIG_ERROR;
@@ -55,5 +60,5 @@
     }
 
-    if (!ppSubCamera(config)) {
+    if (!ppSubCamera(data)) {
         psErrorStackPrint(stderr, "Error setting up camera.\n");
         exitValue = PS_EXIT_CONFIG_ERROR;
@@ -61,11 +66,5 @@
     }
 
-    if (!ppSubArgumentsParse(config)) {
-        psErrorStackPrint(stderr, "Error reading arguments.\n");
-        exitValue = PS_EXIT_CONFIG_ERROR;
-        goto die;
-    }
-
-    if (!ppSubLoop(config)) {
+    if (!ppSubLoop(data)) {
         psErrorStackPrint(stderr, "Error performing subtraction.\n");
         exitValue = PS_EXIT_SYS_ERROR;
@@ -77,6 +76,7 @@
     psTimerStop();
 
+    psFree(data);
+
     pmVisualClose(); //close plot windows, if -visual is set
-    psFree(config);
     pmModelClassCleanup();
     pmConfigDone();
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSub.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSub.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSub.h	(revision 24244)
@@ -14,6 +14,9 @@
 #define PP_SUB_H
 
+#include <stdio.h>
 #include <pslib.h>
 #include <psmodules.h>
+
+#include "ppSubErrorCodes.h"
 
 /// @addtogroup ppSub
@@ -21,70 +24,79 @@
 
 #define PPSUB_RECIPE "PPSUB"            /// Name of the recipe to use
+#define WCS_TOLERANCE 0.001             // Tolerance for WCS
+
+// Output files, for activation/deactivation
+typedef enum {
+    PPSUB_FILES_INPUT    = 0x01,        // Input files
+    PPSUB_FILES_CONV     = 0x02,        // Convolved files (output)
+    PPSUB_FILES_SUB      = 0x04,        // Subtracted files (output)
+    PPSUB_FILES_INV      = 0x08,        // Inverse subtracted files (output)
+    PPSUB_FILES_PSF      = 0x10,        // PSF files (output)
+    PPSUB_FILES_PHOT_SUB = 0x20,        // Subtraction photometry files (output)
+    PPSUB_FILES_PHOT_INV = 0x40,        // Inverse subtraction photometry files (output)
+    PPSUB_FILES_PHOT     = 0x80,        // General photometry files (internal)
+    PPSUB_FILES_ALL      = 0xFF,        // All files
+} ppSubFiles;
+
+/// Data for processing
+typedef struct {
+    pmConfig *config;                   // Configuration
+    psErrorCode quality;                // Quality code; 0 for no problem
+    bool photometry;                    // Perform photometry?
+    bool inverse;                       // Output inverse subtraction as well?
+    psString stamps;                    // Stamps file
+    pmPSF *psf;                         // Point Spread Function
+    psString statsName;                 // Name of statistics file
+    FILE *statsFile;                    // Statistics file
+    psMetadata *stats;                  // Statistics
+} ppSubData;
+
+/// Constructor
+ppSubData *ppSubDataAlloc(pmConfig *config ///< Configuration
+    );
 
 /// Setup the arguments parsing
-bool ppSubArgumentsSetup(int argc, char *argv[], ///< Command-line arguments
-                         pmConfig *config    ///< Configuration
-    );
-
-/// Parse the arguments
-bool ppSubArgumentsParse(pmConfig *config ///< Configuration
+bool ppSubArguments(int argc, char *argv[], ///< Command-line arguments
+                    ppSubData *data ///< Processing data
     );
 
 /// Parse the camera input
-bool ppSubCamera(pmConfig *config       ///< Configuration
+bool ppSubCamera(ppSubData *data        ///< Processing data
     );
 
 /// Loop over the FPA hierarchy
-bool ppSubLoop(pmConfig *config         ///< Configuration
-    );
-
-/// Perform PSF-matched image subtraction on the readout
-bool ppSubReadout(pmConfig *config,     ///< Configuration
-                  psMetadata *stats,    ///< Statistics, for output
-                  const pmFPAview *view ///< View of readout to subtract
+bool ppSubLoop(ppSubData *data          ///< Processing data
     );
 
 /// Generate (if needed) and set or update the masks for input and reference images
-bool ppSubSetMasks(pmConfig *config,     ///< Configuration
-                   const pmFPAview *view ///< View of active readout
+bool ppSubSetMasks(pmConfig *config     ///< Configuration
     );
 
 /// Generate the PSF-matching kernel and convolve the images as needed.  Most of this function involves
 /// looking up the parameters in the recipe and supplying them to the function pmSubtractionMatch()
-bool ppSubMatchPSFs(pmConfig *config,    ///< Configuration
-                    const pmFPAview *view ///< View of active readout
+bool ppSubMatchPSFs(ppSubData *data     ///< Processing data
     );
 
 /// Generate the output readout and pass the kernel info to the header
-bool ppSubDefineOutput(pmConfig *config, ///< Configuration
-                       const pmFPAview *view ///< View of active readout
+bool ppSubDefineOutput(const char *name,///< Name of output to define
+                       pmConfig *config ///< Configuration
     );
 
 /// Photometry stage 1: measure the PSF from the minuend image
-bool ppSubMakePSF(pmConfig *config,       ///< Configuration
-                  const pmFPAview *view ///< View of active readout
+bool ppSubMakePSF(ppSubData *data       ///< Processing data
     );
 
 /// Perform the actual image subtraction, update output concepts
-bool ppSubReadoutSubtract(pmConfig *config,       ///< Configuration
-                          const pmFPAview *view ///< View of active readout
+bool ppSubReadoutSubtract(pmConfig *config ///< Configuration
     );
 
 
 /// Photometry stage 2: find and measure sources on the subtracted image
-bool ppSubReadoutPhotometry(pmConfig *config,     ///< Configuration
-                            psMetadata *stats,    ///< Statistics, for output
-                            const pmFPAview *view ///< View of active readout
-    );
-
-/// Renormalize, update headers and generate JPEGs
-bool ppSubReadoutUpdate(pmConfig *config, ///< Configuration
-                        psMetadata *stats, ///< Statistics for output, or NULL
-                        const pmFPAview *view ///< View of active readout
+bool ppSubReadoutPhotometry(const char *name, ///< Name of file to photometer
+                            ppSubData *data ///< Processing data
     );
 
 /// Higher-order background subtraction
-bool ppSubBackground(pmConfig *config,  ///< Configuration
-                     const pmFPAview *view ///< View to readout
+bool ppSubBackground(pmConfig *config   ///< Configuration
     );
 
@@ -96,4 +108,44 @@
 void ppSubVersionPrint(void);
 
+/// Mark the data quality as bad and prepare to suspend processing
+void ppSubDataQuality(ppSubData *data,  ///< Processing data
+                      psErrorCode error,///< Error code
+                      ppSubFiles files  ///< Files to deactivate
+    );
+
+
+/// Activate or deactivate files
+void ppSubFilesActivate(pmConfig *config, // Configuration
+                        ppSubFiles files, // File to activate/deactivate
+                        bool state      // Activation state
+    );
+
+/// Generate a view suitable for a readout
+///
+/// Assumes we're working with skycells
+pmFPAview *ppSubViewReadout(void);
+
+/// Iterate down the FPA hierarchy, opening files
+bool ppSubFilesIterateDown(pmConfig *config, // Configuration
+                           ppSubFiles files // Files to open
+    );
+
+/// Iterate up the FPA hierarchy, closing files
+bool ppSubFilesIterateUp(pmConfig *config, // Configuration
+                         ppSubFiles files // Files to open
+    );
+
+/// Collect statistics
+bool ppSubReadoutStats(ppSubData *data  // Processing data
+    );
+
+/// Generate JPEG images
+bool ppSubReadoutJpeg(pmConfig *config  // Configuration
+    );
+
+/// Generate inverse subtraction
+bool ppSubReadoutInverse(pmConfig *config // Configuration
+    );
+
 
 // Copy every instance of a single keyword from one metadata to another
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubArguments.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubArguments.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubArguments.c	(revision 24244)
@@ -41,136 +41,4 @@
 }
 
-// Get a float-point value from the command-line or recipe, and add it to the arguments
-#define VALUE_ARG_RECIPE_FLOAT(ARGNAME, RECIPENAME, TYPE) { \
-    ps##TYPE value = psMetadataLookup##TYPE(NULL, config->arguments, ARGNAME); \
-    if (isnan(value)) { \
-        bool mdok; \
-        value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \
-        if (!mdok) { \
-            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", \
-                RECIPENAME, PPSUB_RECIPE); \
-            goto ERROR; \
-        } \
-    } \
-    psMetadataAdd##TYPE(recipe, PS_LIST_TAIL, RECIPENAME, PS_META_REPLACE, NULL, value); \
-}
-
-// Get an integer value from the command-line or recipe, and add it to the arguments
-#define VALUE_ARG_RECIPE_INT(ARGNAME, RECIPENAME, TYPE, UNSET) { \
-    ps##TYPE value = psMetadataLookup##TYPE(NULL, config->arguments, ARGNAME); \
-    if (value == UNSET) { \
-        bool mdok; \
-        value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \
-        if (!mdok) { \
-            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", \
-                RECIPENAME, PPSUB_RECIPE); \
-            goto ERROR; \
-        } \
-    } \
-    psMetadataAdd##TYPE(recipe, PS_LIST_TAIL, RECIPENAME, PS_META_REPLACE, NULL, value); \
-}
-
-/**
- * Get a string value from the command-line and add it to the target
- */
-static bool valueArgStr(psMetadata *arguments, // Command-line arguments
-                        const char *argName, // Argument name in the command-line arguments
-                        const char *mdName, // Name for value in the metadata
-                        psMetadata *target // Target metadata to which to add value
-                        )
-{
-    psString value = psMetadataLookupStr(NULL, arguments, argName); // Value of interest
-    if (value && strlen(value) > 0) {
-        return psMetadataAddStr(target, PS_LIST_TAIL, mdName, PS_META_REPLACE, NULL, value);
-    }
-    return false;
-}
-
-/**
- * Get a string value from the command-line or recipe and add it to the target
- */
-static bool valueArgRecipeStr(psMetadata *arguments, // Command-line arguments
-                              psMetadata *recipe, // Recipe
-                              const char *argName, // Argument name in the command-line arguments
-                              const char *mdName, // Name for value in the metadata and recipe
-                              psMetadata *target // Target metadata to which to add value
-                              )
-{
-    bool mdok;                          // Status of MD lookup
-    psString value = psMetadataLookupStr(&mdok, arguments, argName); // Value of interest
-    if (!value) {
-        value = psMetadataLookupStr(&mdok, recipe, mdName);
-        if (!value) {
-            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s",
-                    mdName, PPSUB_RECIPE);
-            return false;
-        }
-    }
-    return psMetadataAddStr(target, PS_LIST_TAIL, mdName, PS_META_REPLACE, NULL, value);
-}
-
-/**
- * Get a vector from the command-line or recipe, and add it to the target
- */
-static bool vectorArgRecipe(psMetadata *arguments, // Command-line arguments
-                            const char *argName, // Argument name in the command-line arguments
-                            const psMetadata *recipe, // Recipe
-                            const char *recipeName, // Name for value in the recipe
-                            psMetadata *target, // Target to which to add value
-                            psElemType type // Type for vector
-    )
-{
-    psVector *vector;                   // Vector
-    psString string = psMetadataLookupStr(NULL, arguments, argName); // String from arguments
-    if (string) {
-        psArray *array = psStringSplitArray(string, ", ", false); // Array of strings
-        vector = psVectorAlloc(array->n, type);
-        for (int i = 0; i < array->n; i++) {
-            const char *subString = array->data[i]; // String with a value
-            char *end;                  // Ptr to end of string parsed
-
-            switch (type) {
-              case PS_TYPE_F32:
-                vector->data.F32[i] = strtof(subString, &end);
-                break;
-              case PS_TYPE_S32: {
-                  long value = strtol(subString, &end, 10);
-                  if (value > PS_MAX_S32 || value < PS_MIN_S32) {
-                      psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                              "%s list includes value beyond S32 representation: %s",
-                              argName, string);
-                      psFree(vector);
-                      return false;
-                  }
-                  vector->data.S32[i] = value;
-                  break;
-              }
-              default:
-                psAbort("Unsupported type: %x\n", type);
-            }
-            if (end == subString) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to decipher %s list: %s",
-                        argName, string);
-                psFree(vector);
-                return false;
-            }
-        }
-        psFree(array);
-    } else {
-        vector = psMetadataLookupPtr(NULL, recipe, recipeName);
-        if (!psMemCheckVector(vector) || vector->type.type != type) {
-            psError(PS_ERR_BAD_PARAMETER_TYPE, false, "%s in recipe %s is not a vector of type F32.",
-                    recipeName, PPSUB_RECIPE);
-            return false;
-        }
-        psMemIncrRefCounter(vector);
-    }
-
-    psMetadataAddVector(target, PS_LIST_TAIL, recipeName, PS_META_REPLACE, NULL, vector);
-    psFree(vector);                     // Drop reference
-
-    return true;
-}
-
 /**
  * Add a single filename to the arguments as an array, so that it can be used with pmFPAfileBindFromArgs, etc
@@ -189,13 +57,14 @@
 }
 
-bool ppSubArgumentsSetup(int argc, char *argv[], pmConfig *config)
+bool ppSubArguments(int argc, char *argv[], ppSubData *data)
 {
-    //    int argnum;                         // Argument Number
+    assert(data);
+    pmConfig *config = data->config;
     assert(config);
 
-    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
-    if (!recipe) {
-        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSUB_RECIPE);
-        return false;
+    int argNum = psArgumentGet(argc, argv, "-debug"); // Debugging argument number
+    if (argNum) {
+        psArgumentRemove(argNum, &argc, argv);
+        pmSubtractionRegions(true);
     }
 
@@ -212,46 +81,9 @@
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-kernel", 0, "Precalculated kernel to apply", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "Statistics file", NULL);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-region", 0, "Size of iso-kernel region", NAN);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-size", 0, "Kernel half-size (pixels)", 0);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-order", 0, "Spatial polynomial order", -1);
-    psMetadataAddStr(arguments, PS_LIST_TAIL, "-type", 0,
-                     "Kernel type (ISIS|POIS|SPAM|FRIES|GUNK|RINGS)", NULL);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-penalty", 0, "Penalty for wideness", NAN);
-    psMetadataAddStr(arguments, PS_LIST_TAIL, "-isis-widths", 0,
-                     "ISIS Gaussian FWHMs (comma-separated)", NULL);
-    psMetadataAddStr(arguments, PS_LIST_TAIL, "-isis-orders", 0,
-                     "ISIS polynomial orders (comma-separated)", NULL);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-rings-order", 0, "RINGS polynomial order", -1);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-inner", 0, "SPAM and FRIES inner radius", -1);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-spam-binning", 0, "SPAM kernel binning", 0);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-spacing", 0, "Typical stamp spacing (pixels)", NAN);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-footprint", 0, "Stamp footprint half-size (pixels)", -1);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-source-radius", 0, "Source matching radius (pixels)", NAN);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-stride", 0, "Size of convolution patches (pixels)", -1);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-threshold", 0, "Minimum threshold for stamps (ADU)", NAN);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-iter", 0, "Number of rejection iterations", -1);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-rej", 0, "Rejection thresold (sigma)", NAN);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-sys", 0, "Relative systematic error in kernel", NAN);
-    psMetadataAddImageMask(arguments,  PS_LIST_TAIL, "-mask-bad", 0, "Mask value for bad pixels", 0);
-    psMetadataAddImageMask(arguments,  PS_LIST_TAIL, "-mask-poor", 0, "Mask value for poor pixels", 0);
-    psMetadataAddF32(arguments,  PS_LIST_TAIL, "-poor-frac", 0, "Fraction of variance for poor pixels", NAN);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-badfrac", 0, "Maximum fraction of bad pixels to accept", 1.0);
-    psMetadataAddBool(arguments,  PS_LIST_TAIL, "-reverse", 0, "Reverse sense of subtraction?", false);
-    psMetadataAddBool(arguments,  PS_LIST_TAIL, "-generate-mask", 0, "Generate mask if not supplied?", false);
-    psMetadataAddStr(arguments,  PS_LIST_TAIL, "-stamps", 0,
-                     "Stamps filename; file has x,y on each line", NULL);
-    psMetadataAddBool(arguments, PS_LIST_TAIL, "-opt", 0,
-                      "Derive optimum parameters for ISIS kernels?", false);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-opt-min", 0, "Minimum value for optimum kernel search", NAN);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-opt-max", 0, "Minimum value for optimum kernel search", NAN);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-opt-step", 0, "Step value for optimum kernel search", NAN);
-    psMetadataAddF32(arguments, PS_LIST_TAIL, "-opt-tol", 0, "Tolerance for optimum kernel search", NAN);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-opt-order", 0, "Maximum order for optimum kernel search", -1);
-    psMetadataAddBool(arguments, PS_LIST_TAIL, "-dual", 0, "Dual convolution", false);
-    psMetadataAddBool(arguments, PS_LIST_TAIL, "-photometry", 0, "Perform photometry?", false);
+    psMetadataAddStr(arguments,  PS_LIST_TAIL, "-stamps", 0, "Stamps filename; x,y on each line", NULL);
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-threads", 0, "Number of threads", 0);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-bin1", 0, "Binning factor for first level", 0);
-    psMetadataAddS32(arguments, PS_LIST_TAIL, "-bin2", 0, "Binning factor for second level", 0);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-dumpconfig", 0, "file to dump configuration to", NULL);
+    psMetadataAddBool(arguments, PS_LIST_TAIL, "-photometry", 0, "Perform photometry?", NULL);
+    psMetadataAddBool(arguments, PS_LIST_TAIL, "-inverse", 0, "Generate inverse subtractions?", NULL);
     psMetadataAddBool(arguments, PS_LIST_TAIL, "-visual", 0, "Show diagnostic plots", NULL);
 
@@ -302,80 +134,16 @@
     }
 
-    if (psMetadataLookupBool(NULL, arguments, "-photometry")) {
-        psMetadataAddBool(recipe, PS_LIST_TAIL, "PHOTOMETRY", PS_META_REPLACE, "Perform photometry?", true);
-    }
+    data->stamps = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-stamps"));
 
-    return true;
-}
-
-bool ppSubArgumentsParse(pmConfig *config)
-{
-    assert(config);
-
-    psMetadata *arguments = config->arguments; // Command-line arguments
-
-    valueArgStr(arguments, "-stats",  "STATS",  arguments);
-    valueArgStr(arguments, "-stamps", "STAMPS", arguments);
-
-    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
-    if (!recipe) {
-        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSUB_RECIPE);
-        goto ERROR;
-    }
-
-    VALUE_ARG_RECIPE_FLOAT("-region",        "REGION.SIZE",     F32);
-    VALUE_ARG_RECIPE_INT("-size",            "KERNEL.SIZE",     S32, 0);
-    VALUE_ARG_RECIPE_INT("-order",           "SPATIAL.ORDER",   S32, -1);
-    VALUE_ARG_RECIPE_FLOAT("-spacing",       "STAMP.SPACING",   F32);
-    VALUE_ARG_RECIPE_INT("-rings-order",     "RINGS.ORDER",     S32, -1);
-    VALUE_ARG_RECIPE_INT("-inner",           "INNER",           S32, -1);
-    VALUE_ARG_RECIPE_INT("-spam-binning",    "SPAM.BINNING",    S32, 0);
-    VALUE_ARG_RECIPE_INT("-footprint",       "STAMP.FOOTPRINT", S32, -1);
-    VALUE_ARG_RECIPE_FLOAT("-source-radius", "SOURCE.RADIUS",   F32);
-    VALUE_ARG_RECIPE_INT("-stride",          "STRIDE",          S32, -1);
-    VALUE_ARG_RECIPE_FLOAT("-threshold",     "STAMP.THRESHOLD", F32);
-    VALUE_ARG_RECIPE_INT("-iter",            "ITER",            S32, -1);
-    VALUE_ARG_RECIPE_FLOAT("-rej",           "REJ",             F32);
-    VALUE_ARG_RECIPE_FLOAT("-sys",           "SYS",             F32);
-    VALUE_ARG_RECIPE_FLOAT("-badfrac",       "BADFRAC",         F32);
-    VALUE_ARG_RECIPE_FLOAT("-penalty",       "PENALTY",         F32);
-    VALUE_ARG_RECIPE_FLOAT("-poor-frac",     "POOR.FRACTION",   F32);
-    VALUE_ARG_RECIPE_INT("-bin1",            "BIN1",            S32, 0);
-    VALUE_ARG_RECIPE_INT("-bin2",            "BIN2",            S32, 0);
-
-    valueArgRecipeStr(arguments, recipe, "-mask-in",   "MASK.IN",  recipe);
-    valueArgRecipeStr(arguments, recipe, "-mask-bad",  "MASK.BAD",  recipe);
-    valueArgRecipeStr(arguments, recipe, "-mask-poor", "MASK.POOR", recipe);
-
-    vectorArgRecipe(arguments, "-isis-widths", recipe, "ISIS.WIDTHS", recipe, PS_TYPE_F32);
-    vectorArgRecipe(arguments, "-isis-orders", recipe, "ISIS.ORDERS", recipe, PS_TYPE_S32);
-
-    psVector *widths = psMetadataLookupPtr(NULL, recipe, "ISIS.WIDTHS"); // ISIS Gaussian widths
-    psVector *orders = psMetadataLookupPtr(NULL, recipe, "ISIS.ORDERS"); // ISIS Polynomial orders
-    if (widths->n != orders->n) {
-        psError(PS_ERR_BAD_PARAMETER_SIZE, true, "Size of vectors for ISIS widths and orders do not match.");
-        goto ERROR;
-    }
-
-    if (psMetadataLookupBool(NULL, arguments, "-opt") || psMetadataLookupBool(NULL, recipe, "OPTIMUM")) {
-        psMetadataAddBool(recipe, PS_LIST_TAIL, "OPTIMUM", PS_META_REPLACE,
-                          "Derive optimum parameters for ISIS kernels?", true);
-        VALUE_ARG_RECIPE_FLOAT("-opt-min", "OPTIMUM.MIN",   F32);
-        VALUE_ARG_RECIPE_FLOAT("-opt-max", "OPTIMUM.MAX",   F32);
-        VALUE_ARG_RECIPE_FLOAT("-opt-step","OPTIMUM.STEP",  F32);
-        VALUE_ARG_RECIPE_FLOAT("-opt-tol", "OPTIMUM.TOL",   F32);
-        VALUE_ARG_RECIPE_INT("-opt-order", "OPTIMUM.ORDER", S32, -1);
-    }
-
-    psMetadataAddBool(arguments, PS_LIST_TAIL, "REVERSE", 0, "Reverse sense of subtraction",
-                      psMetadataLookupBool(NULL, arguments, "-reverse"));
-    psMetadataAddBool(recipe, PS_LIST_TAIL, "MASK.GENERATE", PS_META_REPLACE, "Generate mask if not supplied",
-                      psMetadataLookupBool(NULL, arguments, "-generate-mask"));
-    psMetadataAddBool(recipe, PS_LIST_TAIL, "DUAL", PS_META_REPLACE, "Dual convolution?",
-                      psMetadataLookupBool(NULL, arguments, "-dual"));
-
-    // Need to update this because it could have been overwritten by the camera's own recipe
-    if (psMetadataLookupBool(NULL, arguments, "-photometry")) {
-        psMetadataAddBool(recipe, PS_LIST_TAIL, "PHOTOMETRY", PS_META_REPLACE, "Perform photometry?", true);
+    data->statsName = psMemIncrRefCounter(psMetadataLookupStr(NULL, arguments, "-stats"));
+    if (data->statsName && strlen(data->statsName) > 0) {
+        psString resolved = pmConfigConvertFilename(data->statsName, config, true, true); // Resolved filename
+        data->statsFile = fopen(resolved, "w");
+        if (!data->statsFile) {
+            psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved);
+            psFree(resolved);
+            return false;
+        }
+        psFree(resolved);
     }
 
@@ -384,19 +152,5 @@
     }
 
-    // Translate the kernel type
-    psString type = psMetadataLookupStr(NULL, arguments, "-type"); // Name of kernel type
-    if (!type || strlen(type) == 0) {
-        type = psMetadataLookupStr(NULL, recipe, "KERNEL.TYPE");
-        if (!type || strlen(type) == 0) {
-            psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find KERNEL.TYPE specified.");
-            goto ERROR;
-        }
-    }
-    psMetadataAddStr(recipe, PS_LIST_TAIL, "KERNEL.TYPE", PS_META_REPLACE, "Type of kernel", type);
-
-    psTrace("ppSub", 1, "Done reading command-line arguments\n");
-
-    // XXX move this to ppSubArguments
-    int threads = psMetadataLookupS32(NULL, config->arguments, "-threads"); // Number of threads
+    int threads = psMetadataLookupS32(NULL, arguments, "-threads"); // Number of threads
     if (threads > 0) {
         if (!psThreadPoolInit(threads)) {
@@ -406,7 +160,6 @@
     }
 
+    psTrace("ppSub", 1, "Done reading command-line arguments\n");
+
     return true;
-
-ERROR:
-    return false;
 }
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubBackground.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubBackground.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubBackground.c	(revision 24244)
@@ -22,8 +22,7 @@
 #include "ppSub.h"
 
-bool ppSubBackground(pmConfig *config, const pmFPAview *view)
+bool ppSubBackground(pmConfig *config)
 {
     psAssert(config, "Require configuration");
-    psAssert(view, "Require view");
 
     bool mdok; // Status of metadata lookups
@@ -36,4 +35,5 @@
     psImageMaskType maskBad = pmConfigMaskGet("BLANK", config); // Bits to mask
 
+    pmFPAview *view = ppSubViewReadout(); // View to readout
     pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); // Output image
     pmReadout *modelRO = pmFPAfileThisReadout(config->files, view, "PSPHOT.BACKMDL"); // Background model
@@ -44,4 +44,5 @@
         if (!psphotModelBackground(config, view, "PPSUB.OUTPUT")) {
             psError(PS_ERR_UNKNOWN, false, "Unable to model background");
+            psFree(view);
             return false;
         }
@@ -50,7 +51,10 @@
         if (!modelRO) {
             psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find background model");
+            psFree(view);
             return false;
         }
     }
+    psFree(view);
+
     psImageBinning *binning = psMetadataLookupPtr(&mdok, modelRO->analysis,
                                                   "PSPHOT.BACKGROUND.BINNING"); // Binning for model
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubCamera.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubCamera.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubCamera.c	(revision 24244)
@@ -134,6 +134,8 @@
 
 
-bool ppSubCamera(pmConfig *config)
+bool ppSubCamera(ppSubData *data)
 {
+    psAssert(data, "Require processing data");
+    pmConfig *config = data->config;
     psAssert(config, "Require configuration");
 
@@ -147,5 +149,5 @@
     pmFPAfile *inVar = defineInputFile(config, input, "PPSUB.INPUT.VARIANCE", "INPUT.VARIANCE",
                                        PM_FPA_FILE_VARIANCE);
-    defineInputFile(config, input, "PPSUB.INPUT.SOURCES", "INPUT.SOURCES", PM_FPA_FILE_CMF);
+    defineInputFile(config, NULL, "PPSUB.INPUT.SOURCES", "INPUT.SOURCES", PM_FPA_FILE_CMF);
 
     // Reference image
@@ -158,25 +160,5 @@
     pmFPAfile *refVar = defineInputFile(config, ref, "PPSUB.REF.VARIANCE", "REF.VARIANCE",
                                         PM_FPA_FILE_VARIANCE);
-    defineInputFile(config, ref, "PPSUB.REF.SOURCES", "REF.SOURCES", PM_FPA_FILE_CMF);
-
-
-    // Output image
-    pmFPAfile *output = defineOutputFile(config, input, true, "PPSUB.OUTPUT", PM_FPA_FILE_IMAGE);
-    pmFPAfile *outMask = defineOutputFile(config, output, false, "PPSUB.OUTPUT.MASK", PM_FPA_FILE_MASK);
-    if (!output || !outMask) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
-        return false;
-    }
-    output->save = true;
-    outMask->save = true;
-    pmFPAfile *outVar = NULL;
-    if (inVar && refVar) {
-        outVar = defineOutputFile(config, output, false, "PPSUB.OUTPUT.VARIANCE", PM_FPA_FILE_VARIANCE);
-        if (!outVar) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
-            return false;
-        }
-        outVar->save = true;
-    }
+    defineInputFile(config, NULL, "PPSUB.REF.SOURCES", "REF.SOURCES", PM_FPA_FILE_CMF);
 
 
@@ -216,4 +198,66 @@
 
 
+    // Now that the camera has been determined, we can read the recipe
+    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
+    if (!recipe) {
+        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSUB_RECIPE);
+        return false;
+    }
+    if (psMetadataLookupBool(NULL, config->arguments, "-photometry")) {
+        psMetadataAddBool(recipe, PS_LIST_TAIL, "PHOTOMETRY", PS_META_REPLACE,
+                          "Perform photometry?", true);
+    }
+    if (psMetadataLookupBool(NULL, config->arguments, "-inverse")) {
+        psMetadataAddBool(recipe, PS_LIST_TAIL, "INVERSE", PS_META_REPLACE,
+                          "Generate inverse subtractions?", true);
+    }
+
+    data->inverse = psMetadataLookupBool(NULL, recipe, "INVERSE");
+    data->photometry = psMetadataLookupBool(NULL, recipe, "PHOTOMETRY");
+
+
+    // Output image
+    pmFPAfile *output = defineOutputFile(config, inConvImage, true, "PPSUB.OUTPUT", PM_FPA_FILE_IMAGE);
+    pmFPAfile *outMask = defineOutputFile(config, output, false, "PPSUB.OUTPUT.MASK", PM_FPA_FILE_MASK);
+    if (!output || !outMask) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
+        return false;
+    }
+    output->save = true;
+    outMask->save = true;
+    if (inVar && refVar) {
+        pmFPAfile *outVar = defineOutputFile(config, output, false, "PPSUB.OUTPUT.VARIANCE",
+                                             PM_FPA_FILE_VARIANCE);
+        if (!outVar) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
+            return false;
+        }
+        outVar->save = true;
+    }
+
+    pmFPAfile *inverse = NULL;          // Inverse output image
+    if (data->inverse) {
+        // Inverse output image
+        inverse = defineOutputFile(config, output, true, "PPSUB.INVERSE", PM_FPA_FILE_IMAGE);
+        pmFPAfile *invMask = defineOutputFile(config, inverse, false, "PPSUB.INVERSE.MASK",
+                                              PM_FPA_FILE_MASK);
+        if (!inverse || !invMask) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
+            return false;
+        }
+        inverse->save = true;
+        invMask->save = true;
+        if (inVar && refVar) {
+            pmFPAfile *invVar = defineOutputFile(config, inverse, false, "PPSUB.INVERSE.VARIANCE",
+                                                 PM_FPA_FILE_VARIANCE);
+            if (!invVar) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
+                return false;
+            }
+            invVar->save = true;
+        }
+    }
+
+
     // Output JPEGs
     pmFPAfile *jpeg1 = pmFPAfileDefineOutput(config, NULL, "PPSUB.OUTPUT.JPEG1");
@@ -245,25 +289,7 @@
     }
 
-    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
-    if (!recipe) {
-        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSUB_RECIPE);
-        return false;
-    }
-
     // psPhot input
-    if (psMetadataLookupBool(NULL, recipe, "PHOTOMETRY")) {
+    if (data->photometry) {
         psphotModelClassInit();        // load implementation-specific models
-
-        // Internal-ish file for getting the PSF from the minuend
-        pmFPAfile *psf = pmFPAfileDefineOutputFromFile(config, output, "PSPHOT.PSF.LOAD");
-        if (!psf) {
-            psError(PS_ERR_IO, false, "Failed to build FPA from PSPHOT.PSF.LOAD");
-            return false;
-        }
-        if (psf->type != PM_FPA_FILE_PSF) {
-            psError(PS_ERR_IO, true, "PSPHOT.PSF.LOAD is not of type PSF");
-            return false;
-        }
-        pmFPAfileActivate(config->files, false, "PSPHOT.PSF.LOAD");
 
         pmFPAfile *psphot = pmFPAfileDefineFromFPA(config, output->fpa, 1, 1, "PSPHOT.INPUT");
@@ -276,4 +302,17 @@
             return false;
         }
+        pmFPAfileActivate(config->files, false, "PSPHOT.INPUT");
+
+        // Internal-ish file for getting the PSF from the minuend
+        pmFPAfile *psf = pmFPAfileDefineOutputFromFile(config, psphot, "PSPHOT.PSF.LOAD");
+        if (!psf) {
+            psError(PS_ERR_IO, false, "Failed to build FPA from PSPHOT.PSF.LOAD");
+            return false;
+        }
+        if (psf->type != PM_FPA_FILE_PSF) {
+            psError(PS_ERR_IO, true, "PSPHOT.PSF.LOAD is not of type PSF");
+            return false;
+        }
+        pmFPAfileActivate(config->files, false, "PSPHOT.PSF.LOAD");
 
         if (!psphotDefineFiles(config, psphot)) {
@@ -281,6 +320,30 @@
             return false;
         }
+
+        // Deactivate psphot output sources --- we want to define output source files of our own
+        pmFPAfile *psphotOutput = pmFPAfileSelectSingle(config->files, "PSPHOT.OUTPUT", 0);
+        psphotOutput->save = false;
+
+        pmFPAfile *outSources = defineOutputFile(config, output, false, "PPSUB.OUTPUT.SOURCES",
+                                                 PM_FPA_FILE_CMF);
+        if (!outSources) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to set up output source file.");
+            return false;
+        }
+        outSources->save = true;
+
+        if (data->inverse) {
+            pmFPAfile *invSources = defineOutputFile(config, inverse, false, "PPSUB.INVERSE.SOURCES",
+                                                     PM_FPA_FILE_CMF);
+            if (!invSources) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to set up inverse source file.");
+                return false;
+            }
+            invSources->save = true;
+        }
     }
 
     return true;
 }
+
+
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubData.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubData.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubData.c	(revision 24244)
@@ -0,0 +1,74 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+#include <psphot.h>
+
+#include "ppSub.h"
+
+
+static void subDataFree(ppSubData *data)
+{
+    if (data->statsFile) {
+        psString stats = psMetadataConfigFormat(data->stats); // Statistics to output
+        if (!stats || strlen(stats) == 0) {
+            psWarning("Unable to generate statistics file.");
+        } else {
+            fprintf(data->statsFile, "%s", stats);
+        }
+        psFree(stats);
+        fclose(data->statsFile);
+        pmConfigRunFilenameAddWrite(data->config, "STATS", data->statsName);
+    }
+    psFree(data->statsName);
+    psFree(data->stamps);
+    psFree(data->psf);
+    psFree(data->stats);
+
+    psString dump_file = psMetadataLookupStr(NULL, data->config->arguments, "-dumpconfig");
+    if (dump_file) {
+        pmConfigDump(data->config, dump_file);
+    }
+    psFree(data->config);
+
+    return;
+}
+
+ppSubData *ppSubDataAlloc(pmConfig *config)
+{
+    ppSubData *data = psAlloc(sizeof(ppSubData)); // Processing data, to return
+    psMemSetDeallocator(data, (psFreeFunc)subDataFree);
+
+    data->config = config;
+    data->quality = 0;
+    data->photometry = false;
+    data->inverse = false;
+    data->stamps = NULL;
+    data->psf = NULL;
+    data->statsName = NULL;
+    data->statsFile = NULL;
+    data->stats = psMetadataAlloc();
+    psMetadataAddS32(data->stats, PS_LIST_TAIL, "QUALITY", 0, "Data quality", 0);
+
+    return data;
+}
+
+
+void ppSubDataQuality(ppSubData *data, psErrorCode error, ppSubFiles files)
+{
+    psAssert(data, "Require processing data");
+
+    if (psMetadataLookupS32(NULL, data->stats, "QUALITY") == 0) {
+        data->quality = error;
+        psMetadataAddS32(data->stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE, "Data quality", error);
+    }
+
+    ppSubFilesActivate(data->config, files, false);
+
+    psErrorClear();
+
+    return;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubDefineOutput.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubDefineOutput.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubDefineOutput.c	(revision 24244)
@@ -21,25 +21,29 @@
 #include "ppSub.h"
 
-bool ppSubDefineOutput(pmConfig *config, const pmFPAview *view)
+bool ppSubDefineOutput(const char *name, pmConfig *config)
 {
+    psAssert(name, "Require name");
     psAssert(config, "Require configuration");
-    psAssert(view, "Require view");
 
-    pmCell *outCell = pmFPAfileThisCell(config->files, view, "PPSUB.OUTPUT"); // Output cell
+    pmFPAview *view = ppSubViewReadout(); // View to readout
+    pmCell *outCell = pmFPAfileThisCell(config->files, view, name); // Output cell
     pmFPA *outFPA = outCell->parent->parent; // Output FPA
-    pmHDU *outHDU = outFPA->hdu; // Output HDU
+    pmHDU *outHDU = outFPA->hdu;        // Output HDU
     if (!outHDU->header) {
         outHDU->header = psMetadataAlloc();
     }
 
-    // generate an output readout (first check if it's already there by virtue of kernels)
-    pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT.KERNELS"); // RO with kernel
-    if (!outRO) {
-        outRO = pmReadoutAlloc(outCell); // Output readout: subtraction
+    // The output readout may already be present if we read in the convolution kernel
+    pmReadout *outRO = NULL;            // Output readout
+    if (outCell->readouts && outCell->readouts->n > 0 && outCell->readouts->data[0]) {
+        outRO = psMemIncrRefCounter(outCell->readouts->data[0]);
+    } else {
+        outRO = pmReadoutAlloc(outCell);
     }
 
-    // convolved input images
+    // Convolved input images
     pmReadout *inConv = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); // Input readout
     pmReadout *refConv = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); // Reference readout
+    psFree(view);
 
     // Add kernel descrption to header.
@@ -55,6 +59,4 @@
     if (!kernels) {
         psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find SUBTRACTION.KERNEL");
-        psFree(inConv);
-        psFree(refConv);
         psFree(outRO);
         return false;
@@ -63,15 +65,17 @@
                      kernels->description);
 
+    // Add additional data to the header
+    pmFPAfile *refFile = psMetadataLookupPtr(NULL, config->files, "PPSUB.REF"); // Reference file
+    pmFPAfile *inFile = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); // Input file
+    psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.REFERENCE", 0,
+                     "Subtraction reference", refFile->filename);
+    psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.INPUT", 0,
+                     "Subtraction input", inFile->filename);
+    ppSubVersionHeader(outHDU->header);
+
+
     outRO->analysis = psMetadataCopy(outRO->analysis, analysis);
 
-#ifdef TESTING
-    {
-        psImage *kernelImage = psMetadataLookupPtr(&mdok, analysis,
-                                                   "SUBTRACTION.KERNEL.IMAGE"); // Image of kernel
-        psFits *fits = psFitsOpen("kernel.fits", "w");
-        psFitsWriteImage(fits, NULL, kernelImage, 0, NULL);
-        psFitsClose(fits);
-    }
-#endif
+    psFree(outRO);
 
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubErrorCodes.c.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubErrorCodes.c.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubErrorCodes.c.in	(revision 24244)
@@ -0,0 +1,37 @@
+/** @file ppSubErrorCodes.c.in
+ *
+ *  @brief
+ *
+ *  @ingroup ppSub
+ *
+ *  @author IfA
+ *  @version $Revision: 1.3 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-05 20:44:04 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+#include "pslib.h"
+#include "ppSubErrorCodes.h"
+
+/*
+ * The line
+    { PPSUB_ERR_$X{ErrorCode}, "$X{ErrorDescription}"},
+ * (without the Xs)
+ * will be replaced by values from errorCodes.dat
+ */
+void ppSubErrorRegister(void)
+{
+    static psErrorDescription errors[] = {
+       { PPSUB_ERR_BASE, "First value we use; lower values belong to psLib" },
+       { PPSUB_ERR_${ErrorCode}, "${ErrorDescription}"},
+    };
+    static int nerror = PPSUB_ERR_NERROR - PPSUB_ERR_BASE; ///< number of values in enum
+
+    for (int i = 0; i < nerror; i++) {
+       psErrorDescription *tmp = psAlloc(sizeof(psErrorDescription));
+       *tmp = errors[i];
+       psErrorRegister(tmp, 1);
+       psFree(tmp);			/* it's on the internal list */
+    }
+    nerror = 0;			                // don't register more than once
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubErrorCodes.dat	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubErrorCodes.dat	(revision 24244)
@@ -0,0 +1,11 @@
+#
+# This file is used to generate pswarpErrorClasses.h
+#
+BASE = 14000		First value we use; lower values belong to psLib
+UNKNOWN			Unknown ppSub error code
+NOT_IMPLEMENTED		Desired feature is not yet implemented
+ARGUMENTS		Incorrect arguments
+CONFIG			Problem in configure files
+IO			Problem in FITS I/O
+DATA                    Problem in data values
+NO_OVERLAP		No overlap between input and skycell
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubErrorCodes.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubErrorCodes.h.in	(revision 24244)
@@ -0,0 +1,30 @@
+/** @file ppSubErrorCodes.h.in
+ *
+ *  @brief
+ *
+ *  @ingroup ppSub
+ *
+ *  @author IfA
+ *  @version $Revision: 1.2 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-05 20:44:04 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+#if !defined(PPSUB_ERROR_CODES_H)
+#define PPSUB_ERROR_CODES_H
+/*
+ * The line
+ *  PPSUB_ERR_$X{ErrorCode},
+ * (without the X)
+ *
+ * will be replaced by values from errorCodes.dat
+ */
+typedef enum {
+    PPSUB_ERR_BASE = 14000,
+    PPSUB_ERR_${ErrorCode},
+    PPSUB_ERR_NERROR
+} ppSubErrorCode;
+
+void ppSubErrorRegister(void);
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubFiles.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubFiles.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubFiles.c	(revision 24244)
@@ -0,0 +1,188 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+#include <psphot.h>
+
+#include "ppSub.h"
+
+
+// Input files
+static const char *inputFiles[] = { "PPSUB.INPUT", "PPSUB.INPUT.MASK", "PPSUB.INPUT.VARIANCE",
+                                    "PPSUB.INPUT.SOURCES", "PPSUB.REF", "PPSUB.REF.MASK",
+                                    "PPSUB.REF.VARIANCE", "PPSUB.REF.SOURCES", NULL };
+
+// Convolved files
+static const char *convFiles[] = { "PPSUB.INPUT.CONV", "PPSUB.INPUT.CONV.MASK", "PPSUB.INPUT.CONV.VARIANCE",
+                                   "PPSUB.REF.CONV", "PPSUB.REF.CONV.MASK", "PPSUB.REF.CONV.VARIANCE",
+                                   NULL };
+
+// Subtraction files
+static const char *subFiles[] = { "PPSUB.OUTPUT", "PPSUB.OUTPUT.MASK", "PPSUB.OUTPUT.VARIANCE",
+                                  "PPSUB.OUTPUT.JPEG1", "PPSUB.OUTPUT.JPEG2",
+                                  NULL };
+
+// Subtraction photometry
+static const char *subPhotFiles[] = { "PPSUB.OUTPUT.SOURCES", NULL };
+
+// Inverse subtraction files
+static const char *invFiles[] = { "PPSUB.INVERSE", "PPSUB.INVERSE.MASK", "PPSUB.INVERSE.VARIANCE", NULL };
+
+// Inverse subtraction photometry
+static const char *invPhotFiles[] = { "PPSUB.INVERSE.SOURCES", NULL };
+
+// PSF files
+static const char *psfFiles[] = { "PSPHOT.PSF.SAVE", NULL };
+
+// Calculation (may be either input or output) files
+static const char *calcFiles[] = { "PPSUB.OUTPUT.KERNELS", NULL };
+
+
+// Activate/deactivate a list of files
+static void filesActivate(pmConfig *config, // Configuration
+                          const char **files, // List of files
+                          bool state    // Activation status to set
+    )
+{
+    for (int i = 0; files[i]; i++) {
+        pmFPAfileActivate(config->files, state, files[i]);
+    }
+    return;
+}
+
+// Activate/deactivate a list of files depending on their 'save' boolean.
+//  This is so we can activate/deactivate the 'calculation' files, which may be either input or output, which
+// is indicated by their 'save' boolean.
+static void filesActivateSave(pmConfig *config, // Configuration
+                              const char **files, // List of files
+                              bool save, // Activate when this save state is set
+                              bool state // Activation status to set
+    )
+{
+    for (int i = 0; files[i]; i++) {
+        pmFPAfile *file = pmFPAfileSelectSingle(config->files, files[i], 0);
+        if (file && file->save == save) {
+            pmFPAfileActivate(config->files, state, files[i]);
+        }
+    }
+    return;
+}
+
+void ppSubFilesActivate(pmConfig *config, ppSubFiles files, bool state)
+{
+    psAssert(config, "Require configuration");
+
+    if (files & PPSUB_FILES_INPUT) {
+        filesActivate(config, inputFiles, state);
+        filesActivateSave(config, calcFiles, false, state);
+    }
+    if (files & PPSUB_FILES_CONV) {
+        filesActivate(config, convFiles, state);
+    }
+    if (files & PPSUB_FILES_SUB) {
+        filesActivate(config, subFiles, state);
+        filesActivateSave(config, calcFiles, true, state);
+    }
+    if (files & PPSUB_FILES_INV) {
+        filesActivate(config, invFiles, state);
+    }
+    if (files & PPSUB_FILES_PHOT_SUB) {
+        filesActivate(config, subPhotFiles, state);
+    }
+    if (files & PPSUB_FILES_PHOT_INV) {
+        filesActivate(config, invPhotFiles, state);
+    }
+    if (files & PPSUB_FILES_PSF) {
+        filesActivate(config, psfFiles, state);
+    }
+    if (files & PPSUB_FILES_PHOT) {
+        psphotFilesActivate(config, state);
+    }
+
+    return;
+}
+
+
+pmFPAview *ppSubViewReadout(void)
+{
+    pmFPAview *view = pmFPAviewAlloc(0);
+    view->chip = view->cell = view->readout = 0;
+    return view;
+}
+
+bool ppSubFilesIterateDown(pmConfig *config, ppSubFiles files)
+{
+    psAssert(config, "Require configuration");
+
+    ppSubFilesActivate(config, PPSUB_FILES_ALL, false);
+    ppSubFilesActivate(config, files, true);
+
+    pmFPAview *view = pmFPAviewAlloc(0);// View to FPA top
+
+    // FPA
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return false;
+    }
+
+    // Chip
+    view->chip = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return false;
+    }
+
+    // Cell
+    view->cell = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return false;
+    }
+
+    // Readout
+    view->readout = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return false;
+    }
+
+    ppSubFilesActivate(config, PPSUB_FILES_ALL, false);
+
+    return true;
+}
+
+bool ppSubFilesIterateUp(pmConfig *config, ppSubFiles files)
+{
+    psAssert(config, "Require configuration");
+
+    ppSubFilesActivate(config, PPSUB_FILES_ALL, false);
+    ppSubFilesActivate(config, files, true);
+
+    pmFPAview *view = ppSubViewReadout(); // View to readout
+
+    // Readout
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+
+    // Cell
+    view->readout = -1;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+
+    // Chip
+    view->cell = -1;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+
+    // FPA
+    view->chip = -1;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+
+    ppSubFilesActivate(config, PPSUB_FILES_ALL, false);
+
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubKernel.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubKernel.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubKernel.c	(revision 24244)
@@ -29,5 +29,5 @@
     }
 
-    psTraceSetLevel("psModules.imcombine", 7);
+    (void) psTraceSetLevel("psModules.imcombine", 7);
 
     const char *inName = argv[1];       // Input file name
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubLoop.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubLoop.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubLoop.c	(revision 24244)
@@ -17,10 +17,11 @@
 #include <pslib.h>
 #include <psmodules.h>
-#include <ppStats.h>
 
 #include "ppSub.h"
 
-bool ppSubLoop(pmConfig *config)
+bool ppSubLoop(ppSubData *data)
 {
+    psAssert(data, "Require processing data");
+    pmConfig *config = data->config;    // Configuration
     psAssert(config, "Require configuration.");
 
@@ -28,160 +29,145 @@
     pmConfigRecipesCull(config, "PPSUB,PPSTATS,PSPHOT,MASKS,JPEG");
 
-    bool mdok;                          // Status of MD lookup
-    const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics
-    psMetadata *stats = NULL;           // Container for statistics
-    FILE *statsFile = NULL;             // File stream for statistics
-    if (statsName && strlen(statsName) > 0) {
-        psString resolved = pmConfigConvertFilename(statsName, config, true, true); // Resolved filename
-        statsFile = fopen(resolved, "w");
-        if (!statsFile) {
-            psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved);
-            psFree(resolved);
-            return false;
-        } else {
-            stats = psMetadataAlloc();
-        }
-        psFree(resolved);
-    }
 
     pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT");
-    if (!input) {
-        psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find input data!\n");
+    pmFPAfile *reference = psMetadataLookupPtr(NULL, config->files, "PPSUB.REF");
+    pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSUB.OUTPUT");
+    psAssert(input && reference && output, "Require files");
+
+    if (!ppSubFilesIterateDown(config, PPSUB_FILES_INPUT | PPSUB_FILES_CONV)) {
+        psError(PPSUB_ERR_IO, false, "Unable to load files.");
         return false;
     }
 
-    pmFPAfile *reference = psMetadataLookupPtr(NULL, config->files, "PPSUB.REF");
-    if (!reference) {
-        psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find reference data!\n");
+    psTimerStart("PPSUB_MATCH");
+
+    if (!ppSubSetMasks(config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to set masks.");
         return false;
     }
 
-    pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSUB.OUTPUT");
-    if (!output) {
-        psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find output data!\n");
+    if (!ppSubMatchPSFs(data)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to match PSFs.");
+        return false;
+    }
+    if (data->quality) {
+        // Can't do anything at all
+        return true;
+    }
+
+    psMetadataAddF32(data->stats, PS_LIST_TAIL, "TIME_MATCH", 0, "Time to match PSFs",
+                     psTimerClear("PPSUB_MATCH"));
+
+    // Close input files
+    if (!ppSubFilesIterateUp(config, PPSUB_FILES_INPUT)) {
+        psError(PPSUB_ERR_IO, false, "Unable to close input files.");
         return false;
     }
 
-    pmFPAview *view = pmFPAviewAlloc(0); // Pointer into FPA hierarchy
-
-    // Iterate over the FPA hierarchy
-    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+    // Set up subtraction files
+    if (!ppSubFilesIterateDown(config, PPSUB_FILES_SUB)) {
+        psError(PPSUB_ERR_IO, false, "Unable to set up subtraction files.");
         return false;
     }
 
-    pmChip *inChip;                    // Input chip of interest
-    while ((inChip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) {
-        pmChip *refChip = pmFPAviewThisChip(view, reference->fpa); // Reference chip of interest
-        if ((!inChip->file_exists && refChip->file_exists) ||
-            (inChip->file_exists && !refChip->file_exists)) {
-            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "FPA format discrepency between input and reference");
-            psFree(view);
+    if (!ppSubDefineOutput("PPSUB.OUTPUT", config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to define output.");
+        return false;
+    }
+
+    if (!data->quality && !ppSubMakePSF(data)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to generate PSF.");
+        return false;
+    }
+
+    if (!ppSubReadoutSubtract(config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to subtract images.");
+        return false;
+    }
+
+    // Close convolved files
+    if (!ppSubFilesIterateUp(config, PPSUB_FILES_PSF | PPSUB_FILES_CONV)) {
+        psError(PPSUB_ERR_IO, false, "Unable to close input files.");
+        return false;
+    }
+
+    // Higher order background subtraction using psphot
+    if (!ppSubBackground(config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to subtract background.");
+        return false;
+    }
+
+    if (!ppSubFilesIterateDown(config, PPSUB_FILES_PHOT_SUB)) {
+        psError(PPSUB_ERR_IO, false, "Unable to set up photometry files.");
+        return false;
+    }
+
+    if (!data->quality && !ppSubReadoutPhotometry("PPSUB.OUTPUT", data)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry.");
+        return false;
+    }
+
+    if (!ppSubFilesIterateUp(config, PPSUB_FILES_PHOT_SUB)) {
+        psError(PPSUB_ERR_IO, false, "Unable to set up photometry files.");
+        return false;
+    }
+
+    // Perform statistics on the cell
+    if (!ppSubReadoutStats(data)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to collect statistics");
+        return false;
+    }
+
+    if (!ppSubReadoutJpeg(config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to update.");
+        return false;
+    }
+
+    if (data->inverse) {
+        // Set up inverse subtraction files
+        if (!ppSubFilesIterateDown(config, PPSUB_FILES_INV)) {
+            psError(PPSUB_ERR_IO, false, "Unable to set up inverse files.");
             return false;
         }
 
-        if (!inChip->file_exists) {
-            continue;
-        }
-
-        if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        if (data->inverse && !ppSubDefineOutput("PPSUB.INVERSE", config)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define inverse.");
             return false;
         }
 
-        pmCell *inCell;                // Cell of interest
-        while ((inCell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) {
-            pmCell *refCell = pmFPAviewThisCell(view, reference->fpa); // Reference cell of interest
-            if ((!inCell->file_exists && refCell->file_exists) ||
-                (inCell->file_exists && !refCell->file_exists)) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                        "FPA format discrepency between input and reference");
-                psFree(view);
-                return false;
-            }
-            if (!inCell->file_exists) {
-                continue;
-            }
-            if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
-                return false;
-            }
-
-            pmReadout *inRO;           // Readin of interest
-            while ((inRO = pmFPAviewNextReadout(view, input->fpa, 1))) {
-                if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
-                    return false;
-                }
-                pmReadout *refRO = pmFPAviewThisReadout(view, reference->fpa);// Reference readout of interest
-                if (!refRO || (!inRO->data_exists && refRO->data_exists) ||
-                    (inRO->data_exists && !refRO->data_exists)) {
-                    psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                            "FPA format discrepency between input and reference");
-                    psFree(view);
-                    return false;
-                }
-                if (!inRO->data_exists) {
-                    continue;
-                }
-
-                // Perform the analysis
-                if (!ppSubReadout(config, stats, view)) {
-                    psError(PS_ERR_UNKNOWN, false, "Unable to subtract images.\n");
-                    return false;
-                }
-
-                if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
-                    return false;
-                }
-            }
-
-            // Perform statistics on the cell
-            if (stats) {
-                pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSUB.OUTPUT"); // Output file
-                if (!output) {
-                    psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find file PPSUB.OUTPUT.\n");
-                    return false;
-                }
-                psImageMaskType maskValue = pmConfigMaskGet("MASK.VALUE", config);
-                ppStatsFPA(stats, output->fpa, view, maskValue, config);
-            }
-
-            if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
-                return false;
-            }
+        if (!ppSubReadoutInverse(config)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to invert images.");
+            return false;
         }
 
-        if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        // Close subtraction files and open inverse photometry files
+        if (!ppSubFilesIterateUp(config, PPSUB_FILES_SUB)) {
+            psError(PPSUB_ERR_IO, false, "Unable to close subtraction files.");
+            return false;
+        }
+
+        if (!ppSubFilesIterateDown(config, PPSUB_FILES_PHOT_INV)) {
+            psError(PPSUB_ERR_IO, false, "Unable to set up inverse files.");
+            return false;
+        }
+
+        if (!data->quality && !ppSubReadoutPhotometry("PPSUB.INVERSE", data)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry.");
+            return false;
+        }
+
+        // Close inverse subtraction files
+        if (!ppSubFilesIterateUp(config, PPSUB_FILES_INV | PPSUB_FILES_PHOT_INV)) {
+            psError(PPSUB_ERR_IO, false, "Unable to close subtraction files.");
+            return false;
+        }
+    } else {
+        // Close subtraction files
+        if (!ppSubFilesIterateUp(config, PPSUB_FILES_SUB)) {
+            psError(PPSUB_ERR_IO, false, "Unable to close subtraction files.");
             return false;
         }
     }
 
-    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
-        return false;
-    }
-
-    psFree(view);
-
-    // Write out summary statistics
-    if (stats) {
-        psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_SUB", 0, "Time for subtraction completion",
-                         psTimerMark("ppSub"));
-
-        const char *statsMDC = psMetadataConfigFormat(stats);
-        if (!statsMDC || strlen(statsMDC) == 0) {
-            psWarning("Unable to generate statistics MDC file.\n");
-        } else {
-            fprintf(statsFile, "%s", statsMDC);
-        }
-        psFree((void *)statsMDC);
-        fclose(statsFile);
-
-        psFree(stats);
-    }
-
-    psString dump_file = psMetadataLookupStr(&mdok, config->arguments, "-dumpconfig");
-    if (dump_file) {
-
-        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); // Input file
-        pmConfigDump(config, input->fpa, dump_file);
-    }
-
     return true;
 }
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubMakePSF.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubMakePSF.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubMakePSF.c	(revision 24244)
@@ -22,8 +22,13 @@
 #include "ppSub.h"
 
-bool ppSubMakePSF(pmConfig *config, const pmFPAview *view)
+bool ppSubMakePSF(ppSubData *data)
 {
+    psAssert(data, "Require processing data");
+    pmConfig *config = data->config;    // Configuration
     psAssert(config, "Require configuration");
-    psAssert(view, "Require view");
+
+    if (!data->photometry) {
+        return true;
+    }
 
     psTimerStart("PPSUB_PHOT");
@@ -36,7 +41,4 @@
     }
 
-    psMetadata *psphotRecipe = psMetadataLookupMetadata(NULL, config->recipes, PSPHOT_RECIPE);// psphot recipe
-    psAssert(recipe, "We checked this earlier, so it should be here.");
-
     bool reverse = psMetadataLookupBool(NULL, config->arguments, "REVERSE"); // Reverse sense of subtraction?
 
@@ -44,4 +46,5 @@
     pmReadout *minuend = NULL;          // Image that will be positive following subtraction
     pmFPAfile *minuendFile = NULL;      // File for minuend image
+    pmFPAview *view = ppSubViewReadout(); // View to readout
     if (reverse) {
         minuend = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV");
@@ -52,30 +55,14 @@
     }
 
-#if 1
-    pmReadout *template = minuend;
     pmFPAfile *photFile = psMetadataLookupPtr(&mdok, config->files, "PSPHOT.INPUT"); // Photometry file
+    if (!pmFPACopy(photFile->fpa, minuendFile->fpa)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to copy FPA for photometry");
+        psFree(view);
+        return false;
+    }
     pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout to photometer
-    if (!photRO) {
-        pmCell *cell = pmFPAviewThisCell(view, photFile->fpa); // Cell to photometer
-        photRO = pmReadoutAlloc(cell); // Output readout: subtraction
+    if (psMetadataLookup(photRO->analysis, "PSPHOT.SOURCES")) {
+        psMetadataRemoveKey(photRO->analysis, "PSPHOT.SOURCES");
     }
-    photRO->image = psImageCopy(photRO->image, template->image, PS_TYPE_F32);
-    if (template->variance) {
-        photRO->variance = psImageCopy(photRO->variance, template->variance, PS_TYPE_F32);
-    } else {
-        psFree(photRO->variance);
-        photRO->variance = NULL;
-    }
-    if (template->mask) {
-        photRO->mask = psImageCopy(photRO->mask, template->mask, PS_TYPE_IMAGE_MASK);
-    } else {
-        psFree(photRO->mask);
-        photRO->mask = NULL;
-    }
-#else
-    // Supply the minuend pmFPAfile to psphot as PSPHOT.INPUT:
-    psMetadataAddPtr(config->files, PS_LIST_TAIL, "PSPHOT.INPUT", PS_DATA_UNKNOWN | PS_META_REPLACE,
-                     "psphot input: view on another pmFPAfile", minuendFile);
-#endif
 
     // Extract the loaded sources from the associated readout, and generate PSF
@@ -83,36 +70,23 @@
     psArray *sources = psMetadataLookupPtr(&mdok, minuend->analysis, "PSPHOT.SOURCES");
     if (!psphotReadoutFindPSF(config, view, sources)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry on subtracted image.");
-        return false;
+        // This is likely a data quality issue
+        // XXX Split into multiple cases using error codes?
+        psErrorStackPrint(stderr, "Unable to determine PSF");
+        psWarning("Unable to determine PSF --- suspect bad data quality.");
+        ppSubDataQuality(data, PSPHOT_ERR_PSF, PPSUB_FILES_PHOT_SUB | PPSUB_FILES_PHOT_INV);
+        psFree(view);
+        return true;
     }
-
-    // Record the FWHM in the output header
-    pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); // Output readout
-    pmHDU *hdu = pmHDUFromCell(outRO->parent); // HDU with header
-    psMetadataItemSupplement(hdu->header, psphotRecipe, "FWHM_MAJ");
-    psMetadataItemSupplement(hdu->header, psphotRecipe, "FWHM_MIN");
 
     // Get rid of the generated header; it will be regenerated by the real photometry run
     psMetadataRemoveKey(photRO->analysis, "PSPHOT.HEADER");
 
+    if (!data->quality) {
+        data->psf = psMemIncrRefCounter(psMetadataLookupPtr(NULL, photRO->parent->parent->analysis,
+                                                            "PSPHOT.PSF"));
+    }
+
+    psFree(view);
+
     return true;
 }
-
-// XXX we used to need this, is it still needed?
-
-// pmCell *photCell = pmFPAfileThisCell(config->files, view, "PSPHOT.INPUT");
-// pmCellFreeReadouts(photCell);
-
-// if (!pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL") ||
-//     !pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV") ||
-//     !pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND")) {
-//     psError(PS_ERR_UNKNOWN, false, "Unable to drop PSPHOT internal files.");
-//     return false;
-// }
-
-// Blow away the sources psphot found --- they're irrelevant for the subtraction
-// XXX is this still needed?  These are now probably not being set
-// pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout with sources
-// psMetadataRemoveKey(photRO->analysis, "PSPHOT.SOURCES");
-// psMetadataRemoveKey(photRO->analysis, "PSPHOT.HEADER");
-
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubMatchPSFs.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubMatchPSFs.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubMatchPSFs.c	(revision 24244)
@@ -22,12 +22,15 @@
 #include "ppSub.h"
 
-bool ppSubMatchPSFs(pmConfig *config, const pmFPAview *view)
+bool ppSubMatchPSFs(ppSubData *data)
 {
+    psAssert(data, "Require processing data");
+    pmConfig *config = data->config;    // Configuration
     psAssert(config, "Require configuration");
-    psAssert(view, "Require view");
 
     // Look up recipe values
     psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
     psAssert(recipe, "We checked this earlier, so it should be here.");
+
+    pmFPAview *view = ppSubViewReadout(); // View to readout
 
     // Input images
@@ -52,7 +55,13 @@
     pmReadout *kernelRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT.KERNELS"); // RO with kernel
 
+    psFree(view);
+
     // Sources in image, used for stamps: these must be loaded from previous analysis stages
-    psArray *inSources = psMetadataLookupPtr(&mdok, inRO->analysis, "PSPHOT.SOURCES"); // Input source list
-    psArray *refSources = psMetadataLookupPtr(&mdok, refRO->analysis, "PSPHOT.SOURCES"); // Ref source list
+    pmReadout *inSourceRO = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.SOURCES");
+    pmReadout *refSourceRO = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.SOURCES");
+    psArray *inSources = inSourceRO ? psMetadataLookupPtr(&mdok, inSourceRO->analysis, "PSPHOT.SOURCES") :
+        NULL; // Source list from input image
+    psArray *refSources = refSourceRO ? psMetadataLookupPtr(&mdok, refSourceRO->analysis, "PSPHOT.SOURCES") :
+        NULL ; // Source list from reference image
 
     psArray *sources = NULL;            // Merged list of sources
@@ -85,5 +94,4 @@
     float spacing = psMetadataLookupF32(NULL, recipe, "STAMP.SPACING"); // Typical stamp spacing
     float threshold = psMetadataLookupF32(NULL, recipe, "STAMP.THRESHOLD"); // Threshold for stmps
-    const char *stampsName = psMetadataLookupStr(&mdok, config->arguments, "STAMPS"); // Filename for stamps
 
     const char *typeStr = psMetadataLookupStr(NULL, recipe, "KERNEL.TYPE"); // Kernel type
@@ -136,15 +144,27 @@
 
     // Match the PSFs
+    bool success = false;               // Operation was successful?
     if (kernelRO) {
-        if (!pmSubtractionMatchPrecalc(inConv, refConv, inRO, refRO, kernelRO->analysis,
-                                       stride, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to convolve images.");
-            return false;
-        }
+        success = pmSubtractionMatchPrecalc(inConv, refConv, inRO, refRO, kernelRO->analysis,
+                                            stride, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac);
     } else {
-        if (!pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, stride, regionSize, spacing,
-                                threshold, sources, stampsName, type, size, order, widths, orders, inner,
-                                ringsOrder, binning, penalty, optimum, optWidths, optOrder, optThresh, iter,
-                                rej, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac, subMode)) {
+        success = pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, stride, regionSize,
+                                     spacing, threshold, sources, data->stamps, type, size, order,
+                                     widths, orders, inner, ringsOrder, binning, penalty, optimum,
+                                     optWidths, optOrder, optThresh, iter, rej, sys, maskVal,
+                                     maskBad, maskPoor, poorFrac, badFrac, subMode);
+    }
+
+    psFree(optWidths);
+    pmSubtractionThreadsFinalize(inRO, refRO);
+
+    if (!success) {
+        psErrorCode error = psErrorCodeLast(); // Error code
+        if (error == PM_ERR_STAMPS) {
+            psErrorStackPrint(stderr, "Unable to find stamps");
+            psWarning("Unable to find stamps --- suspect bad data quality.");
+            ppSubDataQuality(data, error, PPSUB_FILES_ALL);
+            return true;
+        } else {
             psError(PS_ERR_UNKNOWN, false, "Unable to match images.");
             return false;
@@ -152,19 +172,10 @@
     }
 
-    psFree(optWidths);
-
-    pmSubtractionThreadsFinalize(inRO, refRO);
+    pmConceptsCopyFPA(inConv->parent->parent->parent, inRO->parent->parent->parent, true, true);
+    pmConceptsCopyFPA(refConv->parent->parent->parent, refRO->parent->parent->parent, true, true);
 
     psImageCovarianceTransfer(inConv->variance, inConv->covariance);
     psImageCovarianceTransfer(refConv->variance, refConv->covariance);
 
-    // XXX drop the pixels associated with inRO and refRO (now that we have inConv and refConf)
-#ifdef TESTING
-    psphotSaveImage (NULL, inRO->image, "inRO.fits");
-    psphotSaveImage (NULL, refRO->image, "refRO.fits");
-    psphotSaveImage (NULL, inConv->image, "inConv.fits");
-    psphotSaveImage (NULL, refConv->image, "refConv.fits");
-#endif
-
     return true;
 }
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadout.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadout.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadout.c	(revision 24244)
@@ -21,29 +21,24 @@
 #include "ppSub.h"
 
-bool ppSubReadout(pmConfig *config, psMetadata *stats, const pmFPAview *view)
+bool ppSubReadout(const char *name, bool reverse, ppSubData *data, const pmFPAview *view)
 {
-    psTimerStart("PPSUB_MATCH");
+    psAssert(data, "Require processing data");
+    pmConfig *config = data->config;    // Configuration
+    psAssert(config, "Require configuration");
 
-    if (!ppSubSetMasks(config, view)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to set masks.");
-        return false;
-    }
+    psAssert(name, "Require name");
+    psAssert(view, "Require view");
 
-    if (!ppSubMatchPSFs(config, view)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to match PSFs.");
-        return false;
-    }
-
-    if (!ppSubDefineOutput(config, view)) {
+    if (!ppSubDefineOutput(name, config, data, view)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to define output.");
         return false;
     }
 
-    if (!ppSubMakePSF(config, view)) {
+    if (!data->quality && !ppSubMakePSF(name, config, data, view)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to generate PSF.");
         return false;
     }
 
-    if (!ppSubReadoutSubtract(config, view)) {
+    if (!ppSubReadoutSubtract(name, reverse, config, view)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to subtract images.");
         return false;
@@ -51,19 +46,22 @@
 
     // Higher order background subtraction using psphot
-    if (!ppSubBackground(config, view)) {
+    if (!ppSubBackground(name, config, view)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to subtract background.");
         return false;
     }
 
-    if (!ppSubReadoutPhotometry(config, stats, view)) {
+    if (!data->quality && data->!ppSubReadoutPhotometry(name, config, data, view)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry.");
         return false;
     }
 
-    if (!ppSubReadoutUpdate(config, stats, view)) {
+    if (!ppSubReadoutUpdate(name, config, data, view)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to update.");
         return false;
     }
 
+
+
+
     return true;
 }
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutInverse.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutInverse.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutInverse.c	(revision 24244)
@@ -0,0 +1,44 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <pslib.h>
+#include <psmodules.h>
+
+#include "ppSub.h"
+
+bool ppSubReadoutInverse(pmConfig *config)
+{
+    psAssert(config, "Require configuration");
+
+    pmFPAview *view = ppSubViewReadout(); // View to readout
+    pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT");
+    pmReadout *invRO = pmFPAfileThisReadout(config->files, view, "PPSUB.INVERSE");
+
+    invRO->image = (psImage*)psBinaryOp(invRO->image, outRO->image, "*", psScalarAlloc(-1.0, PS_TYPE_F32));
+    invRO->mask = psMemIncrRefCounter(outRO->mask);
+    invRO->variance = psMemIncrRefCounter(outRO->variance);
+    invRO->covariance = psMemIncrRefCounter(outRO->covariance);
+
+    invRO->data_exists = invRO->parent->data_exists = invRO->parent->parent->data_exists = true;
+
+    pmChip *outChip = outRO->parent->parent;       // Output chip
+    pmFPA *outFPA = outChip->parent;               // Output FPA
+    pmFPA *invFPA = invRO->parent->parent->parent; // Inverse FPA
+
+    pmConceptsCopyFPA(invFPA, outFPA, true, true);
+
+    pmChip *invChip = invRO->parent->parent; // Inverse chip
+    pmHDU *invHDU = invFPA->hdu;          // Inverse HDU
+    if (!pmAstromWriteWCS(invHDU->header, outFPA, outChip, WCS_TOLERANCE)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to write WCS astrometry to PPSUB.INVERSE.");
+        return false;
+    }
+    // Read from newly written astrometry so that it exists in the "inverse" FPA (for sources)
+    if (!pmAstromReadWCS(invFPA, invChip, invHDU->header, 1.0)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to read WCS astrometry.");
+        return false;
+    }
+
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutJpeg.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutJpeg.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutJpeg.c	(revision 24244)
@@ -0,0 +1,50 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+
+#include "ppSub.h"
+
+bool ppSubReadoutJpeg(pmConfig *config)
+{
+    psAssert(config, "Require configuration");
+
+    pmFPAview *view = ppSubViewReadout(); // View to readout
+    pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); // Readout to jpeg
+
+    // Look up recipe values
+    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
+    psAssert(recipe, "We checked this earlier, so it should be here.");
+
+    // Generate binned JPEGs
+    psImageMaskType maskBad = pmConfigMaskGet("BLANK", config); // Bits to mask for bad pixels
+
+    int bin1 = psMetadataLookupS32(NULL, recipe, "BIN1"); // First binning level
+    int bin2 = psMetadataLookupS32(NULL, recipe, "BIN2"); // Second binning level
+
+    // Target cells
+    pmCell *cell1 = pmFPAfileThisCell(config->files, view, "PPSUB.OUTPUT.JPEG1"); // Rebinned cell once
+    pmCell *cell2 = pmFPAfileThisCell(config->files, view, "PPSUB.OUTPUT.JPEG2"); // Rebinned cell twice
+    psFree(view);
+
+    pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts
+    if (!pmReadoutRebin(ro1, outRO, maskBad, bin1, bin1)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to bin output (1st binning)");
+        psFree(ro1);
+        psFree(ro2);
+        return false;
+    }
+    if (!pmReadoutRebin(ro2, ro1, 0, bin2, bin2)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to bin output (2nd binning)");
+        psFree(ro1);
+        psFree(ro2);
+        return false;
+    }
+    psFree(ro1);
+    psFree(ro2);
+
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutPhotometry.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutPhotometry.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutPhotometry.c	(revision 24244)
@@ -22,8 +22,13 @@
 #include "ppSub.h"
 
-bool ppSubReadoutPhotometry (pmConfig *config, psMetadata *stats, const pmFPAview *view)
+bool ppSubReadoutPhotometry(const char *name, ppSubData *data)
 {
+    psAssert(data, "Require processing data");
+    pmConfig *config = data->config;    // Configuration
     psAssert(config, "Require configuration");
-    psAssert(view, "Require view");
+
+    if (!data->photometry) {
+        return true;
+    }
 
     // Look up recipe values
@@ -39,4 +44,5 @@
     // The PSF (measured in ppSubMakePSF) is stored on the chip->analysis of PSPHOT.INPUT
     // In order to use an incoming PSF, it must be stored on the chip->analysis of PSPHOT.PSF.LOAD
+    pmFPAview *view = ppSubViewReadout(); // View to readout
     pmChip *psfInputChip = pmFPAfileThisChip(config->files, view, "PSPHOT.INPUT"); // Chip with PSF
     psAssert (psfInputChip, "should have been generated for ppSubMakePSF");
@@ -45,6 +51,8 @@
     pmPSF *psf = psMetadataLookupPtr(NULL, psfInputChip->analysis, "PSPHOT.PSF"); // PSF for photometry
     if (!psf) {
-        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find PSF from psphot");
-        return false;
+        psErrorStackPrint(stderr, "No PSF available");
+        psWarning("No PSF available --- suspect bad data quality.");
+        ppSubDataQuality(data, psErrorCodeLast(), PPSUB_FILES_PHOT_SUB | PPSUB_FILES_PHOT_INV);
+        return true;
     }
     psMetadataAddPtr(psfLoadChip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN | PS_META_REPLACE,
@@ -56,50 +64,54 @@
     // around the pointers so PSPHOT.INPUT corresponds to the output image; previously, it was
     // equivalent to the minuend image.
-    pmFPAfile *outputFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.OUTPUT"); // Output file
-    pmReadout *outRO = pmFPAviewThisReadout(view, outputFile->fpa); // Readout with the sources
+    pmReadout *inRO = pmFPAfileThisReadout(config->files, view, name); // Readout with image and sources
 
-    // XXX possibly rename this to PPSUB.RESID?
     pmFPAfile *photFile = psMetadataLookupPtr(&mdok, config->files, "PSPHOT.INPUT"); // Photometry file
+    pmFPAfile *inFile = psMetadataLookupPtr(&mdok, config->files, name); // Input file
+    if (!pmFPACopy(photFile->fpa, inFile->fpa)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to copy FPA for photometry");
+        psFree(view);
+        return false;
+    }
     pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout to photometer
-    if (!photRO) {
-        pmCell *cell = pmFPAviewThisCell(view, photFile->fpa); // Cell to photometer
-        photRO = pmReadoutAlloc(cell); // Output readout: subtraction
-    }
-    photRO->image = psImageCopy(photRO->image, outRO->image, PS_TYPE_F32);
-    if (outRO->variance) {
-        photRO->variance = psImageCopy(photRO->variance, outRO->variance, PS_TYPE_F32);
-    } else {
-        psFree(photRO->variance);
-        photRO->variance = NULL;
-    }
-    if (outRO->mask) {
-        photRO->mask = psImageCopy(photRO->mask, outRO->mask, PS_TYPE_IMAGE_MASK);
-    } else {
-        psFree(photRO->mask);
-        photRO->mask = NULL;
+    if (psMetadataLookup(photRO->analysis, "PSPHOT.SOURCES")) {
+        psMetadataRemoveKey(photRO->analysis, "PSPHOT.SOURCES");
     }
 
-#if 0
-    psMetadataAddPtr(config->files, PS_LIST_TAIL, "PSPHOT.INPUT", PS_DATA_UNKNOWN | PS_META_REPLACE,
-                     "psphot input: view on another pmFPAfile", photFile);
-#endif
+    psMetadataAddPtr(photRO->parent->parent->analysis, PS_LIST_TAIL, "PSPHOT.PSF",
+                     PS_META_REPLACE | PS_DATA_UNKNOWN, "Point-spread function", data->psf);
+
+    psFree(photRO->analysis);
+    photRO->analysis = psMetadataAlloc();
 
     if (!psphotReadoutMinimal(config, view)) {
-        psWarning("Unable to perform photometry on subtracted image.");
-        psErrorStackPrint(stderr, "Error stack from photometry:");
-        psErrorClear();
+        // This is likely a data quality issue
+        // XXX Split into multiple cases using error codes?
+        psErrorStackPrint(stderr, "Unable to perform photometry on image");
+        psWarning("Unable to perform photometry on image --- suspect bad data quality.");
+        ppSubDataQuality(data, psErrorCodeLast(), PPSUB_FILES_PHOT_SUB | PPSUB_FILES_PHOT_INV);
     }
-#if 1
-    photRO->data_exists = true;
-    photRO->parent->data_exists = true;
-    photRO->parent->parent->data_exists = true;
-#endif
 
-    if (stats) {
+    if (data->stats) {
         psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources
-        psMetadataAddS32(stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected",
-                         sources ? sources->n : 0);
-        psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_PHOT", 0, "Time to do photometry",
-                         psTimerClear("PPSUB_PHOT"));
+        bool mdok;
+        int numSources = psMetadataLookupS32(&mdok, data->stats, "NUM_SOURCES"); // Number of sources
+        numSources += sources ? sources->n : 0;
+        psMetadataAddS32(data->stats, PS_LIST_TAIL, "NUM_SOURCES", PS_META_REPLACE,
+                         "Total number of sources detected", numSources);
+        float newTime = psTimerClear("PPSUB_PHOT"); // Time for photometry
+        float oldTime = psMetadataLookupF32(&mdok, data->stats, "TIME_PHOT"); // Previous time for photometry
+        psMetadataAddF32(data->stats, PS_LIST_TAIL, "TIME_PHOT", PS_META_REPLACE, "Time to do photometry",
+                         isfinite(oldTime) ? oldTime + newTime : newTime);
+    }
+
+    if (!data->quality) {
+        if (!psMetadataCopySingle(inRO->analysis, photRO->analysis, "PSPHOT.SOURCES")) {
+            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to copy PSPHOT.SOURCES");
+            return false;
+        }
+        if (!psMetadataCopySingle(inRO->analysis, photRO->analysis, "PSPHOT.HEADER")) {
+            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to copy PSPHOT.HEADER");
+            return false;
+        }
     }
 
Index: anches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutRenorm.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutRenorm.c	(revision 24243)
+++ 	(revision )
@@ -1,65 +1,0 @@
-/** @file ppSubReadoutRenorm.c
- *
- *  @brief
- *
- *  @ingroup ppSub
- *
- *  @author IfA
- *  @version $Revision: 1.3 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2009-02-06 03:20:02 $
- *  Copyright 2009 Institute for Astronomy, University of Hawaii
- */
-
-#include "ppSub.h"
-
-bool ppSubReadoutRenormPixels (pmConfig *config, psMetadata *recipe, pmReadout *readout) {
-
-    bool mdok = false;
-
-    if (!psMetadataLookupBool(&mdok, recipe, "RENORM")) return true;
-
-    // Statistics for renormalisation
-    psStatsOptions renormMean = psStatsOptionFromString(psMetadataLookupStr(&mdok, recipe, "RENORM.MEAN"));
-    psStatsOptions renormStdev = psStatsOptionFromString(psMetadataLookupStr(&mdok, recipe, "RENORM.STDEV"));
-    if (renormMean == PS_STAT_NONE || renormStdev == PS_STAT_NONE) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, false,
-                "Unable to parse renormalisation statistics from recipe.");
-        return false;
-    }
-    psImageMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask
-    if (!pmReadoutVarianceRenormPixels(readout, maskValue, renormMean, renormStdev, NULL)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to renormalise variances.");
-        return false;
-    }
-    return true;
-}
-
-/**
- * old-style variance renormalization
- */
-bool ppSubReadoutRenormPhot (pmConfig *config, psMetadata *recipe, pmReadout *readout) {
-
-    bool mdok = false;
-
-    if (!psMetadataLookupBool(&mdok, recipe, "RENORM")) return true;
-
-    // Statistics for renormalisation
-    psStatsOptions renormMean = psStatsOptionFromString(psMetadataLookupStr(&mdok, recipe, "RENORM.MEAN"));
-    psStatsOptions renormStdev = psStatsOptionFromString(psMetadataLookupStr(&mdok, recipe, "RENORM.STDEV"));
-    if (renormMean == PS_STAT_NONE || renormStdev == PS_STAT_NONE) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, false,
-                "Unable to parse renormalisation statistics from recipe.");
-        return false;
-    }
-
-    // other renorm parameters
-    int renormNum = psMetadataLookupS32(&mdok, recipe, "RENORM.NUM"); // Number of samples for renormalisation
-    float renormWidth = psMetadataLookupS32(&mdok, recipe, "RENORM.WIDTH"); // Width of Gaussian phot
-
-    psImageMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask
-    if (!pmReadoutVarianceRenormPhot(readout, maskValue, renormNum, renormWidth, renormMean, renormStdev, NULL)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to renormalise variances.");
-        return false;
-    }
-    return true;
-}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutStats.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutStats.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutStats.c	(revision 24244)
@@ -0,0 +1,50 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <pslib.h>
+#include <psmodules.h>
+#include <ppStats.h>
+
+#include "ppSub.h"
+
+
+bool ppSubReadoutStats(ppSubData *data)
+{
+    psAssert(data, "Require processing data");
+    pmConfig *config = data->config;    // Configuration
+    psAssert(config, "Require configuration");
+
+    if (!data->statsFile) {
+        // Nothing to do
+        return true;
+    }
+
+    pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSUB.OUTPUT"); // Output file
+    if (!output) {
+        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find file PPSUB.OUTPUT.\n");
+        return false;
+    }
+    psImageMaskType maskValue = pmConfigMaskGet("MASK.VALUE", config);
+    pmFPAview *view = ppSubViewReadout(); // View to readout
+    ppStatsFPA(data->stats, output->fpa, view, maskValue, config);
+
+    pmReadout *outRO = pmFPAviewThisReadout(view, output->fpa); // Readout of interest
+    psFree(view);
+
+    // Statistics on the matching
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MODE);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_MEAN);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_RMS);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_NORM);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_BGDIFF);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MX);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MY);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXX);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXY);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MYY);
+    psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DECONV_MAX);
+
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutSubtract.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutSubtract.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutSubtract.c	(revision 24244)
@@ -21,11 +21,7 @@
 #include "ppSub.h"
 
-#define WCS_TOLERANCE 0.001             // Tolerance for WCS
-
-bool ppSubReadoutSubtract(pmConfig *config, const pmFPAview *view)
+bool ppSubReadoutSubtract(pmConfig *config)
 {
     psAssert(config, "Require configuration");
-    psAssert(view, "Require view");
-
 
     // Look up recipe values
@@ -35,4 +31,6 @@
 
     bool reverse = psMetadataLookupBool(&mdok, config->arguments, "REVERSE"); // Reverse sense of subtraction?
+
+    pmFPAview *view = ppSubViewReadout(); // View to readout
 
     // Subtraction is: minuend - subtrahend
@@ -74,5 +72,5 @@
     covars->data[0] = psMemIncrRefCounter(minuend->covariance);
     covars->data[1] = psMemIncrRefCounter(subtrahend->covariance);
-    outRO->covariance = psImageCovarianceSum(covars);
+    outRO->covariance = psImageCovarianceAverage(covars);
     psFree(covars);
 
@@ -91,4 +89,5 @@
         psError(PS_ERR_UNKNOWN, false, "Unable to copy concepts from input to output.");
         psFree(outRO);
+        psFree(view);
         return false;
     }
@@ -99,4 +98,5 @@
     pmHDU *outHDU = outFPA->hdu;        // Output HDU
     pmChip *outChip = pmFPAfileThisChip(config->files, view, "PPSUB.OUTPUT"); // Output chip
+    psFree(view);
     if (!outHDU || !inHDU) {
         psError(PS_ERR_UNKNOWN, false, "Unable to find HDU at FPA level to copy astrometry.");
Index: anches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutUpdate.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutUpdate.c	(revision 24243)
+++ 	(revision )
@@ -1,107 +1,0 @@
-/** @file ppSubReadoutUpdate.c
- *
- *  @brief
- *
- *  @ingroup ppSub
- *
- *  @author IfA
- *  @version $Revision: 1.4 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2009-02-18 00:31:20 $
- *  Copyright 2009 Institute for Astronomy, University of Hawaii
- */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <stdio.h>
-#include <pslib.h>
-#include <psmodules.h>
-
-#include "ppSub.h"
-
-bool ppSubReadoutUpdate(pmConfig *config, psMetadata *stats, const pmFPAview *view)
-{
-    psAssert(config, "Require configuration");
-    psAssert(view, "Require view");
-
-    bool mdok = false;                  // Status of MD lookup
-
-    // Look up recipe values
-    psMetadata *recipe = psMetadataLookupMetadata(&mdok, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
-    psAssert(recipe, "We checked this earlier, so it should be here.");
-
-    pmFPAfile *outFile = psMetadataLookupPtr(&mdok, config->files, "PPSUB.OUTPUT"); // Output file
-    pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); // Output image
-    pmFPA *outFPA = outFile->fpa;       // Output FPA
-    pmHDU *outHDU = outFPA->hdu;        // Output HDU
-
-    // Add additional data to the header
-    pmFPAfile *refFile = psMetadataLookupPtr(NULL, config->files, "PPSUB.REF"); // Reference file
-    pmFPAfile *inFile = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); // Input file
-    psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.REFERENCE", 0,
-                     "Subtraction reference", refFile->filename);
-    psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.INPUT", 0,
-                     "Subtraction input", inFile->filename);
-    ppSubVersionHeader(outHDU->header);
-
-    // Statistics on the matching
-    if (stats) {
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MODE);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_MEAN);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_RMS);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_NORM);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_BGDIFF);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MX);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MY);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXX);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXY);
-        psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MYY);
-
-        psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_MATCH", 0, "Time to match PSFs",
-                         psTimerClear("PPSUB_MATCH"));
-    }
-
-    // Generate binned JPEGs
-    {
-        psImageMaskType maskBad = pmConfigMaskGet("BLANK", config); // Bits to mask for bad pixels
-
-        int bin1 = psMetadataLookupS32(NULL, recipe, "BIN1"); // First binning level
-        int bin2 = psMetadataLookupS32(NULL, recipe, "BIN2"); // Second binning level
-
-        // Target cells
-        pmCell *cell1 = pmFPAfileThisCell(config->files, view, "PPSUB.OUTPUT.JPEG1"); // Rebinned cell once
-        pmCell *cell2 = pmFPAfileThisCell(config->files, view, "PPSUB.OUTPUT.JPEG2"); // Rebinned cell twice
-
-        pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts
-        if (!pmReadoutRebin(ro1, outRO, maskBad, bin1, bin1)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to bin output (1st binning)");
-            psFree(ro1);
-            psFree(ro2);
-            return false;
-        }
-        if (!pmReadoutRebin(ro2, ro1, 0, bin2, bin2)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to bin output (2nd binning)");
-            psFree(ro1);
-            psFree(ro2);
-            return false;
-        }
-        psFree(ro1);
-        psFree(ro2);
-    }
-
-#ifdef TESTING
-    // Significance image
-    {
-        psImage *sig = (psImage*)psBinaryOp(NULL, outRO->image, "*", outRO->image);
-        psBinaryOp(sig, sig, "/", outRO->variance);
-        psFits *fits = psFitsOpen("significance.fits", "w");
-        psFitsWriteImage(fits, NULL, sig, 0, NULL);
-        psFitsClose(fits);
-        psFree(sig);
-    }
-#endif
-
-    return true;
-}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubSetMasks.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubSetMasks.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubSetMasks.c	(revision 24244)
@@ -22,8 +22,7 @@
 #include "ppSub.h"
 
-bool ppSubSetMasks(pmConfig *config, const pmFPAview *view)
+bool ppSubSetMasks(pmConfig *config)
 {
     psAssert(config, "Require configuration");
-    psAssert(view, "Require view");
 
     psImageMaskType maskValue, markValue; // Mask values
@@ -50,5 +49,6 @@
     psAssert(lowValue, "LOW or BAD must be non-zero");
 
-    // input images
+    // Input images
+    pmFPAview *view = ppSubViewReadout(); // View to readout
     pmReadout *inRO = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT"); // Input readout
     pmReadout *refRO = pmFPAfileThisReadout(config->files, view, "PPSUB.REF"); // Reference readout
Index: anches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVarianceFactors.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVarianceFactors.c	(revision 24243)
+++ 	(revision )
@@ -1,99 +1,0 @@
-/** @file ppSubVarianceFactors.c
- *
- *  @brief
- *
- *  @ingroup ppSub
- *
- *  @author IfA
- *  @version $Revision: 1.4 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2009-02-18 00:31:20 $
- *  Copyright 2009 Institute for Astronomy, University of Hawaii
- */
-
-#include "ppSub.h"
-
-/**
- * Calculate the variance factor for the output image based on the input images
- */
-bool ppSubVarianceFactors (pmConfig *config, psMetadata *stats, const pmFPAview *view) {
-
-    // Look up recipe values
-    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim
-    psAssert(recipe, "We checked this earlier, so it should be here.");
-
-    // convolved input images
-    pmCell *inCell = pmFPAfileThisCell(config->files, view, "PPSUB.INPUT.CONV"); // Input cell
-    pmCell *refCell = pmFPAfileThisCell(config->files, view, "PPSUB.REF.CONV"); // Reference cell
-
-    pmReadout *inConv = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT.CONV"); // Input cell
-    pmReadout *refConv = pmFPAfileThisReadout(config->files, view, "PPSUB.REF.CONV"); // Reference cell
-
-    pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT"); // Output readout
-
-    // Update variance factors
-    // It's not possible to do this perfectly, since we're combining different images:
-    // vf_sum sigma_sum^2 = vf_1 * sigma_1^2 + vf_2 * sigma_2^2
-    // It's not possible to write vf_sum as a function of vf_1 and vf_2 with no reference to the sigmas.
-    // Instead, we're going to cheat.
-
-    bool mdok;                      // Status of MD lookup
-    pmSubtractionKernels *kernels = psMetadataLookupPtr(&mdok, outRO->analysis, PM_SUBTRACTION_ANALYSIS_KERNEL); // Kernels
-    if (!mdok) {
-        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find subtraction kernels.");
-        return false;
-    }
-
-    float vfIn = psMetadataLookupF32(NULL, inCell->concepts, "CELL.VARFACTOR"); // Variance factor for input
-    if (!isfinite(vfIn)) {
-        vfIn = 1.0;
-    }
-    float vfRef = psMetadataLookupF32(NULL, refCell->concepts, "CELL.VARFACTOR"); // Variance factor for ref
-    if (!isfinite(vfRef)) {
-        vfRef = 1.0;
-    }
-
-    if (kernels->mode == PM_SUBTRACTION_MODE_1) {
-        vfIn *= pmSubtractionVarianceFactor(kernels, 0.0, 0.0, false);
-    }
-    if (kernels->mode == PM_SUBTRACTION_MODE_2) {
-        vfRef *= pmSubtractionVarianceFactor(kernels, 0.0, 0.0, false);
-    }
-    if (kernels->mode == PM_SUBTRACTION_MODE_DUAL) {
-        vfIn *= pmSubtractionVarianceFactor(kernels, 0.0, 0.0, false);
-        vfRef *= pmSubtractionVarianceFactor(kernels, 0.0, 0.0, true);
-    }
-
-    psStats *vfStats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); // Statistics
-    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator
-
-    // mask these values when examining the background
-    psImageMaskType maskVal = pmConfigMaskGet("MASK.VALUE", config); // Bits to mask going in to pmSubtractionMatch
-    psImageMaskType maskBad = pmConfigMaskGet("BLANK", config); // Bits to mask for bad pixels
-
-    // measure the mean of the convolved input variance image
-    if (!psImageBackground(vfStats, NULL, inConv->variance, inConv->mask, maskVal | maskBad, rng)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to measure mean variance for convolved input");
-        psFree(vfStats);
-        psFree(rng);
-        return false;
-    }
-    float inMeanVar = vfStats->robustMedian; // Mean variance of input
-
-    // measure the mean of the convolved reference variance image
-    if (!psImageBackground(vfStats, NULL, refConv->variance, refConv->mask, maskVal | maskBad, rng)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to measure mean variance for convolved reference");
-        psFree(vfStats);
-        psFree(rng);
-        return false;
-    }
-    float refMeanVar = vfStats->robustMedian; // Mean variance of reference
-
-    psFree(rng);
-    psFree(vfStats);
-
-    float vf = (vfIn * inMeanVar + vfRef * refMeanVar) / (inMeanVar + refMeanVar); // Resulting var factor
-    psMetadataItem *vfItem = psMetadataLookup(outRO->parent->concepts, "CELL.VARFACTOR");
-    vfItem->data.F32 = vf;
-
-    return true;
-}
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVersion.c	(revision 24244)
@@ -22,4 +22,5 @@
 
 #include "ppSub.h"
+#include "ppSubVersionDefinitions.h"
 
 #ifndef PPSUB_VERSION
@@ -33,11 +34,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString ppSubVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PPSUB_BRANCH), xstr(PPSUB_VERSION));
+    psStringAppend(&value, "%s@%s", PPSUB_BRANCH, PPSUB_VERSION);
     return value;
 }
@@ -45,5 +43,5 @@
 psString ppSubSource(void)
 {
-    return psStringCopy(xstr(PPSUB_SOURCE));
+    return psStringCopy(PPSUB_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PPSUB_VERSION_DEFINITIONS_H
+#define PPSUB_VERSION_DEFINITIONS_H
+
+#define PPSUB_VERSION @PPSUB_VERSION@ // SVN version
+#define PPSUB_BRANCH  @PPSUB_BRANCH@  // SVN branch
+#define PPSUB_SOURCE  @PPSUB_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/autogen.sh
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/autogen.sh	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/autogen.sh	(revision 24244)
@@ -19,5 +19,5 @@
 fi
 
-ACLOCAL="aclocal $ACLOCAL_FLAGS"
+ACLOCAL="aclocal -I `pwd`/m4 $ACLOCAL_FLAGS"
 AUTOHEADER=autoheader
 AUTOMAKE=automake
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/configure.ac	(revision 24244)
@@ -6,5 +6,6 @@
 
 dnl this enables the building of libtap
-dnl AC_CONFIG_SUBDIRS([test/tap])
+AC_CONFIG_SUBDIRS([test/tap])
+dnl AC_CONFIG_SUBDIRS([test/pstap])
 
 AM_INIT_AUTOMAKE([1.7 foreign dist-bzip2])
@@ -39,6 +40,6 @@
     AC_SUBST(PERL,$PERL)
 
-dnl SUBDIR="etc src share test utils" dnl don't include 'swig', as it is optional
-SUBDIR="etc src share utils" dnl don't include 'swig', as it is optional
+SUBDIR="etc src share test utils" dnl don't include 'swig', as it is optional
+dnl SUBDIR="etc src share utils" dnl don't include 'swig', as it is optional
 
 SRCPATH='${top_srcdir}/src'
@@ -340,4 +341,6 @@
 LDFLAGS=${TMP_LDFLAGS}
 CPPFLAGS=${TMP_CPPFLAGS}
+
+IPP_VERSION
 
 dnl ------- enable -Werror after all of the probes have run ---------
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/m4/ipp_stdopts.m4
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/m4/ipp_stdopts.m4	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/m4/ipp_stdopts.m4	(revision 24244)
@@ -37,2 +37,23 @@
     )
 ])
+
+
+AC_DEFUN([IPP_VERSION],
+[
+	AC_ARG_ENABLE(version,
+		[AS_HELP_STRING(--disable-version,Disable dynamic version information)],
+		[case "${enableval}" in
+		      yes) enable_version=true ;;
+		      no)  enable_version=false ;;
+		      *)   AC_MSG_ERROR(bad value ${enableval} for --disable-version) ;;
+	         esac], [enable_version=true]
+	)
+	
+	AS_IF([test "x$enable_version" = xtrue],
+		[AC_PATH_PROG([SVNVERSION], [svnversion])
+		AC_PATH_PROG([SVN], [svn])]
+	)
+	AC_PROG_SED
+	AM_CONDITIONAL([HAVE_SVNVERSION], [test "x$SVNVERSION" != x])
+	AM_CONDITIONAL([HAVE_SVN], [test "x$SVN" != x])
+])
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/db/psDB.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/db/psDB.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/db/psDB.c	(revision 24244)
@@ -2157,5 +2157,5 @@
 
             // scan through the list and change the logicalOp joining
-            // conditionals together to "OR" if a comment of "==" is found
+            // conditionals together to "OR" if a comment of "==" or "LIKE" is found
             psListIterator *mCursor = psListIteratorAlloc(item->data.list, 0,
                                       false);
@@ -2163,4 +2163,8 @@
             while ((mItem = psListGetAndIncrement(mCursor))) {
                 if (mItem->comment && psStrcasestr(mItem->comment, "==")) {
+                    logicalOp = "OR";
+                    break;
+                }
+                if (mItem->comment && psStrcasestr(mItem->comment, "LIKE")) {
                     logicalOp = "OR";
                     break;
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageCovariance.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageCovariance.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageCovariance.c	(revision 24244)
@@ -130,4 +130,81 @@
 }
 
+psKernel *psImageCovarianceBin(int bin, const psKernel *covariance, bool average)
+{
+    psKernel *covar;                    // Covariance matrix to use
+    if (covariance) {
+        covar = psMemIncrRefCounter((psKernel*)covariance); // Casting away const
+    } else {
+        covar = psImageCovarianceNone();
+    }
+
+    // Check for non-finite elements
+    for (int y = covar->yMin; y <= covar->yMax; y++) {
+        for (int x = covar->xMin; x <= covar->xMax; x++) {
+            if (!isfinite(covar->kernel[y][x])) {
+                psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                        "Non-finite covariance matrix element in covariance matrix at %d,%d", x, y);
+                psFree(covar);
+                return NULL;
+            }
+        }
+    }
+
+    // The following calculation is the same as for psImageCovarianceCalculate(), except that the "convolution
+    // kernel" has a constant value, and the output covariance kernel is binned down, so we unbin it in the
+    // ranges for the inputs.
+
+    // Values for "convolution kernel"
+    int binMin = -(bin - 1) / 2, binMax = bin / 2; // Range of "kernel"
+    int binDiff = binMax - binMin;      // Difference between the max and min
+    float binVal = average ? 1.0 / PS_SQR(bin) : 1.0; // Value of "kernel" pixels
+
+    // Size of the output covariance matrix
+    int xMin = (covar->xMin - binDiff) / bin + 0.5, xMax = (covar->xMax + binDiff) / bin + 0.5;
+    int yMin = (covar->yMin - binDiff) / bin + 0.5, yMax = (covar->yMax + binDiff) / bin + 0.5;
+    psKernel *out = psKernelAlloc(xMin, xMax, yMin, yMax); // Covariance matrix for output
+
+    double total = 0.0;                 // Total covariance
+    for (int y = yMin; y <= yMax; y++) {
+        // Range for v
+        int vMin = PS_MAX(binMin + covar->yMin, bin * y + binMin);
+        int vMax = PS_MIN(binMax + covar->yMax, bin * y + binMax);
+        for (int x = xMin; x <= xMax; x++) {
+            // Range for u
+            int uMin = PS_MAX(binMin + covar->xMin, bin * x + binMin);
+            int uMax = PS_MIN(binMax + covar->xMax, bin * x + binMax);
+
+            double sum = 0.0;           // Sum for value of covariance matrix at (x,y)
+            for (int v = vMin; v <= vMax; v++) {
+                // Range for q
+                int qMin = PS_MAX(v + covar->yMin, binMin);
+                int qMax = PS_MIN(v + covar->yMax, binMax);
+                for (int u = uMin; u <= uMax; u++) {
+                    // Range for p
+                    int pMin = PS_MAX(u + covar->xMin, binMin);
+                    int pMax = PS_MIN(u + covar->xMax, binMax);
+
+                    double xyuvValue = binVal; // Value for (x,y) --> (u,v)
+
+                    double uvpqValue = 0.0; // Value for (u,v) --> (p,q) --> (0,0)
+                    for (int q = qMin; q <= qMax; q++) {
+                        for (int p = pMin; p <= pMax; p++) {
+                            uvpqValue += (double)covar->kernel[q-v][p-u] * (double)binVal;
+                        }
+                    }
+                    sum += xyuvValue * uvpqValue;
+                }
+            }
+            out->kernel[y][x] = sum;
+            total += sum;
+        }
+    }
+    psTrace("psLib.imageops", 3, "Total covariance: %lf ; Central variance: %f\n", total, out->kernel[0][0]);
+
+    psFree(covar);
+
+    return out;
+}
+
 float psImageCovarianceFactor(const psKernel *covariance)
 {
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageCovariance.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageCovariance.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageCovariance.h	(revision 24244)
@@ -35,4 +35,11 @@
     );
 
+/// Calculate the covariance pseudo-matrix for binning
+psKernel *psImageCovarianceBin(
+    int bin,                            ///< Binning factor
+    const psKernel *covariance,         ///< Current covariance pseudo-matrix
+    bool average                        ///< Averaging pixels when binning?
+    );
+
 /// Return the pixel-to-pixel covariance factor
 float psImageCovarianceFactor(
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageGeomManip.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageGeomManip.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageGeomManip.c	(revision 24244)
@@ -110,38 +110,43 @@
     out = psImageRecycle(out, outCols, outRows, in->type.type);
 
-    #define PS_IMAGE_REBIN_CASE(TYPE) \
-case PS_TYPE_##TYPE: { \
-        ps##TYPE *outRowData; \
-        ps##TYPE *vecData = vec->data.TYPE; \
-        psImageMaskType *inRowMask = NULL; \
-        for (psS32 row = 0; row < outRows; row++) { \
-            outRowData = out->data.TYPE[row]; \
-            psS32 inCurrentRow = row * scale; \
-            psS32 inNextRow = (row + 1) * scale; \
-            for (psS32 col = 0; col < outCols; col++) { \
-                psS32 inCurrentCol = col * scale; \
-                psS32 inNextCol = (col + 1) * scale; \
-                psS32 n = 0; \
+#define PS_IMAGE_REBIN_CASE(TYPE)					\
+    case PS_TYPE_##TYPE: {						\
+        ps##TYPE *outRowData;						\
+        ps##TYPE *vecData = vec->data.TYPE;				\
+        psImageMaskType *inRowMask = NULL;				\
+        for (psS32 row = 0; row < outRows; row++) {			\
+            outRowData = out->data.TYPE[row];				\
+            psS32 inCurrentRow = row * scale;				\
+            psS32 inNextRow = (row + 1) * scale;			\
+            for (psS32 col = 0; col < outCols; col++) {			\
+                psS32 inCurrentCol = col * scale;			\
+                psS32 inNextCol = (col + 1) * scale;			\
+                psS32 n = 0;						\
                 for (psS32 inRow = inCurrentRow; inRow < inNextRow && inRow < inRows; inRow++) { \
-                    ps##TYPE* inRowData = in->data.TYPE[inRow]; \
-                    if (mask != NULL) { \
+                    ps##TYPE* inRowData = in->data.TYPE[inRow];		\
+                    if (mask != NULL) {					\
                         inRowMask = mask->data.PS_TYPE_IMAGE_MASK_DATA[inRow]; \
-                    } \
+                    }							\
                     for (psS32 inCol = inCurrentCol; inCol < inNextCol && inCol < inCols; inCol++) { \
-                        if (maskData != NULL) { \
+                        if (maskData != NULL) {				\
                             maskData[n] = (inRowMask[inCol] & maskVal); \
-                        } \
-                        vecData[n++] = inRowData[inCol]; \
-                    } \
-                } \
-                vec->n = n; \
-                if (maskVec) { \
-                    maskVec->n = n; \
-                } \
-                 psVectorStats(myStats, vec, NULL, maskVec, 0xff); /* the mask vector has only 0 or 1 */ \
-                outRowData[col] = (ps##TYPE)psStatsGetValue(myStats, statistic); \
-            } \
-        } \
-    } \
+                        }						\
+                        vecData[n++] = inRowData[inCol];		\
+                    }							\
+                }							\
+                vec->n = n;						\
+                if (maskVec) {						\
+                    maskVec->n = n;					\
+                }							\
+		if (!psVectorStats(myStats, vec, NULL, maskVec, 0xff)) { /* the mask vector has only 0 or 1 */ \
+		    psError(PS_ERR_UNKNOWN, false, "failure to measure stats"); \
+		    psFree(out);					\
+		    out = NULL;						\
+		    goto escape;					\
+		}							\
+		outRowData[col] = (ps##TYPE)psStatsGetValue(myStats, statistic); \
+	    }								\
+	}								\
+    }									\
     break;
 
@@ -161,7 +166,5 @@
             char* typeStr;
             PS_TYPE_NAME(typeStr,in->type.type);
-            psError(PS_ERR_BAD_PARAMETER_TYPE, true,
-                    _("Specified psImage type, %s, is not supported."),
-                    typeStr);
+            psError(PS_ERR_BAD_PARAMETER_TYPE, true, _("Specified psImage type, %s, is not supported."), typeStr);
             psFree(out);
             out = NULL;
@@ -169,4 +172,5 @@
     }
 
+escape:
     psFree(vec);
     psFree(maskVec);
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageMap.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageMap.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageMap.c	(revision 24244)
@@ -211,18 +211,30 @@
             // XXX need to supply a mask and skip the masked pixels when calculating the centroid
             // this will not in general be properly weighted...
-            if (psVectorStats (map->stats, fCell, dfCell, NULL, 0)) {
+            if (!psVectorStats (map->stats, fCell, dfCell, NULL, 0)) {
+		psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		return false;
+	    }
+
+	    // XXX ensure only one option is selected, or save both position and width
+	    map->map->data.F32[iy][ix] = psStatsGetValue (map->stats, map->stats->options);
+
+	    if (isnan(map->map->data.F32[iy][ix])) {
+                mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] = 1;
+	    } else {
                 mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] = 0;
-                // XXX ensure only one option is selected, or save both position and width
-                map->map->data.F32[iy][ix] = psStatsGetValue (map->stats, map->stats->options);
 
                 // calculate the mean position and save:
                 psStatsInit (meanStat);
-                psVectorStats (meanStat, xCell, NULL, NULL, 0);
+                if (!psVectorStats (meanStat, xCell, NULL, NULL, 0)) {
+		    psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		    return false;
+		}
                 xCoord->data.F32[iy][ix] = psStatsGetValue (meanStat, meanStat->options);
                 psStatsInit (meanStat);
-                psVectorStats (meanStat, yCell, NULL, NULL, 0);
+                if (!psVectorStats (meanStat, yCell, NULL, NULL, 0)) {
+		    psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		    return false;
+		}
                 yCoord->data.F32[iy][ix] = psStatsGetValue (meanStat, meanStat->options);
-            } else {
-                mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] = 1;
             }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageMapFit.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageMapFit.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageMapFit.c	(revision 24244)
@@ -73,5 +73,8 @@
 
         // XXX does ROBUST_MEDIAN work with weight?
-        psVectorStats(map->stats, f, NULL, mask, maskValue);
+        if (!psVectorStats(map->stats, f, NULL, mask, maskValue)) {
+	    psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	    return false;
+	}
 
         map->map->data.F32[0][0]   = psStatsGetValue(map->stats, mean);
@@ -305,6 +308,5 @@
 # endif
 
-    if (!psMatrixGJSolveF32(A, B)) {
-        psAbort ("failed on linear equations");
+    if (!psMatrixGJSolve(A, B)) {
         psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.  Returning NULL.\n");
         psFree (A);
@@ -574,9 +576,9 @@
     }
 
-    if (!psMatrixGJSolveF32(A, B)) {
-        psAbort ("failed on linear equations");
-        psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.  Returning NULL.\n");
+    if (!psMatrixGJSolve(A, B)) {
+        psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.\n");
         psFree (A);
         psFree (B);
+	psFree (Empty);
         return false;
     }
@@ -722,9 +724,9 @@
     }
 
-    if (!psMatrixGJSolveF32(A, B)) {
-        psAbort ("failed on linear equations");
-        psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.  Returning NULL.\n");
+    if (!psMatrixGJSolve(A, B)) {
+        psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations..\n");
         psFree (A);
         psFree (B);
+	psFree (Empty);
         return false;
     }
@@ -739,7 +741,6 @@
 
     for (int m = 0; m < Nx; m++) {
-        int I = m; // XXX I'm not entirely sure about this; it wasn't set for this scope --- PAP.
-        map->map->data.F32[0][m] = B->data.F32[I];
-        map->error->data.F32[0][m] = sqrt(A->data.F32[I][I]);
+        map->map->data.F32[0][m] = B->data.F32[m];
+        map->error->data.F32[0][m] = sqrt(A->data.F32[m][m]);
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImagePixelExtract.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImagePixelExtract.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImagePixelExtract.c	(revision 24244)
@@ -419,35 +419,40 @@
         }
 
-#define PSIMAGE_CUT_VERTICAL(TYPE) \
-    case PS_TYPE_##TYPE: { \
-            psVectorMaskType* maskVecData = NULL; \
-            for (psS32 c = col0; c < col1; c++) { \
-                ps##TYPE *imgData = input->data.TYPE[row0] + c; \
-                ps##TYPE *imgVecData = imgVec->data.TYPE; \
-                if (maskVec != NULL) { \
+#define PSIMAGE_CUT_VERTICAL(TYPE)					\
+	case PS_TYPE_##TYPE: {						\
+            psVectorMaskType* maskVecData = NULL;			\
+            for (psS32 c = col0; c < col1; c++) {			\
+                ps##TYPE *imgData = input->data.TYPE[row0] + c;		\
+                ps##TYPE *imgVecData = imgVec->data.TYPE;		\
+                if (maskVec != NULL) {					\
                     maskVecData = maskVec->data.PS_TYPE_VECTOR_MASK_DATA; \
                     maskData = &mask->data.PS_TYPE_IMAGE_MASK_DATA[row0][c]; /* XXX double check this... */ \
                     /** old entry: maskData = (psMaskType* )(mask->data.PS_TYPE_IMAGE_MASK_DATA[row0]) + c; */ \
-                } \
-                for (psS32 r = row0; r < row1; r++) { \
-                   *imgVecData = *imgData; \
-                    imgVecData ++; \
-                    imgData += inCols; \
-                    if (maskVecData != NULL) { \
-                        *maskVecData = (*maskData & maskVal); \
-                        maskVecData ++; \
-                        maskData += inCols; \
-                    } \
-                } \
-                psVectorStats(myStats, imgVec, NULL, maskVec, 0xff); \
-                *outData = psStatsGetValue(myStats, statistic); \
-                if (outPosition != NULL) { \
-                    outPosition->x = c; \
-                    outPosition->y = row0; \
-                    outPosition += delta; \
-                } \
-                outData += delta; \
-            } \
-            break; \
+                }							\
+                for (psS32 r = row0; r < row1; r++) {			\
+		    *imgVecData = *imgData;				\
+                    imgVecData ++;					\
+                    imgData += inCols;					\
+                    if (maskVecData != NULL) {				\
+                        *maskVecData = (*maskData & maskVal);		\
+                        maskVecData ++;					\
+                        maskData += inCols;				\
+                    }							\
+                }							\
+                if (!psVectorStats(myStats, imgVec, NULL, maskVec, 0xff)) { \
+		    psError(PS_ERR_UNKNOWN, false, "failure to measure stats"); \
+		    psFree(out);					\
+		    out = NULL;						\
+		    break;						\
+		}							\
+                *outData = psStatsGetValue(myStats, statistic);		\
+                if (outPosition != NULL) {				\
+                    outPosition->x = c;					\
+                    outPosition->y = row0;				\
+                    outPosition += delta;				\
+                }							\
+                outData += delta;					\
+            }								\
+            break;							\
         }
 
@@ -466,7 +471,5 @@
                 char* typeStr;
                 PS_TYPE_NAME(typeStr,type);
-                psError(PS_ERR_BAD_PARAMETER_TYPE, true,
-                        _("Specified psImage type, %s, is not supported."),
-                        typeStr);
+                psError(PS_ERR_BAD_PARAMETER_TYPE, true, _("Specified psImage type, %s, is not supported."), typeStr);
                 psFree(out);
                 out = NULL;
@@ -534,5 +537,10 @@
 	    }
 
-            psVectorStats(myStats, imgVec, NULL, maskVec, 0xff);
+            if (!psVectorStats(myStats, imgVec, NULL, maskVec, 0xff)) {
+		psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		psFree (out);
+		out = NULL;
+		break;
+	    }
             *outData = psStatsGetValue(myStats, statistic);
             if (outPosition != NULL) {
@@ -933,5 +941,10 @@
 
     for (psS32 r = 0; r < numOut; r++) {
-        psVectorStats(myStats, buffer[r], NULL, bufferMask[r], 0xff);
+        if (!psVectorStats(myStats, buffer[r], NULL, bufferMask[r], 0xff)){
+	    psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	    psFree(out);
+	    out = NULL;
+	    break;
+	}
         outData[r] = psStatsGetValue(myStats, statistic);
     }
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageStats.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageStats.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageStats.c	(revision 24244)
@@ -104,5 +104,9 @@
     }
 
-    psVectorStats(stats, junkData, NULL, junkMask, 0xff);
+    if (!psVectorStats(stats, junkData, NULL, junkMask, 0xff)) {
+	psFree(junkMask);
+	psFree(junkData);
+	return false;
+    }
 
     psFree(junkMask);
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/jpeg/psImageJpeg.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/jpeg/psImageJpeg.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/jpeg/psImageJpeg.c	(revision 24244)
@@ -202,10 +202,13 @@
             if (isfinite(row[i])) {
                 pixel = PS_JPEG_SCALEVALUE(row[i],zero,scale);
+		outPix[0] = Rpix[pixel];
+		outPix[1] = Gpix[pixel];
+		outPix[2] = Bpix[pixel];
             } else {
-                pixel = 0;
-            }
-            outPix[0] = Rpix[pixel];
-            outPix[1] = Gpix[pixel];
-            outPix[2] = Bpix[pixel];
+		// XXX NAN value should be set per-color map
+		outPix[0] = 0xff;
+		outPix[1] = 0x00;
+		outPix[2] = 0xff;
+	    }
         }
         jpeg_write_scanlines(&cinfo, jpegLineList, 1);
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMatrix.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMatrix.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMatrix.c	(revision 24244)
@@ -199,5 +199,5 @@
 /*****************************************************************************/
 
-psImage* psMatrixLUD(psImage* out,
+psImage* psMatrixLUDecomposition(psImage* out,
                      psVector** perm,
                      const psImage* in)
@@ -210,16 +210,16 @@
 
 
-    #define psMatrixLUD_EXIT {psFree(out); return NULL;}
+    #define psMatrixLUDecomposition_EXIT {psFree(out); return NULL;}
 
     // Error checks
-    PS_ASSERT_GENERAL_IMAGE_NON_NULL(in, psMatrixLUD_EXIT);
-    PS_CHECK_POINTERS(in, out, psMatrixLUD_EXIT);
-    PS_CHECK_DIMEN_AND_TYPE(in, PS_DIMEN_IMAGE, psMatrixLUD_EXIT);
-    PS_ASSERT_GENERAL_PTR_NON_NULL(perm, psMatrixLUD_EXIT);
+    PS_ASSERT_GENERAL_IMAGE_NON_NULL(in, psMatrixLUDecomposition_EXIT);
+    PS_CHECK_POINTERS(in, out, psMatrixLUDecomposition_EXIT);
+    PS_CHECK_DIMEN_AND_TYPE(in, PS_DIMEN_IMAGE, psMatrixLUDecomposition_EXIT);
+    PS_ASSERT_GENERAL_PTR_NON_NULL(perm, psMatrixLUDecomposition_EXIT);
 
     out = psImageRecycle(out, in->numCols, in->numRows, in->type.type);
 
-    PS_CHECK_SQUARE(in, psMatrixLUD_EXIT); // gsl_linalg_LU_decomp would fail on non-square input.
-    PS_CHECK_SQUARE(out, psMatrixLUD_EXIT);
+    PS_CHECK_SQUARE(in, psMatrixLUDecomposition_EXIT); // gsl_linalg_LU_decomp would fail on non-square input.
+    PS_CHECK_SQUARE(out, psMatrixLUDecomposition_EXIT);
 
     // Initialize data
@@ -237,5 +237,5 @@
                 "Failed to allocate the permutation vector; "
                 "could not determine the cooresponding data type.");
-        psMatrixLUD_EXIT;
+        psMatrixLUDecomposition_EXIT;
     }
 
@@ -259,5 +259,5 @@
 }
 
-psVector* psMatrixLUSolve(psVector* out,
+psVector* psMatrixLUSolution(psVector* out,
                           const psImage* LU,
                           const psVector* RHS,
@@ -316,7 +316,54 @@
 }
 
-// This used to be "a temporary gauss-jordan solver based on gene's version based on the Numerical Recipes
-// version".  However, it's been removed due to copyright, and replaced with LU Decomposition solving.
-bool psMatrixGJSolve(psImage *a,
+psImage *psMatrixLUInvert(psImage *out,
+                          const psImage* LU,
+                          const psVector* perm)
+{
+    psS32 numRows = 0;
+    psS32 numCols = 0;
+    gsl_matrix *lu, *inverse;
+    gsl_permutation permGSL;
+
+    #define LUSOLVE_CLEANUP {psFree(out); return NULL;}
+
+    // Error checks
+    PS_ASSERT_GENERAL_IMAGE_NON_NULL(LU, LUSOLVE_CLEANUP);
+    PS_CHECK_DIMEN_AND_TYPE(LU, PS_DIMEN_IMAGE, LUSOLVE_CLEANUP);
+    PS_ASSERT_GENERAL_IMAGE_NON_EMPTY(LU, LUSOLVE_CLEANUP);
+    PS_ASSERT_GENERAL_VECTOR_NON_NULL(perm, LUSOLVE_CLEANUP);
+
+    out = psImageRecycle(out, LU->numCols, LU->numRows, LU->type.type);
+
+    // Initialize data
+    numRows = LU->numRows;
+    numCols = LU->numCols;
+
+    // Initialize GSL data
+    lu = gsl_matrix_alloc(numRows, numCols);
+    psImageToGslMatrix(lu, LU);
+
+    permGSL.size = perm->n;
+    permGSL.data = (psPtr)(perm->data.U8);
+
+    inverse = gsl_matrix_alloc(numRows, numCols);
+
+    // Solve for {x} in equation: {b} = [A]{x}
+    gsl_linalg_LU_invert(lu, &permGSL, inverse);
+
+    // Copy GSL vector data to psVector data
+    gslMatrixToPsImage(out, inverse);
+
+    // Free GSL data
+    gsl_matrix_free(lu);
+    gsl_matrix_free(inverse);
+
+    return out;
+}
+
+// This is the LU Decomposition version of the matrix equation solver.  It solves the equation
+// Ax = B, where A is a square matrix (NxN) and B is a vector of length N.  This solver only
+// yields the solution for x, which is returned to the vector B.  This now DOES calculate the
+// inverse of A.  A and B may be F32 or F64  XXX can they differ?
+bool psMatrixLUSolve(psImage *a,
                      psVector *b
                     )
@@ -328,25 +375,4 @@
     PS_ASSERT_INT_EQUAL(a->numCols, a->numRows, false);
     PS_ASSERT_VECTOR_SIZE(b, (long int)a->numCols, false);
-
-# if (0)
-    // XXX expand this out explicitly below
-    // Check for non-finite entries
-#define MATRIX_CHECK_NONFINITE_CASE(TYPE,MATRIX) \
-  case PS_TYPE_##TYPE: { \
-      ps##TYPE **values = MATRIX->data.TYPE; /* Dereference */ \
-      int numCols = (MATRIX)->numCols, numRows = (MATRIX)->numRows; /* Size of matrix */ \
-      for (int i = 0; i < numRows; i++) { \
-          for (int j = 0; j < numCols; j++) { \
-              if (!isfinite(values[i][j])) { \
-                  // psError(PS_ERR_BAD_PARAMETER_VALUE, 3,		
-		  // "Input matrix contains non-finite elements: matrix[%d][%d] is %.2f\n", 
-		  // i, j, values[i][j]);				
-                  return false; \
-              } \
-          } \
-      } \
-      break; \
-  }
-# endif
 
     // Check for non-finite entries in matrix
@@ -390,5 +416,5 @@
     // Decompose the matrix and solve
     psVector *perm = NULL;              // Permutation vector
-    psImage *lu = psMatrixLUD(NULL, &perm, a); // LU decomposed matrix
+    psImage *lu = psMatrixLUDecomposition(NULL, &perm, a); // LU decomposed matrix
     if (!lu) {
         psError(PS_ERR_UNKNOWN, false, "Unable to generate LU decomposed matrix");
@@ -396,5 +422,9 @@
         return false;
     }
-    psVector *ans = psMatrixLUSolve(NULL, lu, b, perm); // Answer
+    psVector *ans = psMatrixLUSolution(NULL, lu, b, perm); // Answer
+
+    // invert the matrix : check here for an ill-conditioned matrix?
+    psMatrixLUInvert (a, lu, perm);
+
     psFree(lu);
     psFree(perm);
@@ -410,4 +440,233 @@
 }
 
+// This is the Gauss-Jordan elimination version of the matrix equation solver.  It solves the
+// equation Ax = B, where A is a square matrix (NxN) and B is a vector of length N.  This
+// solver calculates both the solution for x, which is returned to the vector B and the inverse
+// of A (returned in A).  A and B may be F32 or F64, but must match.
+
+// Gauss-Jordan elimination using full pivots based on William Kahan's BASIC example and Press
+// et al's description.  Substantially reworked for psLib style: major modifications to conform
+// to C indexing, use a boolean to track the completed pivot rows and catch the singular matrix
+// early on.  Also, much cleaner control loops than the Press implementation.
+
+// (based on version by William Kahan -- see Ohana/src/libohana/doc/kahan-gji.pdf)
+
+# define MAX_RANGE 1.0e7
+// MAX_RANGE is used to test for ill-conditioned input matrices.  For an ill-conditioned
+// matrix, one or more of the pivots trends towards zero, and growth goes to infinity.  Rather
+// than allow this to go to the numerical precision, I am raising an error if |growth| > MAX_RANGE
+bool psMatrixGJSolve(psImage *a,
+                     psVector *b
+    )
+{
+    PS_ASSERT_IMAGE_NON_NULL(a, false);
+    PS_ASSERT_VECTOR_NON_NULL(b, false);
+    PS_ASSERT_IMAGE_TYPE_F32_OR_F64(a, false);
+    PS_ASSERT_VECTOR_TYPE_F32_OR_F64(b, false);
+    psAssert (a->type.type == b->type.type, "types must match for now");
+    PS_ASSERT_INT_EQUAL(a->numCols, a->numRows, false);
+    PS_ASSERT_VECTOR_SIZE(b, (long int)a->numCols, false);
+
+    // Check for non-finite entries in matrix
+    switch (a->type.type) {
+      case PS_TYPE_F32: {
+	  psF32 **values = a->data.F32; /* Dereference */
+	  int numCols = a->numCols, numRows = a->numRows; /* Size of matrix */
+	  for (int i = 0; i < numRows; i++) {
+	      for (int j = 0; j < numCols; j++) {
+		  if (!isfinite(values[i][j])) {
+		      return false;
+		  }
+	      }
+	  }
+	  break;
+      }
+      case PS_TYPE_F64: {
+	  psF64 **values = a->data.F64; /* Dereference */
+	  int numCols = a->numCols, numRows = a->numRows; /* Size of matrix */
+	  for (int i = 0; i < numRows; i++) {
+	      for (int j = 0; j < numCols; j++) {
+		  if (!isfinite(values[i][j])) {
+		      return false;
+		  }
+	      }
+	  }
+	  break;
+      }
+      default:
+	psAbort("Should never get here.");
+    }
+  
+    // Following the algorithm laid out by Press et al., we loop along the matrix diagonal, but
+    // we do not operate on the diagonal elements in order.  Instead, we are looking for the
+    // current max element and operating on that diagonal element.  This is effectively column
+    // pivoting.  Row pivoting is perfomed explicitly.
+
+    int nSquare = a->numCols;
+
+    psVector *colIndexV = psVectorAlloc(nSquare, PS_TYPE_S32);
+    psVector *rowIndexV = psVectorAlloc(nSquare, PS_TYPE_S32);
+    psVector *pivotV    = psVectorAlloc(nSquare, PS_TYPE_S32);
+    psVectorInit (pivotV, 0.0);
+
+    if (a->type.type == PS_TYPE_F32) {
+	psF32 **A = a->data.F32;
+	psF32  *B = b->data.F32;
+	int *colIndex = colIndexV->data.S32;
+	int *rowIndex = rowIndexV->data.S32;
+	int *pivot    = pivotV->data.S32;
+	psF32 growth = 1.0;
+
+	for (int diag = 0; diag < nSquare; diag++) {
+
+	    psF32 maxval = 0.0;
+	    int maxrow = 0;
+	    int maxcol = 0;
+
+	    // search for the next pivot
+	    for (int row = 0; row < nSquare; row++) {
+		if (!isfinite(A[row][diag])) goto escape;
+
+		// if we have already operated on this row (pivot[row] is true), skip it
+		if (pivot[row]) continue;
+
+		// if we have not yet operated on this row (pivot[row] is false), look for pivot for this row
+		for (int col = 0; col < nSquare; col++) {
+		    if (pivot[col]) continue;
+		    if (fabs (A[row][col]) < maxval) continue;
+		    maxval = fabs (A[row][col]);
+		    maxrow = row;
+		    maxcol = col;
+		}
+	    }
+
+	    // if pivot[maxcol] is set, we have already done this row: this implies a singular matrix
+	    if (pivot[maxcol]) goto escape;
+	    pivot[maxcol] = 1;
+
+	    // if the selected pivot is off the diagonal, do a row swap
+	    if (maxrow != maxcol) {
+		for (int col = 0; col < nSquare; col++) PS_SWAP (A[maxrow][col], A[maxcol][col]);
+		PS_SWAP (B[maxrow], B[maxcol]);
+	    }
+	    rowIndex[diag] = maxrow;
+	    colIndex[diag] = maxcol;
+	    if (A[maxcol][maxcol] == 0.0) goto escape;
+	    // Kahan replaces the 0.0 pivot with epsilon*(largest element in column) + underFlow.
+	    // Here we are going to raise an error if the dynamic range is too large.
+
+	    /* rescale by pivot reciprocal */
+	    psF32 tmpval = 1.0 / A[maxcol][maxcol];
+	    A[maxcol][maxcol] = 1.0;
+	    for (int col = 0; col < nSquare; col++) A[maxcol][col] *= tmpval;
+	    B[maxcol] *= tmpval;
+
+	    // check for ill-conditioned matrix: measure the pivot growth and trigger on over/under flow
+	    growth *= tmpval;
+	    psTrace ("psLib.math", 4, "growth : %e\n", growth);
+	    if (fabs(growth) > MAX_RANGE) goto escape;
+
+	    /* adjust the elements above the pivot */
+	    for (int row = 0; row < nSquare; row++) {
+		if (row == maxcol) continue;
+		tmpval = A[row][maxcol];
+		A[row][maxcol] = 0.0;
+		for (int col = 0; col < nSquare; col++) A[row][col] -= A[maxcol][col]*tmpval;
+		B[row] -= B[maxcol]*tmpval;
+	    }
+	}
+
+	// swap back the inverse matrix based on the row swaps above
+	for (int col = nSquare - 1; col >= 0; col--) {
+	    if (rowIndex[col] != colIndex[col]) {
+		for (int row = 0; row < nSquare; row++) PS_SWAP (A[row][rowIndex[col]], A[row][colIndex[col]]);
+	    }
+	}
+    } else {
+	psF64 **A = a->data.F64;
+	psF64  *B = b->data.F64;
+	int *colIndex = colIndexV->data.S32;
+	int *rowIndex = rowIndexV->data.S32;
+	int *pivot    = pivotV->data.S32;
+	psF64 growth = 1.0;
+
+	for (int diag = 0; diag < nSquare; diag++) {
+
+	    psF64 maxval = 0.0;
+	    int maxrow = 0;
+	    int maxcol = 0;
+
+	    // search for the next pivot
+	    for (int row = 0; row < nSquare; row++) {
+		if (!isfinite(A[row][diag])) goto escape;
+
+		// if we have already operated on this row (pivot[row] is true), skip it
+		if (pivot[row]) continue;
+
+		// if we have not yet operated on this row (pivot[row] is false), look for pivot for this row
+		for (int col = 0; col < nSquare; col++) {
+		    if (pivot[col]) continue;
+		    if (fabs (A[row][col]) < maxval) continue;
+		    maxval = fabs (A[row][col]);
+		    maxrow = row;
+		    maxcol = col;
+		}
+	    }
+
+	    // if pivot[maxcol] is set, we have already done this row: this implies a singular matrix
+	    if (pivot[maxcol]) goto escape;
+	    pivot[maxcol] = 1;
+
+	    // if the selected pivot is off the diagonal, do a row swap
+	    if (maxrow != maxcol) {
+		for (int col = 0; col < nSquare; col++) PS_SWAP (A[maxrow][col], A[maxcol][col]);
+		PS_SWAP (B[maxrow], B[maxcol]);
+	    }
+	    rowIndex[diag] = maxrow;
+	    colIndex[diag] = maxcol;
+	    if (A[maxcol][maxcol] == 0.0) goto escape;
+	    // Kahan replaces the 0.0 pivot with epsilon*(largest element in column) + underFlow.
+	    // Here we are going to raise an error if the dynamic range is too large.
+
+	    /* rescale by pivot reciprocal */
+	    psF64 tmpval = 1.0 / A[maxcol][maxcol];
+	    A[maxcol][maxcol] = 1.0;
+	    for (int col = 0; col < nSquare; col++) A[maxcol][col] *= tmpval;
+	    B[maxcol] *= tmpval;
+
+	    // check for ill-conditioned matrix: measure the pivot growth and trigger on over/under flow
+	    growth *= tmpval;
+	    psTrace ("psLib.math", 4, "growth : %e\n", growth);
+	    if (fabs(growth) > MAX_RANGE) goto escape;
+
+	    /* adjust the elements above the pivot */
+	    for (int row = 0; row < nSquare; row++) {
+		if (row == maxcol) continue;
+		tmpval = A[row][maxcol];
+		A[row][maxcol] = 0.0;
+		for (int col = 0; col < nSquare; col++) A[row][col] -= A[maxcol][col]*tmpval;
+		B[row] -= B[maxcol]*tmpval;
+	    }
+	}
+
+	// swap back the inverse matrix based on the row swaps above
+	for (int col = nSquare - 1; col >= 0; col--) {
+	    if (rowIndex[col] != colIndex[col]) {
+		for (int row = 0; row < nSquare; row++) PS_SWAP (A[row][rowIndex[col]], A[row][colIndex[col]]);
+	    }
+	}
+    }
+
+    psFree (pivotV);
+    psFree (rowIndexV);
+    psFree (colIndexV);
+    return true;
+
+escape:
+    psFree (pivotV);
+    psFree (rowIndexV);
+    psFree (colIndexV);
+    return false;
+}
 
 psImage* psMatrixInvert(psImage* out,
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMatrix.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMatrix.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMatrix.h	(revision 24244)
@@ -39,5 +39,5 @@
  *  @return  psImage* : Pointer to LU decomposed psImage.
  */
-psImage* psMatrixLUD(
+psImage *psMatrixLUDecomposition(
     psImage* out,                      ///< Image to return, or NULL.
     psVector** perm,                   ///< Output permutation vector used by psMatrixLUSolve.
@@ -54,5 +54,5 @@
  *  @return  psVector* : Pointer to psVector solution of matrix equation.
  */
-psVector* psMatrixLUSolve(
+psVector *psMatrixLUSolution(
     psVector* out,                     ///< Vector to return, or NULL.
     const psImage* LU,                 ///< LU-decomposed matrix.
@@ -61,5 +61,24 @@
 );
 
-/** Used to be a Gauss-Jordan numerical solver, but now uses LU decomposition.
+/** LU Decomposition-based Matrix inversion
+ *
+ *  @return  psImage * : Pointer to psImage inverse of matrix
+ */
+psImage *psMatrixLUInvert(
+    psImage *out,		   ///< place result here if not NULL
+    const psImage* LU,		   ///< LU-decomposed matrix.
+    const psVector* perm	   ///< Permutation vector resulting from psMatrixLUD function.
+);
+
+/** LU Decomposition-based solver for Ax = B.
+ *
+ *  @return bool:   True if successful.
+ */
+bool psMatrixLUSolve(
+    psImage *A,                   ///< Matrix to be solved
+    psVector *b                   ///< Vector of values
+);
+
+/** Gauss-Jordan-based solver for Ax = B.
  *
  *  @return bool:   True if successful.
@@ -69,8 +88,4 @@
     psVector *b                   ///< Vector of values
 );
-
-/** Gauss-Jordan numerical solver for F32 input data */
-#define psMatrixGJSolveF32(A,B) psMatrixGJSolve(A,B)
-
 
 /** Invert psImage matrix.
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizeLMM.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizeLMM.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizeLMM.c	(revision 24244)
@@ -80,4 +80,9 @@
     assert (alpha->numCols == alpha->numRows);
 
+# define TESTGJ 0
+# if (TESTGJ)
+    lambda = 0.0;
+#endif
+
     // set new guess values, applying (1+lambda) scaling to pivots
     Beta = psVectorCopy(Beta, beta, PS_TYPE_F32);
@@ -88,8 +93,35 @@
 
     // error and clear above if kept?
-    if (false == psMatrixGJSolveF32(Alpha, Beta)) {
+    if (!psMatrixGJSolve(Alpha, Beta)) {
         psTrace ("psLib.math", 4, "singular matrix in Guess ABP\n");
         return(false);
     }
+
+// XXX check that the GJ solver works:
+# if (TESTGJ) 
+    psImage *out = psImageAlloc (alpha->numRows, alpha->numCols, PS_TYPE_F32);
+    for (int oy = 0; oy < out->numRows; oy++) {
+	for (int ox = 0; ox < out->numCols; ox++) {
+	    float value = 0;
+	    for (int i = 0; i < alpha->numCols; i++) {
+		value += alpha->data.F32[i][ox]*Alpha->data.F32[oy][i];
+	    }
+	    out->data.F32[oy][ox] = value;
+	}
+    }
+
+    psVector *vect = psVectorAlloc (beta->n, PS_TYPE_F32);
+    for (int oy = 0; oy < vect->n; oy++) {
+	float value = 0;
+	for (int i = 0; i < alpha->numCols; i++) {
+	    value += alpha->data.F32[oy][i]*Beta->data.F32[i];
+	}
+	vect->data.F32[oy] = value;
+    }
+    
+    psFree (out);
+    psFree (vect);
+
+# endif
 
     // check for non-finite entries in result
@@ -191,4 +223,5 @@
     if (isnan(chisq)) {
         psTrace ("psLib.math", 5, "psMinLM_SetABX() returned a NAN chisq.\n");
+	psVectorInit (delta, NAN);
         retValue = false;
     }
@@ -205,4 +238,5 @@
     if (!status) {
         psTrace ("psLib.math", 5, "psMinLM_GuessABP() returned FALSE.\n");
+	psVectorInit (delta, NAN);
         retValue = false;
     }
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizeLMM_ND.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizeLMM_ND.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizeLMM_ND.c	(revision 24244)
@@ -265,5 +265,5 @@
 
     // error and clear above if kept?
-    if (false == psMatrixGJSolveF32(Alpha, Beta)) {
+    if (!psMatrixGJSolve(Alpha, Beta)) {
         psTrace ("psLib.math", 4, "singular matrix in Guess ABP\n");
         return(false);
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizePolyFit.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizePolyFit.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizePolyFit.c	(revision 24244)
@@ -15,5 +15,5 @@
  *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
  *
- *  XXX: psMatrixLUSolve() does not return error codes when the results are NANs.
+ *  XXX: psMatrixLUSolution() does not return error codes when the results are NANs.
  *
  *  XXX: For clip-fit functions, what should we do if the mask is NULL?
@@ -403,57 +403,30 @@
     }
 
+    bool status = false;
     if (USE_GAUSS_JORDAN) {
-        // GaussJordan version
-        if (!psMatrixGJSolve(A, B)) {
-            psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.  Returning NULL.\n");
-            goto escape_GJ;
-        } else {
-            // the first nTerm entries in B correspond directly to the desired
-            // polynomial coefficients.  this is only true for the 1D case
-            for (psS32 k = 0; k < numTerms; k++) {
-                myPoly->coeff[k] = B->data.F64[k];
-                myPoly->coeffErr[k] = sqrt(A->data.F64[k][k]);
-            }
-            // The constant needs to be multiplied by 2, because it's half the a_0.
-            myPoly->coeff[0] *= 2.0;
-            myPoly->coeffErr[0] *= 2.0;
-        }
+        status = psMatrixGJSolve(A, B);
     } else {
-        // LUD version of the fit
-        psImage *ALUD = NULL;
-        psVector* outPerm = NULL;
-        psVector* coeffs = NULL;
-
-        ALUD = psImageAlloc(numTerms, numTerms, PS_TYPE_F64);
-        ALUD = psMatrixLUD(ALUD, &outPerm, A);
-        if (ALUD == NULL) {
-            psError(PS_ERR_UNKNOWN, false, "Could not do LUD decomposition on matrix.  Returning NULL.\n");
-            goto escape_LUD;
-        } else {
-            coeffs = psMatrixLUSolve(coeffs, ALUD, B, outPerm);
-            if (coeffs == NULL) {
-                psError(PS_ERR_UNKNOWN, false, "Could not solve LUD matrix.  Returning NULL.\n");
-                goto escape_LUD;
-            } else {
-                for (psS32 k = 0; k < numTerms; k++) {
-                    myPoly->coeff[k] = coeffs->data.F64[k];
-                }
-            }
-        }
-        psFree(ALUD);
-        psFree(coeffs);
-        psFree(outPerm);
-    }
+        status = psMatrixLUSolve(A, B);
+    }
+    if (!status) {
+	psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.\n");
+	goto escape;
+    } 
+
+    // the first nTerm entries in B correspond directly to the desired
+    // polynomial coefficients.  this is only true for the 1D case
+    for (psS32 k = 0; k < numTerms; k++) {
+	myPoly->coeff[k] = B->data.F64[k];
+	myPoly->coeffErr[k] = sqrt(A->data.F64[k][k]);
+    }
+    // The constant needs to be multiplied by 2, because it's half the a_0.
+    myPoly->coeff[0] *= 2.0;
+    myPoly->coeffErr[0] *= 2.0;
+
     psFree(A);
     psFree(B);
     return true;
 
-escape_LUD:
-    // XXX drop the LUD version!
-    psFree(A);
-    psFree(B);
-    return false;
-
-escape_GJ:
+escape:
     psFree(A);
     psFree(B);
@@ -628,64 +601,32 @@
     }
 
-    // XXX: rel10_ifa used psMatrixGJSolve().  However, this failed tests.  So, I'm using psMatrixLUD().
+    bool status = false;
     if (USE_GAUSS_JORDAN) {
-        // GaussJordan version
-        if (!psMatrixGJSolve(A, B)) {
-            psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.  Returning NULL.\n");
-            goto escape_GJ;
-        } else {
-            // the first nTerm entries in B correspond directly to the desired
-            // polynomial coefficients.  this is only true for the 1D case
-            for (psS32 k = 0; k < nTerm; k++) {
-		if (coeffMask[k] & PS_POLY_MASK_FIT) continue;
-		myPoly->coeff[k] = B->data.F64[k];
-		myPoly->coeffErr[k] = sqrt(A->data.F64[k][k]);
-            }
-        }
+        status = psMatrixGJSolve(A, B);
     } else {
-        // LUD version of the fit
-        psImage *ALUD = NULL;
-        psVector* outPerm = NULL;
-        psVector* coeffs = NULL;
-
-        ALUD = psImageAlloc(nTerm, nTerm, PS_TYPE_F64);
-        ALUD = psMatrixLUD(ALUD, &outPerm, A);
-        if (ALUD == NULL) {
-            psError(PS_ERR_UNKNOWN, false, "Could not do LUD decomposition on matrix.  Returning NULL.\n");
-            goto escape_LUD;
-        } else {
-            coeffs = psMatrixLUSolve(coeffs, ALUD, B, outPerm);
-            if (coeffs == NULL) {
-                psError(PS_ERR_UNKNOWN, false, "Could not solve LUD matrix.  Returning NULL.\n");
-                goto escape_LUD;
-            } else {
-                for (psS32 k = 0; k < nTerm; k++) {
-		    if (coeffMask[k] & PS_POLY_MASK_FIT) continue;
-                    myPoly->coeff[k] = coeffs->data.F64[k];
-                    // XXX LUD does not give inverse of A
-                }
-            }
-        }
-        psFree(ALUD);
-        psFree(coeffs);
-        psFree(outPerm);
-    }
+        status = psMatrixLUSolve(A, B);
+    }
+    if (!status) {
+	psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.\n");
+	goto escape;
+    } 
+
+    // the first nTerm entries in B correspond directly to the desired
+    // polynomial coefficients.  this is only true for the 1D case
+    for (psS32 k = 0; k < nTerm; k++) {
+	if (coeffMask[k] & PS_POLY_MASK_FIT) continue;
+	myPoly->coeff[k] = B->data.F64[k];
+	myPoly->coeffErr[k] = sqrt(A->data.F64[k][k]);
+    }
+
     psFree(A);
     psFree(B);
-
-    psTrace("psLib.math", 4, "---- %s() End ----\n", __func__);
     return true;
 
-escape_LUD:
-    psFree(A);
-    psFree(B);
-    return false;
-
-escape_GJ:
+escape:
     psFree(A);
     psFree(B);
     return false;
 }
-
 
 /******************************************************************************
@@ -1129,26 +1070,31 @@
     }
 
-    if (!psMatrixGJSolve(A, B)) {
-        psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.  Returning NULL.\n");
-        psFree(A);
-        psFree(B);
-        return false;
-    }
-
-    // select the appropriate solution entries
+    bool status = false;
+    if (USE_GAUSS_JORDAN) {
+        status = psMatrixGJSolve(A, B);
+    } else {
+        status = psMatrixLUSolve(A, B);
+    }
+    if (!status) {
+	psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.\n");
+	goto escape;
+    } 
+
+    // select the appropriate solution entries (retain the incoming values if masked on the fit)
     for (int i = 0; i < nTerm; i++) {
-        int l = i / nYterm;         // x index
-        int m = i % nYterm;         // y index
-
-	// retain the incoming values if masked on the fit
-	if (coeffMask[l][m] & PS_POLY_MASK_FIT) continue;
-	myPoly->coeff[l][m] = B->data.F64[i];
-	myPoly->coeffErr[l][m] = sqrt(A->data.F64[i][i]);
+        int ix = i / nYterm;         // x index
+        int iy = i % nYterm;         // y index
+	if (coeffMask[ix][iy] & PS_POLY_MASK_FIT) continue;
+	myPoly->coeff[ix][iy] = B->data.F64[i];
+	myPoly->coeffErr[ix][iy] = sqrt(A->data.F64[i][i]);
     }
     psFree(A);
     psFree(B);
-
-    psTrace("psLib.math", 6, "---- %s() end ----\n", __func__);
     return true;
+
+escape:
+    psFree (A);
+    psFree (B);
+    return false;
 }
 
@@ -1523,70 +1469,29 @@
 
 
-    // XXX: rel10_ifa used psMatrixGJSolve().  However, this failed tests.  So, I'm using psMatrixLUD().
+    bool status = false;
     if (USE_GAUSS_JORDAN) {
-        // does the solution in place
-        // The matrices were overflowing, so I switched to LUD.
-        if (!psMatrixGJSolve(A, B)) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to perform GaussJordan elimination.\n");
-            goto escape_GJ;
-        }
-        // select the appropriate solution entries
-        for (int i = 0; i < nTerm; i++) {
-            int ix = i / nYZterm; // x index
-            int iy = (i % nYZterm) / nZterm; // y index
-            int iz = (i % nYZterm) % nZterm; // z index
-	    if (coeffMask[ix][iy][iz] & PS_POLY_MASK_FIT) continue;
-            myPoly->coeff[ix][iy][iz] = B->data.F64[i];
-            myPoly->coeffErr[ix][iy][iz] = sqrt(A->data.F64[i][i]);
-        }
+        status = psMatrixGJSolve(A, B);
     } else {
-        // LUD version of the fit
-        psImage *ALUD = NULL;
-        psVector* outPerm = NULL;
-        psVector* coeffs = NULL;
-
-        ALUD = psImageAlloc(nTerm, nTerm, PS_TYPE_F64);
-        ALUD = psMatrixLUD(ALUD, &outPerm, A);
-        if (ALUD == NULL) {
-            psError(PS_ERR_UNKNOWN, false, "Could not do LUD decomposition on matrix.  Returning NULL.\n");
-            goto escape_LUD;
-        } else {
-            coeffs = psMatrixLUSolve(coeffs, ALUD, B, outPerm);
-            if (coeffs == NULL) {
-                psError(PS_ERR_UNKNOWN, false, "Could not solve LUD matrix.  Returning NULL.\n");
-                goto escape_LUD;
-            } else {
-                // select the appropriate solution entries
-                for (psS32 ix = 0; ix < nXterm; ix++) {
-                    for (psS32 iy = 0; iy < nYterm; iy++) {
-                        for (psS32 iz = 0; iz < nZterm; iz++) {
-                            psS32 nx = ix+iy*nXterm+iz*nXterm*nYterm;
-			    if (coeffMask[ix][iy][iz] & PS_POLY_MASK_FIT) continue;
-                            myPoly->coeff[ix][iy][iz] = coeffs->data.F64[nx];
-                            // XXX myPoly->coeffErr[ix][iy][iz] = sqrt(A->data.F64[nx][nx]);
-                            // XXX this is wrong: A is not replaced with inverse(A), as for GaussJordan
-                        }
-                    }
-                }
-            }
-        }
-
-        psFree(ALUD);
-        psFree(coeffs);
-        psFree(outPerm);
+        status = psMatrixLUSolve(A, B);
+    }
+    if (!status) {
+	psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.\n");
+	goto escape;
+    } 
+
+    // select the appropriate solution entries
+    for (int i = 0; i < nTerm; i++) {
+	int ix = i / nYZterm; // x index
+	int iy = (i % nYZterm) / nZterm; // y index
+	int iz = (i % nYZterm) % nZterm; // z index
+	if (coeffMask[ix][iy][iz] & PS_POLY_MASK_FIT) continue;
+	myPoly->coeff[ix][iy][iz] = B->data.F64[i];
+	myPoly->coeffErr[ix][iy][iz] = sqrt(A->data.F64[i][i]);
     }
     psFree(A);
     psFree(B);
-
-    psTrace("psLib.math", 4, "---- %s() end ----\n", __func__);
     return true;
 
-escape_LUD:
-    psFree(A);
-    psFree(B);
-    return false;
-
-escape_GJ:
-
+escape:
     psFree(A);
     psFree(B);
@@ -1986,72 +1891,30 @@
     }
 
-    // XXX: rel10_ifa used psMatrixGJSolve().  However, this failed tests.  So, I'm using psMatrixLUD().
+    bool status = false;
     if (USE_GAUSS_JORDAN) {
-        // does the solution in place
-        // The GaussJordan version was overflowing, so I'm using LUD.
-        if (!psMatrixGJSolve(A, B)) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to perform GaussJordan elimination.\n");
-            goto escape_GJ;
-        }
-
-        // select the appropriate solution entries
-        for (int i = 0; i < nTerm; i++) {
-            int ix = i / nYZTterm; // x index
-            int iy = (i % nYZTterm) / nZTterm; // y index
-            int iz = ((i % nYZTterm) % nZTterm) / nTterm; // z index
-            int it = ((i % nYZTterm) % nZTterm) % nTterm; // t index
-	    if (coeffMask[ix][iy][iz][it] & PS_POLY_MASK_FIT) continue;
-            myPoly->coeff[ix][iy][iz][it] = B->data.F64[i];
-            myPoly->coeffErr[ix][iy][iz][it] = sqrt(A->data.F64[i][i]);
-        }
+        status = psMatrixGJSolve(A, B);
     } else {
-        // LUD version of the fit
-        psImage *ALUD = NULL;
-        psVector* outPerm = NULL;
-        psVector* coeffs = NULL;
-
-        ALUD = psImageAlloc(nTerm, nTerm, PS_TYPE_F64);
-        ALUD = psMatrixLUD(ALUD, &outPerm, A);
-        if (ALUD == NULL) {
-            psError(PS_ERR_UNKNOWN, false, "Could not do LUD decomposition on matrix.  Returning NULL.\n");
-            goto escape_LUD;
-        } else {
-            coeffs = psMatrixLUSolve(coeffs, ALUD, B, outPerm);
-            if (coeffs == NULL) {
-                psError(PS_ERR_UNKNOWN, false, "Could not solve LUD matrix.  Returning NULL.\n");
-                goto escape_LUD;
-            } else {
-                // select the appropriate solution entries
-                for (psS32 ix = 0; ix < nXterm; ix++) {
-                    for (psS32 iy = 0; iy < nYterm; iy++) {
-                        for (psS32 iz = 0; iz < nZterm; iz++) {
-                            for (psS32 it = 0; it < nTterm; it++) {
-                                psS32 nx = ix+iy*nXterm+iz*nXterm*nYterm+it*nXterm*nYterm*nZterm;
-				if (coeffMask[ix][iy][iz][it] & PS_POLY_MASK_FIT) continue;
-                                myPoly->coeff[ix][iy][iz][it] = coeffs->data.F64[nx];
-                                // myPoly->coeffErr[ix][iy][iz][it] = sqrt(A->data.F64[nx][nx]);
-                                // XXX this is wrong: LUD does not supply inverse(A)
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        psFree(ALUD);
-        psFree(coeffs);
-        psFree(outPerm);
+        status = psMatrixLUSolve(A, B);
+    }
+    if (!status) {
+	psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.\n");
+	goto escape;
+    } 
+
+    // select the appropriate solution entries
+    for (int i = 0; i < nTerm; i++) {
+	int ix = i / nYZTterm; // x index
+	int iy = (i % nYZTterm) / nZTterm; // y index
+	int iz = ((i % nYZTterm) % nZTterm) / nTterm; // z index
+	int it = ((i % nYZTterm) % nZTterm) % nTterm; // t index
+	if (coeffMask[ix][iy][iz][it] & PS_POLY_MASK_FIT) continue;
+	myPoly->coeff[ix][iy][iz][it] = B->data.F64[i];
+	myPoly->coeffErr[ix][iy][iz][it] = sqrt(A->data.F64[i][i]);
     }
     psFree(A);
     psFree(B);
-
-    psTrace("psLib.math", 4, "---- %s() end ----\n", __func__);
     return true;
 
-escape_LUD:
-    psFree(A);
-    psFree(B);
-    return false;
-
-escape_GJ:
+escape:
     psFree(A);
     psFree(B);
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psPolynomialMD.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psPolynomialMD.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psPolynomialMD.c	(revision 24244)
@@ -304,5 +304,5 @@
     polynomialMDFill(poly->fitMatrix);
 
-    poly->LU = psMatrixLUD(poly->LU, &poly->LUperm, poly->fitMatrix); // LU-decomposed matrix
+    poly->LU = psMatrixLUDecomposition(poly->LU, &poly->LUperm, poly->fitMatrix); // LU-decomposed matrix
     if (!poly->LU) {
         psError(PS_ERR_UNKNOWN, false, "Unable to LU-Decompose least-squares matrix.");
@@ -310,5 +310,5 @@
     }
 
-    poly->coeff = psMatrixLUSolve(poly->coeff, poly->LU, poly->fitVector, poly->LUperm);
+    poly->coeff = psMatrixLUSolution(poly->coeff, poly->LU, poly->fitVector, poly->LUperm);
     if (!poly->coeff) {
         psError(PS_ERR_UNKNOWN, false, "Unable to solve least-squares equation.");
@@ -447,10 +447,10 @@
 
     // Solve least-squares equation
-    *lu = psMatrixLUD(*lu, perm, matrix);
+    *lu = psMatrixLUDecomposition(*lu, perm, matrix);
     if (!*lu) {
         psError(PS_ERR_UNKNOWN, false, "Unable to LU-Decompose least-squares matrix.");
         return false;
     }
-    poly->coeff = psMatrixLUSolve(poly->coeff, *lu, vector, *perm);
+    poly->coeff = psMatrixLUSolution(poly->coeff, *lu, vector, *perm);
     if (!poly->coeff) {
         psError(PS_ERR_UNKNOWN, false, "Unable to solve least-squares equation.");
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psStats.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psStats.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psStats.c	(revision 24244)
@@ -24,6 +24,13 @@
 // unity, even though the standard deviation is not defined in that case (NAN).
 
+// reworking the return values and failure conditions: it should only be an error if the
+// inputs imply a programming error: eg, NULL data vectors, non-sensical input
+// parameters.  If the statistic cannot be calculated (0 length vector, 0 range, no
+// unmasked data values), the statistic should be reported as NAN, but an error should not
+// be raised.  (TBD: do we need to have a unique field in psStats or a return parameter
+// that can be checked for an invalid result?)
+
 #ifdef HAVE_CONFIG_H
-# include "config.h"
+#include "config.h"
 #endif
 
@@ -42,5 +49,4 @@
 #include "psMemory.h"
 #include "psAbort.h"
-//#include "psImage.h"
 #include "psVector.h"
 #include "psTrace.h"
@@ -69,4 +75,6 @@
 #define PS_POLY_MEDIAN_MAX_ITERATIONS 30
 
+#define TRACE "psLib.math"
+
 #define MASK_MARK 0x80   // XXX : can we change this? bit to use internally to mark data as bad
 #define PS_ROBUST_MAX_ITERATIONS 20     // Maximum number of iterations for robust statistics
@@ -87,5 +95,5 @@
             break; \
           case PS_BINARY_DISECT_OUTSIDE_RANGE: \
-            psTrace ("psLib.math", 6, "selected bin outside range"); \
+            psTrace(TRACE, 6, "selected bin outside range"); \
             if (USE_END == -1) { RESULT = 0; } \
             if (USE_END == +1) { RESULT = VECTOR->n - 1; } \
@@ -116,9 +124,7 @@
         } \
         Xt = PS_MIN (BOUNDS->data.F32[BIN+1], PS_MAX(BOUNDS->data.F32[BIN], Xt)); \
-        psTrace("psLib.math", 6, "(Xo, Yo, dX, dY, Xt, Yt) is (%.2f %.2f %.2f %.2f %.2f %.2f)\n", \
+        psTrace(TRACE, 6, "(Xo, Yo, dX, dY, Xt, Yt) is (%.2f %.2f %.2f %.2f %.2f %.2f)\n", \
                 Xo, Yo, dX, dY, Xt, VALUE); \
         RESULT = Xt; }
-
-#define TRACE "psLib.math"
 
 /*****************************************************************************/
@@ -156,5 +162,6 @@
 static psF32 minimizeLMChi2Gauss1D(psVector *deriv, const psVector *params, const psVector *coords);
 // static psF32 fitQuadraticSearchForYThenReturnX(const psVector *xVec, psVector *yVec, psS32 binNum, psF32 yVal);
-static psF32 fitQuadraticSearchForYThenReturnXusingValues(const psVector *xVec, psVector *yVec, psS32 binNum, psF32 yVal);
+// static psF32 fitQuadraticSearchForYThenReturnXusingValues(const psVector *xVec, psVector *yVec, psS32 binNum, psF32 yVal);
+static psF32 fitQuadraticSearchForYThenReturnBin(const psVector *xVec, psVector *yVec, psS32 binNum, psF32 yVal);
 
 /******************************************************************************
@@ -182,6 +189,4 @@
                                  psStats* stats)
 {
-    psTrace(TRACE, 4, "---- %s() begin ----\n", __func__);
-
     long count = 0;                     // Number of points contributing to this mean
     psF32 mean = 0.0;                   // The mean
@@ -222,12 +227,8 @@
     }
     stats->sampleMean = mean;
-    if (isnan(mean)) {
-        // XXX raise an error here?
-        psTrace(TRACE, 4, "---- %s(false) end ----\n", __func__);
-        return true;
-    }
-
-    stats->results |= PS_STAT_SAMPLE_MEAN;
-    psTrace(TRACE, 4, "---- %s(true) end ----\n", __func__);
+
+    if (!isnan(mean)) {
+	stats->results |= PS_STAT_SAMPLE_MEAN;
+    }
     return true;
 }
@@ -252,5 +253,4 @@
         )
 {
-    psTrace(TRACE, 4, "---- %s() begin ----\n", __func__);
     psF32 max = -PS_MAX_F32;            // The calculated maximum
     psF32 min = PS_MAX_F32;             // The calculated minimum
@@ -289,5 +289,4 @@
         stats->results |= PS_STAT_MAX;
     }
-    psTrace(TRACE, 4, "---- %s(%d) end ----\n", __func__, numValid);
     return numValid;
 }
@@ -303,6 +302,4 @@
                                psStats* stats)
 {
-    psTrace(TRACE, 4, "---- %s() begin ----\n", __func__);
-
     bool useRange = stats->options & PS_STAT_USE_RANGE;
     psVectorMaskType *maskData = (maskVector == NULL) ? NULL : maskVector->data.PS_TYPE_VECTOR_MASK_DATA; // Dereference the vector
@@ -334,5 +331,5 @@
 
     if (count == 0) {
-        psError(PS_ERR_BAD_PARAMETER_SIZE, true, "No valid data in input vector.\n");
+        psLogMsg(TRACE, PS_LOG_DETAIL, "No valid data in input vector.\n");
         stats->sampleUQ = NAN;
         stats->sampleLQ = NAN;
@@ -343,4 +340,5 @@
     // Sort the temporary vector.
     if (!psVectorSort(outVector, outVector)) { // Sort in-place (since it's a copy, it's OK)
+	// an error in psVectorSort is a serious error
         psError(PS_ERR_UNEXPECTED_NULL, false, _("Failed to sort input data."));
         stats->sampleUQ = NAN;
@@ -364,6 +362,4 @@
     stats->results |= PS_STAT_SAMPLE_QUARTILE;
 
-    // Return "true" on success.
-    psTrace(TRACE, 4, "---- %s(true) end ----\n", __func__);
     return true;
 }
@@ -390,6 +386,4 @@
                               psStats* stats)
 {
-    psTrace(TRACE, 4, "---- %s() begin ----\n", __func__);
-
     // This procedure requires the mean.  If it has not been already
     // calculated, then call vectorSampleMean()
@@ -400,6 +394,5 @@
     // If the mean is NAN, then generate a warning and set the stdev to NAN.
     if (isnan(stats->sampleMean)) {
-        psTrace(TRACE, 5, "WARNING: vectorSampleStdev(): sample mean is NAN.  "
-                "Setting stats->sampleStdev = NAN.\n");
+        psLogMsg(TRACE, PS_LOG_DETAIL, "WARNING: vectorSampleStdev(): sample mean is NAN. Setting stats->sampleStdev = NAN.\n");
         stats->sampleStdev = NAN;
         return true;
@@ -445,11 +438,10 @@
         // Assume that the user knows what he's doing when he masks out everything --> no error.
         stats->sampleStdev = NAN;
-        psTrace(TRACE, 5, "WARNING: vectorSampleStdev(): no valid psVector elements (%ld).  "
-                "Setting stats->sampleStdev = NAN.\n", count);
+        psLogMsg(TRACE, PS_LOG_DETAIL, "WARNING: vectorSampleStdev(): no valid psVector elements (%ld). Setting stats->sampleStdev = NAN.\n", count);
         return true;
     }
     if (count == 1) {
         stats->sampleStdev = 0.0;
-        psTrace(TRACE, 5, "WARNING: vectorSampleStdev(): only one valid psVector elements (%ld).  "
+        psLogMsg(TRACE, PS_LOG_DETAIL, "WARNING: vectorSampleStdev(): only one valid psVector elements (%ld).  "
                 "Setting stats->sampleStdev = 0.0.\n", count);
         return true;
@@ -462,6 +454,4 @@
     }
     stats->results |= PS_STAT_SAMPLE_STDEV;
-
-    psTrace(TRACE, 4, "---- %s() end ----\n", __func__);
 
     return true;
@@ -473,6 +463,4 @@
                                 psStats* stats)
 {
-    psTrace(TRACE, 4, "---- %s() begin ----\n", __func__);
-
     // This procedure requires the mean and standard deviation
     if (!(stats->results & PS_STAT_SAMPLE_MEAN)) {
@@ -480,5 +468,5 @@
     }
     if (isnan(stats->sampleMean)) {
-        psTrace(TRACE, 5, "WARNING: vectorSampleMoments(): sample mean is NAN.\n");
+        psLogMsg(TRACE, PS_LOG_DETAIL, "WARNING: vectorSampleMoments(): sample mean is NAN.\n");
         goto SAMPLE_MOMENTS_BAD;
     }
@@ -487,5 +475,5 @@
     }
     if (isnan(stats->sampleStdev) || stats->sampleStdev == 0.0) {
-        psTrace(TRACE, 5, "WARNING: vectorSampleMoments(): sample stdev is NAN or 0.\n");
+        psLogMsg(TRACE, PS_LOG_DETAIL, "WARNING: vectorSampleMoments(): sample stdev is NAN or 0.\n");
         goto SAMPLE_MOMENTS_BAD;
     }
@@ -534,6 +522,4 @@
     stats->results |= PS_STAT_SAMPLE_SKEWNESS | PS_STAT_SAMPLE_KURTOSIS;
 
-    psTrace(TRACE, 4, "---- %s() end ----\n", __func__);
-
     return true;
 
@@ -542,5 +528,4 @@
     stats->sampleSkewness = NAN;
     stats->sampleKurtosis = NAN;
-    stats->results |= PS_STAT_SAMPLE_SKEWNESS | PS_STAT_SAMPLE_KURTOSIS;
     return true;
 }
@@ -565,7 +550,4 @@
     )
 {
-    psTrace(TRACE, 4, "---- %s() begin ----\n", __func__);
-    psTrace(TRACE, 4, "Trace level is %d\n", psTraceGetLevel("psLib.math"));
-
     // Ensure that stats->clipIter is within the proper range.
     PS_ASSERT_INT_WITHIN_RANGE(stats->clipIter,
@@ -601,7 +583,6 @@
     vectorSampleMedian(myVector, tmpMask, maskVal, stats);
     if (isnan(stats->sampleMedian)) {
-        psTrace(TRACE, 5, "Call to vectorSampleMedian returned NAN\n");
-        psTrace(TRACE, 4, "---- %s(false) end ----\n", __func__);
-        return false;
+        psLogMsg(TRACE, PS_LOG_DETAIL, "Call to vectorSampleMedian returned NAN\n");
+        return true;
     }
     psTrace(TRACE, 6, "The initial sample median is %f\n", stats->sampleMedian);
@@ -610,7 +591,6 @@
     vectorSampleStdev(myVector, errors, tmpMask, maskVal, stats);
     if (isnan(stats->sampleStdev)) {
-        psTrace(TRACE, 5, "Call to vectorSampleStdev returned NAN\n");
-        psTrace(TRACE, 4, "---- %s(false) end ----\n", __func__);
-        return false;
+        psLogMsg(TRACE, PS_LOG_DETAIL, "Call to vectorSampleStdev returned NAN\n");
+        return true;
     }
     psTrace(TRACE, 6, "The initial sample stdev is %f\n", stats->sampleStdev);
@@ -669,6 +649,8 @@
         if (isnan(stats->sampleMean) || isnan(stats->sampleStdev)) {
             iter = stats->clipIter;
-            psError(PS_ERR_UNKNOWN, true, "vectorSampleMean() or vectorSampleStdev() returned a NAN.\n");
-            return false;
+            psLogMsg(TRACE, PS_LOG_DETAIL, "vectorSampleMean() or vectorSampleStdev() returned a NAN.\n");
+            clippedMean = NAN;
+            clippedStdev = NAN;
+            return true;
         } else {
             clippedMean = stats->sampleMean;
@@ -693,5 +675,4 @@
     psTrace(TRACE, 6, "The final clipped stdev is %f\n", clippedStdev);
 
-    psTrace(TRACE, 4, "---- %s(true) end ----\n", __func__);
     return true;
 }
@@ -722,5 +703,4 @@
                               psStats* stats)
 {
-    psTrace(TRACE, 4, "---- %s() begin ----\n", __func__);
     if (psTraceGetLevel("psLib.math") >= 8) {
         PS_VECTOR_PRINT_F32(myVector);
@@ -767,5 +747,5 @@
         if (numValid == 0 || isnan(min) || isnan(max)) {
             // Data range calculation failed
-            psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n");
+            psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to calculate the min/max of the input vector.\n");
             goto escape;
         }
@@ -779,9 +759,9 @@
             stats->robustLQ = min;
             stats->robustN50 = numValid;
+	    // XXX this is sort of an invalid / out-of-bounds result: to set or not to set these bits:
             stats->results |= PS_STAT_ROBUST_MEDIAN;
             stats->results |= PS_STAT_ROBUST_STDEV;
             stats->results |= PS_STAT_ROBUST_QUARTILE;
-            psTrace(TRACE, 5, "All data points have the same value: %f.\n", min);
-            psTrace(TRACE, 4, "---- %s(0) end  ----\n", __func__);
+            psLogMsg(TRACE, PS_LOG_DETAIL, "All data points have the same value: %f.\n", min);
             psFree(mask);
             psFree(statsMinMax);
@@ -809,6 +789,12 @@
         // XXXXX we need to consider this step if errors -> variance
         if (!psVectorHistogram(histogram, myVector, errors, mask, maskVal)) {
+	    // if psVectorHistogram returns false, we have a programming error
             psError(PS_ERR_UNKNOWN, false, "Unable to generate histogram for robust statistics.\n");
-            goto escape;
+	    psFree(histogram);
+	    psFree(cumulative);
+	    psFree(statsMinMax);
+	    psFree(mask);
+
+	    return false;
         }
         if (psTraceGetLevel("psLib.math") >= 8) {
@@ -837,12 +823,14 @@
         PS_BIN_FOR_VALUE(binMedian, cumulative->nums, totalDataPoints/2.0, 0);
 
-        psTrace(TRACE, 6, "The median bin is %ld (%.2f to %.2f)\n", binMedian,
-                cumulative->bounds->data.F32[binMedian], cumulative->bounds->data.F32[binMedian+1]);
-
-        // ADD step 3: Interpolate to the exact 50% position: this is the robust histogram median.
-        stats->robustMedian = fitQuadraticSearchForYThenReturnXusingValues(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0);
+        psTrace(TRACE, 6, "The median bin is %ld (%.2f to %.2f)\n", binMedian, cumulative->bounds->data.F32[binMedian], cumulative->bounds->data.F32[binMedian+1]);
+
+        // ADD step 3: Interpolate to the exact 50% position in bin units
+	stats->robustMedian = fitQuadraticSearchForYThenReturnBin(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0);
+	// float robustBin = fitQuadraticSearchForYThenReturnXusingValues(cumulative->bounds, cumulative->nums, binMedian, totalDataPoints/2.0);
+	// fprintf (stderr, "robustBin : %f vs %f\n", robustBin, stats->robustMedian);
+
+	// convert bin to bin value: this is the robust histogram median.
         if (isnan(stats->robustMedian)) {
-            psError(PS_ERR_UNKNOWN, false,
-                    "Failed to fit a quadratic and calculate the 50-percent position.\n");
+            psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to fit a quadratic and calculate the 50-percent position.\n");
             goto escape;
         }
@@ -866,5 +854,5 @@
 
         if ((binLo < 0) || (binHi < 0)) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to calculate the 15.8655%% and 84.1345%% data points.\n");
+            psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to calculate the 15.8655%% and 84.1345%% data points.\n");
             goto escape;
         }
@@ -962,9 +950,8 @@
                 stats->results |= PS_STAT_ROBUST_STDEV;
                 stats->results |= PS_STAT_ROBUST_QUARTILE;
-                psTrace(TRACE, 5, "Maximum number of iterations (%d) exceeded.", PS_ROBUST_MAX_ITERATIONS);
-                psTrace(TRACE, 4, "---- %s(0) end  ----\n", __func__);
+                psLogMsg(TRACE, PS_LOG_DETAIL, "Maximum number of iterations (%d) exceeded.", PS_ROBUST_MAX_ITERATIONS);
                 psFree(mask);
                 psFree(statsMinMax);
-                return false;
+                return true;
             }
         } else {
@@ -988,9 +975,8 @@
     // ADD step 8: Interpolate to find these two positions exactly: these are the upper and lower quartile
     // positions.
-    psF32 binLo25F32 = fitQuadraticSearchForYThenReturnXusingValues(cumulative->bounds, cumulative->nums, binLo25, totalDataPoints * 0.25f);
-    psF32 binHi25F32 = fitQuadraticSearchForYThenReturnXusingValues(cumulative->bounds, cumulative->nums, binHi25, totalDataPoints * 0.75f);
+    psF32 binLo25F32 = fitQuadraticSearchForYThenReturnBin(cumulative->bounds, cumulative->nums, binLo25, totalDataPoints * 0.25f);
+    psF32 binHi25F32 = fitQuadraticSearchForYThenReturnBin(cumulative->bounds, cumulative->nums, binHi25, totalDataPoints * 0.75f);
     if (isnan(binLo25F32) || isnan(binHi25F32)) {
-        psError(PS_ERR_UNKNOWN, false,
-                "could not determine the robustUQ: fitQuadraticSearchForYThenReturnX() returned a NAN.\n");
+        psLogMsg(TRACE, PS_LOG_DETAIL, "could not determine the robustUQ: fitQuadraticSearchForYThenReturnBin() returned a NAN.\n");
         goto escape;
     }
@@ -1020,5 +1006,4 @@
     stats->results |= PS_STAT_ROBUST_QUARTILE;
 
-    psTrace(TRACE, 4, "---- %s(0) end  ----\n", __func__);
     return true;
 
@@ -1033,6 +1018,4 @@
     stats->results |= PS_STAT_ROBUST_QUARTILE;
 
-    psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-
     psFree(histogram);
     psFree(cumulative);
@@ -1040,5 +1023,5 @@
     psFree(mask);
 
-    return false;
+    return true;
 }
 
@@ -1057,5 +1040,8 @@
     // calculated, then call vectorSampleMean()
     if (!(stats->results & PS_STAT_ROBUST_MEDIAN)) {
-        vectorRobustStats(myVector, errors, mask, maskVal, stats);
+        if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) {
+            psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n");
+	    return false;
+	}
     }
 
@@ -1064,6 +1050,5 @@
         stats->fittedStdev = NAN;
         stats->fittedStdev = NAN;
-        psTrace(TRACE, 4, "---- %s() end ----\n", __func__);
-        return false;
+        return true;
     }
 
@@ -1096,8 +1081,7 @@
         float max = statsMinMax->max;
         if (numValid == 0 || isnan(min) || isnan(max)) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n");
+            psTrace(TRACE, 5, "Failed to calculate the min/max of the input vector.\n");
             psFree(statsMinMax);
-            psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-            return false;
+            return true;
         }
 
@@ -1111,4 +1095,5 @@
         psHistogram *histogram = psHistogramAlloc(min, max, numBins); // A new histogram (without outliers)
         if (!psVectorHistogram(histogram, myVector, errors, mask, maskVal)) {
+	    // if psVectorHistogram returns false, we have a programming error
             psError(PS_ERR_UNKNOWN, false, "Unable to generate histogram for fitted statistics.\n");
             psFree(histogram);
@@ -1175,4 +1160,6 @@
             PS_VECTOR_PRINT_F32(y);
         }
+	
+	// psMinimizeLMChi2 can return false for bad data as well as for serious failures
         if (!psMinimizeLMChi2(minimizer, NULL, params, NULL, x, y, NULL, minimizeLMChi2Gauss1D)) {
             psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n");
@@ -1183,6 +1170,5 @@
             psFree(histogram);
             psFree(statsMinMax);
-            psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-            return false;
+            return true;
         }
         if (psTraceGetLevel("psLib.math") >= 8) {
@@ -1235,5 +1221,8 @@
     // calculated, then call vectorSampleMean()
     if (!(stats->results & PS_STAT_ROBUST_MEDIAN)) {
-        vectorRobustStats(myVector, errors, mask, maskVal, stats);
+        if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) {
+            psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n");
+	    return false;
+	}
     }
 
@@ -1243,5 +1232,5 @@
         stats->fittedStdev = NAN;
         psTrace(TRACE, 4, "---- %s() end ----\n", __func__);
-        return false;
+        return true;
     }
 
@@ -1274,8 +1263,8 @@
         float max = statsMinMax->max;
         if (numValid == 0 || isnan(min) || isnan(max)) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n");
+            psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to calculate the min/max of the input vector.\n");
             psFree(statsMinMax);
             psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-            return false;
+            return true;
         }
 
@@ -1354,4 +1343,5 @@
         // psVectorInit (fitMask, 0);
 
+	// XXX not sure if these should result in errors or not...
         if (!psVectorFitPolynomial1D (poly, NULL, 0, y, NULL, x)) {
             psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n");
@@ -1366,6 +1356,5 @@
 
         if (poly->coeff[2] >= 0.0) {
-            psTrace(TRACE, 6, "Parabolic fit results: %f + %f x + %f x^2\n",
-                    poly->coeff[0], poly->coeff[1], poly->coeff[2]);
+            psTrace(TRACE, 6, "Parabolic fit results: %f + %f x + %f x^2\n", poly->coeff[0], poly->coeff[1], poly->coeff[2]);
             psError(PS_ERR_UNKNOWN, false, "fit did not converge\n");
             psFree(x);
@@ -1429,5 +1418,8 @@
     // calculated, then call vectorSampleMean()
     if (!(stats->results & PS_STAT_ROBUST_MEDIAN)) {
-        vectorRobustStats(myVector, errors, mask, maskVal, stats);
+        if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) {
+            psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n");
+	    return false;
+	}
     }
 
@@ -1437,5 +1429,5 @@
         stats->fittedStdev = NAN;
         psTrace(TRACE, 4, "---- %s() end ----\n", __func__);
-        return false;
+        return true;
     }
 
@@ -1468,8 +1460,8 @@
         float max = statsMinMax->max;
         if (numValid == 0 || isnan(min) || isnan(max)) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n");
+            psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to calculate the min/max of the input vector.\n");
             psFree(statsMinMax);
             psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-            return false;
+            return true;
         }
 
@@ -1516,8 +1508,7 @@
         PS_BIN_FOR_VALUE (binMax, histogram->bounds, guessMean + maxFitSigma*guessStdev, 0);
         if (binMin == binMax) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n");
+            psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to calculate the min/max of the input vector.\n");
             psFree(statsMinMax);
-            psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-            return false;
+            return true;
         }
 
@@ -1724,5 +1715,8 @@
     // calculated, then call vectorSampleMean()
     if (!(stats->results & PS_STAT_ROBUST_MEDIAN)) {
-        vectorRobustStats(myVector, errors, mask, maskVal, stats);
+        if (!vectorRobustStats(myVector, errors, mask, maskVal, stats)) {
+            psError(PS_ERR_UNKNOWN, false, "failure to measure robust stats\n");
+	    return false;
+	}
     }
 
@@ -1731,6 +1725,5 @@
         stats->fittedStdev = NAN;
         stats->fittedStdev = NAN;
-        psTrace(TRACE, 4, "---- %s() end ----\n", __func__);
-        return false;
+        return true;
     }
 
@@ -1763,8 +1756,9 @@
         float max = statsMinMax->max;
         if (numValid == 0 || isnan(min) || isnan(max)) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n");
+            psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to calculate the min/max of the input vector.\n");
             psFree(statsMinMax);
-            psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-            return false;
+	    stats->fittedStdev = NAN;
+	    stats->fittedStdev = NAN;
+            return true;
         }
 
@@ -1780,8 +1774,10 @@
         psHistogram *histogram = psHistogramAlloc(min, max, numBins); // A new histogram (without outliers)
         if (!psVectorHistogram(histogram, myVector, errors, mask, maskVal)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to generate histogram for fitted statistics v4.\n");
+            psLogMsg(TRACE, PS_LOG_DETAIL, "Unable to generate histogram for fitted statistics v4.\n");
             psFree(histogram);
             psFree(statsMinMax);
-            return false;
+	    stats->fittedStdev = NAN;
+	    stats->fittedStdev = NAN;
+            return true;
         }
         if (psTraceGetLevel("psLib.math") >= 8) {
@@ -1813,8 +1809,9 @@
         PS_BIN_FOR_VALUE (binMax, histogram->bounds, guessMean + maxFitSigma*guessStdev, 0);
         if (binMin == binMax) {
-            psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n");
+            psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to calculate the min/max of the input vector.\n");
             psFree(statsMinMax);
-            psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-            return false;
+	    stats->fittedStdev = NAN;
+	    stats->fittedStdev = NAN;
+            return true;
         }
 
@@ -1877,4 +1874,6 @@
 
             // fit 2nd order polynomial to ln(y) = -(x-xo)^2/2sigma^2
+	    // XXX this fit may fail with an error for an ill-conditioned matrix (bad data)
+	    // we probably should be able to handle the data errors gracefully
             psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2);
             bool status = psVectorFitPolynomial1D (poly, NULL, 0, y, NULL, x);
@@ -1883,15 +1882,15 @@
 
             if (!status) {
-                psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n");
+                psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to fit a gaussian to the robust histogram.\n");
                 psFree(poly);
                 psFree(histogram);
                 psFree(statsMinMax);
-                psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-                return false;
+		stats->fittedStdev = NAN;
+		stats->fittedStdev = NAN;
+                return true;
             }
 
             if (poly->coeff[2] >= 0.0) {
-                psTrace(TRACE, 6, "Failed parabolic fit: %f + %f x + %f x^2\n",
-                        poly->coeff[0], poly->coeff[1], poly->coeff[2]);
+                psLogMsg(TRACE, PS_LOG_MINUTIA, "Failed parabolic fit: %f + %f x + %f x^2\n", poly->coeff[0], poly->coeff[1], poly->coeff[2]);
                 psFree(poly);
                 psFree(histogram);
@@ -1907,7 +1906,8 @@
                 }
 
-                psError(PS_ERR_UNKNOWN, false, "fit did not converge\n");
-                psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-                return false;
+                psLogMsg(TRACE, PS_LOG_DETAIL, "fit did not converge\n");
+		stats->fittedStdev = NAN;
+		stats->fittedStdev = NAN;
+                return true;
             }
 
@@ -1975,10 +1975,11 @@
 
             if (!status) {
-                psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n");
+                psLogMsg(TRACE, PS_LOG_DETAIL, "Failed to fit a gaussian to the robust histogram.\n");
                 psFree(poly);
                 psFree(histogram);
                 psFree(statsMinMax);
-                psTrace(TRACE, 4, "---- %s(false) end  ----\n", __func__);
-                return false;
+		stats->fittedStdev = NAN;
+		stats->fittedStdev = NAN;
+                return true;
             }
 
@@ -2155,6 +2156,4 @@
 psStats* p_psStatsAlloc(const char *file, unsigned int lineno, const char *func, psStatsOptions options)
 {
-    psTrace(TRACE, 3,"---- %s() begin  ----\n", __func__);
-
     psStats *stats = p_psAlloc(file, lineno, func, sizeof(psStats));
     psMemSetDeallocator(stats, (psFreeFunc)statsFree);
@@ -2172,5 +2171,4 @@
     stats->tmpMask = NULL;
 
-    psTrace(TRACE, 3, "---- %s() end  ----\n", __func__);
     return stats;
 }
@@ -2231,5 +2229,4 @@
                    psVectorMaskType maskVal)
 {
-    psTrace(TRACE, 3,"---- %s() begin  ----\n", __func__);
     PS_ASSERT_PTR_NON_NULL(stats, false);
     PS_ASSERT_VECTOR_NON_NULL(in, false);
@@ -2380,5 +2377,4 @@
     psFree(errorsF32);
     psFree(maskVector);
-    psTrace(TRACE, 3,"---- %s() end  ----\n", __func__);
     return status;
 }
@@ -2745,5 +2741,4 @@
     return tmpFloat;
 }
-# endif
 
 /******************************************************************************
@@ -2873,4 +2868,135 @@
     return tmpFloat;
 }
+# endif
+
+/******************************************************************************
+fitQuadraticSearchForYThenReturnXusingValues(*xVec, *yVec, binNum, yVal): A general routine
+which fits a quadratic to three points and returns the x bin value corresponding to the input
+y-value.  This routine takes psVectors of x/y pairs as input, and fits a quadratic to the 3
+points surrounding element binNum in the vectors.  This version uses the values of x[i] for the
+x coordinates (not the midpoints).  This is appropriate for a cumulative histogram.  It then
+determines for what value x does that quadratic f(x) = yVal (the input parameter).
+
+XXX this function is used a fair amount in an inner loop: the polynomial fitting and evaluation
+could easily be done with statically allocated doubles, skipping the psLib versions of
+polynomial fitting, etc.
+
+*****************************************************************************/
+static psF32 fitQuadraticSearchForYThenReturnBin(const psVector *xVec,
+						 psVector *yVec,
+						 psS32 binNum,
+						 psF32 yVal
+    )
+{
+    psTrace(TRACE, 5, "binNum, yVal is (%d, %f)\n", binNum, yVal);
+    if (psTraceGetLevel("psLib.math") >= 8) {
+        PS_VECTOR_PRINT_F32(xVec);
+        PS_VECTOR_PRINT_F32(yVec);
+    }
+
+    PS_ASSERT_VECTOR_NON_NULL(xVec, NAN);
+    PS_ASSERT_VECTOR_NON_NULL(yVec, NAN);
+    PS_ASSERT_VECTOR_TYPE(xVec, PS_TYPE_F32, NAN);
+    PS_ASSERT_VECTOR_TYPE(yVec, PS_TYPE_F32, NAN);
+    PS_ASSERT_INT_WITHIN_RANGE(binNum, 0, (int)(xVec->n - 1), NAN);
+    PS_ASSERT_INT_WITHIN_RANGE(binNum, 0, (int)(yVec->n - 1), NAN);
+
+    psVector *x = psVectorAlloc(3, PS_TYPE_F64);
+    psVector *y = psVectorAlloc(3, PS_TYPE_F64);
+    psF32 tmpFloat = 0.0f;
+
+    if ((binNum >= 1) && (binNum <= (yVec->n - 2)) && (binNum <= (xVec->n - 2))) {
+        // The general case.  We have all three points.
+        x->data.F64[0] = binNum - 1;
+        x->data.F64[1] = binNum;
+        x->data.F64[2] = binNum + 1;
+        y->data.F64[0] = yVec->data.F32[binNum - 1];
+        y->data.F64[1] = yVec->data.F32[binNum];
+        y->data.F64[2] = yVec->data.F32[binNum + 1];
+        psTrace(TRACE, 6, "x vec (orig) is (%f %f %f %f)\n", xVec->data.F32[binNum - 1], xVec->data.F32[binNum], xVec->data.F32[binNum+1], xVec->data.F32[binNum+2]);
+        psTrace(TRACE, 6, "x data is (%f %f %f)\n", x->data.F64[0], x->data.F64[1], x->data.F64[2]);
+        psTrace(TRACE, 6, "y data is (%f %f %f)\n", y->data.F64[0], y->data.F64[1], y->data.F64[2]);
+
+        // Ensure that the y value lies within range of the y values.
+        if (! (((y->data.F64[0] <= yVal) && (yVal <= y->data.F64[2])) ||
+               ((y->data.F64[2] <= yVal) && (yVal <= y->data.F64[0]))) ) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                    _("Specified yVal, %g, is not within y-range, %g to %g."),
+                    (psF64)yVal, y->data.F64[0], y->data.F64[2]);
+            return NAN;
+        }
+
+        // Ensure that the y values are monotonic.
+        if (((y->data.F64[0] < y->data.F64[1]) && !(y->data.F64[1] <= y->data.F64[2])) ||
+            ((y->data.F64[0] > y->data.F64[1]) && !(y->data.F64[1] >= y->data.F64[2]))) {
+            psError(PS_ERR_UNKNOWN, true,
+                    "This routine must be called with monotonically increasing or decreasing data points.\n");
+            psFree(x);
+            psFree(y);
+            return NAN;
+        }
+
+        // Determine the coefficients of the polynomial.
+        psPolynomial1D *myPoly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2);
+        if (!psVectorFitPolynomial1D(myPoly, NULL, 0, y, NULL, x)) {
+            psError(PS_ERR_UNEXPECTED_NULL, false,
+                    _("Failed to fit a 1-dimensional polynomial to the three specified data points.  "
+                      "Returning NAN."));
+            psFree(x);
+            psFree(y);
+            return NAN;
+        }
+
+        psTrace(TRACE, 6, "myPoly->coeff[0] is %f\n", myPoly->coeff[0]);
+        psTrace(TRACE, 6, "myPoly->coeff[1] is %f\n", myPoly->coeff[1]);
+        psTrace(TRACE, 6, "myPoly->coeff[2] is %f\n", myPoly->coeff[2]);
+        psTrace(TRACE, 6, "Fitted y vec is (%f %f %f)\n",
+                (psF32) psPolynomial1DEval(myPoly, (psF64) x->data.F64[0]),
+                (psF32) psPolynomial1DEval(myPoly, (psF64) x->data.F64[1]),
+                (psF32) psPolynomial1DEval(myPoly, (psF64) x->data.F64[2]));
+
+        psTrace(TRACE, 6, "We fit the polynomial, now find x such that f(x) equals %f\n", yVal);
+        float binValue = QuadraticInverse(myPoly->coeff[2], myPoly->coeff[1], myPoly->coeff[0], yVal, x->data.F64[0], x->data.F64[2]);
+        psFree(myPoly);
+
+        if (isnan(binValue)) {
+            psError(PS_ERR_UNEXPECTED_NULL,
+                    false, _("Failed to determine the median of the fitted polynomial.  Returning NAN."));
+            psFree(x);
+            psFree(y);
+            return(NAN);
+        }
+
+	// I believe that mathematically the fitted bin position must be between binNum - 1 and binNum + 1
+	assert (binValue >= binNum - 1);
+	assert (binValue <= binNum + 1);
+
+	int fitBin = binValue;
+	float dX = xVec->data.F32[fitBin+1] - xVec->data.F32[fitBin];
+	float dY = binValue - fitBin;
+	tmpFloat = xVec->data.F32[fitBin] + dY * dX;
+    } else {
+        // These are special cases where the bin is at the beginning or end of the vector.
+        if (binNum == 0) {
+            // We have two points only at the beginning of the vectors x and y.
+	    // X = (dX/dY)(Y - Yo) + Xo
+	    float dX = xVec->data.F32[1] - xVec->data.F32[0];
+	    float dY = yVec->data.F32[1] - yVec->data.F32[0];	    
+	    tmpFloat = (yVal - yVec->data.F32[0]) * (dX / dY) + xVec->data.F32[0];
+        } else if (binNum == (xVec->n - 1)) {
+            // We have two points only at the end of the vectors x and y.
+	    // X = (dX/dY)(Y - Yo) + Xo
+	    float dX = xVec->data.F32[binNum] - xVec->data.F32[binNum-1];
+	    float dY = yVec->data.F32[binNum] - yVec->data.F32[binNum-1];	    
+	    tmpFloat = (yVal - yVec->data.F32[binNum-1]) * (dX / dY) + xVec->data.F32[binNum-1];
+        } 
+    }
+
+    psTrace(TRACE, 6, "FIT: return %f\n", tmpFloat);
+    psFree(x);
+    psFree(y);
+
+    return tmpFloat;
+}
 
 /******************************************************************************
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/Makefile.am	(revision 24244)
@@ -3,14 +3,26 @@
 noinst_LTLIBRARIES = libpslibsys.la
 
-# PSLIB_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-# PSLIB_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-# PSLIB_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+if HAVE_SVNVERSION
+PSLIB_VERSION=`$(SVNVERSION) ../..`
+else
+PSLIB_VERSION="UNKNOWN"
+endif
+
+if HAVE_SVN
+PSLIB_BRANCH=`$(SVN) info ../.. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PSLIB_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PSLIB_BRANCH="UNKNOWN"
+PSLIB_SOURCE="UNKNOWN"
+endif
 
 # Force recompilation of psConfigure.c, since it gets the version information
-psConfigure.c: FORCE
-	touch psConfigure.c
+psConfigure.c: psVersionDefinitions.h
+psVersionDefinitions.h: psVersionDefinitions.h.in FORCE
+	-$(RM) psVersionDefinitions.h
+	$(SED) -e "s|@PSLIB_VERSION@|\"$(PSLIB_VERSION)\"|" -e "s|@PSLIB_BRANCH@|\"$(PSLIB_BRANCH)\"|" -e "s|@PSLIB_SOURCE@|\"$(PSLIB_SOURCE)\"|" psVersionDefinitions.h.in > psVersionDefinitions.h
 FORCE: ;
 
-libpslibsys_la_CPPFLAGS = $(SRCINC) $(PSLIB_CFLAGS) $(CFITSIO_CFLAGS) -DPSLIB_VERSION=$(SVN_VERSION) -DPSLIB_BRANCH=$(SVN_BRANCH) -DPSLIB_SOURCE=$(SVN_SOURCE)
+libpslibsys_la_CPPFLAGS = $(SRCINC) $(PSLIB_CFLAGS) $(CFITSIO_CFLAGS)
 libpslibsys_la_SOURCES = \
 	psAbort.c \
@@ -28,7 +40,7 @@
 	strcasestr.c
 
-EXTRA_DIST = sys.i psErrorCodes.c.in psErrorCodes.h.in
+EXTRA_DIST = sys.i psErrorCodes.c.in psErrorCodes.h.in psVersionDefinitions.h
 
-BUILT_SOURCES = psErrorCodes.c
+BUILT_SOURCES = psErrorCodes.c psVersionDefinitions.h
 
 psErrorCodes.c: ../psErrorCodes_$(PS_LANG).dat psErrorCodes.c.in psErrorCodes.h
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psConfigure.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psConfigure.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psConfigure.c	(revision 24244)
@@ -44,4 +44,6 @@
 #include "psMemory.h"
 
+#include "psVersionDefinitions.h"
+
 static char *memCheckName = NULL;       // Filename to which to write results of mem check
 static FILE *memCheckFile = NULL;       // File to which to write results of mem check
@@ -57,11 +59,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString psLibVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PSLIB_BRANCH), xstr(PSLIB_VERSION));
+    psStringAppend(&value, "%s@%s", PSLIB_BRANCH, PSLIB_VERSION);
     return value;
 }
@@ -69,5 +68,5 @@
 psString psLibSource(void)
 {
-    return psStringCopy(xstr(PSLIB_SOURCE));
+    return psStringCopy(PSLIB_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PS_VERSION_DEFINITIONS_H
+#define PS_VERSION_DEFINITIONS_H
+
+#define PSLIB_VERSION @PSLIB_VERSION@ // SVN version
+#define PSLIB_BRANCH  @PSLIB_BRANCH@  // SVN branch
+#define PSLIB_SOURCE  @PSLIB_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/types/psArguments.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/types/psArguments.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/types/psArguments.h	(revision 24244)
@@ -78,4 +78,54 @@
 );
 
+/** Macro to implement some generic command line arguments.  A package-
+ *  specific routine called pkgnameVersionLong() is presumed to exist.
+ */
+#define PSARGUMENTS_INSTANTIATE_GENERICS( pkgname, config, argc, argv )   \
+  { int N= psArgumentGet (argc, argv, "-version");                        \
+    if (N) {                                                              \
+      psString version;                                                   \
+      version = pkgname ## VersionLong();                                 \
+      fprintf (stdout, "%s\n", version);                                  \
+      psFree (version);                                                   \
+      version = psModulesVersionLong();                                   \
+      fprintf (stdout, "%s\n", version);                                  \
+      psFree (version);                                                   \
+      version = psLibVersionLong();                                       \
+      fprintf (stdout, "%s\n", version); psFree (version);                \
+      exit (0);                                                           \
+    }                                                                     \
+    N = psArgumentGet(argc, argv, "-dumpconfig");                         \
+    if (N) {                                                              \
+      if (N<argc-1) {							  \
+        psArgumentRemove(N, &argc, argv);                                 \
+        psMetadataAddStr(config->arguments,                               \
+                         PS_LIST_TAIL, "DUMP_CONFIG", PS_META_REPLACE,    \
+                         "Filename for configuration dump", argv[N]);     \
+        psArgumentRemove(N, &argc, argv);                                 \
+      }                                                                   \
+      else {                                                              \
+        fprintf(stderr,"-dumpconfig requires a filename argument\n");     \
+        exit(-1);                                                         \
+      }                                                                   \
+    }                                                                     \
+  }
+
+/** Macro to implement handling of '-threads' argument.  A package-
+ *  specific routine called pkgnameSetThreads(int nThreads) is
+ *  presumed to exist.
+ */
+#define PSARGUMENTS_INSTANTIATE_THREADSARG( pkgname, config, argc, argv )   \
+  { int N= psArgumentGet(argc, argv, "-threads");                           \
+    if (N) {                                                                \
+      psArgumentRemove(N, &argc, argv);                                     \
+      int nThreads = atoi(argv[N]);                                         \
+      psMetadataAddS32(config->arguments, PS_LIST_TAIL, "NTHREADS", 0, "number of "#pkgname" threads", nThreads); \
+      psArgumentRemove(N, &argc, argv);                                     \
+      psThreadPoolInit (nThreads);                                          \
+    }                                                                       \
+    pkgname ## SetThreads();                                                \
+  }
+
+
 /// @}
 #endif // #ifndef PS_ARGUMENTS_H
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/src/types/psMetadataConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/src/types/psMetadataConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/src/types/psMetadataConfig.c	(revision 24244)
@@ -180,20 +180,4 @@
 
     return cleaned;
-}
-
-// Count repeat occurances of a single character within a line. The input string must be null terminated.
-static psS32 repeatedChars(char *inString,
-                           char ch)
-{
-    psS32 count = 0;
-
-    while(*inString!='\0') {
-        if(*inString == ch) {
-            count++;
-        }
-        inString++;
-    }
-
-    return count;
 }
 
@@ -1071,13 +1055,4 @@
     *notBlank = true;
 
-    // Check for more than one '@' in a line
-    // XXX this logic is wrong -- there's no reason a @ can't appear in a
-    // comment
-    if (repeatedChars(linePtr, '@') > 1) {
-        psError(PS_ERR_IO, true,
-                _("More than one '@' character is not allowed"));
-        return false;
-    }
-
     // Get metadata item name
     psS32               status = 0;
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/test/astro/tap_psTime_01.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/astro/tap_psTime_01.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/test/astro/tap_psTime_01.c	(revision 24244)
@@ -842,7 +842,7 @@
     {
         psMemId id = psMemGetId();
-        psTime *time2 = psTimeConvert(NULL, PS_TIME_TAI);
-
-        ok(time2 == NULL, "psTimeConvert(NULL, PS_TIME_TAI) returned NULL");
+        bool status = psTimeConvert(NULL, PS_TIME_TAI);
+
+        ok(status == false, "psTimeConvert(NULL, PS_TIME_TAI) returned NULL");
         ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
 
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/Makefile.am	(revision 24244)
@@ -22,4 +22,5 @@
 	tap_psMatrix06 \
 	tap_psMatrix07 \
+	tap_psMatrix08 \
 	tap_psPolyFit1D \
 	tap_psPolyFit2D \
@@ -47,8 +48,9 @@
 	tap_psMatrixVectorArithmetic01 \
 	tap_psMatrixVectorArithmetic04 \
-	tap_psRandom \
 	tap_psMinimizePowell \
 	tap_psSpline1D \
 	tap_psPolynomialMD
+
+#	tap_psRandom
 
 if BUILD_TESTS
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/data/Agj.fits
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/data/Agj.fits	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/data/Agj.fits	(revision 24244)
@@ -0,0 +1,3 @@
+SIMPLE  =                    T / file does conform to FITS standard             BITPIX  =                  -32 / number of bits per data pixel                  NAXIS   =                    2 / number of data axes                            NAXIS1  =                    9 / length of data axis 1                          NAXIS2  =                    9 / length of data axis 2                          EXTEND  =                    T / FITS dataset may contain extensions            COMMENT   FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT   and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H BZERO   =   0.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK          BSCALE  =   1.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK          END                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             =þÓŒE¢u>~Cºœ¥NOŒE¢v;€}&œ¥NP<Öñ?€>~C·œ¥NN@žÈÿ¿$
+kÀÙcÈ>FŸèœ¥NN<Öñ¿$
+k?õhŸ>Š_‡>FŸèÀN›>¿Œµ7>Š_‡?$*¿Œµ7ÀzþÀÙcÇ>FŸæBMI«>GK>FŸîÀN›A¿Œµ8>GOA奈@ÄŸ­¿Œµ8Àzÿ@ÄŸ®A€Ö
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/data/Bgj.fits
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/data/Bgj.fits	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/data/Bgj.fits	(revision 24244)
@@ -0,0 +1,2 @@
+SIMPLE  =                    T / file does conform to FITS standard             BITPIX  =                  -32 / number of bits per data pixel                  NAXIS   =                    2 / number of data axes                            NAXIS1  =                    1 / length of data axis 1                          NAXIS2  =                    9 / length of data axis 2                          EXTEND  =                    T / FITS dataset may contain extensions            COMMENT   FITS (Flexible Image Transport System) format is defined in 'AstronomyCOMMENT   and Astrophysics', volume 376, page 359; bibcode: 2001A&A...376..359H BZERO   =   0.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK          BSCALE  =   1.000000000000E+00 / Scaling: TRUE = BZERO + BSCALE * DISK          END                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             ?ÃÒ+ŸþžšBÄTÖÀ
+B+ÁŒWECtššClBÒ?
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/data/input
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/data/input	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/data/input	(revision 24244)
@@ -0,0 +1,9 @@
+
+macro go
+  rd A Agj.fits 
+  rd Bv Bgj.fits 
+  mget Bv B -y 0
+  set Ai = A
+  set Bi = B
+  gaussj A B
+end
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psMatrix03.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psMatrix03.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psMatrix03.c	(revision 24244)
@@ -106,8 +106,8 @@
         psVector *inVector = (psVector*) psVectorAlloc(3, PS_TYPE_F64);
 
-        luImage = psMatrixLUD(luImage, &perm, inImage);
-        ok(luImage != NULL, "psMatrixLUD() produced a non-NULL LU matrix");
+        luImage = psMatrixLUDecomposition(luImage, &perm, inImage);
+        ok(luImage != NULL, "psMatrixLUDecomposition() produced a non-NULL LU matrix");
         skip_start(luImage == NULL, 6, "Skipping tests because LU matrix was NULL");
-        ok(!checkMatrix(luImage), "psMatrixLUD() produced the correct LU matrix");
+        ok(!checkMatrix(luImage), "psMatrixLUDecomposition() produced the correct LU matrix");
         ok(luImage->type.dimen == PS_DIMEN_IMAGE, "The LU matrix has the correct ->dimen member");
         ok(luImage == tempImage, "The LU matrix was not created from scratch");
@@ -119,6 +119,6 @@
         inVector->n = 3;
 
-        outVector = psMatrixLUSolve(outVector, luImage, inVector, perm);
-        ok(!checkVector(outVector), "psMatrixLUSolve() correctly solved the equations");
+        outVector = psMatrixLUSolution(outVector, luImage, inVector, perm);
+        ok(!checkVector(outVector), "psMatrixLUSolution() correctly solved the equations");
         ok(outVector->type.dimen == PS_DIMEN_VECTOR, "The output vector hasthe correct ->dimen member");
         ok(outVector == tempVector, "The output vector was not created from scratch");
@@ -159,8 +159,8 @@
         inVector32->n = 3;
 
-        luImage32 = psMatrixLUD(luImage32, &perm32, inImage32);
-        ok(luImage32 != NULL, "psMatrixLUD() produced a non-NULL LU matrix");
+        luImage32 = psMatrixLUDecomposition(luImage32, &perm32, inImage32);
+        ok(luImage32 != NULL, "psMatrixLUDecomposition() produced a non-NULL LU matrix");
         skip_start(luImage32 == NULL, 6, "Skipping tests because LU matrix was NULL");
-        ok(!checkMatrix(luImage32), "psMatrixLUD() produced the correct LU matrix");
+        ok(!checkMatrix(luImage32), "psMatrixLUDecomposition() produced the correct LU matrix");
         ok(luImage32->type.dimen == PS_DIMEN_IMAGE, "The LU matrix has the correct ->dimen member");
         ok(luImage32 == tempImage32, "The LU matrix was not created from scratch");
@@ -168,6 +168,6 @@
         // Determine solution to matrix equation
 
-        outVector32 = psMatrixLUSolve(outVector32, luImage32, inVector32, perm32);
-        ok(!checkVector(outVector32), "psMatrixLUSolve() correctly solved the equations");
+        outVector32 = psMatrixLUSolution(outVector32, luImage32, inVector32, perm32);
+        ok(!checkVector(outVector32), "psMatrixLUSolution() correctly solved the equations");
         ok(outVector32->type.dimen == PS_DIMEN_VECTOR, "The output vector hasthe correct ->dimen member");
         ok(outVector32 == tempVector32, "The output vector was not created from scratch");
@@ -182,47 +182,44 @@
     }
 
-
     // Attempt to use null image input argument
     // XXX: This test should generate an error or warning
     // XXX: This seg-faults
-    if (0) {
-        psMemId id = psMemGetId();
-        psImage *imageTest = (psImage*)psImageAlloc(3, 3, PS_TYPE_F64);
-        psMatrixLUD(imageTest, NULL, NULL);
-        psFree(imageTest);
-        ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
-    }
-
+    { 
+	psMemId id = psMemGetId();
+	psImage *imageTest = (psImage*)psImageAlloc(3, 3, PS_TYPE_F64);
+	psMatrixLUDecomposition(imageTest, NULL, NULL);
+	psFree(imageTest);
+	ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
+    }
 
     // Attempt to use null input vector argument
     // XXX: This test should generate an error or warning
     // XXX: This seg-faulta
-    if (0) {
-        psMemId id = psMemGetId();
-        psVector *vectorBad = NULL;
-        psVector *vectorBadOut = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-        psVector *permBad = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-        psImage *imageTest = (psImage*)psImageAlloc(3, 3, PS_TYPE_F64);
-        psMatrixLUSolve(vectorBadOut, imageTest, vectorBad, permBad);
-        psFree(vectorBadOut);
-        psFree(permBad);
-        psFree(imageTest);
-        ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
-    }
-
+    { 
+	psMemId id = psMemGetId();
+	psVector *vectorBad = NULL;
+	psVector *vectorBadOut = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
+	psVector *permBad = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
+	psImage *imageTest = (psImage*)psImageAlloc(3, 3, PS_TYPE_F64);
+	psMatrixLUSolution(vectorBadOut, imageTest, vectorBad, permBad);
+	psFree(vectorBadOut);
+	psFree(permBad);
+	psFree(imageTest);
+	ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
+    }
 
     // Attempt to use null LU image argument
     // XXX: This test should generate an error or warning, but we don't know how to test that.
     // XXX: This seg-faulta
-    if (0) {
-        psMemId id = psMemGetId();
-        psVector *vectorBadOut = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-        psVector *vectorBad = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-        psVector *permBad = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-        psMatrixLUSolve(vectorBadOut, NULL, vectorBad, permBad);
-        psFree(vectorBadOut);
-        psFree(vectorBad);
-        psFree(permBad);
-        ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
+    {
+	psMemId id = psMemGetId();
+	psVector *vectorBadOut = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
+	psVector *vectorBad = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
+	psVector *permBad = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
+	psMatrixLUSolution(vectorBadOut, NULL, vectorBad, permBad);
+	psFree(vectorBadOut);
+	psFree(vectorBad);
+	psFree(permBad);
+	ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
     }
 }
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psMatrix08.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psMatrix08.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psMatrix08.c	(revision 24244)
@@ -0,0 +1,212 @@
+/** @file  tap_psMatrix_08.c
+*
+*  @brief psMatrixLUSolve, psMatrixGJSolve tests (for ill-conditioned matrix)
+*  @author  Eugene Magnier, IfA
+*
+*  @version $Revision: 1.2 $  $Name: not supported by cvs2svn $
+*  @date  $Date: 2007-05-02 04:20:06 $
+*
+*  Copyright 2004-2005 Institute for Astronomy, University of Hawaii
+*
+*/
+#include <stdio.h>
+#include <string.h>
+#include <pslib.h>
+#include "tap.h"
+#include "pstap.h"
+
+# define DEBUG 1
+
+psS32 main( psS32 argc, char* argv[] )
+{
+    plan_tests(23);
+    // psTraceSetLevel("psLib.math.psMatrixGJSolve", 4);
+
+    // Transpose input image into output image
+    {
+        psMemId id = psMemGetId();
+
+	psFits *fits = NULL;
+
+	// we have a specific image and vector pair which gave us trouble elsewhere:
+	// XXX this is an ill-conditioned matrix.  LU Decomposition does not inform us that it is ill-conditioned.  
+	// the result solves the equation, but what are the errors on the values?
+	fits = psFitsOpen ("data/Agj.fits", "r");
+        ok(fits, "opened test image Agj.fits");
+
+	psImage *Aimage = psFitsReadImage (fits, psRegionSet(0,0,0,0), 0);
+        ok(Aimage, "loaded test image Agj.fits");
+
+	psImage *aimage = psImageCopy (NULL, Aimage, Aimage->type.type);
+        ok(aimage, "copied test image Agj.fits");
+
+	psFitsClose (fits);
+
+	fits = psFitsOpen ("data/Bgj.fits", "r");
+        ok(fits, "opened test image Bgj.fits");
+
+	psImage *Bimage = psFitsReadImage (fits, psRegionSet(0,0,0,0), 0);
+        ok(Aimage, "loaded test image Bgj.fits");
+
+	psFitsClose (fits);
+
+	psVector *Bvector = psVectorAlloc (Bimage->numRows, Bimage->type.type);
+        ok(Bvector, "allocated B vector");
+
+	for (int i = 0; i < Bimage->numRows; i++) {
+	    double value = psImageGet (Bimage, 0, i);
+	    psVectorSet (Bvector, i, value);
+	}
+
+	bool status;
+	status = psMatrixLUSolve(Aimage, Bvector);
+        ok(!status, "psMatrixLUSolve correctly returns false for ill-conditioned matrix");
+
+# if (DEBUG)
+	fprintf (stderr, "LU Solution:\n");
+	for (int i = 0; i < Bimage->numRows; i++) {
+	    double value = psVectorGet (Bvector, i);
+	    double valerr = psImageGet (Aimage, i, i);
+	    fprintf (stderr, "%f +/- %f\n", value, valerr);
+	}
+
+	// calculate Ax and compare with B:
+	fprintf (stderr, "result:\n");
+	for (int i = 0; i < Bimage->numRows; i++) {
+	    double value = 0;
+	    for (int j = 0; j < Bvector->n; j++) {
+		double tmpV = psVectorGet (Bvector, j);
+		double tmpI = psImageGet (aimage, j, i);
+		value += tmpV*tmpI;
+	    }
+	    double actual = psImageGet (Bimage, 0, i);
+	    fprintf (stderr, "%f vs %f (delta: %f)\n", value, actual, actual - value);
+	}
+# endif
+
+        psFree(Aimage);
+        psFree(Bimage);
+        psFree(Bvector);
+        ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
+    }
+
+    // Transpose input image into output image
+    {
+        psMemId id = psMemGetId();
+
+	psFits *fits = NULL;
+
+	// we have a specific ill-conditioned matrix in Agj.fits. psMatrixGJSolve detects this and reports a failure.
+	fits = psFitsOpen ("data/Agj.fits", "r");
+        ok(fits, "opened test image Agj.fits");
+
+	psImage *Aimage = psFitsReadImage (fits, psRegionSet(0,0,0,0), 0);
+        ok(Aimage, "loaded test image Agj.fits");
+
+	psFitsClose (fits);
+
+	fits = psFitsOpen ("data/Bgj.fits", "r");
+        ok(fits, "opened test image Bgj.fits");
+
+	psImage *Bimage = psFitsReadImage (fits, psRegionSet(0,0,0,0), 0);
+        ok(Bimage, "loaded test image Bgj.fits");
+
+	psFitsClose (fits);
+
+	psVector *Bvector = psVectorAlloc (Bimage->numRows, Bimage->type.type);
+        ok(Bvector, "allocated B vector");
+
+	for (int i = 0; i < Bimage->numRows; i++) {
+	    double value = psImageGet (Bimage, 0, i);
+	    psVectorSet (Bvector, i, value);
+	}
+
+	bool status;
+	status = psMatrixGJSolve(Aimage, Bvector);
+        ok(!status, "psMatrixGJSolve correctly returns false for ill-conditioned matrix");
+
+# if (DEBUG)
+	fprintf (stderr, "GJ Solution:\n");
+	for (int i = 0; i < Bimage->numRows; i++) {
+	    double value = psVectorGet (Bvector, i);
+	    fprintf (stderr, "%f\n", value);
+	}
+# endif
+
+        psFree(Aimage);
+        psFree(Bimage);
+        psFree(Bvector);
+        ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
+    }
+
+    // Transpose input image into output image
+    {
+        psMemId id = psMemGetId();
+
+	psFits *fits = NULL;
+
+	// we have a specific ill-conditioned matrix in Agj.fits. psMatrixGJSolve detects this and reports a failure.
+	fits = psFitsOpen ("data/Agj.fits", "r");
+        ok(fits, "opened test image Agj.fits");
+
+	psImage *aimage = psFitsReadImage (fits, psRegionSet(0,0,0,0), 0);
+        ok(aimage, "loaded test image Agj.fits");
+
+	psImage *Aimage = psImageCopy (NULL, aimage, PS_TYPE_F64);
+        ok(Aimage, "converted test image to F64");
+
+	psFitsClose (fits);
+
+	fits = psFitsOpen ("data/Bgj.fits", "r");
+        ok(fits, "opened test image Bgj.fits");
+
+	psImage *bimage = psFitsReadImage (fits, psRegionSet(0,0,0,0), 0);
+        ok(bimage, "loaded test image Bgj.fits");
+
+	psImage *Bimage = psImageCopy (NULL, bimage, PS_TYPE_F64);
+        ok(Bimage, "converted test image to F64");
+
+	psFitsClose (fits);
+
+	psVector *Bvector = psVectorAlloc (Bimage->numRows, Bimage->type.type);
+        ok(Bvector, "allocated B vector");
+
+	for (int i = 0; i < Bimage->numRows; i++) {
+	    double value = psImageGet (Bimage, 0, i);
+	    psVectorSet (Bvector, i, value);
+	}
+
+	bool status;
+	status = psMatrixGJSolve(Aimage, Bvector);
+        ok(!status, "psMatrixGJSolve correctly returns false for ill-conditioned matrix");
+
+# if (DEBUG)	
+	fprintf (stderr, "GJ Solution:\n");
+	for (int i = 0; i < Bimage->numRows; i++) {
+	    double value = psVectorGet (Bvector, i);
+	    double valerr = psImageGet (Aimage, i, i);
+	    fprintf (stderr, "%f +/- %f\n", value, valerr);
+	}
+	
+	// calculate Ax and compare with B:
+	fprintf (stderr, "result:\n");
+	for (int i = 0; i < Bimage->numRows; i++) {
+	    double value = 0;
+	    for (int j = 0; j < Bvector->n; j++) {
+		double tmpV = psVectorGet (Bvector, j);
+		double tmpI = psImageGet (aimage, j, i);
+		value += tmpV*tmpI;
+	    }
+	    double actual = psImageGet (Bimage, 0, i);
+	    fprintf (stderr, "%f vs %f (delta: %f)\n", value, actual, actual - value);
+	}
+# endif
+
+        psFree(Aimage);
+        psFree(Bimage);
+        psFree(aimage);
+        psFree(bimage);
+        psFree(Bvector);
+        ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
+    }
+}
Index: anches/cnb_branches/cnb_branch_20090301/psLib/test/math/tst_psMatrix03.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tst_psMatrix03.c	(revision 24243)
+++ 	(revision )
@@ -1,235 +1,0 @@
-/** @file  tst_psMatrix_03.c
- *
- *  @brief Test driver for psMatrix LU functions
- *
- *  This test driver contains the following tests for psMatrix test point 3:
- *     A)  Create input and output images and vectors
- *     B)  Calculate LU matrix
- *     C)  Determine solution to matrix equation
- *     D)  Free input and output images and vectors
- *     E)  Attempt to use null image input argument
- *     F)  Attempt to use null input vector argument
- *     G)  ttempt to use null LU image argument
- *
- *  @author  Ross Harman, MHPCC
- *
- *  @version $Revision: 1.2 $  $Name: not supported by cvs2svn $
- *  @date  $Date: 2005-08-24 01:24:24 $
- *
- *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
- *
- */
-
-#include "pslib_strict.h"
-#include "psTest.h"
-
-#define TOLERANCE 0.000001
-
-#define CHECK_MATRIX(IMAGE)                                                                                  \
-for(psU32 i=0; i<IMAGE->numRows; i++) {                                                                  \
-    for(psU32 j=0; j<IMAGE->numCols; j++) {                                                              \
-        if(IMAGE->type.type == PS_TYPE_F64) {                                                            \
-            if(fabs(IMAGE->data.F64[i][j]-truthMatrix[i][j]) > TOLERANCE) {                              \
-                printf("Matrix values at element %d, %d don't agree %lf vs %lf\n", i, j,                 \
-                       IMAGE->data.F64[i][j], truthMatrix[i][j]);                                        \
-            }                                                                                            \
-        } else if(IMAGE->type.type == PS_TYPE_F32){                                                      \
-            if(fabs(IMAGE->data.F32[i][j]-truthMatrix[i][j]) > TOLERANCE) {                              \
-                printf("Matrix values at element %d, %d don't agree %f vs %lf\n", i, j,                  \
-                       IMAGE->data.F32[i][j], truthMatrix[i][j]);                                        \
-            }                                                                                            \
-        }                                                                                                \
-    }                                                                                                    \
-}
-
-#define CHECK_VECTOR(VECTOR)                                                                                 \
-for(psU32 i=0; i<VECTOR->n; i++) {                                                                       \
-    if(VECTOR->type.type == PS_TYPE_F64) {                                                               \
-        if(fabs(VECTOR->data.F64[i]-truthVector[i]) > TOLERANCE) {                                       \
-            printf("Vector values at element %d don't agree %lf vs %lf\n", i,                            \
-                   VECTOR->data.F64[i], truthVector[i]);                                                 \
-        }                                                                                                \
-    } else if(VECTOR->type.type == PS_TYPE_F32){                                                         \
-        if(fabs(VECTOR->data.F32[i]-truthVector[i]) > TOLERANCE) {                                       \
-            printf("Vector values at element %d don't agree %f vs %lf\n", i,                             \
-                   VECTOR->data.F32[i], truthVector[i]);                                                 \
-        }                                                                                                \
-    }                                                                                                    \
-}
-
-
-psS32 main(psS32 argc, char* argv[])
-{
-    psLogSetFormat("HLNM");
-    psImage *luImage = NULL;
-    psImage *inImage = NULL;
-    psImage *tempImage = NULL;
-    psVector *tempVector = NULL;
-    psVector *perm = NULL;
-    psVector *outVector = NULL;
-    psVector *inVector = NULL;
-    psImage *luImage32 = NULL;
-    psImage *inImage32 = NULL;
-    psImage *tempImage32 = NULL;
-    psVector *tempVector32 = NULL;
-    psVector *perm32 = NULL;
-    psVector *outVector32 = NULL;
-    psVector *inVector32 = NULL;
-
-    double truthVector[3] = {
-                                4.000000,
-                                -2.000000,
-                                3.000000
-                            };
-
-    double truthMatrix[3][3] = {{4.000000,  5.000000,  6.000000},
-                                {0.750000, -2.750000, -6.500000},
-                                {0.500000, -0.545455, -0.545455}};
-
-
-    // Test A - Create input and output images and vectors
-    printPositiveTestHeader(stdout, "psMatrix", "Create input and output images and vectors");
-    luImage = (psImage*)psImageAlloc(3, 3, PS_TYPE_F64);
-    perm = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-    outVector = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-    inVector = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-    inImage = (psImage*)psImageAlloc(3, 3, PS_TYPE_F64);
-    inImage->data.F64[0][0] =  2;
-    inImage->data.F64[0][1] =  4;
-    inImage->data.F64[0][2] =  6;
-    inImage->data.F64[1][0] =  4;
-    inImage->data.F64[1][1] =  5;
-    inImage->data.F64[1][2] =  6;
-    inImage->data.F64[2][0] =  3;
-    inImage->data.F64[2][1] =  1;
-    inImage->data.F64[2][2] = -2;
-    inVector->data.F64[0] = 18.0;
-    inVector->data.F64[1] = 24.0;
-    inVector->data.F64[2] =  4.0;
-    inVector->n = 3;
-    luImage32 = (psImage*)psImageAlloc(3, 3, PS_TYPE_F32);
-    outVector32 = (psVector*)psVectorAlloc(3, PS_TYPE_F32);
-    inVector32 = (psVector*)psVectorAlloc(3, PS_TYPE_F32);
-    inImage32 = (psImage*)psImageAlloc(3, 3, PS_TYPE_F32);
-    inImage32->data.F32[0][0] =  2;
-    inImage32->data.F32[0][1] =  4;
-    inImage32->data.F32[0][2] =  6;
-    inImage32->data.F32[1][0] =  4;
-    inImage32->data.F32[1][1] =  5;
-    inImage32->data.F32[1][2] =  6;
-    inImage32->data.F32[2][0] =  3;
-    inImage32->data.F32[2][1] =  1;
-    inImage32->data.F32[2][2] = -2;
-    inVector32->data.F32[0] = 18.0;
-    inVector32->data.F32[1] = 24.0;
-    inVector32->data.F32[2] =  4.0;
-    inVector32->n = 3;
-    printFooter(stdout, "psMatrix", "Create input and output images and vectors", true);
-
-
-    // Test B - Calculate LU matrix
-    printPositiveTestHeader(stdout, "psMatrix", "Calculate LU matrix");
-    tempImage = luImage;
-    luImage = psMatrixLUD(luImage, &perm, inImage);
-    CHECK_MATRIX(luImage);
-    if(luImage->type.dimen != PS_DIMEN_IMAGE) {
-        printf("Error: Resulting image is not PS_DIMEN_IMAGE\n");
-    } else if(luImage != tempImage) {
-        printf("Error: Return pointer not equal to output argument pointer\n");
-    }
-
-    tempImage32 = luImage32;
-    luImage32 = psMatrixLUD(luImage32, &perm32, inImage32);
-    CHECK_MATRIX(luImage32);
-    if(luImage32->type.dimen != PS_DIMEN_IMAGE) {
-        printf("Error: Resulting image is not PS_DIMEN_IMAGE\n");
-    } else if(luImage32 != tempImage32) {
-        printf("Error: Return pointer not equal to output argument pointer\n");
-    }
-    printFooter(stdout, "psMatrix", "Calculate LU matrix", true);
-
-    // Test C - Determine solution to matrix equation
-    printPositiveTestHeader(stdout, "psMatrix", "Determine solution to matrix equation");
-    tempVector = outVector;
-    outVector = psMatrixLUSolve(outVector, luImage, inVector, perm);
-    CHECK_VECTOR(outVector);
-    if(outVector->type.dimen != PS_DIMEN_VECTOR) {
-        printf("Error: Resulting image is not PS_DIMEN_VECTOR\n");
-    } else if(outVector != tempVector) {
-        printf("Error: Return pointer not equal to output argument pointer\n");
-    }
-
-    tempVector32 = outVector32;
-    outVector32 = psMatrixLUSolve(outVector32, luImage32, inVector32, perm32);
-    CHECK_VECTOR(outVector32);
-    if(outVector32->type.dimen != PS_DIMEN_VECTOR) {
-        printf("Error: Resulting image is not PS_DIMEN_VECTOR\n");
-    } else if(outVector32 != tempVector32) {
-        printf("Error: Return pointer not equal to output argument pointer\n");
-    }
-    printFooter(stdout, "psMatrix", "Determine solution to matrix equation", true);
-
-
-    // Test D - Free input and output images and vectors
-    printPositiveTestHeader(stdout, "psMatrix", "Free input and output images and vectors");
-    psFree(inImage);
-    psFree(luImage);
-    psFree(perm);
-    psFree(outVector);
-    psFree(inVector);
-    psFree(inImage32);
-    psFree(luImage32);
-    psFree(perm32);
-    psFree(outVector32);
-    psFree(inVector32);
-    if( psMemCheckLeaks(0, NULL, stdout, false) ) {
-        psError(PS_ERR_UNKNOWN,true,"Memory leaks detected");
-        return 10;
-    }
-    psS32 nBad = psMemCheckCorruption(0);
-    if(nBad) {
-        printf("ERROR: Found %d bad memory blocks\n", nBad);
-    }
-    printFooter(stdout, "psMatrix" ,"Free input and output images and vectors", true);
-
-
-    // Test E - Attempt to use null image input argument
-    printNegativeTestHeader(stdout,"psMatrix", "Attempt to use null image input argument",
-                            "Invalid operation: inImage or its data is NULL.", 0);
-    psImage *imageTest = (psImage*)psImageAlloc(3, 3, PS_TYPE_F64);
-    psMatrixLUD(imageTest, NULL, NULL);
-    printFooter(stdout, "psMatrix", "Attempt to use null image input argument", true);
-
-
-    // Test F - Attempt to use null input vector argument
-    printNegativeTestHeader(stdout,"psMatrix", "Attempt to use null input vector argument",
-                            "Invalid operation: inVector or its data is NULL.", 0);
-    psVector *vectorBad = NULL;
-    psVector *vectorBadOut = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-    psVector *permBad = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-    imageTest = (psImage*)psImageAlloc(3, 3, PS_TYPE_F64);
-    psMatrixLUSolve(vectorBadOut, imageTest, vectorBad, permBad);
-    printFooter(stdout, "psMatrix", "Attempt to use null input vector argument", true);
-
-
-    // Test G - Attempt to use null LU image argument
-    printNegativeTestHeader(stdout,"psMatrix", "Attempt to use null LU image argument",
-                            "Invalid operation: inImage or its data is NULL.", 0);
-    vectorBadOut = (psVector*)psVectorAlloc(3, PS_TYPE_F64);
-    psMatrixLUSolve(vectorBadOut, NULL, vectorBad, permBad);
-    printFooter(stdout, "psMatrix", "Attempt to use null LU image argument", true);
-
-    psFree(permBad);
-    psFree(imageTest);
-
-    if( psMemCheckLeaks(0, NULL, stdout, false) ) {
-        psError(PS_ERR_UNKNOWN,true,"Memory leaks detected");
-        return 10;
-    }
-    nBad = psMemCheckCorruption(0);
-    if(nBad) {
-        printf("ERROR: Found %d bad memory blocks\n", nBad);
-    }
-
-    return 0;
-}
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/test/sys/tap_psMemory.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/sys/tap_psMemory.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/test/sys/tap_psMemory.c	(revision 24244)
@@ -512,5 +512,7 @@
     
         psSpline1D *spline;
-        spline = psSpline1DAlloc(2, 1, 0, 2);
+	// XXX API changed : 
+        // spline = psSpline1DAlloc(2, 1, 0, 2);
+        spline = psSpline1DAlloc();
         okay = psMemCheckType(PS_DATA_SPLINE1D, spline);
         if (!okay )
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/test/sys/tap_psString.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/test/sys/tap_psString.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/test/sys/tap_psString.c	(revision 24244)
@@ -420,5 +420,6 @@
 
         //psStringAppend should return 0 for NULL input format
-        outSize = psStringAppend(test, nullTest);
+	// note that only a string literal is allowed for the NULL due to gcc change
+        outSize = psStringAppend(test, NULL);
         ok(outSize == 0, "psStringAppend to return 0 for NULL input format");
 
@@ -428,5 +429,6 @@
 
         //psStringPrepend should return 0 for NULL input format
-        outSize = psStringPrepend(test, nullTest);
+	// note that only a string literal is allowed for the NULL due to gcc change
+        outSize = psStringPrepend(test, NULL);
         ok(outSize == 0, "psStringPrepend to return 0 for NULL input format");
 
Index: /branches/cnb_branches/cnb_branch_20090301/psLib/utils/psParseErrorCodes
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psLib/utils/psParseErrorCodes	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psLib/utils/psParseErrorCodes	(revision 24244)
@@ -4,20 +4,18 @@
 use Getopt::Long;
 
-my @ErrorCodes        = ();
-my @ErrorDescriptions = ();
-
-my $data = "psErrorCodes.dat";
-my $outdir = ".";
+my $data;                       # Input data
+my $outdir = ".";               # Output directory
 
 # Assign variables based on the presence of command line options to the script
 GetOptions(
-    "data=s"  => \$data,
-    "outdir=s" => \$outdir,
-    "verbose" => \$verbose,
-    "help"    => \$help
-);
+           "data=s"   => \$data,
+           "outdir=s" => \$outdir,
+           "verbose"  => \$verbose,
+           "help"     => \$help
+           );
 
-if ($help) {
-    print "Usage: parseErrorCodes ", "[--data=dataFile] ", "[--help] ",
+if ($help
+    or not defined $data) {
+    print "Usage: parseErrorCodes ", "[--data=dataFile] ", "[--outdir=directory] ", "[--help] ",
         "[--verbose] filename\n\n";
     exit(0);
@@ -28,4 +26,7 @@
     die "Can not open data file $data.";
 }
+
+my @ErrorCodes        = ();
+my @ErrorDescriptions = ();
 
 print "Datafile:\n" if $verbose;
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/configure.ac	(revision 24244)
@@ -305,4 +305,8 @@
 echo "PSMODULES_CFLAGS: $PSMODULES_CFLAGS"
 echo "PSMODULE_LIBS: $PSMODULES_LIBS"
+
+IPP_VERSION(PSMODULES)
+AM_CONDITIONAL([HAVE_SVNVERSION], [test "x$SVNVERSION" != x])
+AM_CONDITIONAL([HAVE_SVN], [test "x$SVN" != x])
 
 dnl ------- enable -Werror after all of the probes have run ---------
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryDistortion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryDistortion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryDistortion.c	(revision 24244)
@@ -151,8 +151,14 @@
 
             // also measure the L and M median positions as a representative coordinate
-            psVectorStats (stats, L, NULL, NULL, 0);
+            if (!psVectorStats (stats, L, NULL, NULL, 0)) {
+		psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		goto skip;
+	    }
             grad->FP.x = stats->sampleMedian;
 
-            psVectorStats (stats, M, NULL, NULL, 0);
+            if (!psVectorStats (stats, M, NULL, NULL, 0)) {
+		psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		goto skip;
+	    }
             grad->FP.y = stats->sampleMedian;
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryObjects.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryObjects.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryObjects.c	(revision 24244)
@@ -83,7 +83,12 @@
 {
     psArray *matches = psArrayAllocEmpty(x1->n);
+    psVector *found1 = psVectorAlloc(x1->n, PS_TYPE_S8);
+    psVector *found2 = psVectorAlloc(x2->n, PS_TYPE_S8);
 
     const double RADIUS_SQR = PS_SQR(RADIUS);
     double dX, dY, dR;
+
+    psVectorInit (found1, 0);
+    psVectorInit (found2, 0);
 
     int jStart;
@@ -100,6 +105,15 @@
         }
 
+	if (found1->data.S8[i]) {
+	    i++;
+	    continue;
+	}
+	if (found2->data.S8[j]) {
+	    j++;
+	    continue;
+	}
+
         jStart = j;
-        while (fabs(dX) < RADIUS && j < x2->n) {
+        while ((fabs(dX) < RADIUS) && (j < x2->n)) {
 
             dX = x1->data.F64[i] - x2->data.F64[j];
@@ -111,4 +125,8 @@
                 continue;
             }
+	    if (found2->data.S8[j]) {
+		j++;
+		continue;
+	    }
 
             // got a match; add to output list
@@ -117,4 +135,7 @@
             psFree (match);
 
+	    found1->data.S8[i] = 1;
+	    found2->data.S8[j] = 1;
+
             j++;
         }
@@ -122,4 +143,7 @@
         i++;
     }
+    psFree (found1);
+    psFree (found2);
+
     return (matches);
 }
@@ -420,4 +444,5 @@
     obj->sky  = psSphereAlloc();
     obj->Mag  = 0;
+    obj->Color= 0;
     obj->dMag = 0;
 
@@ -447,4 +472,5 @@
     *obj->sky  = *old->sky;
     obj->Mag   =  old->Mag;
+    obj->Color =  old->Color;
     obj->dMag  =  old->dMag;
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryObjects.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryObjects.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryObjects.h	(revision 24244)
@@ -33,12 +33,13 @@
 typedef struct
 {
-    psPlane *pix;   ///< the position in the pmReadout frame
-    psPlane *cell;   ///< the position in the pmCell frame
-    psPlane *chip;   ///< the position in the pmChip frame
-    psPlane *FP;   ///< the position in the pmFPA frame
-    psPlane *TP;   ///< the position in the tangent plane
-    psSphere *sky;        ///< the position on the Celestial Sphere.
-    double Mag;                         ///< object magnitude XXX what filter?
-    double dMag;                        ///< error on object magnitude
+    psPlane *pix;			///< the position in the pmReadout frame
+    psPlane *cell;			///< the position in the pmCell frame
+    psPlane *chip;			///< the position in the pmChip frame
+    psPlane *FP;			///< the position in the pmFPA frame
+    psPlane *TP;			///< the position in the tangent plane
+    psSphere *sky;			///< the position on the Celestial Sphere.
+    float Mag;				///< object magnitude in extracted filter
+    float Color;			///< object color 
+    float dMag;				///< error on object magnitude
 }
 pmAstromObj;
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryUtils.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryUtils.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryUtils.c	(revision 24244)
@@ -46,7 +46,7 @@
 
         /* this loop uses the Newton-Raphson method to solve for Xo,Yo
-        * it needs the high order terms to be small 
-        * Xo,Yo are in pixels;
-        */
+	 * it needs the high order terms to be small 
+	 * Xo,Yo are in pixels;
+	 */
         double dPos = tol + 1;
         for (int i = 0; (dPos > tol) && (i < 20); i++) {
@@ -60,12 +60,12 @@
             Beta->data.F32[1] = psPolynomial2DEval (trans->y, Xo, Yo);
 
-            if (!psMatrixGJSolveF32 (Alpha, Beta)) {
-	      psError(PS_ERR_UNKNOWN, false, "Unable to solve for center.");
-	      psFree (Alpha);
-	      psFree (Beta);
-	      psFree (XdX);
-	      psFree (XdY);
-	      psFree (YdX);
-	      psFree (YdY);
+            if (!psMatrixGJSolve (Alpha, Beta)) {
+		psError(PS_ERR_UNKNOWN, false, "Unable to solve for center.");
+		psFree (Alpha);
+		psFree (Beta);
+		psFree (XdX);
+		psFree (XdY);
+		psFree (YdX);
+		psFree (YdY);
 		return NULL;
 	    }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryWCS.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryWCS.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryWCS.c	(revision 24244)
@@ -96,4 +96,9 @@
 
     if (!status1 || !status2) {
+        Nx = psMetadataLookupS32 (&status1, header, "IMNAXIS1");
+        Ny = psMetadataLookupS32 (&status2, header, "IMNAXIS2");
+    }
+
+    if (!status1 || !status2) {
         Nx = psMetadataLookupS32 (&status1, header, "ZNAXIS1");
         Ny = psMetadataLookupS32 (&status2, header, "ZNAXIS2");
@@ -143,6 +148,6 @@
     int Nx = region->x1 - region->x0;
     int Ny = region->y1 - region->y0;
-    psMetadataAddS32 (header, PS_LIST_TAIL, "NAXIS1", PS_META_REPLACE, "Mosaic Dimensions", Nx);
-    psMetadataAddS32 (header, PS_LIST_TAIL, "NAXIS2", PS_META_REPLACE, "Mosaic Dimensions", Ny);
+    psMetadataAddS32 (header, PS_LIST_TAIL, "IMNAXIS1", PS_META_REPLACE, "Mosaic Dimensions", Nx);
+    psMetadataAddS32 (header, PS_LIST_TAIL, "IMNAXIS2", PS_META_REPLACE, "Mosaic Dimensions", Ny);
 
     pmAstromWCStoHeader (header, wcs);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPA.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPA.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPA.c	(revision 24244)
@@ -298,7 +298,5 @@
 
 
-pmCell *pmCellAlloc(
-    pmChip *chip,
-    const char *name)
+pmCell *pmCellAlloc(pmChip *chip, const char *name)
 {
     pmCell *tmpCell = (pmCell *) psAlloc(sizeof(pmCell));
@@ -323,5 +321,5 @@
     }
     tmpCell->conceptsRead = PM_CONCEPT_SOURCE_NONE;
-    // XXX does this work?  moved to conceptsRead... pmConceptsBlankCell(tmpCell);
+    pmConceptsBlankCell(tmpCell);
 
     return tmpCell;
@@ -362,5 +360,5 @@
     }
     tmpChip->conceptsRead = PM_CONCEPT_SOURCE_NONE;
-    // XXX does this work?  moved to conceptsRead... pmConceptsBlankChip(tmpChip);
+    pmConceptsBlankChip(tmpChip);
     return tmpChip;
 }
@@ -393,5 +391,5 @@
     }
     tmpFPA->conceptsRead = PM_CONCEPT_SOURCE_NONE;
-    // XXX does this work?  moved to conceptsRead... pmConceptsBlankFPA(tmpFPA);
+    pmConceptsBlankFPA(tmpFPA);
 
     // this may be somewhat pedantic, but it makes these things consistent
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPABin.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPABin.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPABin.c	(revision 24244)
@@ -28,10 +28,17 @@
     int numColsOut = binning->nXruff, numRowsOut = binning->nYruff; // Size of output image
 
-    // Output image
-    psImage *outImage;
+
+    psImage *outImage;                  // Output image
     if (out->image && out->image->numCols >= numColsOut && out->image->numRows >= numRowsOut) {
         outImage = out->image;
     } else {
         outImage = out->image = psImageRecycle(out->image,  numColsOut, numRowsOut, PS_TYPE_F32);
+    }
+
+    psImage *outMask;                   // Output mask
+    if (out->mask && out->mask->numCols >= numColsOut && out->mask->numRows >= numRowsOut) {
+        outMask = out->mask;
+    } else {
+        outMask = out->mask = psImageRecycle(out->mask,  numColsOut, numRowsOut, PS_TYPE_IMAGE_MASK);
     }
 
@@ -58,5 +65,14 @@
             }
 
-            outImage->data.F32[yOut][xOut] = numPix > 0 ? sum / numPix : NAN;
+            float imageValue, maskValue;// Values to set
+            if (numPix > 0) {
+                imageValue = sum / numPix;
+                maskValue = 0;
+            } else {
+                imageValue = NAN;
+                maskValue = maskVal;
+            }
+            outImage->data.F32[yOut][xOut] = imageValue;
+            outMask->data.PS_TYPE_IMAGE_MASK_DATA[yOut][xOut] = maskValue;
             xStart = xStop;
         }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPACopy.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPACopy.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPACopy.c	(revision 24244)
@@ -399,7 +399,5 @@
     if (targetHDU) {
         pmHDU *sourceHDU = pmHDUFromCell(source); // The source HDU
-        if (!sourceHDU) {
-            psWarning("Unable to copy header: no source header.");
-        } else if (sourceHDU->header) {
+        if (sourceHDU && sourceHDU->header) {
             targetHDU->header = psMetadataCopy(targetHDU->header, sourceHDU->header);
         }
@@ -481,18 +479,15 @@
     if (targetFPA && sourceFPA) {
         if (targetFPA->toSky) {
-            psAssert (targetFPA->toSky == sourceFPA->toSky, "chips within FPA have inconsistent astrometry references");
-        } else {
-            targetFPA->toSky = psMemIncrRefCounter (sourceFPA->toSky);
-        }
+            psFree(targetFPA->toSky);
+        }
+        targetFPA->toSky = psMemIncrRefCounter (sourceFPA->toSky);
         if (targetFPA->toTPA) {
-            psAssert (targetFPA->toTPA == sourceFPA->toTPA, "chips within FPA have inconsistent astrometry references");
-        } else {
-            targetFPA->toTPA = psMemIncrRefCounter (sourceFPA->toTPA);
-        }
+            psFree(targetFPA->toTPA);
+        }
+        targetFPA->toTPA = psMemIncrRefCounter (sourceFPA->toTPA);
         if (targetFPA->fromTPA) {
-            psAssert (targetFPA->fromTPA == sourceFPA->fromTPA, "chips within FPA have inconsistent astrometry references");
-        } else {
-            targetFPA->fromTPA = psMemIncrRefCounter (sourceFPA->fromTPA);
-        }
+            psFree(targetFPA->fromTPA);
+        }
+        targetFPA->fromTPA = psMemIncrRefCounter (sourceFPA->fromTPA);
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAMaskWeight.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAMaskWeight.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAMaskWeight.c	(revision 24244)
@@ -482,5 +482,5 @@
         return false;
     }
-    float stdev = psStatsGetValue(stdevStats, stdevStat); // Stadard deviation of fluxes
+    float stdev = psStatsGetValue(stdevStats, stdevStat); // Standard deviation of fluxes
     psFree(stdevStats);
     psFree(noise);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPARead.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPARead.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPARead.c	(revision 24244)
@@ -64,5 +64,5 @@
 
 // Set the "thisXXXScan" value in the readout for the appropriate image type
-static int readoutSetThisScan(pmReadout *readout, // Readout of interest
+static void readoutSetThisScan(pmReadout *readout, // Readout of interest
                               fpaReadType type, // Type of image
                               int thisScan // Starting scan number
@@ -72,15 +72,14 @@
       case FPA_READ_TYPE_IMAGE:
         readout->thisImageScan = thisScan;
-        return readout->lastImageScan;
+        return;
       case FPA_READ_TYPE_MASK:
         readout->thisMaskScan = thisScan;
-        return readout->lastMaskScan;
+        return;
       case FPA_READ_TYPE_VARIANCE:
         readout->thisVarianceScan = thisScan;
-        return readout->lastVarianceScan;
+        return;
       default:
         psAbort("Unknown read type: %x\n", type);
     }
-    return false;
 }
 
@@ -103,5 +102,5 @@
 
 // Set the "lastXXXScan" value in the readout for the appropriate image type
-static int readoutSetLastScan(pmReadout *readout, // Readout of interest
+static void readoutSetLastScan(pmReadout *readout, // Readout of interest
                               fpaReadType type, // Type of image
                               int lastScan // Last scan number
@@ -111,15 +110,14 @@
       case FPA_READ_TYPE_IMAGE:
         readout->lastImageScan = lastScan;
-        return readout->lastImageScan;
+        return;
       case FPA_READ_TYPE_MASK:
         readout->lastMaskScan = lastScan;
-        return readout->lastMaskScan;
+        return;
       case FPA_READ_TYPE_VARIANCE:
         readout->lastVarianceScan = lastScan;
-        return readout->lastVarianceScan;
+        return;
       default:
         psAbort("Unknown read type: %x\n", type);
     }
-    return false;
 }
 
@@ -253,5 +251,5 @@
     int readdir = psMetadataLookupS32(&mdok, cell->concepts, "CELL.READDIR"); // Read direction
     if (!mdok || readdir == 0 || (readdir != 1 && readdir != 2)) {
-        psError(PS_ERR_IO, true, "CELL.READDIR is not set to -1 or +1.\n");
+        psError(PS_ERR_IO, true, "CELL.READDIR is not set to 1 or 2.\n");
         return false;
     }
@@ -592,6 +590,5 @@
     }
 
-    // Determine the number of scans to read
-    int lastScan = readoutSetLastScan(readout, type, thisScan + numScans);
+    int origThisScan = thisScan;        // Original value of thisScan (starting point for read)
     if (thisScan == 0) {
         overlap = 0;
@@ -601,5 +598,4 @@
         thisScan = 0;
     }
-    readoutSetThisScan(readout, type, thisScan);
 
     // Calculate limits, adjust readout->row0,col0
@@ -620,4 +616,8 @@
         }
     }
+    int lastScan = origThisScan + numScans; // Last scan to read
+
+    readoutSetThisScan(readout, type, thisScan);
+    readoutSetLastScan(readout, type, lastScan);
 
     // Blow away existing data.
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfile.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfile.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfile.c	(revision 24244)
@@ -205,11 +205,14 @@
     }
     if (strstr(newName, "{CHIP.NAME}")) {
+        const char *name = NULL;        // Name of chip
         pmChip *chip = pmFPAviewThisChip(view, fpa);
         if (chip) {
-            char *name = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME");
-            if (name) {
-                psStringSubstitute(&newName, name, "{CHIP.NAME}");
-            }
-        }
+            name = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME");
+            psAssert(name, "All chips should have a name");
+        } else {
+            name = "fpa";
+        }
+        psStringSubstitute(&newName, name, "{CHIP.NAME}");
+
     }
     if (strstr(newName, "{CHIP.ID}")) {
@@ -255,11 +258,13 @@
     }
     if (strstr(newName, "{CELL.NAME}")) {
+        const char *name = NULL;        // Name of cell
         pmCell *cell = pmFPAviewThisCell(view, fpa);
         if (cell) {
-            char *name = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME");
-            if (name) {
-                psStringSubstitute(&newName, name, "{CELL.NAME}");
-            }
-        }
+            name = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME");
+            psAssert(name, "All cells should have a name");
+        } else {
+            name = "chip";
+        }
+        psStringSubstitute(&newName, name, "{CELL.NAME}");
     }
     if (strstr(newName, "{CELL.N}")) {
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmReadoutFake.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmReadoutFake.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmReadoutFake.c	(revision 24244)
@@ -29,4 +29,6 @@
 #define MAX_AXIS_RATIO 20.0             // Maximum axis ratio for PSF model
 #define SOURCE_MASK (PM_SOURCE_MODE_DEFECT | PM_SOURCE_MODE_CR_LIMIT) // Mask to apply to input sources
+#define MODEL_MASK (PM_MODEL_STATUS_NONCONVERGE | PM_MODEL_STATUS_OFFIMAGE | \
+                    PM_MODEL_STATUS_BADARGS | PM_MODEL_STATUS_LIMITS) // Mask to apply to models
 
 
@@ -80,21 +82,4 @@
 
     int numSources = sources->n;          // Number of stars
-
-    float flux0 = NAN;                  // Flux for central intensity of 1.0
-    if (normalisePeak) {
-        pmModel *fakeModel = pmModelFromPSFforXY(psf, (float)numCols / 2.0, (float)numRows / 2.0,
-                                                 1.0); // Fake model, with central intensity of 1.0
-        psAssert(fakeModel, "failed to generate model: should this be an error or not?");
-
-        if (circularise && !circulariseModel(fakeModel)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to circularise PSF model.");
-            psFree(fakeModel);
-            return false;
-        }
-
-        flux0 = fakeModel->modelFlux(fakeModel->params); // Flux for central intensity of 1.0
-        psFree(fakeModel);
-    }
-
     for (int i = 0; i < numSources; i++) {
         pmSource *source = sources->data[i]; // Source of interest
@@ -118,10 +103,25 @@
 
         float flux = powf(10.0, -0.4 * source->psfMag); // Flux of source
+
         if (normalisePeak) {
-            flux /= flux0;
+            // Normalise flux
+            pmModel *normModel = pmModelFromPSFforXY(psf, x, y, 1.0); // Model for normalisation
+            if (!normModel || (normModel->flags & MODEL_MASK)) {
+                psFree(normModel);
+                continue;
+            }
+            if (circularise && !circulariseModel(normModel)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to circularise PSF model.");
+                psFree(normModel);
+                return false;
+            }
+
+            flux /= normModel->modelFlux(normModel->params);
+            psFree(normModel);
         }
 
         pmModel *fakeModel = pmModelFromPSFforXY(psf, x, y, flux);
-        if (!fakeModel) {
+        if (!fakeModel || (fakeModel->flags & MODEL_MASK)) {
+            psFree(fakeModel);
             continue;
         }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConcepts.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConcepts.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConcepts.c	(revision 24244)
@@ -421,24 +421,28 @@
 
 // Interpolate the concept.  Generalises the FPA/Chip/Cell
-#define CONCEPT_INTERPOLATE(SOURCE, NAME) \
+#define CONCEPT_INTERPOLATE(SOURCE, NAME, DEFAULT) \
     if (strncmp(concept, NAME, strlen(NAME)) == 0) { \
-        if (!(SOURCE)) { \
-            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Cannot interpolate %s because %s not provided", \
-                    concept, NAME); \
-            psFree(string); \
-            return NULL; \
-        } \
-        psMetadataItem *item = psMetadataLookup((SOURCE)->concepts, concept); /* Item with concept */ \
-        if (!item) { \
-            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Can't find concept %s in %s", concept, NAME); \
-            psFree(string); \
-            return NULL; \
-        } \
-        \
-        psString value = psMetadataItemParseString(item); /* Value of concept */ \
-        if (!value) { \
-            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to parse concept %s", concept); \
-            psFree(string); \
-            return NULL; \
+        psString value = NULL; /* Value of concept */ \
+        if (SOURCE) { \
+            psMetadataItem *item = psMetadataLookup((SOURCE)->concepts, concept); /* Item with concept */ \
+            if (!item) { \
+                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Can't find concept %s in %s", concept, NAME); \
+                psFree(string); \
+                return NULL; \
+            } \
+            \
+            value = psMetadataItemParseString(item); /* Value of concept */ \
+            if (!value) { \
+                psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to parse concept %s", concept); \
+                psFree(string); \
+                return NULL; \
+            } \
+        } else { \
+            if (!(DEFAULT)) { \
+                psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to determine concept %s", concept); \
+                psFree(string); \
+                return NULL; \
+            } \
+            value = psStringCopy(DEFAULT); \
         } \
         \
@@ -483,7 +487,7 @@
         psTrace("psModules.concepts", 7, "Interpolating concept %s", concept);
 
-        CONCEPT_INTERPOLATE(fpa,  "FPA");
-        CONCEPT_INTERPOLATE(chip, "CHIP");
-        CONCEPT_INTERPOLATE(cell, "CELL");
+        CONCEPT_INTERPOLATE(fpa,  "FPA", NULL);
+        CONCEPT_INTERPOLATE(chip, "CHIP", "fpa");
+        CONCEPT_INTERPOLATE(cell, "CELL", "chip");
 
         psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unrecognised concept: %s", concept);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConceptsStandard.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConceptsStandard.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConceptsStandard.c	(revision 24244)
@@ -33,4 +33,20 @@
 break;
 
+
+// Format type for time
+typedef enum {
+  TIME_FORMAT_YYYYMMDD,			// Date stored in YYYY-MM-DD order (ISO-standard)
+  TIME_FORMAT_DDMMYYYY,			// Date stored in DD-MM-YYYY order
+  TIME_FORMAT_MMDDYYYY,			// Date stored in MM-DD-YYYY order
+  TIME_FORMAT_JD,			// Date stored as JD
+  TIME_FORMAT_MJD,			// Date stored as MJD
+} conceptTimeFormatType;
+
+// Format for time
+typedef struct {
+    conceptTimeFormatType format;       // Format type for time
+    bool separate;                      // Date and time stored separately?
+    bool pre2000;                       // Year is pre-2000 (two digits only)?
+} conceptTimeFormat;
 
 
@@ -630,4 +646,55 @@
 }
 
+static conceptTimeFormat conceptGetTimeFormat(const char *name, // Concept name ("CELL.TIME" or "FPA.TIME")
+                                              const psMetadata *cameraFormat // Camera format
+    )
+{
+    conceptTimeFormat time;               // Time format, to return
+    time.format = TIME_FORMAT_YYYYMMDD;
+    time.separate = false;
+    time.pre2000 = false;
+
+    bool mdok = true;                   // Status of MD lookup
+    psMetadata *formats = psMetadataLookupMetadata(&mdok, cameraFormat, "FORMATS"); // The formats
+    if (mdok && formats) {
+        psString format = psMetadataLookupStr(&mdok, formats, name); // The formats for eg, CELL.TIME
+        if (mdok && format && strlen(format) > 0) {
+            psList *formatList = psStringSplit(format, " ,;", false); // List of formats specified
+            psListIterator *formatListIter = psListIteratorAlloc(formatList, PS_LIST_HEAD, false); // Iterator
+            while ((format = psListGetAndIncrement(formatListIter))) {
+                if (strcasecmp(format, "SEPARATE") == 0) {
+                    time.separate = true;
+                } else if (strcasecmp(format, "YYYYMMDD") == 0) {
+                    time.format = TIME_FORMAT_YYYYMMDD;
+                } else if (strcasecmp(format, "MMDDYYYY") == 0) {
+                    time.format = TIME_FORMAT_MMDDYYYY;
+                } else if (strcasecmp(format, "DDMMYYYY") == 0) {
+                    time.format = TIME_FORMAT_DDMMYYYY;
+                } else if (strcasecmp(format, "ISO") == 0) {
+                    time.format = TIME_FORMAT_YYYYMMDD;
+                } else if (strcasecmp(format, "YEAR.FIRST") == 0) {
+                    time.format = TIME_FORMAT_YYYYMMDD;
+                } else if (strcasecmp(format, "USA") == 0) {
+                    time.format = TIME_FORMAT_MMDDYYYY;
+                } else if (strcasecmp(format, "BACKWARDS") == 0) {
+                    time.format = TIME_FORMAT_DDMMYYYY;
+                } else if (strcasecmp(format, "PRE2000") == 0) {
+                    time.pre2000 = true;
+                } else if (strcasecmp(format, "MJD") == 0) {
+                    time.format = TIME_FORMAT_MJD;
+                } else if (strcasecmp(format, "JD") == 0) {
+                    time.format = TIME_FORMAT_JD;
+                } else {
+                    psWarning("Unrecognised FORMATS option for %s: %s --- ignored.", name, format);
+                }
+            }
+            psFree(formatListIter);
+            psFree(formatList);
+        }
+    }
+
+    return time;
+}
+
 // Determine the corresponding TIMESYS for one of the TIME concepts
 static psTimeType conceptGetTimesysForTime(const char *name, // Concept name ("CELL.TIME" or "FPA.TIME")
@@ -734,81 +801,10 @@
     psTimeType timeSys = conceptGetTimesysForTime(pattern->name, fpa, chip, cell); // Time system
 
-    // Work out how the time is represented
-    bool separateTime = false;
-    bool usaTime = false;               // Is the time specified in USA (MM-DD-YYYY) order?
-    bool backwardsTime = false;         // Is the time specified in backwards (DD-MM-YYYY) order?
-    bool yearFirst = false;            // Is the time specified in yearFirst (YYYY-MM-DD) order?
-    bool pre2000Time = false;           // Is the time specified pre-2000 (where the year only has two digits)
-    bool mjdTime = false;               // Is the time specified a MJD?
-    bool jdTime = false;                // Is the time specified a JD?
-
-    // Get format
-    bool mdok;                          // Status of MD lookup
-    psMetadata *formats = psMetadataLookupMetadata(&mdok, cameraFormat, "FORMATS");
-    if (!mdok || !formats) {
-        psError(PS_ERR_UNKNOWN, true, "Unable to find FORMATS in camera configuration.\n");
-        return NULL;
-    }
-
-    psString timeFormat = psMetadataLookupStr(&mdok, formats, pattern->name);
-    if (!mdok || strlen(timeFormat) == 0) {
-        psError(PS_ERR_UNKNOWN, true, "Unable to find %s in FORMATS.\n", pattern->name);
-        return NULL;
-    }
-
-    // Parse the time format
-    // why should more than one be allowed??
-    psList *timeFormats = psStringSplit(timeFormat, " ,;", false); // List of the format options
-    psListIterator *timeFormatsIter = psListIteratorAlloc(timeFormats, PS_LIST_HEAD, false); // Iter
-    while ((timeFormat = psListGetAndIncrement(timeFormatsIter))) {
-        if (strcasecmp(timeFormat, "SEPARATE") == 0) {
-            separateTime = true;
-        } else if (strcasecmp(timeFormat, "USA") == 0) {
-            usaTime = true;
-            backwardsTime = false;
-            yearFirst = false;
-            jdTime = false;
-            mjdTime = false;
-        } else if (strcasecmp(timeFormat, "BACKWARDS") == 0) {
-            backwardsTime = true;
-            usaTime = false;
-            yearFirst = false;
-            jdTime = false;
-            mjdTime = false;
-        } else if (strcasecmp(timeFormat, "YEAR.FIRST") == 0) {
-            yearFirst = true;
-            usaTime = false;
-            backwardsTime = false;
-            jdTime = false;
-            mjdTime = false;
-        } else if (strcasecmp(timeFormat, "PRE2000") == 0) {
-            pre2000Time = true;
-        } else if (strcasecmp(timeFormat, "MJD") == 0) {
-            mjdTime = true;
-            jdTime = false;
-            yearFirst = false;
-            backwardsTime = false;
-            usaTime = false;
-        } else if (strcasecmp(timeFormat, "JD") == 0) {
-            jdTime = true;
-            mjdTime = false;
-            yearFirst = false;
-            backwardsTime = false;
-            usaTime = false;
-        } else {
-            psError(PS_ERR_UNKNOWN, true, "Unrecognised FORMATS option for %s: %s --- "
-                    "ignored.\n", pattern->name, timeFormat);
-            psFree(timeFormatsIter);
-            psFree(timeFormats);
-            return NULL;
-        }
-    }
-    psFree(timeFormatsIter);
-    psFree(timeFormats);
+    conceptTimeFormat timeFormat = conceptGetTimeFormat(pattern->name, cameraFormat); // Format for time
 
     psTime *time = NULL;                // The time
     switch (concept->type) {
       case PS_DATA_LIST: {
-          if (!separateTime) {
+          if (!timeFormat.separate) {
               psWarning ("DATE and TIME stored separately, but not specified in format\n");
           }
@@ -837,19 +833,25 @@
               return NULL;
           }
-          if (backwardsTime) {
-              // Need to switch days and years
-              int temp = day;
-              day = year;
-              year = temp;
-          }
-          if (usaTime) {
-              // Need to switch everything around.... Yanks!
-              int temp = day;
-              day = month;
-              month = year;
-              year = temp;
+          switch (timeFormat.format) {
+            case TIME_FORMAT_DDMMYYYY: {
+                // Need to switch days and years
+                int temp = day;
+                day = year;
+                year = temp;
+                break;
+            }
+            case TIME_FORMAT_MMDDYYYY: {
+                // Need to switch everything around.... Yanks!
+                int temp = day;
+                day = month;
+                month = year;
+                year = temp;
+                break;
+            }
+            default:
+              break;
           }
           if (year < 100) {
-              if (pre2000Time) {
+              if (timeFormat.pre2000) {
                   year += 1900;
               } else {
@@ -898,26 +900,38 @@
       case PS_DATA_STRING: {
           psString timeString = concept->data.V;   // String with the time
-          if (jdTime) {
-              double timeValue = strtod (timeString, NULL);
-              time = psTimeFromJD(timeValue);
-          } else if (mjdTime) {
-              double timeValue = strtod (timeString, NULL);
-              time = psTimeFromMJD(timeValue);
-          } else {
-              // It's ISO
-              time = psTimeFromISO(timeString, timeSys);
-          } // Interpreting the time string
+          switch (timeFormat.format) {
+            case TIME_FORMAT_JD: {
+                double timeValue = strtod (timeString, NULL);
+                time = psTimeFromJD(timeValue);
+                break;
+            }
+            case TIME_FORMAT_MJD: {
+                double timeValue = strtod (timeString, NULL);
+                time = psTimeFromMJD(timeValue);
+                break;
+            }
+            case TIME_FORMAT_YYYYMMDD: {
+                // this is ISO-standard
+                time = psTimeFromISO(timeString, timeSys);
+                break;
+            }
+            default:
+              psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to interpret time string: %s", timeString);
+              return NULL;
+          }
           break;
       }
       case PS_TYPE_F32: {
           double timeValue = (double)concept->data.F32;
-          if (jdTime) {
+          switch (timeFormat.format) {
+            case TIME_FORMAT_JD:
               time = psTimeFromJD(timeValue);
-          } else if (mjdTime) {
+              break;
+            case TIME_FORMAT_MJD:
               time = psTimeFromMJD(timeValue);
-          } else {
-              psError(PS_ERR_UNKNOWN, true, "Not sure how to parse %s (%f) --- trying JD\n",
-                      pattern->name, timeValue);
-              time = psTimeFromJD(timeValue);
+              break;
+            default:
+              psError(PS_ERR_UNKNOWN, true, "Unable to interpret time %s (%f)", pattern->name, timeValue);
+              return NULL;
           }
           break;
@@ -925,12 +939,14 @@
       case PS_TYPE_F64: {
           double timeValue = (double)concept->data.F64;
-          if (jdTime) {
+          switch (timeFormat.format) {
+            case TIME_FORMAT_JD:
               time = psTimeFromJD(timeValue);
-          } else if (mjdTime) {
+              break;
+            case TIME_FORMAT_MJD:
               time = psTimeFromMJD(timeValue);
-          } else {
-              psError(PS_ERR_UNKNOWN, true, "Not sure how to parse %s (%f) --- trying JD\n",
-                      pattern->name, timeValue);
-              time = psTimeFromJD(timeValue);
+              break;
+            default:
+              psError(PS_ERR_UNKNOWN, true, "Unable to interpret time %s (%f)", pattern->name, timeValue);
+              return NULL;
           }
           break;
@@ -946,8 +962,13 @@
     }
 
-    if (jdTime || mjdTime) {
+    // Set the time system appropriately
+    switch (timeFormat.format) {
+      case TIME_FORMAT_JD:
+      case TIME_FORMAT_MJD:
         conceptSetTimesysForTime(pattern->name, fpa, chip, cell, PS_TIME_TAI);
-    } else {
+        break;
+      default:
         time->type = timeSys;
+        break;
     }
 
@@ -1176,6 +1197,24 @@
                                           const pmCell *cell)
 {
+    psString timeName = psStringCopy(concept->name); // Name of corresponding TIME concept
+    psStringSubstitute(&timeName, "TIME", "TIMESYS");
+
+    conceptTimeFormat timeFormat = conceptGetTimeFormat(timeName, cameraFormat); // Format for time
+    psFree(timeName);
+
+    psTimeType timesys = concept->data.S32; // Time system
+
+    // JD and MJD are converted to TAI before writing
+    switch (timeFormat.format) {
+      case TIME_FORMAT_JD:
+      case TIME_FORMAT_MJD:
+        timesys = PS_TIME_TAI;
+        break;
+      default:
+        break;
+    }
+
     psString sys = NULL;            // String to store
-    switch (concept->data.S32) {
+    switch (timesys) {
       case PS_TIME_TAI:
         sys = psStringCopy("TAI");
@@ -1211,64 +1250,7 @@
     psTimeConvert(time, timeSys);
 
-    // Work out the format
-    bool separateTime = false;          // Are the date and time stored separately?
-    bool pre2000Time = false;           // Is the year in pre-2000 format (two digits only)?
-    bool backwardsTime = false;         // Is the date stored backwards (DD-MM-YYYY)?
-    bool usaTime = false;               // Is the date stored in USA order (MM-DD-YYYY)?
-    bool jdTime = false;                // Is the date stored as a JD?
-    bool mjdTime = false;               // Is the date stored as a MJD?
-    bool yearFirst = false;
-
-    bool mdok = true;                   // Status of MD lookup
-    psMetadata *formats = psMetadataLookupMetadata(&mdok, cameraFormat, "FORMATS"); // The formats
-    if (mdok && formats) {
-        psString format = psMetadataLookupStr(&mdok, formats, concept->name); // The formats for eg, CELL.TIME
-        if (mdok && format && strlen(format) > 0) {
-            psList *formatList = psStringSplit(format, " ,;", false); // List of formats specified
-            psListIterator *formatListIter = psListIteratorAlloc(formatList, PS_LIST_HEAD, false); // Iterator
-            while ((format = psListGetAndIncrement(formatListIter))) {
-                if (strcasecmp(format, "SEPARATE") == 0) {
-                    separateTime = true;
-                } else if (strcasecmp(format, "USA") == 0) {
-                    usaTime = true;
-                    backwardsTime = false;
-                    jdTime = false;
-                    mjdTime = false;
-                } else if (strcasecmp(format, "BACKWARDS") == 0) {
-                    backwardsTime = true;
-                    usaTime = false;
-                    mjdTime = false;
-                    jdTime = false;
-                } else if (strcasecmp(format, "YEAR.FIRST") == 0) {
-                    yearFirst = true;
-                    usaTime = false;
-                    backwardsTime = false;
-                    jdTime = false;
-                    mjdTime = false;
-                } else if (strcasecmp(format, "PRE2000") == 0) {
-                    pre2000Time = true;
-                } else if (strcasecmp(format, "MJD") == 0) {
-                    mjdTime = true;
-                    usaTime = false;
-                    backwardsTime = false;
-                    jdTime = false;
-                    separateTime = false;
-                } else if (strcasecmp(format, "JD") == 0) {
-                    jdTime = true;
-                    usaTime = false;
-                    backwardsTime = false;
-                    mjdTime = false;
-                    separateTime = false;
-                } else {
-                    psWarning("Unrecognised FORMATS option for %s: %s --- "
-                              "ignored.\n", concept->name, format);
-                }
-            }
-            psFree(formatListIter);
-            psFree(formatList);
-        }
-    }
-
-    if (separateTime) {
+    conceptTimeFormat timeFormat = conceptGetTimeFormat(concept->name, cameraFormat); // Format for time
+
+    if (timeFormat.separate) {
         // We're working with two separate headers --- construct a list with the date and time separately
         psString dateTimeString = psTimeToISO(time); // String representation
@@ -1280,24 +1262,28 @@
         // Need to format the strings....
         // XXX: Couldn't be bothered doing these right now
-        if (pre2000Time) {
+        if (timeFormat.pre2000) {
             psError(PS_ERR_UNKNOWN, true, "Don't you realise it's the twenty-first century?\n");
             return NULL;
         }
-        if (backwardsTime) {
-            int day, month, year;
-            psTrace ("psModules.concepts", 5, "ISO time has year first, convert to DD-MM-YYYY");
-            sscanf (dateString, "%d-%d-%d", &year, &month, &day);
-            sprintf (dateString, "%02d-%02d-%04d", day, month, year);
-            // XXX fix this for str length
-        }
-        if (usaTime) {
-            int day, month, year;
-            psTrace ("psModules.concepts", 5, "ISO time has year first, convert to MM-DD-YYYY");
-            sscanf (dateString, "%d-%d-%d", &year, &month, &day);
-            sprintf (dateString, "%02d-%02d-%04d", month, day, year);
-            // XXX fix this for str length
-        }
-        if (yearFirst) {
-            psTrace ("psModules.concepts", 5, "ISO time has year first, no adjustment needed");
+
+        switch (timeFormat.format) {
+          case TIME_FORMAT_DDMMYYYY: {
+              int day, month, year;
+              psTrace ("psModules.concepts", 5, "ISO time has year first, convert to DD-MM-YYYY");
+              sscanf (dateString, "%d-%d-%d", &year, &month, &day);
+              sprintf (dateString, "%02d-%02d-%04d", day, month, year);
+              // XXX fix this for str length
+              break;
+          }
+          case TIME_FORMAT_MMDDYYYY: {
+              int day, month, year;
+              psTrace ("psModules.concepts", 5, "ISO time has year first, convert to MM-DD-YYYY");
+              sscanf (dateString, "%d-%d-%d", &year, &month, &day);
+              sprintf (dateString, "%02d-%02d-%04d", month, day, year);
+              // XXX fix this for str length
+              break;
+          }
+          default:
+            break;
         }
 
@@ -1313,25 +1299,31 @@
         psListAdd(dateTime, PS_LIST_TAIL, timeItem);
 
-        psMetadataItem *item = psMetadataItemAllocPtr(concept->name, PS_DATA_LIST, concept->comment,
-                                                      dateTime);
-        psFree (dateItem);
-        psFree (timeItem);
-        psFree (dateTime);
+        psMetadataItem *item = psMetadataItemAllocPtr(concept->name, PS_DATA_LIST,
+                                                      concept->comment, dateTime);
+        psFree(dateItem);
+        psFree(timeItem);
+        psFree(dateTime);
         return item;
     }
-    if (jdTime) {
-        double jd = psTimeToMJD(time);
-        return psMetadataItemAllocF64(concept->name, concept->comment, jd);
-    }
-    if (mjdTime) {
-        double mjd = psTimeToMJD(time);
-        return psMetadataItemAllocF64(concept->name, concept->comment, mjd);
-    }
-
-    // If we've gotten this far, so it's straight ISO.
-    psString dateTimeString = psTimeToISO(time); // String representation
-    psMetadataItem *item = psMetadataItemAllocStr(concept->name, concept->comment, dateTimeString);
-    psFree(dateTimeString);
-    return item;
+
+    switch (timeFormat.format) {
+      case TIME_FORMAT_JD: {
+          double jd = psTimeToMJD(time);
+          return psMetadataItemAllocF64(concept->name, concept->comment, jd);
+      }
+      case TIME_FORMAT_MJD: {
+          double mjd = psTimeToMJD(time);
+          return psMetadataItemAllocF64(concept->name, concept->comment, mjd);
+      }
+      default: {
+          // If we've gotten this far, it's straight ISO.
+          psString dateTimeString = psTimeToISO(time); // String representation
+          psMetadataItem *item = psMetadataItemAllocStr(concept->name, concept->comment, dateTimeString);
+          psFree(dateTimeString);
+          return item;
+      }
+    }
+
+    return NULL;
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConceptsWrite.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConceptsWrite.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConceptsWrite.c	(revision 24244)
@@ -62,5 +62,5 @@
         while ((cItem = psListGetAndIncrement(cIter))) {
             if (cItem->type != PS_DATA_STRING) {
-		psWarning("psMetadataItem from list is of type %x instead of %x (PS_DATA_STRING) --- can't interpret.\n", cItem->type, PS_DATA_STRING);
+                psWarning("psMetadataItem from list is of type %x instead of %x (PS_DATA_STRING) --- can't interpret.\n", cItem->type, PS_DATA_STRING);
                 psFree(cIter);
                 psFree(sIter);
@@ -606,5 +606,5 @@
         for (long i = 0; i < chips->n; i++) {
             pmChip *chip = chips->data[i];  // Chip of interest
-            if (chip && !chip->hdu) {
+            if (chip) {
                 success &= pmConceptsWriteChip(chip, false, true, config);
             }
@@ -622,5 +622,5 @@
     pmFPA *fpa = chip->parent;          // FPA to which the chip belongs
     bool success = conceptsWrite(&conceptsChip, fpa, chip, NULL, config, chip->concepts);
-    if (propagateUp && !fpa->hdu) {
+    if (propagateUp) {
         psMetadata *conceptsFPA = pmConceptsSpecs(PM_FPA_LEVEL_FPA); // Concept specifications
         success &= conceptsWrite(&conceptsFPA, fpa, chip, NULL, config, fpa->concepts);
@@ -630,5 +630,5 @@
         for (long i = 0; i < cells->n; i++) {
             pmCell *cell = cells->data[i];  // Cell of interest
-            if (cell && !cell->hdu) {
+            if (cell) {
                 success &= pmConceptsWriteCell(cell, false, config);
             }
@@ -649,12 +649,8 @@
     bool success = conceptsWrite(&conceptsCell, fpa, chip, cell, config, cell->concepts);
     if (propagateUp) {
-        if (!chip->hdu) {
-            psMetadata *conceptsChip = pmConceptsSpecs(PM_FPA_LEVEL_CHIP); // Concept specifications
-            success &= conceptsWrite(&conceptsChip, fpa, chip, cell, config, chip->concepts);
-            if (!fpa->hdu) {
-                psMetadata *conceptsFPA = pmConceptsSpecs(PM_FPA_LEVEL_FPA); // Concept specifications
-                success &= conceptsWrite(&conceptsFPA, fpa, chip, cell, config, fpa->concepts);
-            }
-        }
+        psMetadata *conceptsChip = pmConceptsSpecs(PM_FPA_LEVEL_CHIP); // Concept specifications
+        success &= conceptsWrite(&conceptsChip, fpa, chip, cell, config, chip->concepts);
+        psMetadata *conceptsFPA = pmConceptsSpecs(PM_FPA_LEVEL_FPA); // Concept specifications
+        success &= conceptsWrite(&conceptsFPA, fpa, chip, cell, config, fpa->concepts);
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/Makefile.am	(revision 24244)
@@ -1,14 +1,26 @@
 noinst_LTLIBRARIES = libpsmodulesconfig.la
 
-# PSMODULES_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-# PSMODULES_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-# PSMODULES_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+if HAVE_SVNVERSION
+PSMODULES_VERSION=`$(SVNVERSION) ../..`
+else
+PSMODULES_VERSION="UNKNOWN"
+endif
+
+if HAVE_SVN
+PSMODULES_BRANCH=`$(SVN) info ../.. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PSMODULES_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PSMODULES_BRANCH="UNKNOWN"
+PSMODULES_SOURCE="UNKNOWN"
+endif
 
 # Force recompilation of pmVersion.c, since it gets the version information
-# pmVersion.c: FORCE
-# 	touch pmVersion.c
-# FORCE: ;
+pmVersion.c: pmVersionDefinitions.h
+pmVersionDefinitions.h: pmVersionDefinitions.h.in FORCE
+	-$(RM) pmVersionDefinitions.h
+	$(SED) -e "s|@PSMODULES_VERSION@|\"$(PSMODULES_VERSION)\"|" -e "s|@PSMODULES_BRANCH@|\"$(PSMODULES_BRANCH)\"|" -e "s|@PSMODULES_SOURCE@|\"$(PSMODULES_SOURCE)\"|" pmVersionDefinitions.h.in > pmVersionDefinitions.h
+FORCE: ;
 
-libpsmodulesconfig_la_CPPFLAGS = $(SRCINC) $(PSMODULES_CFLAGS) -DPSMODULES_VERSION=$(SVN_VERSION) -DPSMODULES_BRANCH=$(SVN_BRANCH) -DPSMODULES_SOURCE=$(SVN_SOURCE)
+libpsmodulesconfig_la_CPPFLAGS = $(SRCINC) $(PSMODULES_CFLAGS)
 libpsmodulesconfig_la_LDFLAGS  = -release $(PACKAGE_VERSION)
 libpsmodulesconfig_la_SOURCES  = \
@@ -35,6 +47,6 @@
 
 # Error codes.
-BUILT_SOURCES = pmErrorCodes.h pmErrorCodes.c
-CLEANFILES = *~ pmErrorCodes.h pmErrorCodes.c
+BUILT_SOURCES = pmErrorCodes.h pmErrorCodes.c pmVersionDefinitions.h
+CLEANFILES = *~ pmErrorCodes.h pmErrorCodes.c pmVersionDefinitions.h
 
 pmErrorCodes.h : pmErrorCodes.dat pmErrorCodes.h.in
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfig.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfig.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfig.c	(revision 24244)
@@ -551,4 +551,5 @@
                 psWarning("Unable to resolve log destination: %s --- ignored", logDest);
             } else {
+                pmConfigRunFilenameAddWrite(config, "LOG", logDest);
                 config->logFD = psMessageDestination(resolved);
             }
@@ -609,4 +610,5 @@
                 psWarning("Unable to resolve trace destination: %s --- ignored", traceDest);
             } else {
+                pmConfigRunFilenameAddWrite(config, "TRACE", traceDest);
                 config->traceFD = psMessageDestination(resolved);
             }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigDump.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigDump.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigDump.c	(revision 24244)
@@ -138,5 +138,5 @@
 
 
-bool pmConfigDump(const pmConfig *config, const pmFPA *source, const char *filename)
+bool pmConfigDump(const pmConfig *config, const char *filename)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigDump.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigDump.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigDump.h	(revision 24244)
@@ -31,5 +31,4 @@
 ///
 bool pmConfigDump(const pmConfig *config, ///< Configuration to dump
-                  const pmFPA *source,    ///< Source FPA, defines the level for the file rule
                   const char *filename    ///< Output file name
     );
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRecipes.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRecipes.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRecipes.c	(revision 24244)
@@ -81,5 +81,5 @@
             psTrace ("psModules.config", 3, "read recipes from camera config");
         } else {
-            psLogMsg ("psModules.config", PS_LOG_DETAIL, "no recipe supplied by camera config");
+            psTrace ("psModules.config", PS_LOG_DETAIL, "no recipe supplied by camera config");
         }
     }
@@ -97,5 +97,5 @@
             psTrace ("psModules.config", 3, "read recipes from symbolic references");
         } else {
-            psLogMsg ("psModules.config", PS_LOG_DETAIL, "no recipe supplied by symbolic reference");
+            psTrace ("psModules.config", PS_LOG_DETAIL, "no recipe supplied by symbolic reference");
         }
 
@@ -105,5 +105,5 @@
             return false;
         }
-        psLogMsg ("psModules.config", PS_LOG_DETAIL, "merged camera recipes with system recipes");
+        psTrace ("psModules.config", PS_LOG_DETAIL, "merged camera recipes with system recipes");
 
         // load recipe-files specified on the command line
@@ -115,5 +115,5 @@
             psTrace ("psModules.config", 3, "read recipes from command-line arguments");
         } else {
-            psLogMsg ("psModules.config", PS_LOG_DETAIL, "no recipe supplied on command-line arguments");
+            psTrace ("psModules.config", PS_LOG_DETAIL, "no recipe supplied on command-line arguments");
         }
 
@@ -126,5 +126,5 @@
             psTrace ("psModules.config", 3, "read recipes from symbolic references");
         } else {
-            psLogMsg ("psModules.config", PS_LOG_DETAIL, "no recipe supplied by symbolic reference");
+            psTrace ("psModules.config", PS_LOG_DETAIL, "no recipe supplied by symbolic reference");
         }
 
@@ -137,5 +137,5 @@
             psTrace ("psModules.config", 3, "read recipes from command-line arguments");
         } else {
-            psLogMsg ("psModules.config", PS_LOG_DETAIL, "no recipe supplied on command-line arguments");
+            psTrace ("psModules.config", PS_LOG_DETAIL, "no recipe supplied on command-line arguments");
         }
     }
@@ -348,5 +348,5 @@
             psFree(recipesIter);
             return false;
-	}
+        }
     }
     psFree(recipesIter);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRun.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRun.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRun.c	(revision 24244)
@@ -41,10 +41,13 @@
 // Add a file to a nominated metadata in the RUN information
 static bool configRunFileAdd(pmConfig *config, // Configuration
-                             const pmFPAfile *file, // File to add
+                             const char *description, // Description of file
+                             const char *name, // Name of file
                              const char *target // Name of metadata to which to add
                              )
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
-    PS_ASSERT_PTR_NON_NULL(file, false);
+    PS_ASSERT_STRING_NON_EMPTY(description, false);
+    PS_ASSERT_STRING_NON_EMPTY(name, false);
+    PS_ASSERT_STRING_NON_EMPTY(target, false);
 
     psMetadata *run = configRun(config);// RUN information
@@ -53,9 +56,6 @@
     psAssert(files, "Require list of files");
 
-    const char *name = file->name;      // Name of symbolic file
-    const char *value = file->origname ? file->origname : file->filename; // The file (system) name
-
     psString regex = NULL;              // Regular expression for iteration
-    psStringAppend(&regex, "^%s$", name);
+    psStringAppend(&regex, "^%s$", description);
     psMetadataIterator *iter = psMetadataIteratorAlloc(files, PS_LIST_HEAD, regex);
     psFree(regex);
@@ -63,5 +63,5 @@
     while ((item = psMetadataGetAndIncrement(iter))) {
         psAssert(item->type == PS_DATA_STRING, "We only put STRING types here.");
-        if (strcmp(item->data.str, value) == 0) {
+        if (strcmp(item->data.str, name) == 0) {
             // It's already present
             psFree(iter);
@@ -71,5 +71,5 @@
     psFree(iter);
 
-    return psMetadataAddStr(files, PS_LIST_TAIL, name, PS_META_DUPLICATE_OK, NULL, value);
+    return psMetadataAddStr(files, PS_LIST_TAIL, description, PS_META_DUPLICATE_OK, NULL, name);
 }
 
@@ -79,5 +79,15 @@
     PS_ASSERT_PTR_NON_NULL(file, false);
 
-    return configRunFileAdd(config, file, "FILES.INPUT");
+    return configRunFileAdd(config, file->name, file->origname ? file->origname : file->filename,
+                            "FILES.INPUT");
+}
+
+bool pmConfigRunFilenameAddRead(pmConfig *config, const char *description, const char *name)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    PS_ASSERT_STRING_NON_EMPTY(description, false);
+    PS_ASSERT_STRING_NON_EMPTY(name, false);
+
+    return configRunFileAdd(config, description, name, "FILES.INPUT");
 }
 
@@ -87,5 +97,15 @@
     PS_ASSERT_PTR_NON_NULL(file, false);
 
-    return configRunFileAdd(config, file, "FILES.OUTPUT");
+    return configRunFileAdd(config, file->name, file->origname ? file->origname : file->filename,
+                            "FILES.OUTPUT");
+}
+
+bool pmConfigRunFilenameAddWrite(pmConfig *config, const char *description, const char *name)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    PS_ASSERT_STRING_NON_EMPTY(description, false);
+    PS_ASSERT_STRING_NON_EMPTY(name, false);
+
+    return configRunFileAdd(config, description, name, "FILES.OUTPUT");
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRun.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRun.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRun.h	(revision 24244)
@@ -12,8 +12,22 @@
     );
 
+/// Add a filename to the list of files read in the run-time information
+bool pmConfigRunFilenameAddRead(
+    pmConfig *config,                   ///< Configuration
+    const char *description,            ///< Description of file
+    const char *name                    ///< Name of file
+    );
+
 /// Add a file to the list of files written in the run-time information
 bool pmConfigRunFileAddWrite(
     pmConfig *config,                   ///< Configuration
     const pmFPAfile *file               ///< File to add
+    );
+
+/// Add a filename to the list of files written in the run-time information
+bool pmConfigRunFilenameAddWrite(
+    pmConfig *config,                   ///< Configuration
+    const char *description,            ///< Description of file
+    const char *name                    ///< Name of file
     );
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmErrorCodes.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmErrorCodes.dat	(revision 24244)
@@ -11,4 +11,5 @@
 OBJECTS			Problem in objects
 SKY			Problem in sky
+STAMPS			Unable to select stamps for PSF-matching
 # these errors correspond to standard exit conditions
 ARGUMENTS               Incorrect arguments
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmErrorCodes.h.in	(revision 24244)
@@ -9,5 +9,5 @@
  */
 typedef enum {
-    PM_ERR_BASE = 1200,
+    PM_ERR_BASE = 1000,
     PM_ERR_${ErrorCode},
     PM_ERR_NERROR
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmVersion.c	(revision 24244)
@@ -7,4 +7,5 @@
 #include <pslib.h>
 #include "pmVersion.h"
+#include "pmVersionDefinitions.h"
 
 #ifndef PSMODULES_VERSION
@@ -18,11 +19,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString psModulesVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PSMODULES_BRANCH), xstr(PSMODULES_VERSION));
+    psStringAppend(&value, "%s@%s", PSMODULES_BRANCH, PSMODULES_VERSION);
     return value;
 }
@@ -30,5 +28,5 @@
 psString psModulesSource(void)
 {
-    return psStringCopy(xstr(PSMODULES_SOURCE));
+    return psStringCopy(PSMODULES_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PM_VERSION_DEFINITIONS_H
+#define PM_VERSION_DEFINITIONS_H
+
+#define PSMODULES_VERSION @PSMODULES_VERSION@ // SVN version
+#define PSMODULES_BRANCH  @PSMODULES_BRANCH@  // SVN branch
+#define PSMODULES_SOURCE  @PSMODULES_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmDetrendDB.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmDetrendDB.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmDetrendDB.c	(revision 24244)
@@ -175,4 +175,5 @@
     if (!pipe) {
         psError (PS_ERR_IO, false, "error calling command %s", line);
+        psLogMsg ("psModules.detrend", PS_LOG_ERROR, "detselect command:\n %s\n", line);
         goto failure;
     }
@@ -188,4 +189,10 @@
         psError (PS_ERR_IO, false, "error running detselect");
         psLogMsg ("psModules.detrend", PS_LOG_ERROR, "detselect command:\n %s\n", line);
+        goto failure;
+    }
+
+    if (!buffer->data || strlen(buffer->data) == 0) {
+        psError(PS_ERR_IO, true, "Unable to find suitable detrend");
+        psLogMsg("psModules.detrend", PS_LOG_ERROR, "detselect command:\n %s\n", line);
         goto failure;
     }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmFringeStats.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmFringeStats.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmFringeStats.c	(revision 24244)
@@ -821,5 +821,5 @@
 
     // Solve the least-squares equation
-    if (! psMatrixGJSolve(A, B)) {
+    if (!psMatrixGJSolve(A, B)) {
         psError(PS_ERR_UNKNOWN, false, "Could not solve linear equations.  Returning NULL.\n");
         return false;
@@ -882,5 +882,8 @@
 
     psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_QUARTILE); // Statistics
-    psVectorStats(stats, diffs, NULL, mask, 1);
+    if (!psVectorStats(stats, diffs, NULL, mask, 1)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return 0;
+    }
     float middle = stats->sampleMedian; // The middle of the distribution
     float thresh = rej * 0.74 * (stats->sampleUQ - stats->sampleLQ); // The rejection threshold
@@ -969,9 +972,15 @@
 
     // Get rid of the extreme outliers by assuming most of the points are somewhat clustered
-    psVectorStats(median, science->f, NULL, NULL, 0);
+    if (!psVectorStats(median, science->f, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return NULL;
+    }
     scale->coeff->data.F32[0] = median->sampleMedian;
     for (int i = 0; i < fringes->n; i++) {
         pmFringeStats *fringe = fringes->data[i]; // The fringe of interest
-        psVectorStats(median, fringe->f, NULL, NULL, 0);
+        if (!psVectorStats(median, fringe->f, NULL, NULL, 0)) {
+	    psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	    return NULL;
+	}
         scale->coeff->data.F32[0] -= median->sampleMedian;
         scale->coeff->data.F32[i] = 0.0;
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmMaskBadPixels.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmMaskBadPixels.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmMaskBadPixels.c	(revision 24244)
@@ -81,6 +81,6 @@
 
 
-bool pmMaskFlagSuspectPixels(pmReadout *output, const pmReadout *readout, float median, float stdev,
-                             float rej, psImageMaskType maskVal)
+bool pmMaskFlagSuspectPixelsBySigma(pmReadout *output, const pmReadout *readout, float median, float stdev,
+				    float rej, psImageMaskType maskVal)
 {
     PS_ASSERT_PTR_NON_NULL(readout, false);
@@ -115,5 +115,6 @@
         // If we get down here and the statistics are missing, then we should go and mask the entire image
         psWarning("Missing statistics --- flagging entire image as suspect.");
-        return (psImage*)psBinaryOp(suspect, suspect, "+", psScalarAlloc(1.0, PS_TYPE_F32));
+        psBinaryOp (suspect, suspect, "+", psScalarAlloc(1.0, PS_TYPE_F32));
+        return true;
     }
 
@@ -128,4 +129,59 @@
         for (int x = 0; x < image->numCols; x++) {
             if (fabs((image->data.F32[y][x] - median) / stdev) < rej) continue;
+	    if (mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskVal)) continue;
+	    suspect->data.F32[y][x] += 1.0;
+        }
+    }
+    psFree(suspect);                    // Drop reference
+
+    psMetadataItem *numItem = psMetadataLookup(output->analysis, PM_MASK_ANALYSIS_NUM); // Item with number
+    assert(numItem);
+    numItem->data.S32++;
+
+    return true;
+}
+
+bool pmMaskFlagSuspectPixelsByValue(pmReadout *output, const pmReadout *readout, 
+				    float min, float max, psImageMaskType maskVal)
+{
+    PS_ASSERT_PTR_NON_NULL(readout, false);
+    PS_ASSERT_IMAGE_NON_NULL(readout->image, false);
+    PS_ASSERT_IMAGE_NON_EMPTY(readout->image, false);
+    PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, false);
+    if (readout->mask) {
+        PS_ASSERT_IMAGE_NON_EMPTY(readout->mask, false);
+        PS_ASSERT_IMAGES_SIZE_EQUAL(readout->image, readout->mask, false);
+        PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_IMAGE_MASK, false);
+    }
+    PS_ASSERT_PTR_NON_NULL(output, false);
+
+    bool mdok;                          // Status of MD lookup
+    psImage *suspect = psMetadataLookupPtr(&mdok, output->analysis, PM_MASK_ANALYSIS_SUSPECT); // Suspect img
+    if (suspect) {
+        PS_ASSERT_IMAGE_NON_EMPTY(suspect, false);
+        PS_ASSERT_IMAGE_TYPE(suspect, PS_TYPE_F32, false);
+        PS_ASSERT_IMAGES_SIZE_EQUAL(readout->image, suspect, false);
+        psMemIncrRefCounter(suspect);
+    } else {
+        suspect = psImageAlloc(readout->image->numCols, readout->image->numRows, PS_TYPE_F32);
+        psImageInit(suspect, 0);
+        psMetadataAddImage(output->analysis, PS_LIST_TAIL, PM_MASK_ANALYSIS_SUSPECT, PS_META_REPLACE,
+                           "Suspect pixels", suspect);
+        psMetadataAddS32(output->analysis, PS_LIST_TAIL, PM_MASK_ANALYSIS_NUM, PS_META_REPLACE,
+                         "Number of input images", 0);
+    }
+
+    psImage *image = readout->image;    // Image of interest
+    psImage *mask = readout->mask;      // Corresponding mask
+
+    psTrace ("psModules.detrend", 3, "suspect: < %f or > %f\n", min, max);
+
+    // XXX this loop could result in pixels with suspect = 0.0 but no valid input pixels (all
+    // masked).  need to track the number of good as well as suspect pixels?
+    for (int y = 0; y < image->numRows; y++) {
+        for (int x = 0; x < image->numCols; x++) {
+	    bool above = image->data.F32[y][x] > max;
+	    bool below = image->data.F32[y][x] < min;
+	    if (!above && !below) continue;
 	    if (mask && (mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskVal)) continue;
 	    suspect->data.F32[y][x] += 1.0;
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmMaskBadPixels.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmMaskBadPixels.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmMaskBadPixels.h	(revision 24244)
@@ -50,10 +50,24 @@
 /// high value in the suspect pixels image, allowing them to be identified.  The suspect pixels
 /// image is of type S32.  The relevant median and standard deviation must be supplied in the
-/// readout->analysis metadata as READOUT.MEDIAN, READOUT.STDEVe
-bool pmMaskFlagSuspectPixels(pmReadout *output, ///< Output readout, optionally with suspect pixels image
+/// readout->analysis metadata as READOUT.MEDIAN, READOUT.STDEV
+bool pmMaskFlagSuspectPixelsBySigma(pmReadout *output, ///< Output readout, optionally with suspect pixels image
                              const pmReadout *readout, ///< Readout to inspect
                              float median, ///< Image median
                              float stdev, ///< Image standard deviation
                              float rej, ///< Rejection threshold (standard deviations)
+                             psImageMaskType maskVal ///< Mask value for statistics
+    );
+
+/// Find out-of-range pixels and flag them
+///
+/// Pixels great > max or < min have the corresponding pixel in the "suspect pixels" image
+/// incremented.  After accumulating over a suitable sample of images, bad pixels should have a
+/// high value in the suspect pixels image, allowing them to be identified.  The suspect pixels
+/// image is of type S32.  The relevant median and standard deviation must be supplied in the
+/// readout->analysis metadata as READOUT.MEDIAN, READOUT.STDEV
+bool pmMaskFlagSuspectPixelsByValue(pmReadout *output, ///< Output readout, optionally with suspect pixels image
+                             const pmReadout *readout, ///< Readout to inspect
+                             float min, ///< Image min acceptable value
+                             float max, ///< Image max acceptable value
                              psImageMaskType maskVal ///< Mask value for statistics
     );
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmOverscan.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmOverscan.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmOverscan.c	(revision 24244)
@@ -74,5 +74,8 @@
             mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = 0;
             ordinate->data.F32[i] = 2.0*(float)i/(float)pixels->n - 1.0; // Scale to [-1,1]
-            psVectorStats(myStats, values, NULL, NULL, 0);
+            if (!psVectorStats(myStats, values, NULL, NULL, 0)) {
+		psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		return false;
+	    }
             reduced->data.F32[i] = psStatsGetValue(myStats, statistic);
         } else if (overscanOpts->fitType == PM_FIT_NONE) {
@@ -255,4 +258,5 @@
     psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK,
 		     comment, "");
+    psTrace ("psModules.detrend", 4, "%s\n", comment);
     psFree(comment);
 
@@ -274,5 +278,8 @@
 	psFree(iter);
 
-	(void)psVectorStats(stats, pixels, NULL, NULL, 0);
+	if (!psVectorStats(stats, pixels, NULL, NULL, 0)) {
+	    psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	    return false;
+	}
 	psFree(pixels);
 	double reduced = psStatsGetValue(stats, statistic); // Result of statistics
@@ -344,4 +351,22 @@
 	}
 
+	// generate stats of overscan vector for header
+	{ 
+	  psString comment = NULL;    // Comment to add
+	  psStats *vectorStats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV);
+	  if (!psVectorStats (vectorStats, reduced, NULL, NULL, 0)) {
+	      psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	      return false;
+	  }
+	  psStringAppend(&comment, "Mean Overscan value: %f", vectorStats->sampleMean);
+	  psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, "");
+	  psFree(comment);
+
+	  // write metadata header value
+	  psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan mean", vectorStats->sampleMean);
+	  psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", vectorStats->sampleStdev);
+	  psFree (vectorStats);
+	}
+
 	// Subtract row by row
 	for (int i = 0; i < image->numRows; i++) {
@@ -392,7 +417,25 @@
 	}
 
-	// Subtract column by column
-	for (int i = 0; i < image->numCols; i++) {
-	    for (int j = 0; j < image->numRows; j++) {
+	// generate stats of overscan vector for header
+	{ 
+	  psString comment = NULL;    // Comment to add
+	  psStats *vectorStats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV);
+	  if (!psVectorStats (vectorStats, reduced, NULL, NULL, 0)) {
+	      psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	      return false;
+	  }
+	  psStringAppend(&comment, "Mean Overscan value: %f", vectorStats->sampleMean);
+	  psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, "");
+	  psFree(comment);
+
+	  // write metadata header value
+	  psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_VAL", PS_META_REPLACE, "Overscan mean", vectorStats->sampleMean);
+	  psMetadataAddF32(hdu->header, PS_LIST_TAIL, "OVER_SIG", PS_META_REPLACE, "Overscan stdev", vectorStats->sampleStdev);
+	  psFree (vectorStats);
+	}
+
+	// Subtract column by column 
+	for (int j = 0; j < image->numRows; j++) {
+	  for (int i = 0; i < image->numCols; i++) {
 		image->data.F32[j][i] -= reduced->data.F32[i];
 	    }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmRemnance.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmRemnance.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmRemnance.c	(revision 24244)
@@ -66,6 +66,8 @@
             values->n = numValues;
             if (!psVectorStats(stats, values, NULL, NULL, 0)) {
-                // Can't do anything about it
-                psErrorClear();
+		psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		return false;
+	    }
+	    if (isnan(stats->sampleMedian)) {
                 maxMask = max;
                 continue;
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmShutterCorrection.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmShutterCorrection.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmShutterCorrection.c	(revision 24244)
@@ -350,10 +350,17 @@
     psStats *rawStats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
     psStats *resStats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
-    psVectorStats (rawStats, counts, NULL, NULL, 0);
-    psVectorStats (resStats, resid, NULL, NULL, 0);
+    if (!psVectorStats (rawStats, counts, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return NULL;
+    }
+    if (!psVectorStats (resStats, resid, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return NULL;
+    }
 
     // XXX temporary hard-wired minimum stdev improvement factor
     psTrace("psModules.detrend", 3, "raw scatter %f vs res scatter %f\n", rawStats->sampleStdev, resStats->sampleStdev);
     if (rawStats->sampleStdev / resStats->sampleStdev < 1.5) corr->valid = false;
+    if (isnan(rawStats->sampleStdev) || isnan(resStats->sampleStdev)) corr->valid = false;
 
     psFree (rawStats);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmSkySubtract.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmSkySubtract.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmSkySubtract.c	(revision 24244)
@@ -410,9 +410,9 @@
     // XXX: How do we know if these matrix operations were successful?
     //
-    Aout = psMatrixLUD(Aout, &outPerm, A);
+    Aout = psMatrixLUDecomposition(Aout, &outPerm, A);
     PS_ASSERT_IMAGE_NON_NULL(Aout, NULL);
     PS_ASSERT_IMAGE_NON_EMPTY(Aout, NULL);
     psVector *C = psVectorAlloc(localPolyTerms, PS_TYPE_F64);
-    psMatrixLUSolve(C, Aout, B, outPerm);
+    psMatrixLUSolution(C, Aout, B, outPerm);
 
     //
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/extras/pmVisual.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/extras/pmVisual.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/extras/pmVisual.c	(revision 24244)
@@ -281,6 +281,12 @@
     psStats *statsX = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
     psStats *statsY = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
-    psVectorStats (statsX, xVec, NULL, NULL, 0);
-    psVectorStats (statsY, yVec, NULL, NULL, 0);
+    if (!psVectorStats (statsX, xVec, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
+    if (!psVectorStats (statsY, yVec, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
 
     float xhi  = statsX->sampleMedian + 3 *statsX->sampleStdev;
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmImageCombine.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmImageCombine.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmImageCombine.c	(revision 24244)
@@ -132,4 +132,8 @@
         // Combine all the pixels, using the specified stat.
         if (!psVectorStats(stats, pixelData, pixelErrors, pixelMasks, 0xff)) {
+	    psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	    return false;
+	}
+	if (isnan(stats->sampleMean)) {
             combine->data.F32[y][x] = NAN;
             psFree(buffer);
@@ -137,5 +141,5 @@
         }
         float combinedPixel = stats->sampleMean; // Value of the combination
-
+	
         if (iter == 0) {
             // Save the value produced with no rejection, since it may be useful later
@@ -364,5 +368,7 @@
     // Get the median
     psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN);
-    psVectorStats(stats, pixels, NULL, mask, 1);
+    if (!psVectorStats(stats, pixels, NULL, mask, 1)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+    }
     float median = stats->sampleMedian;
     psFree(stats);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmPSFEnvelope.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmPSFEnvelope.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmPSFEnvelope.c	(revision 24244)
@@ -33,5 +33,5 @@
 
 
-//#define TESTING                         // Enable test output
+// #define TESTING                         // Enable test output
 #define PEAK_FLUX 1.0e4                 // Peak flux for each source
 #define SKY_VALUE 0.0e0                 // Sky value for fake image
@@ -40,4 +40,6 @@
 #define PSF_STATS PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV // Statistics options for measuring PSF
 #define SOURCE_FIT_ITERATIONS 100       // Number of iterations for source fitting
+#define MODEL_MASK (PM_MODEL_STATUS_NONCONVERGE | PM_MODEL_STATUS_OFFIMAGE | \
+                    PM_MODEL_STATUS_BADARGS | PM_MODEL_STATUS_LIMITS) // Mask to apply to models
 
 
@@ -112,5 +114,7 @@
     psImageInit(envelope, SKY_VALUE);
     pmReadout *fakeRO = pmReadoutAlloc(NULL); // Fake readout
-    float maxRadius = 0.0;              // Maximum radius of sources
+    float maxRadius = 0.0;              // Maximum radius for sources
+    psVector *numbers = psVectorAlloc(numFakes, PS_TYPE_S32); // Number of detections for each source
+    psVectorInit(numbers, 0);
     for (int i = 0; i < inputs->n; i++) {
         pmPSF *psf = inputs->data[i];   // PSF of interest
@@ -127,4 +131,5 @@
             psFree(xOffset);
             psFree(fakes);
+            psFree(numbers);
             psf->residuals = resid;
             return NULL;
@@ -140,4 +145,7 @@
 
             double flux = fakeRO->image->data.F32[(int)y][(int)x];
+            if (!isfinite(flux) || flux < 0) {
+                continue;
+            }
             float norm = PEAK_FLUX / flux; // Normalisation for source
             psRegion region = psRegionSet(x - radius, x + radius, y - radius, y + radius); // PSF region
@@ -151,10 +159,17 @@
             // Get the radius
             pmModel *model = pmModelFromPSFforXY(psf, x, y, PEAK_FLUX); // Model for source
-            psAssert (model, "failed to generate model: should this be an error or not?");
+            if (!model || (model->flags & MODEL_MASK)) {
+                continue;
+            }
             float srcRadius = model->modelRadius(model->params, PS_SQR(VARIANCE_VAL)); // Radius for source
+            if (srcRadius == 0) {
+                continue;
+            }
             if (srcRadius > maxRadius) {
                 maxRadius = srcRadius;
             }
-            psFree(model);
+
+            // If we got this far, the source is decent
+            numbers->data.S32[j]++;
         }
 
@@ -175,8 +190,4 @@
     psFree(fakeRO);
 
-    if (maxRadius > radius) {
-        maxRadius = radius;
-    }
-
 #ifdef TESTING
     {
@@ -190,4 +201,5 @@
 
     // Put the fake sources onto a full-size image
+    psArray *goodFakes = psArrayAllocEmpty(numFakes); // Good fake sources
     pmReadout *readout = pmReadoutAlloc(NULL); // Readout to contain envelope pixels
     readout->image = psImageAlloc(numCols, numRows, PS_TYPE_F32);
@@ -195,4 +207,8 @@
     for (int i = 0; i < numFakes; i++) {
         pmSource *source = fakes->data[i]; // Fake source
+        if (numbers->data.S32[i] > 0) {
+            psArrayAdd(goodFakes, goodFakes->n, source);
+        }
+
         // Position of source on fake image
         int xFake = source->peak->x + xOffset->data.S32[i];
@@ -213,4 +229,5 @@
             psFree(yOffset);
             psFree(fakes);
+            psFree(numbers);
             return NULL;
         }
@@ -220,4 +237,16 @@
     psFree(yOffset);
     psFree(envelope);
+    psFree(numbers);
+
+    psFree(fakes);
+    fakes = goodFakes;
+    numFakes = fakes->n;
+
+    if (numFakes == 0.0) {
+        psError(PS_ERR_UNKNOWN, false, "No fake sources are suitable for PSF fitting.");
+        psFree(fakes);
+        psFree(readout);
+        return false;
+    }
 
     // XXX Setting the variance seems to be an art
@@ -232,4 +261,8 @@
     psImageInit(readout->mask, 0);
 
+    if (maxRadius > radius) {
+        maxRadius = radius;
+    }
+
 #ifdef TESTING
     {
@@ -243,4 +276,5 @@
 
     // Reset the sources to point to the new pixels, and measure the moments in preparation for PSF fitting
+    int numMoments = 0;                 // Number of moments measured
     for (int i = 0; i < numFakes; i++) {
         pmSource *source = fakes->data[i]; // Fake source
@@ -257,5 +291,5 @@
         source->maskObj = NULL;
 
-        if (!pmSourceDefinePixels(source, readout, x, y, maxRadius)) {
+        if (!pmSourceDefinePixels(source, readout, x, y, radius)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to define pixels for source.");
             psFree(readout);
@@ -265,9 +299,16 @@
 
         if (!pmSourceMoments(source, maxRadius)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to measure moments for source.");
-            psFree(readout);
-            psFree(fakes);
-            return NULL;
-        }
+            // Can't do anything about it; limp along as best we can
+            psErrorClear();
+            continue;
+        }
+        numMoments++;
+    }
+
+    if (numMoments == 0) {
+        psError(PS_ERR_UNKNOWN, true, "Unable to measure moments for sources.");
+        psFree(fakes);
+        psFree(readout);
+        return NULL;
     }
 
@@ -286,4 +327,5 @@
     options->psfFieldXo = 0;
     options->psfFieldYo = 0;
+    options->chiFluxTrend = false;      // All sources have similar flux, so fitting a trend often fails
 
     pmSourceFitModelInit(SOURCE_FIT_ITERATIONS, 0.01, VARIANCE_VAL, true);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmReadoutCombine.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmReadoutCombine.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmReadoutCombine.c	(revision 24244)
@@ -384,23 +384,27 @@
             // Combination
             if (!psVectorStats(stats, pixels, errors, mask, 1)) {
-                // Can't do much about it, but it's not worth worrying about
-                psErrorClear();
+		psError(PS_ERR_UNKNOWN, false, "error in pixel stats");
+		return false;
+	    }
+	    
+	    outputImage[yOut][xOut] = psStatsGetValue(stats, params->combine);
+
+	    if (isnan(outputImage[yOut][xOut])) {
                 outputImage[yOut][xOut] = NAN;
                 outputMask[yOut][xOut] = params->blank;
+                sigma->data.F32[yOut][xOut] = NAN;
                 if (params->variances) {
                     outputVariance[yOut][xOut] = NAN;
                 }
-                sigma->data.F32[yOut][xOut] = NAN;
-            } else {
-                outputImage[yOut][xOut] = psStatsGetValue(stats, params->combine);
-                outputMask[yOut][xOut] = isfinite(outputImage[yOut][xOut]) ? 0 : params->blank;
-                if (params->variances) {
-                    float stdev = psStatsGetValue(stats, combineStdev);
-                    outputVariance[yOut][xOut] = PS_SQR(stdev); // Variance
-                    // XXXX this is not the correct formal error.
-                    // also, the weighted mean is not obviously the correct thing here
-                }
-                sigma->data.F32[yOut][xOut] = psStatsGetValue(stats, combineStdev);
-            }
+		continue;
+	    }
+	    outputMask[yOut][xOut] = 0;
+	    sigma->data.F32[yOut][xOut] = psStatsGetValue(stats, combineStdev);
+	    if (params->variances) {
+		float stdev = psStatsGetValue(stats, combineStdev);
+		outputVariance[yOut][xOut] = PS_SQR(stdev); // Variance
+		// XXXX this is not the correct formal error.
+		// also, the weighted mean is not obviously the correct thing here
+	    }
         }
     }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmStack.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmStack.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmStack.c	(revision 24244)
@@ -34,6 +34,6 @@
 
 //#define TESTING                         // Enable test output
-//#define TEST_X 2318                     // x coordinate to examine
-//#define TEST_Y 2306                     // y coordinate to examine
+//#define TEST_X 1085                     // x coordinate to examine
+//#define TEST_Y 3371                     // y coordinate to examine
 
 
@@ -265,5 +265,5 @@
 
     int numBad = frac * numGood + 0.5;  // Number of bad values
-    int low = numBad / 2, high = low + numGood; // Indices (modulo masked pixels) delimiting range of interest
+    int low = numBad / 2, high = low + numGood - numBad; // Indices (modulo masked pixels)
 
     sortBuffer = psVectorSortIndex(sortBuffer, values);
@@ -536,8 +536,13 @@
 #ifdef TESTING
                   if (x == TEST_X && y == TEST_Y) {
-                      fprintf(stderr, "Rejection limit: %f\n", limit);
+                      fprintf(stderr, "Rejecting without variance; rejection limit: %f\n", limit);
                   }
 #endif
               } else {
+#ifdef TESTING
+                  if (x == TEST_X && y == TEST_Y) {
+                      fprintf(stderr, "Rejecting with variance...\n");
+                  }
+#endif
                   median = combinationWeightedOlympic(pixelData, pixelWeights, pixelMasks, discard, sort);
               }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtraction.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtraction.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtraction.c	(revision 24244)
@@ -822,4 +822,10 @@
     psFree(mask);
 
+    // XXX raise an error?
+    if (isnan(stats->sampleMean)) {
+        psFree(stats);
+        return -1;
+    }
+
     double mean, rms;                 // Mean and RMS of deviations
     if (numStamps < MIN_SAMPLE_STATS) {
@@ -849,4 +855,12 @@
     float limit = sigmaRej * rms; // Limit on maximum deviation
     psTrace("psModules.imcombine", 1, "Deviation limit: %f\n", limit);
+
+
+    psString ds9name = NULL;            // Filename for ds9 region file
+    static int ds9num = 0;              // File number for ds9 region file
+    psStringAppend(&ds9name, "stamps_reject_%d.ds9", ds9num);
+    FILE *ds9 = pmSubtractionStampsFile(stamps, ds9name, "rejected stamps");
+    psFree(ds9name);
+    ds9num++;
 
     int numRejected = 0;                // Number of stamps rejected
@@ -872,4 +886,5 @@
                     }
                 }
+                pmSubtractionStampPrint(ds9, stamp->x, stamp->y, stamps->footprint, "red");
 
                 // Set stamp for replacement
@@ -897,8 +912,13 @@
                 numGood++;
                 newMean += deviations->data.F32[i];
+                pmSubtractionStampPrint(ds9, stamp->x, stamp->y, stamps->footprint, "green");
             }
         }
     }
     newMean /= numGood;
+
+    if (ds9) {
+        fclose(ds9);
+    }
 
     if (numRejected > 0) {
@@ -1385,4 +1405,19 @@
     }
 
+    // Data exists on the outputs now
+    if (out1) {
+        out1->data_exists = true;
+        if (out1->parent) {
+            out1->parent->data_exists = out1->parent->parent->data_exists = true;
+        }
+    }
+    if (out2) {
+        out2->data_exists = true;
+        if (out2->parent) {
+            out2->parent->data_exists = out2->parent->parent->data_exists = true;
+        }
+    }
+
+
     psLogMsg("psModules.imcombine", PS_LOG_INFO, "Convolve image: %f sec",
              psTimerClear("pmSubtractionConvolve"));
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionAnalysis.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionAnalysis.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionAnalysis.c	(revision 24244)
@@ -201,6 +201,11 @@
         psFree(image);
 
-        psMetadataAddF32(analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_DECONV_MAX,
-                         PS_META_DUPLICATE_OK, "Maximum deconvolution fraction", max);
+        psMetadataItem *item = psMetadataLookup(analysis, PM_SUBTRACTION_ANALYSIS_DECONV_MAX); // Previous
+        if (item) {
+            item->data.F32 = PS_MAX(item->data.F32, max);
+        } else {
+            psMetadataAddF32(analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_DECONV_MAX, 0,
+                             "Maximum deconvolution fraction", max);
+        }
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionEquation.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionEquation.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionEquation.c	(revision 24244)
@@ -749,4 +749,11 @@
     }
 
+    psString ds9name = NULL;            // Filename for ds9 region file
+    static int ds9num = 0;              // File number for ds9 region file
+    psStringAppend(&ds9name, "stamps_solution_%d.ds9", ds9num);
+    FILE *ds9 = pmSubtractionStampsFile(stamps, ds9name, "solution stamps");
+    psFree(ds9name);
+    ds9num++;
+
     if (kernels->mode != PM_SUBTRACTION_MODE_DUAL) {
         // Accumulate the least-squares matricies and vectors
@@ -761,5 +768,8 @@
                 (void)psBinaryOp(sumMatrix, sumMatrix, "+", stamp->matrix1);
                 (void)psBinaryOp(sumVector, sumVector, "+", stamp->vector1);
+                pmSubtractionStampPrint(ds9, stamp->x, stamp->y, stamps->footprint, "green");
                 numStamps++;
+            } else if (stamp->status == PM_SUBTRACTION_STAMP_REJECTED) {
+                pmSubtractionStampPrint(ds9, stamp->x, stamp->y, stamps->footprint, "red");
             }
         }
@@ -788,5 +798,5 @@
 
         psVector *permutation = NULL;       // Permutation vector, required for LU decomposition
-        psImage *luMatrix = psMatrixLUD(NULL, &permutation, sumMatrix);
+        psImage *luMatrix = psMatrixLUDecomposition(NULL, &permutation, sumMatrix);
         psFree(sumMatrix);
         if (!luMatrix) {
@@ -797,5 +807,5 @@
             return NULL;
         }
-        kernels->solution1 = psMatrixLUSolve(kernels->solution1, luMatrix, sumVector, permutation);
+        kernels->solution1 = psMatrixLUSolution(kernels->solution1, luMatrix, sumVector, permutation);
 
         psFree(sumVector);
@@ -830,4 +840,5 @@
                 (void)psBinaryOp(sumVector1, sumVector1, "+", stamp->vector1);
                 (void)psBinaryOp(sumVector2, sumVector2, "+", stamp->vector2);
+                pmSubtractionStampPrint(ds9, stamp->x, stamp->y, stamps->footprint, "green");
                 numStamps++;
             }
@@ -990,4 +1001,8 @@
         // XXXXX Free temporary matrices and vectors
 
+    }
+
+    if (ds9) {
+        fclose(ds9);
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionMask.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionMask.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionMask.c	(revision 24244)
@@ -38,5 +38,5 @@
 
 psImage *pmSubtractionMask(const psImage *mask1, const psImage *mask2, psImageMaskType maskVal,
-                           int size, int footprint, float badFrac, bool useFFT)
+                           int size, int footprint, float badFrac, pmSubtractionMode mode)
 {
     PS_ASSERT_IMAGE_NON_NULL(mask1, NULL);
@@ -114,10 +114,4 @@
     }
 
-    // XXX Could do something smarter here --- we will get images that are predominantly masked (where the
-    // skycell isn't overlapped by a large fraction by the observation), so that convolving around every bad
-    // pixel is wasting time.  As a first cut, I've put in a check on the fraction of bad pixels, but we could
-    // imagine looking for the edge of big regions and convolving just at the edge.  As a second cut, allow
-    // use of FFT convolution.
-
     for (int y = 0; y < numRows; y++) {
         for (int x = 0; x < numCols; x++) {
@@ -131,14 +125,12 @@
     }
 
-    // Block out the entire stamp footprint around bad input pixels.
-
     // We want to block out with the CONVOLVE mask anything that would be bad if we convolved with a bad
-    // reference pixel (within 'size').  Then we want to block out with the FOOTPRINT mask everything within a
+    // reference pixel (within 'size').  Then we want to block out with the REJ mask everything within a
     // footprint's distance of those (within 'footprint').
 
     bool oldThreads = psImageConvolveSetThreads(true); // Old value of threading for psImageConvolve
 
-    if (!psImageConvolveMask(mask, mask, PM_SUBTRACTION_MASK_BAD_1,
-                             PM_SUBTRACTION_MASK_CONVOLVE_1,
+    // Pixels that will be bad (or poor) if we convolve with a bad reference pixel
+    if (!psImageConvolveMask(mask, mask, PM_SUBTRACTION_MASK_BAD_1, PM_SUBTRACTION_MASK_CONVOLVE_1,
                              -size, size, -size, size)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to convolve bad pixels from mask 1.");
@@ -146,8 +138,31 @@
         return NULL;
     }
-    if (!psImageConvolveMask(mask, mask, PM_SUBTRACTION_MASK_BAD_2,
-                             PM_SUBTRACTION_MASK_CONVOLVE_2,
+    if (!psImageConvolveMask(mask, mask, PM_SUBTRACTION_MASK_BAD_2, PM_SUBTRACTION_MASK_CONVOLVE_2,
                              -size, size, -size, size)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to convolve bad pixels from mask 2.");
+        psFree(mask);
+        return NULL;
+    }
+
+    // Pixels that should not be chosen as stamps
+    psImageMaskType maskRej = PM_SUBTRACTION_MASK_BAD_1 | PM_SUBTRACTION_MASK_BAD_2 |
+        PM_SUBTRACTION_MASK_BORDER;     // Mask value for rejection
+    switch (mode) {
+      case PM_SUBTRACTION_MODE_1:
+        maskRej |= PM_SUBTRACTION_MASK_CONVOLVE_1;
+        break;
+      case PM_SUBTRACTION_MODE_2:
+        maskRej |= PM_SUBTRACTION_MASK_CONVOLVE_2;
+        break;
+      case PM_SUBTRACTION_MODE_UNSURE:
+      case PM_SUBTRACTION_MODE_DUAL:
+        maskRej |= PM_SUBTRACTION_MASK_CONVOLVE_1 | PM_SUBTRACTION_MASK_CONVOLVE_2;
+        break;
+      default:
+        psAbort("Unsupported subtraction mode: %x", mode);
+    }
+    if (!psImageConvolveMask(mask, mask, maskRej, PM_SUBTRACTION_MASK_REJ,
+                             -footprint, footprint, -footprint, footprint)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to convolve bad pixels.");
         psFree(mask);
         return NULL;
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionMask.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionMask.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionMask.h	(revision 24244)
@@ -11,5 +11,5 @@
                            int footprint, ///< Half-size of the kernel footprint
                            float badFrac, ///< Maximum fraction of bad input pixels to accept
-                           bool useFFT  ///< Use FFT to do convolution?
+                           pmSubtractionMode mode  ///< Subtraction mode
     );
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionMatch.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionMatch.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionMatch.c	(revision 24244)
@@ -18,6 +18,8 @@
 #include "pmSubtractionMask.h"
 #include "pmSubtractionThreads.h"
+#include "pmSubtractionVisual.h"
+#include "pmErrorCodes.h"
+
 #include "pmSubtractionMatch.h"
-#include "pmSubtractionVisual.h"
 
 #define BG_STAT PS_STAT_ROBUST_MEDIAN   // Statistic to use for background
@@ -62,5 +64,6 @@
                       psImage *variance,  // Variance map
                       const psRegion *region, // Region of interest, or NULL
-                      float threshold,  // Threshold for stamp finding
+                      float thresh1,  // Threshold for stamp finding on readout 1
+                      float thresh2,  // Threshold for stamp finding on readout 2
                       float stampSpacing, // Spacing between stamps
                       int size,         // Kernel half-size
@@ -70,8 +73,11 @@
 {
     psTrace("psModules.imcombine", 3, "Finding stamps...\n");
-    *stamps = pmSubtractionStampsFind(*stamps, ro1->image, subMask, region, threshold, footprint,
-                                      stampSpacing, mode);
+
+    psImage *image1 = ro1 ? ro1->image : NULL, *image2 = ro2 ? ro2->image : NULL; // Images of interest
+
+    *stamps = pmSubtractionStampsFind(*stamps, image1, image2, subMask, region, thresh1, thresh2,
+                                      size, footprint, stampSpacing, mode);
     if (!*stamps) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to find stamps.");
+        psError(psErrorCodeLast(), false, "Unable to find stamps.");
         return false;
     }
@@ -238,5 +244,5 @@
 
     psImage *subMask = pmSubtractionMask(ro1->mask, ro2 ? ro2->mask : NULL, maskVal, size, 0,
-                                         badFrac, useFFT); // Subtraction mask
+                                         badFrac, mode); // Subtraction mask
     if (!subMask) {
         psError(PS_ERR_UNKNOWN, false, "Unable to generate subtraction mask.");
@@ -371,5 +377,5 @@
 
     subMask = pmSubtractionMask(ro1->mask, ro2 ? ro2->mask : NULL, maskVal, size, footprint,
-                                badFrac, useFFT);
+                                badFrac, subMode);
     if (!subMask) {
         psError(PS_ERR_UNKNOWN, false, "Unable to generate subtraction mask.");
@@ -390,13 +396,23 @@
     }
 
+    float stampThresh1 = NAN, stampThresh2 = NAN; // Stamp thresholds for images
     {
         psStats *bg = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics for backgroun
-        if (!psImageBackground(bg, NULL, ro1->image, ro1->mask, maskVal, rng)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to measure background statistics.");
-            psFree(bg);
-            psFree(rng);
-            goto MATCH_ERROR;
-        }
-        threshold = bg->robustMedian + threshold * bg->robustStdev;
+        if (ro1) {
+            if (!psImageBackground(bg, NULL, ro1->image, ro1->mask, maskVal, rng)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to measure background statistics.");
+                psFree(bg);
+                goto MATCH_ERROR;
+            }
+            stampThresh1 = bg->robustMedian + threshold * bg->robustStdev;
+        }
+        if (ro2) {
+            if (!psImageBackground(bg, NULL, ro2->image, ro2->mask, maskVal, rng)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to measure background statistics.");
+                psFree(bg);
+                goto MATCH_ERROR;
+            }
+            stampThresh2 = bg->robustMedian + threshold * bg->robustStdev;
+        }
         psFree(bg);
     }
@@ -416,16 +432,16 @@
             }
 
-            if (sources) {
-                stamps = pmSubtractionStampsSetFromSources(sources, ro1->image, subMask, region, footprint,
-                                                           stampSpacing, subMode);
-            } else if (stampsName && strlen(stampsName) > 0) {
-                stamps = pmSubtractionStampsSetFromFile(stampsName, ro1->image, subMask, region, footprint,
-                                                        stampSpacing, subMode);
+            if (stampsName && strlen(stampsName) > 0) {
+                stamps = pmSubtractionStampsSetFromFile(stampsName, ro1->image, subMask, region, size,
+                                                        footprint, stampSpacing, subMode);
+            } else if (sources) {
+                stamps = pmSubtractionStampsSetFromSources(sources, ro1->image, subMask, region, size,
+                                                           footprint, stampSpacing, subMode);
             }
 
             // We get the stamps here; we will also attempt to get stamps at the first iteration, but it
             // doesn't matter.
-            if (!getStamps(&stamps, ro1, ro2, subMask, variance, NULL, threshold, stampSpacing,
-                           size, footprint, subMode)) {
+            if (!getStamps(&stamps, ro1, ro2, subMask, variance, NULL, stampThresh1, stampThresh2,
+                           stampSpacing, size, footprint, subMode)) {
                 goto MATCH_ERROR;
             }
@@ -488,6 +504,6 @@
                 psLogMsg("psModules.imcombine", PS_LOG_INFO, "Iteration %d.", k);
 
-                if (!getStamps(&stamps, ro1, ro2, subMask, variance, region, threshold, stampSpacing,
-                               size, footprint, subMode)) {
+                if (!getStamps(&stamps, ro1, ro2, subMask, variance, region, stampThresh1, stampThresh2,
+                               stampSpacing, size, footprint, subMode)) {
                     goto MATCH_ERROR;
                 }
@@ -565,20 +581,4 @@
             psFree(kernels);
             kernels = NULL;
-
-            // There is data in the readout now
-            if (subMode == PM_SUBTRACTION_MODE_1 || subMode == PM_SUBTRACTION_MODE_DUAL) {
-                conv1->data_exists = true;
-                if (conv1->parent) {
-                    conv1->parent->data_exists = true;
-                    conv1->parent->parent->data_exists = true;
-                }
-            }
-            if (subMode == PM_SUBTRACTION_MODE_2 || subMode == PM_SUBTRACTION_MODE_DUAL) {
-                conv2->data_exists = true;
-                if (conv2->parent) {
-                    conv2->parent->data_exists = true;
-                    conv2->parent->parent->data_exists = true;
-                }
-            }
         }
     }
@@ -830,12 +830,14 @@
     psFree(mask);
 
+    // XXX raise an error here or not?
+    if (isnan(stats->robustMedian)) {
+        psFree(stats);
+        return PM_SUBTRACTION_MODE_ERR;
+    }
+
     psLogMsg("psModules.imcombine", PS_LOG_INFO, "Median width ratio: %lf", stats->robustMedian);
     pmSubtractionMode mode = (stats->robustMedian <= 1.0 ? PM_SUBTRACTION_MODE_1 : PM_SUBTRACTION_MODE_2);
     psFree(stats);
 
-    // XXX EAM : I think Paul left some test code in here.  I've commented these lines out
-    // return PM_SUBTRACTION_MODE_2;
-    // exit(1);
-
     return mode;
 }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionStamps.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionStamps.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionStamps.c	(revision 24244)
@@ -20,5 +20,5 @@
 #include "pmModel.h"
 #include "pmSource.h"
-
+#include "pmErrorCodes.h"
 
 #include "pmSubtraction.h"
@@ -78,72 +78,100 @@
 }
 
-// Is this position unmasked?
-static bool checkStampMask(int x, int y, // Coordinates of stamp
-                           const psImage *mask, // Mask
-                           pmSubtractionMode mode, // Mode for subtraction
-                           int footprint // Footprint to check for Bad Stuff
-                           )
-{
-    if (!mask) {
-        return true;
-    }
-
-    bool clean = true;                  // Is the footprint clean?
-    int numCols = mask->numCols, numRows = mask->numRows; // Size of image
-
-    // Check the footprint bounds
-    if (x < footprint || x >= numCols - footprint || y < footprint || y >= numRows - footprint) {
-        clean = false;
-    }
-
-    // Determine mask value
-    psImageMaskType maskVal = PM_SUBTRACTION_MASK_BORDER | PM_SUBTRACTION_MASK_BAD_1 | PM_SUBTRACTION_MASK_BAD_2;
-    switch (mode) {
-      case PM_SUBTRACTION_MODE_1:
-        maskVal |= PM_SUBTRACTION_MASK_CONVOLVE_1;
-        break;
-      case PM_SUBTRACTION_MODE_2:
-        maskVal |= PM_SUBTRACTION_MASK_CONVOLVE_2;
-        break;
-      case PM_SUBTRACTION_MODE_UNSURE:
-      case PM_SUBTRACTION_MODE_DUAL:
-        maskVal |= PM_SUBTRACTION_MASK_CONVOLVE_1 | PM_SUBTRACTION_MASK_CONVOLVE_2;
-        break;
-      default:
-        psAbort("Unsupported subtraction mode: %x", mode);
-    }
-
-    // Check the immediate pixel
-    if (clean && (mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & (maskVal | PM_SUBTRACTION_MASK_REJ))) {
-        clean = false;
-    }
-
-    int xMin = PS_MAX(x - footprint, 0), xMax = PS_MIN(x + footprint, numCols - 1); // Bounds in x
-    int yMin = PS_MAX(y - footprint, 0), yMax = PS_MIN(y + footprint, numRows - 1); // Bounds in y
-
-    // Check the footprint
-    if (clean) {
-        for (int j = yMin; j <= yMax; j++) {
-            for (int i = xMin; i <= xMax; i++) {
-                if (mask->data.PS_TYPE_IMAGE_MASK_DATA[j][i] & maskVal) {
-                    clean = false;
-                    goto CHECK_STAMP_MASK_DONE;
-                }
+
+// Search a region for a suitable stamp
+bool stampSearch(int *xStamp, int *yStamp, // Coordinates of stamp, to return
+                 float *fluxStamp, // Flux of stamp, to return
+                 const psImage *image1, const psImage *image2, // Images to search
+                 float thresh1, float thresh2, // Thresholds for images
+                 const psImage *subMask, // Subtraction mask
+                 int xMin, int xMax, int yMin, int yMax, // Bounds of search
+                 int numCols, int numRows, // Size of images
+                 int border             // Border around image
+    )
+{
+    bool found = false;                 // Found a suitable stamp?
+    *fluxStamp = -INFINITY;             // Flux of best stamp
+
+    // Ensure we're not going to go outside the bounds of the image
+    xMin = PS_MAX(border, xMin);
+    xMax = PS_MIN(numCols - border - 1, xMax);
+    yMin = PS_MAX(border, yMin);
+    yMax = PS_MIN(numRows - border - 1, yMax);
+
+    for (int y = yMin; y <= yMax; y++) {
+        for (int x = xMin; x <= xMax; x++) {
+            if ((image1 && image1->data.F32[y][x] < thresh1) ||
+                (image2 && image2->data.F32[y][x] < thresh2)) {
+                continue;
             }
-        }
-    }
-CHECK_STAMP_MASK_DONE:
-
-    if (!clean) {
-        // Mask the footprint, so we don't select something near it again
-        for (int j = yMin; j <= yMax; j++) {
-            for (int i = xMin; i <= xMax; i++) {
-                mask->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= PM_SUBTRACTION_MASK_REJ;
+
+            if (subMask && subMask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] &
+                (PM_SUBTRACTION_MASK_BORDER | PM_SUBTRACTION_MASK_REJ)) {
+                return false;
             }
-        }
-    }
-
-    return clean;
-}
+
+            // We take the MIN to attempt to avoid transients in both images
+            float flux = (image1 && image2) ? PS_MIN(image1->data.F32[y][x], image2->data.F32[y][x]) :
+                ((image1) ? image1->data.F32[y][x] : image2->data.F32[y][x]); // Flux at pixel
+            if (flux > *fluxStamp) {
+                *xStamp = x;
+                *yStamp = y;
+                *fluxStamp = flux;
+                found = true;
+            }
+        }
+    }
+
+    return found;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Functions for generating ds9 region files
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static bool ds9regions = false;         // Save ds9 region files?
+
+void pmSubtractionRegions(bool state)
+{
+    ds9regions = state;
+}
+
+FILE *pmSubtractionStampsFile(const pmSubtractionStampList *stamps, const char *filename,
+                              const char *description)
+{
+    if (!ds9regions || !stamps || !filename) {
+        return NULL;
+    }
+
+    psLogMsg("psModules.imcombine", PS_LOG_INFO, "Writing %s to ds9 region file: %s",
+             description, filename);
+
+    FILE *file = fopen(filename, "w");
+
+    // Outline the stamps
+    for (int i = 0; i < stamps->num; i++) {
+        psRegion *region = stamps->regions->data[i]; // Region of interest
+        float xCentre = 0.5 * (region->x0 + region->x1), yCentre = 0.5 * (region->y0 + region->y1);
+        fprintf(file, "image;box(%f,%f,%f,%f,0) # color=blue\nimage;text(%f,%f,{%d}) # color=blue\n",
+                xCentre, yCentre, region->x1 - region->x0, region->y1 - region->y0,
+                xCentre, yCentre, i);
+    }
+
+    return file;
+}
+
+void pmSubtractionStampPrint(FILE *ds9, float x, float y, float size, const char *color)
+{
+    if (!ds9regions || !ds9) {
+        return;
+    }
+    fprintf(ds9, "image;circle(%f,%f,%f)", x, y, size);
+    if (color && strlen(color) > 0) {
+        fprintf(ds9, " # color=%s", color);
+    }
+    fprintf(ds9, "\n");
+    return;
+}
+
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -225,16 +253,42 @@
 
 
-pmSubtractionStampList *pmSubtractionStampsFind(pmSubtractionStampList *stamps, const psImage *image,
-                                                const psImage *subMask, const psRegion *region,
-                                                float threshold, int footprint, float spacing,
+pmSubtractionStampList *pmSubtractionStampsFind(pmSubtractionStampList *stamps, const psImage *image1,
+                                                const psImage *image2, const psImage *subMask,
+                                                const psRegion *region, float thresh1, float thresh2,
+                                                int size, int footprint, float spacing,
                                                 pmSubtractionMode mode)
 {
-    PS_ASSERT_IMAGE_NON_NULL(image, NULL);
-    PS_ASSERT_IMAGE_TYPE(image, PS_TYPE_F32, NULL);
+    if (!image1 && !image2) {
+        psError(PS_ERR_UNEXPECTED_NULL, true, "Must specify an image");
+        return NULL;
+    }
+    int numCols = 0, numRows = 0;       // Size of images
+    if (image1) {
+        PS_ASSERT_IMAGE_NON_NULL(image1, NULL);
+        PS_ASSERT_IMAGE_TYPE(image1, PS_TYPE_F32, NULL);
+        if (subMask) {
+            PS_ASSERT_IMAGES_SIZE_EQUAL(image1, subMask, NULL);
+        }
+        numCols = image1->numCols;
+        numRows = image1->numRows;
+    }
+    if (image2) {
+        PS_ASSERT_IMAGE_NON_NULL(image2, NULL);
+        PS_ASSERT_IMAGE_TYPE(image2, PS_TYPE_F32, NULL);
+        if (subMask) {
+            PS_ASSERT_IMAGES_SIZE_EQUAL(image2, subMask, NULL);
+        }
+        numCols = image2->numCols;
+        numRows = image2->numRows;
+    }
+    if (image1 && image2) {
+        PS_ASSERT_IMAGES_SIZE_EQUAL(image1, image2, NULL);
+    }
     if (subMask) {
         PS_ASSERT_IMAGE_NON_NULL(subMask, NULL);
-        PS_ASSERT_IMAGES_SIZE_EQUAL(image, subMask, NULL);
         PS_ASSERT_IMAGE_TYPE(subMask, PS_TYPE_IMAGE_MASK, NULL);
-    }
+        PS_ASSERT_IMAGE_SIZE(subMask, numCols, numRows, NULL);
+    }
+    PS_ASSERT_INT_POSITIVE(size, NULL);
     PS_ASSERT_INT_NONNEGATIVE(footprint, NULL);
     PS_ASSERT_FLOAT_LARGER_THAN(spacing, 0.0, NULL);
@@ -246,9 +300,9 @@
             return false;
         }
-        if (region->x0 < 0 || region->x1 > image->numCols ||
-            region->y0 < 0 || region->y1 > image->numRows) {
+        if (region->x0 < 0 || region->x1 > numCols ||
+            region->y0 < 0 || region->y1 > numRows) {
             psString string = psRegionToString(*region);
             psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Input region (%s) does not fit in image (%dx%d)",
-                    string, image->numCols, image->numRows);
+                    string, numCols, numRows);
             psFree(string);
             return false;
@@ -256,5 +310,5 @@
     }
 
-    int numRows = image->numRows, numCols = image->numCols; // Size of image
+    int border = size + footprint;      // Border size
 
     if (!stamps) {
@@ -281,7 +335,7 @@
             numSearch++;
 
-            float xStamp = 0, yStamp = 0;   // Coordinates of stamp
-            float fluxStamp = NAN;          // Flux of stamp
-            bool goodStamp = false;         // Found a good stamp?
+            int xStamp = 0, yStamp = 0; // Coordinates of stamp
+            float fluxStamp = -INFINITY; // Flux of stamp
+            bool goodStamp = false;     // Found a good stamp?
 
             // A couple different ways of finding stamps:
@@ -291,36 +345,35 @@
                 psVector *fluxList = stamps->flux->data[i]; // List of stamp fluxes
 
-                // Take stamp off the top of the (sorted) list
-                if (xList->n > 0) {
-                    int index = xList->n - 1; // Index of new stamp
-                    xStamp = xList->data.F32[index];
-                    yStamp = yList->data.F32[index];
-                    fluxStamp = fluxList->data.F32[index];
+                // Take stamps off the top of the (sorted) list
+                for (int j = xList->n - 1; j >= 0 && !goodStamp; j--) {
+                    int xCentre = xList->data.F32[j] + 0.5, yCentre = yList->data.F32[j] + 0.5;// Stamp centre
 
                     // Chop off the top of the list
-                    xList->n = index;
-                    yList->n = index;
-                    fluxList->n = index;
-
-                    goodStamp = (fluxStamp > threshold) ? true : false;
-                } else {
-                    psTrace("psModules.imcombine", 9, "No sources in subregion %d", i);
+                    xList->n = j;
+                    yList->n = j;
+                    fluxList->n = j;
+
+                    // Fish around a bit to see if we can find a pixel that isn't masked
+                    psTrace("psModules.imcombine", 7, "Searching for stamp %d around %d,%d\n",
+                            i, xCentre, yCentre);
+
+                    // Search bounds
+                    int search = footprint - size; // Search radius
+                    int xMin = PS_MAX(border, xCentre - search);
+                    int xMax = PS_MIN(numCols - border -1, xCentre + search);
+                    int yMin = PS_MAX(border, yCentre - search);
+                    int yMax = PS_MIN(numRows - border - 1, yCentre + search);
+
+                    goodStamp = stampSearch(&xStamp, &yStamp, &fluxStamp, image1, image2, thresh1, thresh2,
+                                            subMask, xMin, xMax, yMin, yMax, numCols, numRows, border);
                 }
             } else {
                 // Use a simple method of automatically finding stamps --- take the highest pixel in the
                 // subregion
-                fluxStamp = threshold;
                 psRegion *subRegion = stamps->regions->data[i]; // Sub-region of interest
-                for (int y = subRegion->y0; y <= subRegion->y1; y++) {
-                    for (int x = subRegion->x0; x <= subRegion->x1; x++) {
-                        if (checkStampMask(x, y, subMask, mode, footprint) &&
-                            image->data.F32[y][x] > fluxStamp) {
-                            fluxStamp = image->data.F32[y][x];
-                            xStamp = x;
-                            yStamp = y;
-                            goodStamp = true;
-                        }
-                    }
-                }
+
+                goodStamp = stampSearch(&xStamp, &yStamp, &fluxStamp, image1, image2, thresh1, thresh2,
+                                        subMask, subRegion->x0, subRegion->x1, subRegion->y0, subRegion->y1,
+                                        numCols, numRows, border);
             }
 
@@ -355,4 +408,5 @@
     if (numGood == 0 && numFound == 0) {
         // No good stamps
+        psError(PM_ERR_STAMPS, true, "Unable to find suitable stamps");
         psFree(stamps);
         return NULL;
@@ -363,8 +417,9 @@
 
 
+
 pmSubtractionStampList *pmSubtractionStampsSet(const psVector *x, const psVector *y,
                                                const psImage *image, const psImage *subMask,
-                                               const psRegion *region, int footprint, float spacing,
-                                               pmSubtractionMode mode)
+                                               const psRegion *region, int size, int footprint,
+                                               float spacing, pmSubtractionMode mode)
 
 {
@@ -378,9 +433,8 @@
         PS_ASSERT_IMAGE_NON_NULL(subMask, NULL);
         PS_ASSERT_IMAGE_TYPE(subMask, PS_TYPE_IMAGE_MASK, NULL);
-        if (image) {
-            PS_ASSERT_IMAGE_NON_NULL(image, NULL);
-            PS_ASSERT_IMAGES_SIZE_EQUAL(image, subMask, NULL);
-        }
-    }
+        PS_ASSERT_IMAGES_SIZE_EQUAL(image, subMask, NULL);
+    }
+    PS_ASSERT_INT_POSITIVE(size, NULL);
+    PS_ASSERT_INT_POSITIVE(footprint, NULL);
     PS_ASSERT_FLOAT_LARGER_THAN(spacing, 0.0, NULL);
 
@@ -389,4 +443,10 @@
                                                                  region, footprint, spacing); // Stamp list
     int numStamps = stamps->num;        // Number of stamps
+
+    psString ds9name = NULL;            // Filename for ds9 region file
+    static int ds9num = 0;              // File number for ds9 region file
+    psStringAppend(&ds9name, "stamps_all_%d.ds9", ds9num);
+    FILE *ds9 = pmSubtractionStampsFile(stamps, ds9name, "all stamps"); // ds9 region file
+    ds9num++;
 
     // Initialise the lists
@@ -408,10 +468,5 @@
             psTrace("psModules.imcombine", 9, "Rejecting input stamp (%d,%d) because outside region",
                     xPix, yPix);
-            continue;
-        }
-        if (!checkStampMask(xPix, yPix, subMask, mode, footprint)) {
-            // Not a good stamp
-            psTrace("psModules.imcombine", 9, "Rejecting input stamp (%d,%d) because bad mask",
-                    xPix, yPix);
+            pmSubtractionStampPrint(ds9, xPix, yPix, footprint, "red");
             continue;
         }
@@ -437,4 +492,5 @@
                 psTrace("psModules.imcombine", 9, "Putting input stamp (%d,%d) into subregion %d",
                         xPix, yPix, j);
+                pmSubtractionStampPrint(ds9, xPix, yPix, footprint, "green");
             }
         }
@@ -443,5 +499,10 @@
             psTrace("psModules.imcombine", 9, "Unable to find subregion for stamp (%d,%d)",
                     xPix, yPix);
-        }
+            pmSubtractionStampPrint(ds9, xPix, yPix, footprint, "yellow");
+        }
+    }
+
+    if (ds9) {
+        fclose(ds9);
     }
 
@@ -474,5 +535,4 @@
     }
 
-
     return stamps;
 }
@@ -544,53 +604,7 @@
 }
 
-#if 0
-bool pmSubtractionStampsGenerate(pmSubtractionStampList *stamps, float fwhm, int kernelSize)
-{
-    PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false);
-    PS_ASSERT_FLOAT_LARGER_THAN(fwhm, 0.0, false);
-    PS_ASSERT_INT_NONNEGATIVE(kernelSize, false);
-
-    int size = kernelSize + stamps->footprint; // Size of postage stamps
-    int num = stamps->num;              // Number of stamps
-    float sigma = fwhm / (2.0 * sqrtf(2.0 * log(2.0))); // Gaussian sigma
-
-    for (int i = 0; i < num; i++) {
-        pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest
-        if (!(stamp->status & PM_SUBTRACTION_STAMP_CALCULATE)) {
-            continue;
-        }
-
-        float x = stamp->x, y = stamp->y; // Coordinates of stamp
-        float flux = stamp->flux; // Flux of star
-        if (!isfinite(flux)) {
-            psWarning("Unable to generate PSF for stamp %d --- bad flux.", i);
-            stamp->status = PM_SUBTRACTION_STAMP_REJECTED;
-            continue;
-        }
-
-        float xStamp = x - (int)(x + 0.5); // x coordinate of star in stamp frame
-        float yStamp = y - (int)(y + 0.5); // y coordinate of star in stamp frame
-
-        psFree(stamp->image2);
-        stamp->image2 = psKernelAlloc(-size, size, -size, size);
-        psKernel *target = stamp->image2; // Target stamp
-
-        // Put in a Waussian, just for fun!
-        for (int v = -size; v <= size; v++) {
-            for (int u = -size; u <= size; u++) {
-                float z = (PS_SQR(u + xStamp) + PS_SQR(v + yStamp)) / (2.0 * PS_SQR(sigma));
-                target->kernel[v][u] = flux / sigma * 0.5 * M_2_SQRTPI * M_SQRT1_2 / (1.0 + z + PS_SQR(z));
-            }
-        }
-
-    }
-
-    return true;
-}
-#endif
-
 pmSubtractionStampList *pmSubtractionStampsSetFromSources(const psArray *sources, const psImage *image,
                                                           const psImage *subMask, const psRegion *region,
-                                                          int footprint, float spacing,
+                                                          int size, int footprint, float spacing,
                                                           pmSubtractionMode mode)
 {
@@ -621,5 +635,5 @@
     y->n = numOut;
 
-    pmSubtractionStampList *stamps = pmSubtractionStampsSet(x, y, image, subMask, region,
+    pmSubtractionStampList *stamps = pmSubtractionStampsSet(x, y, image, subMask, region, size,
                                                             footprint, spacing, mode); // Stamps to return
     psFree(x);
@@ -636,5 +650,6 @@
 pmSubtractionStampList *pmSubtractionStampsSetFromFile(const char *filename, const psImage *image,
                                                        const psImage *subMask, const psRegion *region,
-                                                       int footprint, float spacing, pmSubtractionMode mode)
+                                                       int size, int footprint, float spacing,
+                                                       pmSubtractionMode mode)
 {
     PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
@@ -652,5 +667,5 @@
     psBinaryOp(y, y, "-", psScalarAlloc(1.0, PS_TYPE_F32));
 
-    pmSubtractionStampList *stamps = pmSubtractionStampsSet(x, y, image, subMask, region, footprint,
+    pmSubtractionStampList *stamps = pmSubtractionStampsSet(x, y, image, subMask, region, size, footprint,
                                                             spacing, mode);
     psFree(data);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionStamps.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionStamps.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionStamps.h	(revision 24244)
@@ -74,8 +74,11 @@
 /// Find stamps on an image
 pmSubtractionStampList *pmSubtractionStampsFind(pmSubtractionStampList *stamps, ///< Output stamps, or NULL
-                                                const psImage *image, ///< Image for which to find stamps
+                                                const psImage *image1, ///< Image for which to find stamps
+                                                const psImage *image2, ///< Image for which to find stamps
                                                 const psImage *mask, ///< Mask, or NULL
                                                 const psRegion *region, ///< Region to search, or NULL
-                                                float threshold, ///< Threshold for stamps in the image
+                                                float thresh1, ///< Threshold for stamps in image 1
+                                                float thresh2, ///< Threshold for stamps in image 2
+                                                int size, ///< Kernel half-size
                                                 int footprint, ///< Half-size for stamps
                                                 float spacing, ///< Rough spacing for stamps
@@ -89,4 +92,5 @@
                                                const psImage *mask, ///< Mask, or NULL
                                                const psRegion *region, ///< Region to search, or NULL
+                                               int size, ///< Kernel half-size
                                                int footprint, ///< Half-size for stamps
                                                float spacing, ///< Rough spacing for stamps
@@ -100,4 +104,5 @@
     const psImage *subMask,             ///< Mask, or NULL
     const psRegion *region,             ///< Region to search, or NULL
+    int size,                           ///< Kernel half-size
     int footprint,                      ///< Half-size for stamps
     float spacing,                      ///< Rough spacing for stamps
@@ -111,4 +116,5 @@
     const psImage *subMask,             ///< Mask, or NULL
     const psRegion *region,             ///< Region to search, or NULL
+    int size,                           ///< Kernel half-size
     int footprint,                      ///< Half-size for stamps
     float spacing,                      ///< Rough spacing for stamps
@@ -124,3 +130,27 @@
     );
 
+
+/// Turn on/off generation of ds9 region files
+///
+/// Intended for debugging
+void pmSubtractionRegions(bool state    ///< Generate ds9 region files?
+    );
+
+/// Open a file for ds9 regions
+///
+/// Intended for debugging
+FILE *pmSubtractionStampsFile(const pmSubtractionStampList *stamps, ///< List of stamps, for outlines
+                              const char *filename, ///< Filename to write
+                              const char *description ///< Description of file
+    );
+
+/// Print a stamp position to ds9 region file
+///
+/// Intended for debugging
+void pmSubtractionStampPrint(FILE *ds9, ///< ds9 region file
+                             float x, float y, ///< Position of stamp
+                             float size,///< Size of circle
+                             const char *color ///< Colour
+    );
+
 #endif
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/Makefile.am	(revision 24244)
@@ -38,4 +38,5 @@
 	pmSourceIO_PS1_CAL_0.c \
 	pmSourceIO_CMF_PS1_V1.c \
+	pmSourceIO_MatchedRefs.c \
 	pmSourcePlots.c \
 	pmSourcePlotPSFModel.c \
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/models/pmModel_PS1_V1.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/models/pmModel_PS1_V1.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/models/pmModel_PS1_V1.c	(revision 24244)
@@ -159,5 +159,5 @@
             break;
         case PM_PAR_7:
-            params_min =   0.1;
+            params_min =  -1.0;
             break;
         default:
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/models/pmModel_SGAUSS.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/models/pmModel_SGAUSS.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/models/pmModel_SGAUSS.c	(revision 24244)
@@ -325,5 +325,8 @@
 
     psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN);
-    psVectorStats (stats, contour, NULL, NULL, 0);
+    if (!psVectorStats (stats, contour, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
     value = stats->sampleMedian;
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmGrowthCurveGenerate.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmGrowthCurveGenerate.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmGrowthCurveGenerate.c	(revision 24244)
@@ -89,5 +89,8 @@
 	psVectorAppend (values, growth->fitMag);
     }
-    psVectorStats (stats, values, NULL, NULL, 0);
+    if (!psVectorStats (stats, values, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
     psf->growth->fitMag = stats->sampleMedian;
 
@@ -104,5 +107,8 @@
 	    psVectorAppend (values, growth->apMag->data.F32[i]);
 	}
-	psVectorStats (stats, values, NULL, NULL, 0);
+	if (!psVectorStats (stats, values, NULL, NULL, 0)) {
+	    psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	    return false;
+	}
 	psf->growth->apMag->data.F32[i] = stats->sampleMedian;
     }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSF.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSF.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSF.c	(revision 24244)
@@ -75,4 +75,6 @@
     options->poissonErrorsParams  = true;
 
+    options->chiFluxTrend = true;
+
     return options;
 }
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSF.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSF.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSF.h	(revision 24244)
@@ -82,4 +82,5 @@
     bool          poissonErrorsParams; ///< use poission errors for model parameter fitting
     float         radius;
+    bool          chiFluxTrend;         // Fit a trend in Chi2 as a function of flux?
 } pmPSFOptions;
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSFtry.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSFtry.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSFtry.c	(revision 24244)
@@ -43,12 +43,12 @@
 
     for (int j = 0; j < trend->map->map->numRows; j++) {
-	for (int i = 0; i < trend->map->map->numCols; i++) {
-	    fprintf (stderr, "%5.2f  ", trend->map->map->data.F32[j][i]);
-	}
-	fprintf (stderr, "\t\t\t");
-	for (int i = 0; i < trend->map->map->numCols; i++) {
-	    fprintf (stderr, "%5.2f  ", trend->map->error->data.F32[j][i]);
-	}
-	fprintf (stderr, "\n");
+        for (int i = 0; i < trend->map->map->numCols; i++) {
+            fprintf (stderr, "%5.2f  ", trend->map->map->data.F32[j][i]);
+        }
+        fprintf (stderr, "\t\t\t");
+        for (int i = 0; i < trend->map->map->numCols; i++) {
+            fprintf (stderr, "%5.2f  ", trend->map->error->data.F32[j][i]);
+        }
+        fprintf (stderr, "\n");
     }
     return true;
@@ -63,9 +63,9 @@
     float Wt = 0.0;
     for (int j = 0; j < map->map->numRows; j++) {
-	for (int i = 0; i < map->map->numCols; i++) {
-	    if (!isfinite(map->map->data.F32[j][i])) continue;
-	    Sum += map->map->data.F32[j][i] * map->error->data.F32[j][i];
-	    Wt += map->error->data.F32[j][i];
-	}
+        for (int i = 0; i < map->map->numCols; i++) {
+            if (!isfinite(map->map->data.F32[j][i])) continue;
+            Sum += map->map->data.F32[j][i] * map->error->data.F32[j][i];
+            Wt += map->error->data.F32[j][i];
+        }
     }
 
@@ -75,9 +75,9 @@
     // XXX for now, we are just replacing bad pixels with the Mean
     for (int j = 0; j < map->map->numRows; j++) {
-	for (int i = 0; i < map->map->numCols; i++) {
-	    if (isfinite(map->map->data.F32[j][i]) && 
-		(map->error->data.F32[j][i] > 0.0)) continue;
-	    map->map->data.F32[j][i] = Mean;
-	}
+        for (int i = 0; i < map->map->numCols; i++) {
+            if (isfinite(map->map->data.F32[j][i]) &&
+                (map->error->data.F32[j][i] > 0.0)) continue;
+            map->map->data.F32[j][i] = Mean;
+        }
     }
     return true;
@@ -174,12 +174,12 @@
 
         pmSource *source = psfTry->sources->data[i];
-	if (!source->moments) {
+        if (!source->moments) {
             psfTry->mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PSFTRY_MASK_EXT_FAIL;
-	    continue;
-	}
-	if (!source->moments->nPixels) {
+            continue;
+        }
+        if (!source->moments->nPixels) {
             psfTry->mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PSFTRY_MASK_EXT_FAIL;
-	    continue;
-	}
+            continue;
+        }
 
         source->modelEXT = pmSourceModelGuess (source, psfTry->psf->type);
@@ -211,5 +211,5 @@
 
     if (Next == 0) {
-        psError(PS_ERR_UNKNOWN, true, "No sources with good extended fits from which to determine PSF.");
+        psError(PS_ERR_UNKNOWN, false, "No sources with good extended fits from which to determine PSF.");
         psFree(psfTry);
         return NULL;
@@ -282,5 +282,5 @@
 
     if (Npsf == 0) {
-        psError(PS_ERR_UNKNOWN, true, "No sources with good PSF fits after model is built.");
+        psError(PS_ERR_UNKNOWN, false, "No sources with good PSF fits after model is built.");
         psFree(psfTry);
         return NULL;
@@ -311,19 +311,22 @@
 
     // linear clipped fit of chisq trend vs flux
-    bool result = psVectorClipFitPolynomial1D(psfTry->psf->ChiTrend, options->stats, mask, 0xff, chisq, NULL, flux);
-    psStatsOptions meanStat = psStatsMeanOption(options->stats->options); // Statistic for mean
-    psStatsOptions stdevStat = psStatsStdevOption(options->stats->options); // Statistic for stdev
-
-    psLogMsg ("pmPSFtry", 4, "chisq vs flux fit: %f +/- %f\n",
-              psStatsGetValue(stats, meanStat), psStatsGetValue(stats, stdevStat));
-
-    psFree(flux);
-    psFree(mask);
-    psFree(chisq);
-
-    if (!result) {
-        psError(PS_ERR_UNKNOWN, false, "Failed to fit psf->ChiTrend");
-        psFree(psfTry);
-        return NULL;
+    if (options->chiFluxTrend) {
+        bool result = psVectorClipFitPolynomial1D(psfTry->psf->ChiTrend, options->stats,
+                                                  mask, 0xff, chisq, NULL, flux);
+        psStatsOptions meanStat = psStatsMeanOption(options->stats->options); // Statistic for mean
+        psStatsOptions stdevStat = psStatsStdevOption(options->stats->options); // Statistic for stdev
+
+        psLogMsg ("pmPSFtry", 4, "chisq vs flux fit: %f +/- %f\n",
+                  psStatsGetValue(stats, meanStat), psStatsGetValue(stats, stdevStat));
+
+        psFree(flux);
+        psFree(mask);
+        psFree(chisq);
+
+        if (!result) {
+            psError(PS_ERR_UNKNOWN, false, "Failed to fit psf->ChiTrend");
+            psFree(psfTry);
+            return NULL;
+        }
     }
 
@@ -600,7 +603,7 @@
 
     for (int i = 0; i < sources->n; i++) {
-	// skip any masked sources (failed to fit one of the model steps or get a magnitude)
-	if (srcMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) continue;
-	
+        // skip any masked sources (failed to fit one of the model steps or get a magnitude)
+        if (srcMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) continue;
+
         pmSource *source = sources->data[i];
         assert (source->modelEXT); // all unmasked sources should have modelEXT
@@ -628,8 +631,8 @@
         for (int i = 1; i <= PS_MAX (psf->trendNx, psf->trendNy); i++) {
 
-	    // copy srcMask to mask (we do not want the mask values set in pmPSFFitShapeParamsMap to be sticky)
-	    for (int i = 0; i < mask->n; i++) {
-		mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = srcMask->data.PS_TYPE_VECTOR_MASK_DATA[i];
-	    }
+            // copy srcMask to mask (we do not want the mask values set in pmPSFFitShapeParamsMap to be sticky)
+            for (int i = 0; i < mask->n; i++) {
+                mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = srcMask->data.PS_TYPE_VECTOR_MASK_DATA[i];
+            }
             if (!pmPSFFitShapeParamsMap (psf, i, &scatterTotal, mask, x, y, mag, e0, e1, e2, dz)) {
                 break;
@@ -643,12 +646,12 @@
         }
         if (entryMin == -1) {
-            psError (PS_ERR_UNKNOWN, true, "failed to find image map for shape params");
+            psError (PS_ERR_UNKNOWN, false, "failed to find image map for shape params");
             return false;
         }
 
-	// copy srcMask to mask (we do not want the mask values set in pmPSFFitShapeParamsMap to be sticky)
-	for (int i = 0; i < mask->n; i++) {
-	    mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = srcMask->data.PS_TYPE_VECTOR_MASK_DATA[i];
-	}
+        // copy srcMask to mask (we do not want the mask values set in pmPSFFitShapeParamsMap to be sticky)
+        for (int i = 0; i < mask->n; i++) {
+            mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = srcMask->data.PS_TYPE_VECTOR_MASK_DATA[i];
+        }
         if (!pmPSFFitShapeParamsMap (psf, entryMin, &scatterTotal, mask, x, y, mag, e0, e1, e2, dz)) {
             psAbort ("failed pmPSFFitShapeParamsMap on second pass?");
@@ -659,8 +662,8 @@
         psf->trendNy = trend->map->map->numRows;
 
-	// copy mask back to srcMask
-	for (int i = 0; i < mask->n; i++) {
-	    srcMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = mask->data.PS_TYPE_VECTOR_MASK_DATA[i];
-	}
+        // copy mask back to srcMask
+        for (int i = 0; i < mask->n; i++) {
+            srcMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = mask->data.PS_TYPE_VECTOR_MASK_DATA[i];
+        }
 
         psFree (mask);
@@ -686,33 +689,33 @@
         for (int i = 0; i < psf->psfTrendStats->clipIter; i++) {
 
-	    psStatsOptions meanOption = psStatsMeanOption(psf->psfTrendStats->options);
-	    psStatsOptions stdevOption = psStatsStdevOption(psf->psfTrendStats->options);
-
-	    pmTrend2D *trend = NULL;
-	    float mean, stdev;
+            psStatsOptions meanOption = psStatsMeanOption(psf->psfTrendStats->options);
+            psStatsOptions stdevOption = psStatsStdevOption(psf->psfTrendStats->options);
+
+            pmTrend2D *trend = NULL;
+            float mean, stdev;
 
             // XXX we are using the same stats structure on each pass: do we need to re-init it?
             bool status = true;
 
-	    trend = psf->params->data[PM_PAR_E0];
+            trend = psf->params->data[PM_PAR_E0];
             status &= pmTrend2DFit (trend, srcMask, 0xff, x, y, e0, NULL);
-	    mean = psStatsGetValue (trend->stats, meanOption);
-	    stdev = psStatsGetValue (trend->stats, stdevOption);
+            mean = psStatsGetValue (trend->stats, meanOption);
+            stdev = psStatsGetValue (trend->stats, stdevOption);
             psTrace ("psModules.objects", 4, "clipped E0 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e0->n);
-	    pmSourceVisualPSFModelResid (trend, x, y, e0, srcMask);
-
-	    trend = psf->params->data[PM_PAR_E1];
+            pmSourceVisualPSFModelResid (trend, x, y, e0, srcMask);
+
+            trend = psf->params->data[PM_PAR_E1];
             status &= pmTrend2DFit (trend, srcMask, 0xff, x, y, e1, NULL);
-	    mean = psStatsGetValue (trend->stats, meanOption);
-	    stdev = psStatsGetValue (trend->stats, stdevOption);
+            mean = psStatsGetValue (trend->stats, meanOption);
+            stdev = psStatsGetValue (trend->stats, stdevOption);
             psTrace ("psModules.objects", 4, "clipped E1 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e1->n);
-	    pmSourceVisualPSFModelResid (trend, x, y, e1, srcMask);
-
-	    trend = psf->params->data[PM_PAR_E2];
+            pmSourceVisualPSFModelResid (trend, x, y, e1, srcMask);
+
+            trend = psf->params->data[PM_PAR_E2];
             status &= pmTrend2DFit (trend, srcMask, 0xff, x, y, e2, NULL);
-	    mean = psStatsGetValue (trend->stats, meanOption);
-	    stdev = psStatsGetValue (trend->stats, stdevOption);
+            mean = psStatsGetValue (trend->stats, meanOption);
+            stdev = psStatsGetValue (trend->stats, stdevOption);
             psTrace ("psModules.objects", 4, "clipped E2 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e2->n);
-	    pmSourceVisualPSFModelResid (trend, x, y, e2, srcMask);
+            pmSourceVisualPSFModelResid (trend, x, y, e2, srcMask);
 
             if (!status) {
@@ -755,11 +758,11 @@
     if (psf->fieldNx > psf->fieldNy) {
         Nx = scale;
-	float AR = psf->fieldNy / (float) psf->fieldNx;
-	Ny = (int) (Nx * AR + 0.5);
+        float AR = psf->fieldNy / (float) psf->fieldNx;
+        Ny = (int) (Nx * AR + 0.5);
         Ny = PS_MAX (1, Ny);
     } else {
         Ny = scale;
-	float AR = psf->fieldNx / (float) psf->fieldNy;
-	Nx = (int) (Ny * AR + 0.5);
+        float AR = psf->fieldNx / (float) psf->fieldNy;
+        Nx = (int) (Ny * AR + 0.5);
         Nx = PS_MAX (1, Nx);
     }
@@ -809,41 +812,41 @@
 
     if (scale == 1) {
-	x_fit  = psMemIncrRefCounter (x);
-	y_fit  = psMemIncrRefCounter (y);
-	x_tst  = psMemIncrRefCounter (x);
-	y_tst  = psMemIncrRefCounter (y);
-	e0obs_fit = psMemIncrRefCounter (e0obs);
-	e1obs_fit = psMemIncrRefCounter (e1obs);
-	e2obs_fit = psMemIncrRefCounter (e2obs);
-	e0obs_tst = psMemIncrRefCounter (e0obs);
-	e1obs_tst = psMemIncrRefCounter (e1obs);
-	e2obs_tst = psMemIncrRefCounter (e2obs);
+        x_fit  = psMemIncrRefCounter (x);
+        y_fit  = psMemIncrRefCounter (y);
+        x_tst  = psMemIncrRefCounter (x);
+        y_tst  = psMemIncrRefCounter (y);
+        e0obs_fit = psMemIncrRefCounter (e0obs);
+        e1obs_fit = psMemIncrRefCounter (e1obs);
+        e2obs_fit = psMemIncrRefCounter (e2obs);
+        e0obs_tst = psMemIncrRefCounter (e0obs);
+        e1obs_tst = psMemIncrRefCounter (e1obs);
+        e2obs_tst = psMemIncrRefCounter (e2obs);
     } else {
-	x_fit  = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	y_fit  = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	x_tst  = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	y_tst  = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	e0obs_fit = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	e1obs_fit = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	e2obs_fit = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	e0obs_tst = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	e1obs_tst = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	e2obs_tst = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
-	for (int i = 0; i < e0obs_fit->n; i++) {
-	    // e0obs->n ==  8 or 9:
-	    // i = 0, 1, 2, 3 : 2i+0 = 0, 2, 4, 6
-	    // i = 0, 1, 2, 3 : 2i+1 = 1, 3, 5, 7
-	    x_fit->data.F32[i] = x->data.F32[2*i+0];  
-	    x_tst->data.F32[i] = x->data.F32[2*i+1];  
-	    y_fit->data.F32[i] = y->data.F32[2*i+0];  
-	    y_tst->data.F32[i] = y->data.F32[2*i+1];  
-
-	    e0obs_fit->data.F32[i] = e0obs->data.F32[2*i+0];  
-	    e0obs_tst->data.F32[i] = e0obs->data.F32[2*i+1];  
-	    e1obs_fit->data.F32[i] = e1obs->data.F32[2*i+0];
-	    e1obs_tst->data.F32[i] = e1obs->data.F32[2*i+1];
-	    e2obs_fit->data.F32[i] = e2obs->data.F32[2*i+0];
-	    e2obs_tst->data.F32[i] = e2obs->data.F32[2*i+1];
-	}
+        x_fit  = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        y_fit  = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        x_tst  = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        y_tst  = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        e0obs_fit = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        e1obs_fit = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        e2obs_fit = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        e0obs_tst = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        e1obs_tst = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        e2obs_tst = psVectorAlloc (e0obs->n/2, PS_TYPE_F32);
+        for (int i = 0; i < e0obs_fit->n; i++) {
+            // e0obs->n ==  8 or 9:
+            // i = 0, 1, 2, 3 : 2i+0 = 0, 2, 4, 6
+            // i = 0, 1, 2, 3 : 2i+1 = 1, 3, 5, 7
+            x_fit->data.F32[i] = x->data.F32[2*i+0];
+            x_tst->data.F32[i] = x->data.F32[2*i+1];
+            y_fit->data.F32[i] = y->data.F32[2*i+0];
+            y_tst->data.F32[i] = y->data.F32[2*i+1];
+
+            e0obs_fit->data.F32[i] = e0obs->data.F32[2*i+0];
+            e0obs_tst->data.F32[i] = e0obs->data.F32[2*i+1];
+            e1obs_fit->data.F32[i] = e1obs->data.F32[2*i+0];
+            e1obs_tst->data.F32[i] = e1obs->data.F32[2*i+1];
+            e2obs_fit->data.F32[i] = e2obs->data.F32[2*i+0];
+            e2obs_tst->data.F32[i] = e2obs->data.F32[2*i+1];
+        }
     }
 
@@ -852,5 +855,5 @@
     // copy mask values to fitMask as a starting point
     for (int i = 0; i < fitMask->n; i++) {
-	fitMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = mask->data.PS_TYPE_VECTOR_MASK_DATA[i];
+        fitMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = mask->data.PS_TYPE_VECTOR_MASK_DATA[i];
     }
 
@@ -859,42 +862,42 @@
     for (int i = 0; i < nIter; i++) {
         // XXX we are using the same stats structure on each pass: do we need to re-init it?
-	psStatsOptions meanOption = psStatsMeanOption(psf->psfTrendStats->options);
-	psStatsOptions stdevOption = psStatsStdevOption(psf->psfTrendStats->options);
-
-	pmTrend2D *trend = NULL;
-	float mean, stdev;
-
-	// XXX we are using the same stats structure on each pass: do we need to re-init it?
-	bool status = true;
-
-	trend = psf->params->data[PM_PAR_E0];
-	status &= pmTrend2DFit (trend, fitMask, 0xff, x_fit, y_fit, e0obs_fit, NULL);
-	mean = psStatsGetValue (trend->stats, meanOption);
-	stdev = psStatsGetValue (trend->stats, stdevOption);
-	psTrace ("psModules.objects", 4, "clipped E0 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e0obs_fit->n);
-	// printTrendMap (trend);
-	psImageMapCleanup (trend->map);
-	// printTrendMap (trend);
-	pmSourceVisualPSFModelResid (trend, x, y, e0obs, mask);
-
-	trend = psf->params->data[PM_PAR_E1];
-	status &= pmTrend2DFit (trend, fitMask, 0xff, x_fit, y_fit, e1obs_fit, NULL);
-	mean = psStatsGetValue (trend->stats, meanOption);
-	stdev = psStatsGetValue (trend->stats, stdevOption);
-	psTrace ("psModules.objects", 4, "clipped E1 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e1obs_fit->n);
-	// printTrendMap (trend);
-	psImageMapCleanup (trend->map);
-	// printTrendMap (trend);
-	pmSourceVisualPSFModelResid (trend, x, y, e1obs, mask);
-
-	trend = psf->params->data[PM_PAR_E2];
-	status &= pmTrend2DFit (trend, fitMask, 0xff, x_fit, y_fit, e2obs_fit, NULL);
-	mean = psStatsGetValue (trend->stats, meanOption);
-	stdev = psStatsGetValue (trend->stats, stdevOption);
-	psTrace ("psModules.objects", 4, "clipped E2 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e2obs->n);
-	// printTrendMap (trend);
-	psImageMapCleanup (trend->map);
-	// printTrendMap (trend);
-	pmSourceVisualPSFModelResid (trend, x, y, e2obs, mask);
+        psStatsOptions meanOption = psStatsMeanOption(psf->psfTrendStats->options);
+        psStatsOptions stdevOption = psStatsStdevOption(psf->psfTrendStats->options);
+
+        pmTrend2D *trend = NULL;
+        float mean, stdev;
+
+        // XXX we are using the same stats structure on each pass: do we need to re-init it?
+        bool status = true;
+
+        trend = psf->params->data[PM_PAR_E0];
+        status &= pmTrend2DFit (trend, fitMask, 0xff, x_fit, y_fit, e0obs_fit, NULL);
+        mean = psStatsGetValue (trend->stats, meanOption);
+        stdev = psStatsGetValue (trend->stats, stdevOption);
+        psTrace ("psModules.objects", 4, "clipped E0 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e0obs_fit->n);
+        // printTrendMap (trend);
+        psImageMapCleanup (trend->map);
+        // printTrendMap (trend);
+        pmSourceVisualPSFModelResid (trend, x, y, e0obs, mask);
+
+        trend = psf->params->data[PM_PAR_E1];
+        status &= pmTrend2DFit (trend, fitMask, 0xff, x_fit, y_fit, e1obs_fit, NULL);
+        mean = psStatsGetValue (trend->stats, meanOption);
+        stdev = psStatsGetValue (trend->stats, stdevOption);
+        psTrace ("psModules.objects", 4, "clipped E1 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e1obs_fit->n);
+        // printTrendMap (trend);
+        psImageMapCleanup (trend->map);
+        // printTrendMap (trend);
+        pmSourceVisualPSFModelResid (trend, x, y, e1obs, mask);
+
+        trend = psf->params->data[PM_PAR_E2];
+        status &= pmTrend2DFit (trend, fitMask, 0xff, x_fit, y_fit, e2obs_fit, NULL);
+        mean = psStatsGetValue (trend->stats, meanOption);
+        stdev = psStatsGetValue (trend->stats, stdevOption);
+        psTrace ("psModules.objects", 4, "clipped E2 : %f +/- %f keeping %ld of %ld\n", mean, stdev, psf->psfTrendStats->clippedNvalues, e2obs->n);
+        // printTrendMap (trend);
+        psImageMapCleanup (trend->map);
+        // printTrendMap (trend);
+        pmSourceVisualPSFModelResid (trend, x, y, e2obs, mask);
     }
     psf->psfTrendStats->clipIter = nIter; // restore default setting
@@ -941,5 +944,5 @@
     // XXX copy fitMask values back to mask
     for (int i = 0; i < fitMask->n; i++) {
-	mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = fitMask->data.PS_TYPE_VECTOR_MASK_DATA[i];
+        mask->data.PS_TYPE_VECTOR_MASK_DATA[i] = fitMask->data.PS_TYPE_VECTOR_MASK_DATA[i];
     }
     psFree (fitMask);
@@ -958,13 +961,22 @@
     float dEsquare = 0.0;
     psStatsInit (stats);
-    psVectorStats (stats, e0res, NULL, mask, maskValue);
+    if (!psVectorStats (stats, e0res, NULL, mask, maskValue)) {
+        psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+        return false;
+    }
     dEsquare += PS_SQR(psStatsGetValue(stats, stdevOpt));
 
     psStatsInit (stats);
-    psVectorStats (stats, e1res, NULL, mask, maskValue);
+    if (!psVectorStats (stats, e1res, NULL, mask, maskValue)) {
+        psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+        return false;
+    }
     dEsquare += PS_SQR(psStatsGetValue(stats, stdevOpt));
 
     psStatsInit (stats);
-    psVectorStats (stats, e2res, NULL, mask, maskValue);
+    if (!psVectorStats (stats, e2res, NULL, mask, maskValue)) {
+        psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+        return false;
+    }
     dEsquare += PS_SQR(psStatsGetValue(stats, stdevOpt));
 
@@ -998,5 +1010,5 @@
     for (int i = 0; i < nBin; i++) {
         int j;
-	int nValid = 0;
+        int nValid = 0;
         for (j = 0; (j < nGroup) && (n < mag->n); j++, n++) {
             int N = index->data.U32[n];
@@ -1006,7 +1018,7 @@
 
             mkSubset->data.PS_TYPE_VECTOR_MASK_DATA[j]   = mask->data.PS_TYPE_VECTOR_MASK_DATA[N];
-	    if (!mask->data.PS_TYPE_VECTOR_MASK_DATA[N]) nValid ++;
-        }
-	if (nValid < 3) continue;
+            if (!mask->data.PS_TYPE_VECTOR_MASK_DATA[N]) nValid ++;
+        }
+        if (nValid < 3) continue;
 
         dE0subset->n = j;
@@ -1018,13 +1030,20 @@
         float dEsquare = 0.0;
         psStatsInit (statsS);
-        psVectorStats (statsS, dE0subset, NULL, mkSubset, 0xff);
+        if (!psVectorStats (statsS, dE0subset, NULL, mkSubset, 0xff)) {
+        }
         dEsquare += PS_SQR(psStatsGetValue(statsS, stdevOpt));
 
         psStatsInit (statsS);
-        psVectorStats (statsS, dE1subset, NULL, mkSubset, 0xff);
+        if (!psVectorStats (statsS, dE1subset, NULL, mkSubset, 0xff)) {
+            psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+            return false;
+        }
         dEsquare += PS_SQR(psStatsGetValue(statsS, stdevOpt));
 
         psStatsInit (statsS);
-        psVectorStats (statsS, dE2subset, NULL, mkSubset, 0xff);
+        if (!psVectorStats (statsS, dE2subset, NULL, mkSubset, 0xff)) {
+            psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+            return false;
+        }
         dEsquare += PS_SQR(psStatsGetValue(statsS, stdevOpt));
 
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSource.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSource.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSource.c	(revision 24244)
@@ -479,9 +479,15 @@
         stats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV);
 
-        psVectorStats (stats, tmpSx, NULL, NULL, 0);
+        if (!psVectorStats (stats, tmpSx, NULL, NULL, 0)) {
+	    psError(PS_ERR_UNKNOWN, false, "failed to measure Sx stats");
+            return (emptyClump);
+	}
         psfClump.X  = stats->clippedMean;
         psfClump.dX = stats->clippedStdev;
 
-        psVectorStats (stats, tmpSy, NULL, NULL, 0);
+        if (!psVectorStats (stats, tmpSy, NULL, NULL, 0)) {
+	    psError(PS_ERR_UNKNOWN, false, "failed to measure Sy stats");
+            return (emptyClump);
+	}
         psfClump.Y  = stats->clippedMean;
         psfClump.dY = stats->clippedStdev;
@@ -636,6 +642,8 @@
 
         if (!psVectorStats (stats, starsn_moments, NULL, NULL, 0)) {
-            // Don't care about this error
-            psErrorClear();
+	    psError(PS_ERR_UNKNOWN, false, "failed to measure SN / moments stats");
+	    psFree (stats);
+	    psFree (starsn_peaks);
+	    return false;
         }
         psLogMsg ("pmObjects", 3, "SN range (moments): %f - %f\n", stats->min, stats->max);
@@ -648,6 +656,8 @@
         stats = psStatsAlloc (PS_STAT_MIN | PS_STAT_MAX);
         if (!psVectorStats (stats, starsn_peaks, NULL, NULL, 0)) {
-            // Don't care about this error
-            psErrorClear();
+	    psError(PS_ERR_UNKNOWN, false, "failed to measure SN / moments stats");
+	    psFree (stats);
+	    psFree (starsn_peaks);
+	    return false;
         }
         psLogMsg ("psModules.objects", 3, "SN range (peaks)  : %f - %f (%ld)\n",
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceFitModel.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceFitModel.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceFitModel.c	(revision 24244)
@@ -172,10 +172,10 @@
     fitStatus = psMinimizeLMChi2(myMin, covar, params, constraint, x, y, yErr, model->modelFunc);
     for (int i = 0; i < dparams->n; i++) {
-        if (psTraceGetLevel("psModules.objects") >= 4) {
-            fprintf (stderr, "%f ", params->data.F32[i]);
-        }
         if ((constraint->paramMask != NULL) && constraint->paramMask->data.PS_TYPE_VECTOR_MASK_DATA[i])
             continue;
         dparams->data.F32[i] = sqrt(covar->data.F32[i][i]);
+        if (psTraceGetLevel("psModules.objects") >= 4) {
+            fprintf (stderr, "%f +/- %f\n", params->data.F32[i], dparams->data.F32[i]);
+        }
     }
     psTrace ("psModules.objects", 4, "niter: %d, chisq: %f", myMin->iter, myMin->value);
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceIO.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceIO.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceIO.c	(revision 24244)
@@ -562,6 +562,8 @@
 
     // not needed if only one chip
-    if (file->fpa->chips->n == 1) return true;
-
+    if (file->fpa->chips->n == 1) {
+	pmSourceWriteMatchedRefs (file->fits, file->fpa, config);
+	return true;
+    }
 
     // find the FPA phu
@@ -667,4 +669,5 @@
     psFree (outhead);
 
+    pmSourceWriteMatchedRefs (file->fits, file->fpa, config);
     return true;
 }
@@ -1053,2 +1056,3 @@
 }
 
+    
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceIO.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceIO.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceIO.h	(revision 24244)
@@ -75,4 +75,6 @@
 bool pmSourceLocalAstrometry (psSphere *ptSky, float *posAngle, float *pltScale, pmChip *chip, float xPos, float yPos);
 
+bool pmSourceWriteMatchedRefs (psFits *fits, pmFPA *fpa, pmConfig *config);
+
 /// @}
 # endif /* PM_SOURCE_IO_H */
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceIO_MatchedRefs.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceIO_MatchedRefs.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceIO_MatchedRefs.c	(revision 24244)
@@ -0,0 +1,136 @@
+/** @file  pmSourceIO_MatchedRefs.c
+ *
+ *  @author EAM, IfA
+ *
+ *  @version $Revision: 1.3 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-18 02:44:19 $
+ *
+ *  Copyright 2004 Maui High Performance Computing Center, University of Hawaii
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <math.h>
+#include <string.h>
+#include <pslib.h>
+
+#include "pmConfig.h"
+#include "pmDetrendDB.h"
+
+#include "pmHDU.h"
+#include "pmFPA.h"
+#include "pmFPALevel.h"
+#include "pmFPAview.h"
+#include "pmFPAfile.h"
+
+#include "pmSpan.h"
+#include "pmFootprint.h"
+#include "pmPeaks.h"
+#include "pmMoments.h"
+#include "pmGrowthCurve.h"
+#include "pmResiduals.h"
+#include "pmTrend2D.h"
+#include "pmPSF.h"
+#include "pmModel.h"
+#include "pmSource.h"
+#include "pmModelClass.h"
+#include "pmSourceIO.h"
+#include "pmAstrometryObjects.h"
+#include "pmAstrometryWCS.h"
+
+bool pmSourceWriteMatchedRefs (psFits *fits, pmFPA *fpa, pmConfig *config) {
+
+    bool status = true;
+    pmChip *chip = NULL;
+    pmCell *cell = NULL;
+    pmReadout *readout = NULL;
+    pmFPAview *view = pmFPAviewAlloc (0);
+
+    // first, check if there are any matches to be written
+
+    // determine the output table format
+    // XXX move this elsewhere? (psastro recipe? filerules?)
+    psMetadata *recipe = psMetadataLookupMetadata(&status, config->recipes, "PSASTRO");
+    if (!status) {
+	psError(PS_ERR_UNKNOWN, true, "missing recipe PSASTRO in config data");
+	return false;
+    }
+
+    bool REFS_OUTPUT = psMetadataLookupBool(&status, recipe, "PSASTRO.SAVE.REFMATCH");
+    if (!REFS_OUTPUT) return true;
+
+    psArray *table = psArrayAllocEmpty (0x1000);
+
+    // this loop selects the matched stars for all chips
+    while ((chip = pmFPAviewNextChip (view, fpa, 1)) != NULL) {
+        psTrace ("psastro", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
+        if (!chip->process || !chip->file_exists) continue;
+	
+	char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME");
+
+	while ((cell = pmFPAviewNextCell (view, fpa, 1)) != NULL) {
+            psTrace ("psastro", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
+            if (!cell->process || !cell->file_exists) continue;
+
+	    // process each of the readouts
+	    // XXX there can only be one readout per chip, right?
+	    while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) {
+		if (! readout->data_exists) continue;
+
+		// select the raw objects for this readout
+		psArray *rawstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.RAWSTARS");
+		if (rawstars == NULL) continue;
+
+		// select the raw objects for this readout
+		psArray *refstars = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.REFSTARS");
+		if (refstars == NULL) continue;
+		psTrace ("psastro", 4, "Trying %ld refstars\n", refstars->n);
+
+		psArray *matches = psMetadataLookupPtr (NULL, readout->analysis, "PSASTRO.MATCH");
+		if (matches == NULL) continue;
+
+		for (int i = 0; i < matches->n; i++) {
+		    pmAstromMatch *match = matches->data[i];
+
+		    pmAstromObj *raw = rawstars->data[match->raw];
+		    pmAstromObj *ref = refstars->data[match->ref];
+
+		    psMetadata *row = psMetadataAlloc ();
+		    psMetadataAdd (row, PS_LIST_TAIL, "RA",       PS_DATA_F64, "right ascension (deg, J2000)", PM_DEG_RAD*ref->sky->r);
+		    psMetadataAdd (row, PS_LIST_TAIL, "DEC",      PS_DATA_F64, "declination (deg, J2000)",     PM_DEG_RAD*ref->sky->d);
+		    psMetadataAdd (row, PS_LIST_TAIL, "X_CHIP",   PS_DATA_F32, "x coord on chip", 	       raw->chip->x);
+		    psMetadataAdd (row, PS_LIST_TAIL, "Y_CHIP",   PS_DATA_F32, "y coord on chip", 	       raw->chip->y);   
+		    psMetadataAdd (row, PS_LIST_TAIL, "X_FPA",    PS_DATA_F32, "x coord on focal plane",       raw->FP->x);
+		    psMetadataAdd (row, PS_LIST_TAIL, "Y_FPA",    PS_DATA_F32, "y coord on focal plane",       raw->FP->y);
+		    psMetadataAdd (row, PS_LIST_TAIL, "MAG_INST", PS_DATA_F32, "instrumental magnitude",       raw->Mag);
+		    psMetadataAdd (row, PS_LIST_TAIL, "MAG_REF",  PS_DATA_F32, "reference star magnitude",     ref->Mag);
+		    psMetadataAdd (row, PS_LIST_TAIL, "COLOR_REF",PS_DATA_F32, "reference star color",         ref->Color);
+		    psMetadataAdd (row, PS_LIST_TAIL, "CHIP_ID",  PS_DATA_STRING, "chip identifier",           chipName);
+		    // XXX need to add the reference color, but this needs getstar / dvo.photcodes for the reference to be refined.
+
+		    psArrayAdd (table, 100, row);
+		    psFree (row);
+		}
+	    }
+	}
+    }
+    psFree (view);
+
+    if (table->n == 0) {
+	psFree(table);
+	return true;
+    }
+
+    if (!psFitsWriteTable(fits, NULL, table, "MATCHED_REFS")) {
+        psError(PS_ERR_IO, false, "writing MATCHED_REFS\n");
+        psFree(table);
+        return false;
+    }
+
+    psFree(table);
+    return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceMatch.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceMatch.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceMatch.c	(revision 24244)
@@ -473,4 +473,10 @@
         return -1;
     }
+    // XXX handle this case better:
+    if (isnan(stats->clippedMean))  {
+        psError(PS_ERR_UNKNOWN, false, "Unable to perform statistics on transparencies.");
+        psFree(stats);
+        return -1;
+    }
 
     float thresh = stats->clippedMean + photoLevel * stats->clippedStdev; // Threshold for clouds
Index: /branches/cnb_branches/cnb_branch_20090301/psModules/src/psmodules.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psModules/src/psmodules.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psModules/src/psmodules.h	(revision 24244)
@@ -28,4 +28,5 @@
 #include <pmConfigMask.h>
 #include <pmConfigDump.h>
+#include <pmConfigRun.h>
 #include <pmVersion.h>
 
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/configure.ac	(revision 24244)
@@ -193,4 +193,6 @@
 echo "PSASTRO_LIBS: $PSASTRO_LIBS"
 
+IPP_VERSION
+
 AC_SUBST([PSASTRO_CFLAGS])
 AC_SUBST([PSASTRO_LIBS])
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/Makefile.am	(revision 24244)
@@ -1,12 +1,27 @@
 lib_LTLIBRARIES = libpsastro.la
 
-libpsastro_la_CFLAGS = $(PSASTRO_CFLAGS) $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPSASTRO_VERSION=$(SVN_VERSION) -DPSASTRO_BRANCH=$(SVN_BRANCH) -DPSASTRO_SOURCE=$(SVN_SOURCE)
-libpsastro_la_LDFLAGS = $(PSASTRO_LIBS) $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
+if HAVE_SVNVERSION
+PSASTRO_VERSION=`$(SVNVERSION) ..`
+else
+PSASTRO_VERSION="UNKNOWN"
+endif
+
+if HAVE_SVN
+PSASTRO_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PSASTRO_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PSASTRO_BRANCH="UNKNOWN"
+PSASTRO_SOURCE="UNKNOWN"
+endif
 
 # Force recompilation of psastroVersion.c, since it gets the version information
-# can we do this with dependency info?
-# psastroVersion.c: FORCE
-# touch psastroVersion.c
-# FORCE: ;
+psastroVersion.c: psastroVersionDefinitions.h
+psastroVersionDefinitions.h: psastroVersionDefinitions.h.in FORCE
+	-$(RM) psastroVersionDefinitions.h
+	$(SED) -e "s|@PSASTRO_VERSION@|\"$(PSASTRO_VERSION)\"|" -e "s|@PSASTRO_BRANCH@|\"$(PSASTRO_BRANCH)\"|" -e "s|@PSASTRO_SOURCE@|\"$(PSASTRO_SOURCE)\"|" psastroVersionDefinitions.h.in > psastroVersionDefinitions.h
+FORCE: ;
+
+libpsastro_la_CFLAGS = $(PSASTRO_CFLAGS) $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
+libpsastro_la_LDFLAGS = $(PSASTRO_LIBS) $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 
 bin_PROGRAMS = psastro psastroExtract psastroModel psastroModelFit gpcModel
@@ -50,5 +65,4 @@
 	psastroExtractStars.c      \
 	psastroExtractGhosts.c      \
-	psastroExtractFindChip.c      \
 	psastroCleanup.c
 
@@ -73,9 +87,12 @@
 
 libpsastro_la_SOURCES = \
+	psastroMaskUpdates.c        \
+	psastroMaskUtils.c          \
+	psastroLoadGhosts.c         \
+	psastroGhostUtils.c         \
 	psastroErrorCodes.c         \
 	psastroVersion.c            \
 	psastroDefineFiles.c        \
 	psastroAnalysis.c           \
-	psastroMaskUpdates.c        \
 	psastroAstromGuess.c        \
 	psastroLoadRefstars.c       \
@@ -102,4 +119,5 @@
 	psastroMosaicSetAstrom.c    \
 	psastroMosaicSetMatch.c     \
+	psastroFindChip.c           \
 	psastroZeroPoint.c    	    \
 	psastroDemoDump.c           \
@@ -122,5 +140,5 @@
 
 ### Error codes.
-BUILT_SOURCES = psastroErrorCodes.h psastroErrorCodes.c
+BUILT_SOURCES = psastroErrorCodes.h psastroErrorCodes.c psastroVersionDefinitions.h
 CLEANFILES = psastroErrorCodes.h psastroErrorCodes.c
 EXTRA_DIST = psastroErrorCodes.dat psastroErrorCodes.h.in psastroErrorCodes.c.in
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastro.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastro.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastro.c	(revision 24244)
@@ -13,13 +13,11 @@
 # include "psastroStandAlone.h"
 
-static void usage (void) {
-    fprintf (stderr, "USAGE: psastro [-file image(s)] [-list imagelist] (output)\n");
-    exit (2);
+static void usage(void) {
+    fprintf(stderr, "USAGE: psastro [-file image(s)] [-list imagelist] (output)\n");
+    exit(PS_EXIT_CONFIG_ERROR);
 }
 
-int main (int argc, char **argv) {
-
-    pmConfig *config = NULL;
-
+int main (int argc, char **argv)
+{
     psTimerStart ("complete");
 
@@ -28,9 +26,11 @@
     // model inits are needed in pmSourceIO
     // models defined in psphot/src/models are not available in psastro
-    pmModelClassInit ();
+    pmModelClassInit();
 
     // load configuration information
-    config = psastroArguments (argc, argv);
-    if (!config) usage ();
+    pmConfig *config = config = psastroArguments(argc, argv);
+    if (!config) {
+        usage();
+    }
 
     psastroVersionPrint();
@@ -39,5 +39,6 @@
     if (!psastroParseCamera (config)) {
         psErrorStackPrint(stderr, "error setting up the camera\n");
-        exit (1);
+        psFree(config);
+        exit(PS_EXIT_CONFIG_ERROR);
     }
 
@@ -46,22 +47,31 @@
     if (!psastroDataLoad (config)) {
         psErrorStackPrint(stderr, "error loading input data\n");
-        exit (1);
+        psFree(config);
+        exit(PS_EXIT_DATA_ERROR);
     }
 
+    psMetadata *stats = psMetadataAlloc(); // Statistics, for output
+    psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", 0, "No problems", 0);
+
     // run the full astrometry analysis (chip and/or mosaic)
-    if (!psastroAnalysis (config)) {
+    if (!psastroAnalysis(config, stats)) {
         psErrorStackPrint(stderr, "failure in psastro analysis\n");
-        exit (1);
+        psFree(config);
+        psFree(stats);
+        exit(PS_EXIT_SYS_ERROR);
     }
 
     // write out the results
-    if (!psastroDataSave (config)) {
+    if (!psastroDataSave(config, stats)) {
         psErrorStackPrint(stderr, "failed to write out data\n");
-        exit (1);
+        psFree(config);
+        psFree(stats);
+        exit(PS_EXIT_DATA_ERROR);
     }
 
-    psLogMsg ("psastro", 3, "complete psastro run: %f sec\n", psTimerMark ("complete"));
+    psFree (stats);
+    psLogMsg("psastro", 3, "complete psastro run: %f sec\n", psTimerMark ("complete"));
 
-    psastroCleanup (config);
-    exit (EXIT_SUCCESS);
+    psastroCleanup(config);
+    exit(PS_EXIT_SUCCESS);
 }
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastro.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastro.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastro.h	(revision 24244)
@@ -46,8 +46,23 @@
 #endif
 
-bool              psastroDataSave (pmConfig *config);
+/**
+ * this structure defines the parameters to describe a ghost
+ */
+typedef struct {
+    psPlane *srcFP;			///< location in FPA coords of the source star
+    psPlane *FP;			///< location in FPA coords of the ghost center
+    psPlane *chip;			///< location in chip coords of the ghost center
+    double Mag;				///< instrumental magnitude of source star
+    psEllipseAxes inner;		///< inner elliptical annulus boundary
+    psEllipseAxes outer;		///< outer elliptical annulus boundary
+} psastroGhost;
+
+psastroGhost     *psastroGhostAlloc (void);
+bool              psastroLoadGhosts (pmConfig *config);
+
+bool              psastroDataSave (pmConfig *config, psMetadata *stats);
 bool              psastroDefineFiles (pmConfig *config, pmFPAfile *input);
 bool              psastroDefineFile (pmConfig *config, pmFPA *input, char *filerule, char *argname, pmFPAfileType fileType, pmDetrendType detrendType);
-bool              psastroAnalysis (pmConfig *config);
+bool              psastroAnalysis (pmConfig *config, psMetadata *stats);
 
 bool              psastroConvertFPA (pmFPA *fpa, psMetadata *recipe);
@@ -125,5 +140,5 @@
 bool              psastroAstromGuessSetChip (pmFPA *fpa, pmChip *chip, const pmFPAview *view, double pixelScale, bool bilevelAstrometry);
 bool              psastroAstromGuessSetFPA (pmFPA *fpa, bool *bilevelAstrometry);
-bool              psastroMetadataStats (pmConfig *config);
+bool              psastroMetadataStats (pmConfig *config, psMetadata *stats);
 
 bool 		  psastroZeroPoint (pmConfig *config);
@@ -131,15 +146,31 @@
 bool 		  psastroZeroPointFromRecipe (float *zeropt, float *exptime, pmFPA *fpa, psMetadata *recipe);
 
+// masking functions
+pmCell           *pmCellInChip (pmChip *chip, float x, float y);
+bool 		  pmCellCoordsForChip (float *xCell, float *yCell, pmCell *cell, float xChip, float yChip);
+bool 		  pmChipCoordsForCell (float *xChip, float *yChip, pmCell *cell, float xCell, float yCell);
+		  
+bool 		  psastroMaskCircle (psImage *mask, psImageMaskType value, float x0, float y0, float dX, float dY);
+bool 		  psastroMaskEllipse (psImage *mask, psImageMaskType value, float x0, float y0, psEllipseAxes axes);
+bool 		  psastroMaskEllipticalAnnulus (psImage *mask, psImageMaskType value, float x0, float y0, psEllipseAxes eInner, psEllipseAxes eOuter);
+		  
+bool 		  psastroMaskBox (psImage *mask, psImageMaskType value, float x0, float y0, float dL, float dW, float theta);
+void 		  psastroMaskLine (psImage *mask, psImageMaskType value, double x1, double y1, double x2, double y2, int dW);
+void 		  psastroMaskLineBresen (psImage *mask, psImageMaskType value, int X1, int Y1, int X2, int Y2, int dW, int swapcoords);
+void 		  psastroMaskRectangle (psImage *mask, psImageMaskType value, int x0, int y0, int x1, int y1);
+
+pmChip           *psastroFindChip (double *xChip, double *yChip, pmFPA *fpa, double xFPA, double yFPA);
+bool 		  psastroChipBounds (pmFPA *fpa);
+
 // psastroExtract functions
 bool 		  psastroExtractAnalysis (pmConfig *config);
 bool 		  psastroExtractStars (pmConfig *config);
 bool 		  psastroExtractGhosts (pmConfig *config);
-pmChip           *psastroExtractFindChip (float *xChip, float *yChip, pmFPA *fpa, float xFPA, float yFPA);
-bool 		  psastroExtractChipBounds (pmFPA *fpa);
 
-psImage          *psastroExtractStar (psImage *input, float x, float y, float dX, float dY);
+psImage          *psastroExtractStar (psImage *input, double x, double y, double dX, double dY);
 bool 		  psastroExtractParseCamera (pmConfig *config);
 bool 		  psastroExtractDataLoad (pmConfig *config);
 pmConfig         *psastroExtractArguments (int argc, char **argv);
+bool              psastroExtractFreeChipBounds(void);
 
 ///@}
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroAnalysis.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroAnalysis.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroAnalysis.c	(revision 24244)
@@ -1,5 +1,5 @@
 /** @file psastroAnalysis.c
  *
- *  @brief 
+ *  @brief
  *
  *  @ingroup libpsastro
@@ -13,5 +13,21 @@
 # include "psastroInternal.h"
 
-bool psastroAnalysis (pmConfig *config) {
+/// Turn save on/off for a file
+static void fileSave(pmConfig *config,  // Configuration
+                     const char *name,  // Name of file
+                     bool save          // Save file?
+    )
+{
+    pmFPAfile *file = pmFPAfileSelectSingle(config->files, name, 0); // File of interest
+    if (!file) {
+        psErrorClear();
+        return;
+    }
+    file->save = save;
+    return;
+}
+
+
+bool psastroAnalysis (pmConfig *config, psMetadata *stats) {
 
     bool status;
@@ -40,6 +56,15 @@
     }
     if (nStars == 0) {
-        psLogMsg ("psastro", 2, "skipping astrometry analysis : no stars\n");
-        return false;
+        // This is likely a data quality issue
+        psWarning("No stars for astrometry analysis --- suspect bad data quality");
+        if (stats && psMetadataLookupS32(NULL, stats, "QUALITY") == 0) {
+            psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE,
+                             "No stars for astrometry", PSASTRO_ERR_DATA);
+        }
+        fileSave(config, "PSASTRO.OUTPUT", false);
+        fileSave(config, "PSASTRO.OUTPUT.MASK", false);
+        fileSave(config, "PSASTRO.OUT.REFSTARS", false);
+        psErrorClear();
+        return true;
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroAstromGuess.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroAstromGuess.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroAstromGuess.c	(revision 24244)
@@ -382,6 +382,12 @@
     psStats *statsQ = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV);
 
-    psVectorStats (statsP, cornerPd, NULL, cornerMK, 1);
-    psVectorStats (statsQ, cornerQd, NULL, cornerMK, 1);
+    if (!psVectorStats (statsP, cornerPd, NULL, cornerMK, 1)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
+    if (!psVectorStats (statsQ, cornerQd, NULL, cornerMK, 1)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
 
     float angle = atan2 (map->y->coeff[1][0], map->x->coeff[1][0]);
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroChipAstrom.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroChipAstrom.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroChipAstrom.c	(revision 24244)
@@ -130,4 +130,9 @@
     }
 
+    if (numGoodChips == 0) {
+        psError(PSASTRO_ERR_UNKNOWN, false, "Failed to fit any chips");
+        return false;
+    }
+
     if (!psastroFixChips (config, recipe)) {
         psError(PSASTRO_ERR_UNKNOWN, false, "failed to align problematic chips");
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroDataSave.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroDataSave.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroDataSave.c	(revision 24244)
@@ -22,5 +22,5 @@
  * this loop saves the photometry/astrometry data files
  */
-bool psastroDataSave (pmConfig *config) {
+bool psastroDataSave (pmConfig *config, psMetadata *stats) {
 
     pmChip *chip;
@@ -84,5 +84,14 @@
 
     // Write out summary statistics
-    if (!psastroMetadataStats (config)) ESCAPE;
+    if (!psastroMetadataStats (config, stats)) ESCAPE;
+
+    bool status;
+    psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
+    if (dump_file) {
+        pmConfigCamerasCull(config, NULL);
+        pmConfigRecipesCull(config, "PPIMAGE,PPSTATS,PSPHOT,MASKS,PSASTRO");
+
+        pmConfigDump(config, dump_file);
+    }
 
     // activate all files except PSASTRO.OUTPUT
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroErrorCodes.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroErrorCodes.dat	(revision 24244)
@@ -2,6 +2,6 @@
 # This file is used to generate pmErrorClasses.h
 #
-BASE = 300		First value we use; lower values belong to psLib
-UNKNOWN			Unknown PM error code
+BASE = 4000		First value we use; lower values belong to psLib
+UNKNOWN			Unknown psastro error code
 NOT_IMPLEMENTED		Desired feature is not yet implemented
 ARGUMENTS		Incorrect arguments
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroErrorCodes.h.in	(revision 24244)
@@ -21,5 +21,5 @@
  */
 typedef enum {
-    PSASTRO_ERR_BASE = 600,
+    PSASTRO_ERR_BASE = 4000,
     PSASTRO_ERR_${ErrorCode},
     PSASTRO_ERR_NERROR
Index: anches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractFindChip.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractFindChip.c	(revision 24243)
+++ 	(revision )
@@ -1,106 +1,0 @@
-/** @file psastroExtractFindChip.c
- *
- *  @brief calculate chip position for the given FPA coords
- *
- *  @ingroup psastroExtract
- *
- *  @author IfA
- *  @version $Revision: 1.7 $
- *  @date $Date: 2009-02-07 02:03:34 $
- *  Copyright 2009 Institute for Astronomy, University of Hawaii
- */
-
-# include "psastroInternal.h"
-
-static psVector *chipXmin = NULL;
-static psVector *chipXmax = NULL;
-static psVector *chipYmin = NULL;
-static psVector *chipYmax = NULL;
-
-bool psastroExtractChipBounds (pmFPA *fpa) {
-
-    chipXmin = psVectorAlloc (fpa->chips->n, PS_TYPE_F32);
-    chipXmax = psVectorAlloc (fpa->chips->n, PS_TYPE_F32);
-    chipYmin = psVectorAlloc (fpa->chips->n, PS_TYPE_F32);
-    chipYmax = psVectorAlloc (fpa->chips->n, PS_TYPE_F32);
-
-    // this loop selects the matched stars for all chips
-    for (int i = 0; i < fpa->chips->n; i++) {
-
-      pmChip *chip = fpa->chips->data[i];
-      if (!chip->process || !chip->file_exists) { continue; }
-      if (!chip->fromFPA) { continue; }
-
-      // determine RA,DEC of 4 corners, use to find RA_MIN,MAX, DEC_MIN,MAX
-      psRegion *region = pmChipPixels (chip);
-      psPlane ptCH[4], ptFP;
-
-      ptCH[0].x = region->x0;
-      ptCH[0].y = region->y0;
-      ptCH[1].x = region->x1;
-      ptCH[1].y = region->y0;
-      ptCH[2].x = region->x1;
-      ptCH[2].y = region->y1;
-      ptCH[3].x = region->x0;
-      ptCH[3].y = region->y1;
-      psFree (region);
-      
-      double Xmin = +FLT_MAX;
-      double Xmax = -FLT_MAX;
-      double Ymin = +FLT_MAX;
-      double Ymax = -FLT_MAX;
-
-      for (int j = 0; j < 4; j++) {
-	psPlaneTransformApply (&ptFP, chip->toFPA, &ptCH[j]);
-	Xmin = PS_MIN (ptFP.x, Xmin);
-	Xmax = PS_MAX (ptFP.x, Xmax);
-	Ymin = PS_MIN (ptFP.y, Ymin);
-	Ymax = PS_MAX (ptFP.y, Ymax);
-      }
-
-      // fpa-range for the given chip
-      chipXmin->data.F32[i] = Xmin;
-      chipXmax->data.F32[i] = Xmax;
-      chipYmin->data.F32[i] = Ymin;
-      chipYmax->data.F32[i] = Ymax;
-    }
-
-    return true;
-}
-
-pmChip *psastroExtractFindChip (float *xChip, float *yChip, pmFPA *fpa, float xFPA, float yFPA) {
-
-    *xChip = NAN;
-    *yChip = NAN;
-
-    if (!chipXmin) {
-	psastroExtractChipBounds (fpa);
-    }
-
-    for (int i = 0; i < fpa->chips->n; i++) {
-
-	if (xFPA <  chipXmin->data.F32[i]) continue;
-	if (xFPA >= chipXmax->data.F32[i]) continue;
-	if (yFPA <  chipYmin->data.F32[i]) continue;
-	if (yFPA >= chipYmax->data.F32[i]) continue;
-
-	pmChip *chip = fpa->chips->data[i];
-	psRegion *region = pmChipPixels (chip);
-
-	psPlane ptCH, ptFP;
-	ptFP.x = xFPA;
-	ptFP.y = yFPA;
-	psPlaneTransformApply (&ptCH, chip->fromFPA, &ptFP);
-
-	if (ptCH.x <  region->x0) continue;
-	if (ptCH.x >= region->x1) continue;
-	if (ptCH.y <  region->y0) continue;
-	if (ptCH.y >= region->y1) continue;
-
-	*xChip = ptCH.x;
-	*yChip = ptCH.y;
-	return chip;
-    }
-
-    return NULL;
-}
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractGhosts.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractGhosts.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractGhosts.c	(revision 24244)
@@ -89,9 +89,9 @@
 
 		    // XXX can make a more clever model...
-		    float xFPA = -1*ref->FP->x;
-		    float yFPA = -1*ref->FP->y;
+		    double xFPA = -1*ref->FP->x;
+		    double yFPA = -1*ref->FP->y;
 
-		    float xChip, yChip;
-		    pmChip *ghostChip = psastroExtractFindChip (&xChip, &yChip, fpa, xFPA, yFPA);
+		    double xChip, yChip;
+		    pmChip *ghostChip = psastroFindChip (&xChip, &yChip, fpa, xFPA, yFPA);
 		    if (!ghostChip) continue;
 		    if (!ghostChip->cells) continue;
@@ -107,5 +107,8 @@
 		    if (ghosts == NULL) { 
 			ghosts = psArrayAllocEmpty (100);
-			psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSASTRO.GHOSTS", PS_DATA_ARRAY, "astrometry matches", ghosts);
+			if (!psMetadataAdd (ghostReadout->analysis, PS_LIST_TAIL, "PSASTRO.GHOSTS", PS_DATA_ARRAY, "astrometry matches", ghosts)) {
+			  psError(PSASTRO_ERR_CONFIG, false, "failure to add ghosts to readout");
+			  return false;
+			}
 			psFree (ghosts);
 		    }
@@ -127,4 +130,6 @@
     }
 
+    psastroExtractFreeChipBounds();
+
     psFree (view);
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractStar.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractStar.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractStar.c	(revision 24244)
@@ -13,5 +13,5 @@
 # include "psastroInternal.h"
 
-psImage *psastroExtractStar (psImage *input, float x, float y, float dX, float dY) {
+psImage *psastroExtractStar (psImage *input, double x, double y, double dX, double dY) {
 
     // skip if no pixels are on the image
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractStars.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractStars.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroExtractStars.c	(revision 24244)
@@ -76,4 +76,5 @@
 
     int nExt = 0;
+    int nExtGhost = 0;
 
     // open the output file handle: we are just saving a series of extensions to this file
@@ -157,4 +158,5 @@
                     psMetadataAddF32 (header, PS_LIST_TAIL, "FPA_Y",   0, "focal-plane coordinate", ref->FP->y);
                     psMetadataAddF32 (header, PS_LIST_TAIL, "MAG",     0, "magnitude",              ref->Mag);
+                    psMetadataAddF32 (header, PS_LIST_TAIL, "INST_MAG",0, "instrumental magnitude", ref->Mag - MagOffset);
 
 		    snprintf (extname, 80, "extname.%05d", nExt);
@@ -175,5 +177,5 @@
 
 		    pmReadout *inReadout = pmFPAviewThisReadout (view, input->fpa);
-		    psImage *subraster = psastroExtractStar (inReadout->image, ghost->chip->x, ghost->chip->y, 200.0, 200.0);
+		    psImage *subraster = psastroExtractStar (inReadout->image, ghost->chip->x, ghost->chip->y, 400.0, 400.0);
 		    if (!subraster) continue;
 		    
@@ -187,8 +189,9 @@
                     psMetadataAddF32 (header, PS_LIST_TAIL, "FPA_Y0",  0, "star focal-plane coordinate", ghost->TP->y); // note : over-loaded value 
                     psMetadataAddF32 (header, PS_LIST_TAIL, "MAG",     0, "magnitude",              ghost->Mag);
-
-		    snprintf (extname, 80, "extname.%05d", nExt);
-		    psFitsWriteImage (outStars, header, subraster, 0, extname);
-		    nExt ++;
+                    psMetadataAddF32 (header, PS_LIST_TAIL, "INST_MAG",0, "instrumental magnitude", ghost->Mag - MagOffset);
+
+		    snprintf (extname, 80, "extname.%05d", nExtGhost);
+		    psFitsWriteImage (outGhosts, header, subraster, 0, extname);
+		    nExtGhost ++;
 		    
 		    psFree (header);
@@ -204,4 +207,5 @@
 
     psFitsClose (outStars);
+    psFitsClose (outGhosts);
     psFree (view);
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroFindChip.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroFindChip.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroFindChip.c	(revision 24244)
@@ -0,0 +1,119 @@
+/** @file psastroFindChip.c
+ *
+ *  @brief calculate chip position for the given FPA coords
+ *
+ *  @ingroup psastroExtract
+ *
+ *  @author IfA
+ *  @version $Revision: 1.7 $
+ *  @date $Date: 2009-02-07 02:03:34 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+# include "psastroInternal.h"
+
+static psVector *chipXmin = NULL;
+static psVector *chipXmax = NULL;
+static psVector *chipYmin = NULL;
+static psVector *chipYmax = NULL;
+
+bool psastroChipBounds (pmFPA *fpa) {
+
+    chipXmin = psVectorAlloc (fpa->chips->n, PS_TYPE_F32);
+    chipXmax = psVectorAlloc (fpa->chips->n, PS_TYPE_F32);
+    chipYmin = psVectorAlloc (fpa->chips->n, PS_TYPE_F32);
+    chipYmax = psVectorAlloc (fpa->chips->n, PS_TYPE_F32);
+
+    // this loop selects the matched stars for all chips
+    for (int i = 0; i < fpa->chips->n; i++) {
+
+      pmChip *chip = fpa->chips->data[i];
+      if (!chip->process || !chip->file_exists) { continue; }
+      if (!chip->fromFPA) { continue; }
+
+      // determine RA,DEC of 4 corners, use to find RA_MIN,MAX, DEC_MIN,MAX
+      psRegion *region = pmChipPixels (chip);
+      psPlane ptCH[4], ptFP;
+
+      ptCH[0].x = region->x0;
+      ptCH[0].y = region->y0;
+      ptCH[1].x = region->x1;
+      ptCH[1].y = region->y0;
+      ptCH[2].x = region->x1;
+      ptCH[2].y = region->y1;
+      ptCH[3].x = region->x0;
+      ptCH[3].y = region->y1;
+      psFree (region);
+      
+      double Xmin = +FLT_MAX;
+      double Xmax = -FLT_MAX;
+      double Ymin = +FLT_MAX;
+      double Ymax = -FLT_MAX;
+
+      for (int j = 0; j < 4; j++) {
+	psPlaneTransformApply (&ptFP, chip->toFPA, &ptCH[j]);
+	Xmin = PS_MIN (ptFP.x, Xmin);
+	Xmax = PS_MAX (ptFP.x, Xmax);
+	Ymin = PS_MIN (ptFP.y, Ymin);
+	Ymax = PS_MAX (ptFP.y, Ymax);
+      }
+
+      // fpa-range for the given chip
+      chipXmin->data.F32[i] = Xmin;
+      chipXmax->data.F32[i] = Xmax;
+      chipYmin->data.F32[i] = Ymin;
+      chipYmax->data.F32[i] = Ymax;
+    }
+
+    return true;
+}
+
+pmChip *psastroFindChip (double *xChip, double *yChip, pmFPA *fpa, double xFPA, double yFPA) {
+
+    *xChip = NAN;
+    *yChip = NAN;
+
+    if (!chipXmin) {
+	psastroChipBounds (fpa);
+    }
+
+    for (int i = 0; i < fpa->chips->n; i++) {
+
+	if (xFPA <  chipXmin->data.F32[i]) continue;
+	if (xFPA >= chipXmax->data.F32[i]) continue;
+	if (yFPA <  chipYmin->data.F32[i]) continue;
+	if (yFPA >= chipYmax->data.F32[i]) continue;
+
+	pmChip *chip = fpa->chips->data[i];
+	psRegion *region = pmChipPixels (chip);
+
+	psPlane ptCH, ptFP;
+	ptFP.x = xFPA;
+	ptFP.y = yFPA;
+	psPlaneTransformApply (&ptCH, chip->fromFPA, &ptFP);
+
+	if (ptCH.x <  region->x0) goto next_chip;
+	if (ptCH.x >= region->x1) goto next_chip;
+	if (ptCH.y <  region->y0) goto next_chip;
+	if (ptCH.y >= region->y1) goto next_chip;
+	psFree (region);
+
+	*xChip = ptCH.x;
+	*yChip = ptCH.y;
+	return chip;
+
+    next_chip:
+	psFree (region);
+    }
+
+    return NULL;
+}
+
+bool psastroExtractFreeChipBounds () {
+  
+  psFree (chipXmin);
+  psFree (chipXmax);
+  psFree (chipYmin);
+  psFree (chipYmax);
+  return true;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroFixChipsTest.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroFixChipsTest.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroFixChipsTest.c	(revision 24244)
@@ -140,5 +140,8 @@
     psStats *stats;
     stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV);
-    psVectorStats (stats, dX, NULL, NULL, 0);
+    if (!psVectorStats (stats, dX, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
     Xo = stats->robustMedian;
     fprintf (stderr, "offset x: %f +/- %f\n", stats->robustMedian, stats->robustStdev);
@@ -146,5 +149,8 @@
 
     stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV);
-    psVectorStats (stats, dY, NULL, NULL, 0);
+    if (!psVectorStats (stats, dY, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
     Yo = stats->robustMedian;
     fprintf (stderr, "offset y: %f +/- %f\n", stats->robustMedian, stats->robustStdev);
@@ -168,5 +174,8 @@
     }
     stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV);
-    psVectorStats (stats, dT, NULL, NULL, 0);
+    if (!psVectorStats (stats, dT, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
     float To = stats->robustMedian;
     fprintf (stderr, "offset t: %f +/- %f\n", stats->robustMedian, stats->robustStdev);
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroGhostUtils.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroGhostUtils.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroGhostUtils.c	(revision 24244)
@@ -0,0 +1,47 @@
+/** @file psastroGhostUtils.c
+ *
+ *  @brief Ghost support functions
+ *
+ *  @ingroup libpsastro
+ *
+ *  @author IfA
+ *  @version $Revision: 1.19 $
+ *  @date $Date: 2009-02-09 21:25:34 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+# include "psastroInternal.h"
+
+static void psastroGhostFree (psastroGhost *ghost) {
+
+    if (ghost == NULL) return;
+
+    psFree (ghost->srcFP);
+    psFree (ghost->FP);
+    psFree (ghost->chip);
+
+    return;
+}
+
+psastroGhost *psastroGhostAlloc (void) {
+
+    psastroGhost *ghost = (psastroGhost *) psAlloc(sizeof(psastroGhost));
+    psMemSetDeallocator(ghost, (psFreeFunc) psastroGhostFree);
+
+    ghost->srcFP = psPlaneAlloc();
+    ghost->FP    = psPlaneAlloc();
+    ghost->chip  = psPlaneAlloc();
+    
+    ghost->Mag   = 0.0;
+
+    ghost->inner.major = 0.0;
+    ghost->inner.minor = 0.0;
+    ghost->inner.theta = 0.0;
+
+    ghost->outer.major = 0.0;
+    ghost->outer.minor = 0.0;
+    ghost->outer.theta = 0.0;
+
+    return ghost;
+}
+
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroLoadGhosts.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroLoadGhosts.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroLoadGhosts.c	(revision 24244)
@@ -0,0 +1,220 @@
+/** @file psastroLoadGhosts.c
+ *
+ *  @brief calculate ghost FPA and Chip positions for the stars loaded on the FPA based on the model
+ *
+ *  @ingroup psastro
+ *
+ *  @author IfA
+ *  @version $Revision: 1.7 $
+ *  @date $Date: 2009-02-07 02:03:34 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+# include "psastroInternal.h"
+
+# define GET_2D_POLY(NAME,OUT) \
+    md = psMetadataLookupMetadata (&status, ghostModel, NAME); \
+    if (!md) { \
+	psError(PSASTRO_ERR_CONFIG, true, "Missing %s in model file %s", NAME, ghostFile); \
+	goto escape; \
+    } \
+    OUT = psPolynomial2DfromMetadata(md); \
+    if (!OUT) { \
+	psError(PSASTRO_ERR_CONFIG, true, "Trouble interpretting %s in model file %s", NAME, ghostFile); \
+	goto escape; \
+    }
+
+# define GET_1D_POLY(NAME,OUT) \
+    md = psMetadataLookupMetadata (&status, ghostModel, NAME); \
+    if (!md) { \
+	psError(PSASTRO_ERR_CONFIG, true, "Missing %s in model file %s", NAME, ghostFile); \
+	goto escape; \
+    } \
+    OUT = psPolynomial1DfromMetadata(md);	\
+    if (!OUT) { \
+	psError(PSASTRO_ERR_CONFIG, true, "Trouble interpretting %s in model file %s", NAME, ghostFile); \
+	goto escape; \
+    }
+
+/**
+ * calculate ghost FPA and Chip positions for the stars loaded on the FPA
+ */
+bool psastroLoadGhosts (pmConfig *config) {
+
+    bool status;
+    pmChip *chip = NULL;
+    pmCell *cell = NULL;
+    pmReadout *readout = NULL;
+    float zeropt, exptime;
+    psMetadata *md = NULL;
+    psPolynomial2D *centerX = NULL;
+    psPolynomial2D *centerY = NULL;
+    psPolynomial1D *outerMajor = NULL;
+    psPolynomial1D *outerMinor = NULL;
+    psPolynomial1D *innerMajor = NULL;
+    psPolynomial1D *innerMinor = NULL;
+    psMetadata *ghostModel = NULL;
+
+    psLogMsg ("psastro", PS_LOG_INFO, "determine ghost positions");
+
+    // select the current recipe
+    psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, PSASTRO_RECIPE);
+    if (!recipe) {
+        psError(PSASTRO_ERR_CONFIG, true, "Can't find PSASTRO recipe");
+        return false;
+    }
+
+    bool REFSTAR_MASK_GHOST = psMetadataLookupBool (&status, recipe, "REFSTAR_MASK_GHOST");
+    if (!REFSTAR_MASK_GHOST) return true;
+
+    char *ghostFile = psMetadataLookupStr (&status, recipe, "GHOST_MODEL");
+    if (!strcasecmp(ghostFile, "NONE")) return true;
+
+    double MAX_MAG = psMetadataLookupF32 (&status, recipe, "GHOST_MAX_MAG");
+
+    if (!pmConfigFileRead (&ghostModel, ghostFile, "GHOST MODEL")) {
+	psError(PSASTRO_ERR_CONFIG, true, "Trouble loading ghost model");
+        return false;
+    }
+
+    pmFPAview *view = pmFPAviewAlloc (0);
+
+    GET_2D_POLY ("GHOST.CENTER.X", centerX);
+    GET_2D_POLY ("GHOST.CENTER.Y", centerY);
+
+    GET_1D_POLY ("GHOST.OUTER.MAJOR", outerMajor);
+    GET_1D_POLY ("GHOST.OUTER.MINOR", outerMinor);
+    GET_1D_POLY ("GHOST.INNER.MAJOR", innerMajor);
+    GET_1D_POLY ("GHOST.INNER.MINOR", innerMinor);
+
+    // select the input astrometry data (also carries the refstars)
+    pmFPAfile *astrom = psMetadataLookupPtr (NULL, config->files, "PSASTRO.INPUT");
+    if (!astrom) {
+        psError(PSASTRO_ERR_CONFIG, true, "Can't find input data");
+	goto escape;
+    }
+    pmFPA *fpa = astrom->fpa;
+
+    // really error-out here?  or just skip?
+    if (!psastroZeroPointFromRecipe (&zeropt, &exptime, fpa, recipe)) {
+        psLogMsg ("psastro", PS_LOG_INFO, "failed to load zeropt data from recipe");
+	goto escape;
+    }
+
+    // recipe values are given in instrumental magnitudes
+    // use the zero point and exposure time to convert to apparent mags: M_ap = M_inst + C_0 + 2.5*log(exptime)
+    float MagOffset = zeropt + 2.5*log10(exptime);
+    MAX_MAG += MagOffset;
+
+    // this loop selects the matched stars for all chips
+    while ((chip = pmFPAviewNextChip (view, fpa, 1)) != NULL) {
+        psTrace ("psastro", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
+        if (!chip->process || !chip->file_exists) { continue; }
+        if (!chip->fromFPA) { continue; }
+
+        while ((cell = pmFPAviewNextCell (view, fpa, 1)) != NULL) {
+            psTrace ("psastro", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
+            if (!cell->process || !cell->file_exists) { continue; }
+
+            // process each of the readouts
+            while ((readout = pmFPAviewNextReadout (view, fpa, 1)) != NULL) {
+                if (! readout->data_exists) { continue; }
+
+                // select the raw objects for this readout (loaded in psastroExtract.c)
+                psArray *refstars = psMetadataLookupPtr (&status, readout->analysis, "PSASTRO.REFSTARS");
+                if (refstars == NULL) { continue; }
+
+                // identify the bright stars of interest
+                for (int i = 0; i < refstars->n; i++) {
+                    pmAstromObj *ref = refstars->data[i];
+                    if (ref->Mag > MAX_MAG) continue;
+
+		    // Ghost model:
+		    // X(ghost) = -X + dX(X,Y) --> GHOST.CENTER.X
+		    // Y(ghost) = -Y + dY(X,Y) --> GHOST.CENTER.Y
+		    // R1inner  = R1inner_0 + r dR1inner
+		    // R2inner  = R2inner_0 + r dR1inner
+		    // R1outer  = R1outer_0 + r dR1inner
+		    // R2outer  = R2outer_0 + r dR1inner
+
+		    psastroGhost *ghost = psastroGhostAlloc ();
+		    ghost->srcFP->x = ref->FP->x; 
+		    ghost->srcFP->y = ref->FP->y;
+
+		    // XXX it is stupid that this takes -X_fpa,-Y_fpa --> the analysis script is reporting the guess FPA position, and the fit is using that...
+		    ghost->FP->x = -ref->FP->x + psPolynomial2DEval(centerX, -ref->FP->x, -ref->FP->y);
+		    ghost->FP->y = -ref->FP->y + psPolynomial2DEval(centerY, -ref->FP->x, -ref->FP->y);
+
+		    float rSrc = hypot (ref->FP->x, ref->FP->y);
+
+		    ghost->inner.major = psPolynomial1DEval (innerMajor, rSrc);
+		    ghost->inner.minor = psPolynomial1DEval (innerMinor, rSrc);
+		    ghost->inner.theta = atan2(ref->FP->y, ref->FP->x);
+
+		    ghost->outer.major = psPolynomial1DEval (outerMajor, rSrc);
+		    ghost->outer.minor = psPolynomial1DEval (outerMinor, rSrc);
+		    ghost->outer.theta = atan2(ref->FP->y, ref->FP->x);
+
+		    // report instrumental ghost star mags
+		    ghost->Mag = ref->Mag - MagOffset;
+		    
+		    // XXX this code yields a single chip: we need to provide results for any chips
+		    // which encompass the full size of the ghost
+		    pmChip *ghostChip = psastroFindChip (&ghost->chip->x, &ghost->chip->y, fpa, -ghost->srcFP->x, -ghost->srcFP->y);
+		    fprintf (stderr, "raw chip position: %f, %f ", ghost->chip->x, ghost->chip->y);
+
+		    ghostChip = psastroFindChip (&ghost->chip->x, &ghost->chip->y, fpa, ghost->FP->x, ghost->FP->y);
+		    fprintf (stderr, "-> model chip position: %f, %f\n", ghost->chip->x, ghost->chip->y);
+
+		    if (!ghostChip) goto skip;
+		    if (!ghostChip->cells) goto skip;
+		    if (!ghostChip->cells->n) goto skip;
+		    pmCell *ghostCell = ghostChip->cells->data[0];
+		    if (!ghostCell) goto skip;
+		    if (!ghostCell->readouts) goto skip;
+		    if (!ghostCell->readouts->n) goto skip;
+		    pmReadout *ghostReadout = ghostCell->readouts->data[0];
+		    if (!ghostReadout) goto skip;
+
+		    psArray *ghosts = psMetadataLookupPtr (&status, ghostReadout->analysis, "PSASTRO.GHOSTS");
+		    if (ghosts == NULL) { 
+			ghosts = psArrayAllocEmpty (100);
+			if (!psMetadataAdd (ghostReadout->analysis, PS_LIST_TAIL, "PSASTRO.GHOSTS", PS_DATA_ARRAY, "astrometry matches", ghosts)) {
+			  psError(PSASTRO_ERR_CONFIG, false, "failure to add ghosts to readout");
+			  goto escape;
+			}
+			psFree (ghosts);
+		    }
+
+		    psArrayAdd (ghosts, 100, ghost);
+
+		skip:
+		    psFree (ghost);
+                }
+            }
+        }
+    }
+
+    psastroExtractFreeChipBounds();
+
+    psFree (centerX);
+    psFree (centerY);
+    psFree (innerMajor);
+    psFree (innerMinor);
+    psFree (outerMajor);
+    psFree (outerMinor);
+    psFree (ghostModel);
+    psFree (view);
+    return true;
+
+escape:
+    psFree (centerX);
+    psFree (centerY);
+    psFree (innerMajor);
+    psFree (innerMinor);
+    psFree (outerMajor);
+    psFree (outerMinor);
+    psFree (ghostModel);
+    psFree (view);
+    return false;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroLoadRefstars.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroLoadRefstars.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroLoadRefstars.c	(revision 24244)
@@ -181,8 +181,10 @@
             ref->sky->d   = RAD_DEG*psMetadataLookupF32 (&status, row, "DEC");
             ref->Mag      = 0.001*psMetadataLookupS32 (&status, row, "MAG");  // ELIXIR uses millimags
+	    ref->Color    = 0.0;
         } else {
             ref->sky->r   = RAD_DEG*psMetadataLookupF64 (&status, row, "RA");
             ref->sky->d   = RAD_DEG*psMetadataLookupF64 (&status, row, "DEC");
             ref->Mag      = psMetadataLookupF32 (&status, row, "MAG"); // PANSTARRS uses mags
+	    ref->Color    = 0.0;
         }
 
@@ -219,4 +221,12 @@
         ref->sky->d   = RAD_DEG*psMetadataLookupF32 (&status, row, "DEC");
         ref->Mag      = psMetadataLookupF32 (&status, row, "MAG");
+        float MagC1   = psMetadataLookupF32 (&status, row, "MAG_C1");
+        float MagC2   = psMetadataLookupF32 (&status, row, "MAG_C2");
+	if (!isnan(MagC1) && !isnan(MagC2)) {
+	    ref->Color = MagC1 - MagC2;
+	} else {
+	    // XXX save the color and the slope in the table header?
+	    ref->Color = 0.0;
+	}
 
         // XXX VERY temporary hack to avoid M31 bulge
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMaskMisc.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMaskMisc.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMaskMisc.c	(revision 24244)
@@ -0,0 +1,51 @@
+/** storage : I don't really need these print functions any more ***/
+# if (0)
+    bool REFSTAR_MASK_REGIONS              = psMetadataLookupBool (&status, recipe, "REFSTAR_MASK_REGIONS");
+
+        // text region files for testing
+        FILE *f = NULL;
+        if (REFSTAR_MASK_REGIONS) {
+          char *filename = NULL;
+          char *chipname = psMetadataLookupStr (&status, chip->concepts, "CHIP.NAME");
+          psStringAppend (&filename, "refstars.mask.%s.dat", chipname);
+          FILE *f = fopen (filename, "w");
+          if (!f) {
+            psWarning ("cannot create refstar mask file %s\n", filename);
+            continue;
+          }
+          psFree (filename);
+        }
+
+                    if (REFSTAR_MASK_REGIONS) {
+                      fprintf (f, "CIRCLE %f %f  %f %f\n", ref->chip->x, ref->chip->y, radius, radius);
+                    }
+
+                        if (REFSTAR_MASK_REGIONS) {
+                          // lower side
+                          x0 = ref->chip->x + spikeWidth*sin(Theta);
+                          y0 = ref->chip->y - spikeWidth*cos(Theta);
+                          x1 = ref->chip->x + spikeLength*cos(Theta) + spikeWidth*sin(Theta);
+                          y1 = ref->chip->y + spikeLength*sin(Theta) - spikeWidth*cos(Theta);
+                          dx = x1 - x0;
+                          dy = y1 - y0;
+
+                          fprintf (f, "LINE %f %f  %f %f\n", x0, y0, dx, dy);
+
+                          // upper side
+                          x0 = ref->chip->x - spikeWidth*sin(Theta);
+                          y0 = ref->chip->y + spikeWidth*cos(Theta);
+                          x1 = ref->chip->x + spikeLength*cos(Theta) - spikeWidth*sin(Theta);
+                          y1 = ref->chip->y + spikeLength*sin(Theta) + spikeWidth*cos(Theta);
+                          dx = x1 - x0;
+                          dy = y1 - y0;
+                          fprintf (f, "LINE %f %f  %f %f\n", x0, y0, dx, dy);
+                        }
+
+			if (REFSTAR_MASK_REGIONS) {
+			    fprintf (f, "LINE %f %f  %f %f\n", ref->chip->x, ref->chip->y, 0.0, -100.0);
+			}
+
+        if (REFSTAR_MASK_REGIONS) {
+          fclose (f);
+        }
+# endif
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMaskUpdates.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMaskUpdates.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMaskUpdates.c	(revision 24244)
@@ -19,14 +19,4 @@
 }
 
-pmCell *pmCellInChip (pmChip *chip, float x, float y);
-bool pmCellCoordsForChip (float *xCell, float *yCell, pmCell *cell, float xChip, float yChip);
-bool pmChipCoordsForCell (float *xChip, float *yChip, pmCell *cell, float xCell, float yCell);
-
-bool psastroMaskCircle (psImage *mask, psImageMaskType value, float x0, float y0, float dX, float dY);
-bool psastroMaskBox (psImage *mask, psImageMaskType value, float x0, float y0, float dL, float dW, float theta);
-void psastroMaskLine (psImage *mask, psImageMaskType value, double x1, double y1, double x2, double y2, int dW);
-void psastroMaskLineBresen (psImage *mask, psImageMaskType value, int X1, int Y1, int X2, int Y2, int dW, int swapcoords);
-void psastroMaskRectangle (psImage *mask, psImageMaskType value, int x0, int y0, int x1, int y1);
-
 /**
  * create a mask or mask regions based on the collection of reference stars that * are in the vicinity of each chip
@@ -40,5 +30,7 @@
     float zeropt, exptime;
 
-    psImageMaskType maskValue  = pmConfigMaskGet("GHOST", config); // Mask value for ghost pixels
+    psImageMaskType ghostMaskValue = pmConfigMaskGet("GHOST", config); // Mask value for ghost pixels
+    psImageMaskType spikeMaskValue = pmConfigMaskGet("SPIKE", config); // Mask value for ghost pixels
+    psImageMaskType starMaskValue  = pmConfigMaskGet("STARCORE", config); // Mask value for ghost pixels
 
     // psImageMaskType maskBlank  = pmConfigMaskGet("BLANK", config); // Mask value for blank pixels
@@ -55,7 +47,16 @@
     if (!REFSTAR_MASK) return true;
 
+    // convert star positions to ghost positions and add to the readout->analysis data
+    if (!psastroLoadGhosts (config)) {
+        psError(PSASTRO_ERR_CONFIG, false, "Error loading ghosts");
+        return false;
+    }
+    bool COUNT_GHOSTS = psMetadataLookupF32 (&status, recipe, "REFSTAR_COUNT_GHOSTS");
+    double GHOST_MAX_MAG = psMetadataLookupF32 (&status, recipe, "GHOST_MAX_MAG");
+    int nGhosts = 0;
+
     psLogMsg ("psastro", PS_LOG_INFO, "generating a bright-star mask");
 
-    bool REFSTAR_MASK_REGIONS              = psMetadataLookupBool (&status, recipe, "REFSTAR_MASK_REGIONS");
+    bool REFSTAR_MASK_BLEED                = psMetadataLookupBool (&status, recipe, "REFSTAR_MASK_BLEED");
 
     double REFSTAR_MASK_MAX_MAG            = psMetadataLookupF32 (&status, recipe, "REFSTAR_MASK_MAX_MAG");
@@ -90,4 +91,5 @@
     REFSTAR_MASK_SATSPIKE_MAG_MAX += MagOffset;
     REFSTAR_MASK_BLEED_MAG_MAX += MagOffset;
+    GHOST_MAX_MAG += MagOffset;
 
     // select the output mask image :: we mosaic to chip mosaic format
@@ -132,18 +134,4 @@
 
         if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
-
-        // text region files for testing
-        FILE *f = NULL;
-        if (REFSTAR_MASK_REGIONS) {
-          char *filename = NULL;
-          char *chipname = psMetadataLookupStr (&status, chip->concepts, "CHIP.NAME");
-          psStringAppend (&filename, "refstars.mask.%s.dat", chipname);
-          FILE *f = fopen (filename, "w");
-          if (!f) {
-            psWarning ("cannot create refstar mask file %s\n", filename);
-            continue;
-          }
-          psFree (filename);
-        }
 
         pmChip *refChip  = pmFPAviewThisChip (view, refMask->fpa);
@@ -172,9 +160,9 @@
 
             // we mask pixels on the input mask image (chip-mosaic)
-            pmCell *cellMask = pmFPAviewThisCell(view, outMask->fpa);
-            pmReadout *readoutMask = NULL;
-            if (cellMask->readouts->n) {
-                readoutMask = cellMask->readouts->data[0];
-            }
+            // pmCell *cellMask = pmFPAviewThisCell(view, outMask->fpa);
+            // pmReadout *readoutMask = NULL;
+            // if (cellMask->readouts->n) {
+            //     readoutMask = cellMask->readouts->data[0];
+            // }
 
             // process each of the readouts
@@ -183,7 +171,11 @@
                 if (! readout->data_exists) { continue; }
 
+		// XXX why not do this? 
+		pmReadout *readoutMask = pmFPAviewThisReadout (view, outMask->fpa);
+		if (!readoutMask) continue;
+
                 // select the raw objects for this readout
                 psArray *refstars = psMetadataLookupPtr (&status, readout->analysis, "PSASTRO.REFSTARS");
-                if (refstars == NULL) { continue; }
+                if (!refstars) continue;
 
                 // we need to generate the following masks regions:
@@ -191,87 +183,91 @@
                 // 2) diffraction spikes in direction ROT - ROTo
                 // 3) bleed trail in the direction of the readout
-
-                // XXX for the moment, let's just generate mana region-file objects in chip pixel space
+		//    update: with 'burntool' applied to the data, the bleed trail mask is not needed.
+
                 for (int i = 0; i < refstars->n; i++) {
                     pmAstromObj *ref = refstars->data[i];
+		    
+		    if (COUNT_GHOSTS) {
+			if (ref->Mag > GHOST_MAX_MAG) {
+			    nGhosts ++;
+			}
+		    }
+
                     if (ref->Mag > REFSTAR_MASK_MAX_MAG) continue;
 
-                    // XXX convert ref->Mag to instrumental mags
+		    // the reference magnitudes have been converted from the instrumental
+		    // values supplied in the recipe to apparent mags (above)
 
                     // CIRCLE around the stars (scaled by magnitude)
                     float radius = REFSTAR_MASK_SATSTAR_MAG_SLOPE * (REFSTAR_MASK_SATSTAR_MAG_MAX - ref->Mag);
 
-                    if (REFSTAR_MASK_REGIONS) {
-                      fprintf (f, "CIRCLE %f %f  %f %f\n", ref->chip->x, ref->chip->y, radius, radius);
+                    // XXX for now, assume cell binning is 1x1 relative to chip
+		    psastroMaskCircle (readoutMask->mask, starMaskValue, ref->chip->x, ref->chip->y, radius, radius);
+
+                    for (float theta = 0.0; theta < 2*M_PI; theta += M_PI / 2.0) {
+
+                        float Theta = theta - ROTANGLE - REFSTAR_MASK_SATSTAR_POS_ZERO;
+
+			// LINE for boundaries of the saturation spikes (scaled by magnitude)
+			// float MAG_MAX = (theta == 0.0) || (theta == M_PI) ? REFSTAR_MASK_SATSPIKE_MAG_MAX1 : REFSTAR_MASK_SATSPIKE_MAG_MAX2;
+
+			float MAG_MAX = REFSTAR_MASK_SATSPIKE_MAG_MAX;
+			float spikeLength = REFSTAR_MASK_SATSPIKE_MAG_SLOPE * (MAG_MAX - ref->Mag);
+			float spikeWidth = 0.5*REFSTAR_MASK_SATSPIKE_WIDTH;
+			// XXX we can make the width depend on the spike as well...
+			// The length should also be a function of the image background level
+
+			psastroMaskBox (readoutMask->mask, spikeMaskValue, ref->chip->x, ref->chip->y, spikeLength, spikeWidth, Theta);
                     }
 
-                    // XXX for now, assume cell binning is 1x1 relative to chip
-                    if (readoutMask) {
-                        psastroMaskCircle (readoutMask->mask, maskValue, ref->chip->x, ref->chip->y, radius, radius);
-                    }
-
-                    // LINE for boundaries of the saturation spikes (scaled by magnitude)
-                    float spikeLength = REFSTAR_MASK_SATSPIKE_MAG_SLOPE * (REFSTAR_MASK_SATSPIKE_MAG_MAX - ref->Mag);
-                    float spikeWidth = REFSTAR_MASK_SATSPIKE_WIDTH;
-
-                    for (float theta = 0.0; theta < 2*M_PI; theta += M_PI / 2.0) {
-                        float x0, y0, x1, y1, dx, dy;
-
-                        float Theta = theta - ROTANGLE - REFSTAR_MASK_SATSTAR_POS_ZERO;
-
-                        if (REFSTAR_MASK_REGIONS) {
-                          // lower side
-                          x0 = ref->chip->x + spikeWidth*sin(Theta);
-                          y0 = ref->chip->y - spikeWidth*cos(Theta);
-                          x1 = ref->chip->x + spikeLength*cos(Theta) + spikeWidth*sin(Theta);
-                          y1 = ref->chip->y + spikeLength*sin(Theta) - spikeWidth*cos(Theta);
-                          dx = x1 - x0;
-                          dy = y1 - y0;
-
-                          fprintf (f, "LINE %f %f  %f %f\n", x0, y0, dx, dy);
-
-                          // upper side
-                          x0 = ref->chip->x - spikeWidth*sin(Theta);
-                          y0 = ref->chip->y + spikeWidth*cos(Theta);
-                          x1 = ref->chip->x + spikeLength*cos(Theta) - spikeWidth*sin(Theta);
-                          y1 = ref->chip->y + spikeLength*sin(Theta) + spikeWidth*cos(Theta);
-                          dx = x1 - x0;
-                          dy = y1 - y0;
-                          fprintf (f, "LINE %f %f  %f %f\n", x0, y0, dx, dy);
-                        }
-
-                        if (readoutMask) {
-                            psastroMaskBox (readoutMask->mask, maskValue, ref->chip->x, ref->chip->y, spikeLength, spikeWidth, Theta);
-                        }
-                    }
-
-                    // convert x,y chip coordinates to cells in maskChip
-                    pmCell *refCell = pmCellInChip (refChip, ref->chip->x, ref->chip->y);
-
-                    // LINE for boundaries of the bleed lines
-                    if (REFSTAR_MASK_REGIONS) {
-                      fprintf (f, "LINE %f %f  %f %f\n", ref->chip->x, ref->chip->y, 0.0, -100.0);
-                    }
-
-                    if (readoutMask && refCell) {
-                        float xCell = 0.0;
-                        float yCell = 0.0;
-                        float xEnd = 0.0;
-                        float yEnd = 0.0;
-                        // find coordinate of star on cell
-                        pmCellCoordsForChip (&xCell, &yCell, refCell, ref->chip->x, ref->chip->y);
-                        // find coordinate of end-point on chip
-
-                        int ySize = psMetadataLookupS32(NULL, refCell->concepts, "CELL.YSIZE");
-                        pmChipCoordsForCell (&xEnd, &yEnd, refCell, xCell, ySize);
-
-                        float width = REFSTAR_MASK_BLEED_MAG_SLOPE*(REFSTAR_MASK_BLEED_MAG_MAX - ref->Mag);
-                        psastroMaskRectangle (readoutMask->mask, maskValue, (int) ref->chip->x-0.5*width, (int) ref->chip->y, (int) ref->chip->x+0.5*width+1, yEnd);
-                    }
+		    if (REFSTAR_MASK_BLEED) {
+
+			// convert x,y chip coordinates to cells in maskChip
+			pmCell *refCell = pmCellInChip (refChip, ref->chip->x, ref->chip->y);
+
+			// LINE for boundaries of the bleed lines
+			if (refCell) {
+			    float xCell = 0.0;
+			    float yCell = 0.0;
+			    float xEnd = 0.0;
+			    float yEnd = 0.0;
+			    // find coordinate of star on cell
+			    pmCellCoordsForChip (&xCell, &yCell, refCell, ref->chip->x, ref->chip->y);
+			    // find coordinate of end-point on chip
+
+			    int ySize = psMetadataLookupS32(NULL, refCell->concepts, "CELL.YSIZE");
+			    pmChipCoordsForCell (&xEnd, &yEnd, refCell, xCell, ySize);
+
+			    float width = REFSTAR_MASK_BLEED_MAG_SLOPE*(REFSTAR_MASK_BLEED_MAG_MAX - ref->Mag);
+			    psastroMaskRectangle (readoutMask->mask, spikeMaskValue, (int) ref->chip->x-0.5*width, (int) ref->chip->y, (int) ref->chip->x+0.5*width+1, yEnd);
+			}
+		    }
                 }
+
+                // select the raw objects for this readout (loaded in psastroExtract.c)
+                psArray *ghosts = psMetadataLookupPtr (&status, readout->analysis, "PSASTRO.GHOSTS");
+                if (ghosts == NULL) { continue; }
+
+                // identify the bright stars of interest
+                for (int i = 0; i < ghosts->n; i++) {
+                    psastroGhost *ghost = ghosts->data[i];
+		    // XXX bright vs faint ghost bits? (OR with SUSPECT) 
+		    psastroMaskEllipticalAnnulus (readoutMask->mask, ghostMaskValue, ghost->chip->x, ghost->chip->y, ghost->inner, ghost->outer);
+                }
+
+		// select the raw objects for this readout, flag is they fall in a mask
+		psArray *rawstars = psMetadataLookupPtr (&status, readout->analysis, "PSASTRO.RAWSTARS");
+		if (rawstars == NULL) return false;
+		
+		// XXX finish this:
+		for (int i = 0; i < rawstars->n; i++) {
+		    pmAstromObj *raw = rawstars->data[i];
+		    psImageMaskType value = readoutMask->mask->data.PS_TYPE_IMAGE_MASK_DATA[(int)(raw->chip->x)][(int)(raw->chip->y)];
+		    if (value) continue;
+		}
             }
         }
 
-        // output sequence for mask corresponding to this chip
+        // output sequence for mask corresponding to this chip (XXX this may not be needed...)
         *viewMask = *view;
         while ((cell = pmFPAviewNextCell (viewMask, outMask->fpa, 1)) != NULL) {
@@ -285,11 +281,18 @@
             if (!pmFPAfileIOChecks (config, viewMask, PM_FPA_AFTER)) ESCAPE;
         }
-
         if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
-        if (REFSTAR_MASK_REGIONS) {
-          fclose (f);
-        }
     }
     if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+
+    if (COUNT_GHOSTS) {
+	// save nGhosts to update header.
+	psMetadata *updates = psMetadataLookupMetadata (&status, fpa->analysis, "PSASTRO.HEADER");
+	if (!updates) {
+	    updates = psMetadataAlloc ();
+	    psMetadataAddMetadata (fpa->analysis, PS_LIST_TAIL, "PSASTRO.HEADER",  PS_META_REPLACE, "psastro header stats", updates);
+	    psFree (updates);
+	}
+	psMetadataAddS32 (updates, PS_LIST_TAIL, "NGHOSTS", PS_META_REPLACE, "total expected ghosts", nGhosts);
+    }
 
     // deactivate all files
@@ -301,220 +304,2 @@
 }
 
-// XXX this is going to be very slow...
-pmCell *pmCellInChip (pmChip *chip, float x, float y) {
-
-# if (0)
-    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
-    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
-    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
-    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
-
-    // XXX fix the binning : currently not selected from concepts
-    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
-    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
-    int xBin = 1;
-    int yBin = 1;
-
-    // Position on the cell
-    float xCell = PM_CHIP_TO_CELL(xChip, x0Cell, xParityCell, xBin);
-    float yCell = PM_CHIP_TO_CELL(yChip, y0Cell, yParityCell, yBin);
-# endif
-
-    for (int i = 0; i < chip->cells->n; i++) {
-
-        pmCell *cell = chip->cells->data[i];
-        psRegion *region = pmCellExtent (cell);
-
-        if (x < region->x0) goto skip;
-        if (x > region->x1) goto skip;
-        if (y < region->y0) goto skip;
-        if (y > region->y1) goto skip;
-
-        psFree (region);
-        return cell;
-
-    skip:
-        psFree (region);
-    }
-    return NULL;
-}
-
-/**
- * convert chip coords to cell coords, given known cell
- */
-bool pmCellCoordsForChip (float *xCell, float *yCell, pmCell *cell, float xChip, float yChip) {
-
-    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
-    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
-    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
-    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
-
-    // XXX fix the binning : currently not selected from concepts
-    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
-    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
-    int xBin = 1;
-    int yBin = 1;
-
-    // Position on the cell
-    // ((pos)*(binning)*(cellParity) + (cell0))
-    // XXX this is probably totally wrong now....
-    // ((pos) - (cell0))*(cellParity)/(binning))
-    *xCell = (xChip - x0Cell)*xParityCell/xBin;
-    *yCell = (yChip - y0Cell)*yParityCell/yBin;
-
-    return true;
-}
-
-/**
- * convert chip coords to cell coords, given known cell
- */
-bool pmChipCoordsForCell (float *xChip, float *yChip, pmCell *cell, float xCell, float yCell) {
-
-    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
-    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
-    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
-    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
-
-    // XXX fix the binning : currently not selected from concepts
-    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
-    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
-    int xBin = 1;
-    int yBin = 1;
-
-    // Position on the cell
-    // ((pos)*(binning)*(cellParity) + (cell0))
-    // XXX this is probably totally wrong now....
-    // ((pos) - (cell0))*(cellParity)/(binning))
-    *xChip = xCell*xBin*xParityCell + x0Cell;
-    *yChip = yCell*yBin*yParityCell + y0Cell;
-
-    return true;
-}
-
-// XXX should be doing an OR
-bool psastroMaskCircle (psImage *mask, psImageMaskType value, float x0, float y0, float dX, float dY) {
-
-    // XXX need to worry about row0, col0
-    for (int ix = -dX; ix <= +dX; ix++) {
-        int jx = ix + x0;
-        if (jx < 0) continue;
-        if (jx >= mask->numCols) continue;
-        for (int iy = -dY; iy <= +dY; iy++) {
-            int jy = iy + y0;
-            if (jy < 0) continue;
-            if (jy >= mask->numRows) continue;
-
-            double r2 = PS_SQR(ix/dX) + PS_SQR(iy/dY);
-            if (r2 > 1.0) continue;
-
-            mask->data.PS_TYPE_IMAGE_MASK_DATA[jy][jx] |= value;
-        }
-    }
-    return true;
-}
-
-// XXX should be doing an OR
-bool psastroMaskBox (psImage *mask, psImageMaskType value, float x0, float y0, float dL, float dW, float theta) {
-
-    // draw a series of lines (from -0.5*dW to +0.5*dW) of length dL, starting at x0, y0, angle theta
-
-    float xs = x0;
-    float ys = y0;
-
-    float xe = xs + dL*cos(theta);
-    float ye = ys + dL*sin(theta);
-
-    psastroMaskLine (mask, value, xs, ys, xe, ye, (int) dW);
-    return true;
-}
-
-/**
- * identify the quadrant and draw the correct line
- */
-void psastroMaskLine (psImage *mask, psImageMaskType value, double x1, double y1, double x2, double y2, int dW) {
-
-  int FlipDirect, FlipCoords;
-  int X1, Y1, X2, Y2, dX, dY;
-
-  /* rather than draw the line from float positions, we find the closest
-     integer end-points and draw the line between those pixels */
-
-  X1 = ROUND(x1);
-  Y1 = ROUND(y1);
-  X2 = ROUND(x2);
-  Y2 = ROUND(y2);
-
-  dX = X2 - X1;
-  dY = Y2 - Y1;
-
-  FlipCoords = (abs(dX) < abs(dY));
-  FlipDirect = FlipCoords ? (y1 > y2) : (x1 > x2);
-
-  if (!FlipDirect && !FlipCoords) psastroMaskLineBresen (mask, value, X1, Y1, X2, Y2, dW, FALSE);
-  if ( FlipDirect && !FlipCoords) psastroMaskLineBresen (mask, value, X2, Y2, X1, Y1, dW, FALSE);
-  if (!FlipDirect &&  FlipCoords) psastroMaskLineBresen (mask, value, Y1, X1, Y2, X2, dW, TRUE);
-  if ( FlipDirect &&  FlipCoords) psastroMaskLineBresen (mask, value, Y2, X2, Y1, X1, dW, TRUE);
-
-  return;
-}
-
-/**
- * use the Bresenham line drawing technique
- * integer-only Bresenham line-draw version which is fast
- */
-void psastroMaskLineBresen (psImage *mask, psImageMaskType value, int X1, int Y1, int X2, int Y2, int dW, int swapcoords) {
-
-    int X, Y, dX, dY;
-    int e, e2;
-
-    dX = X2 - X1;
-    dY = Y2 - Y1;
-
-    Y = Y1;
-    e = 0;
-    for (X = X1; X <= X2; X++) {
-        if (X > 0) {
-            if (swapcoords) {
-                if (X >= mask->numRows) continue;
-                for (int y = Y - dW; y <= Y + dW; y++) {
-                    if (y < 0) continue;
-                    if (y >= mask->numCols) continue;
-                    mask->data.PS_TYPE_IMAGE_MASK_DATA[X][y] |= value;
-                }
-            } else {
-                if (X >= mask->numCols) continue;
-                for (int y = Y - dW; y <= Y + dW; y++) {
-                    if (y < 0) continue;
-                    if (y >= mask->numRows) continue;
-                    mask->data.PS_TYPE_IMAGE_MASK_DATA[y][X] |= value;
-                }
-            }
-        }
-        e += dY;
-        e2 = 2 * e;
-        if (e2 > dX) {
-            Y++;
-            e -= dX;
-        }
-        if (e2 < -dX) {
-            Y--;
-            e += dX;
-        }
-    }
-    return;
-}
-
-void psastroMaskRectangle (psImage *mask, psImageMaskType value, int x0, int y0, int x1, int y1) {
-
-    int xs = PS_MAX (0, PS_MIN (mask->numCols, PS_MIN (x0, x1)));
-    int xe = PS_MAX (0, PS_MIN (mask->numCols, PS_MAX (x0, x1)));
-    int ys = PS_MAX (0, PS_MIN (mask->numRows, PS_MIN (y0, y1)));
-    int ye = PS_MAX (0, PS_MIN (mask->numRows, PS_MAX (y0, y1)));
-
-    for (int iy = ys; iy < ye; iy++) {
-        for (int ix = xs; ix < xe; ix++) {
-            mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= value;
-        }
-    }
-}
-
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMaskUtils.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMaskUtils.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMaskUtils.c	(revision 24244)
@@ -0,0 +1,350 @@
+/** @file psastroMaskUpdates.c
+ *
+ *  @brief 
+ *
+ *  @ingroup libpsastro
+ *
+ *  @author IfA
+ *  @version $Revision: 1.7 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-07 02:03:34 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+# include "psastroInternal.h"
+
+# define ESCAPE { \
+  psError(PS_ERR_UNKNOWN, false, "I/O failure in psastroMaskUpdate"); \
+  psFree (view); \
+  return false; \
+}
+
+// XXX this is going to be very slow...
+pmCell *pmCellInChip (pmChip *chip, float x, float y) {
+
+# if (0)
+    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
+    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
+    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
+    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
+
+    // XXX fix the binning : currently not selected from concepts
+    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
+    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
+    int xBin = 1;
+    int yBin = 1;
+
+    // Position on the cell
+    float xCell = PM_CHIP_TO_CELL(xChip, x0Cell, xParityCell, xBin);
+    float yCell = PM_CHIP_TO_CELL(yChip, y0Cell, yParityCell, yBin);
+# endif
+
+    for (int i = 0; i < chip->cells->n; i++) {
+
+        pmCell *cell = chip->cells->data[i];
+        psRegion *region = pmCellExtent (cell);
+
+        if (x < region->x0) goto skip;
+        if (x > region->x1) goto skip;
+        if (y < region->y0) goto skip;
+        if (y > region->y1) goto skip;
+
+        psFree (region);
+        return cell;
+
+    skip:
+        psFree (region);
+    }
+    return NULL;
+}
+
+/**
+ * convert chip coords to cell coords, given known cell
+ */
+bool pmCellCoordsForChip (float *xCell, float *yCell, pmCell *cell, float xChip, float yChip) {
+
+    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
+    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
+    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
+    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
+
+    // XXX fix the binning : currently not selected from concepts
+    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
+    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
+    int xBin = 1;
+    int yBin = 1;
+
+    // Position on the cell
+    // ((pos)*(binning)*(cellParity) + (cell0))
+    // XXX this is probably totally wrong now....
+    // ((pos) - (cell0))*(cellParity)/(binning))
+    *xCell = (xChip - x0Cell)*xParityCell/xBin;
+    *yCell = (yChip - y0Cell)*yParityCell/yBin;
+
+    return true;
+}
+
+/**
+ * convert chip coords to cell coords, given known cell
+ */
+bool pmChipCoordsForCell (float *xChip, float *yChip, pmCell *cell, float xCell, float yCell) {
+
+    int x0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.X0");
+    int y0Cell = psMetadataLookupS32(NULL, cell->concepts, "CELL.Y0");
+    int xParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.XPARITY");
+    int yParityCell = psMetadataLookupS32(NULL, cell->concepts, "CELL.YPARITY");
+
+    // XXX fix the binning : currently not selected from concepts
+    // int xBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.XBIN"); // Binning in x and y
+    // int yBin = psMetadataLookupS32(NULL, cell->concepts, "CELL.YBIN"); // Binning in x and y
+    int xBin = 1;
+    int yBin = 1;
+
+    // Position on the cell
+    // ((pos)*(binning)*(cellParity) + (cell0))
+    // XXX this is probably totally wrong now....
+    // ((pos) - (cell0))*(cellParity)/(binning))
+    *xChip = xCell*xBin*xParityCell + x0Cell;
+    *yChip = yCell*yBin*yParityCell + y0Cell;
+
+    return true;
+}
+
+bool psastroMaskCircle (psImage *mask, psImageMaskType value, float x0, float y0, float dX, float dY) {
+
+    for (int ix = -dX; ix <= +dX; ix++) {
+        int jx = ix + x0;
+        if (jx < 0) continue;
+        if (jx >= mask->numCols) continue;
+        for (int iy = -dY; iy <= +dY; iy++) {
+            int jy = iy + y0;
+            if (jy < 0) continue;
+            if (jy >= mask->numRows) continue;
+
+            double r2 = PS_SQR(ix/dX) + PS_SQR(iy/dY);
+            if (r2 > 1.0) continue;
+
+            mask->data.PS_TYPE_IMAGE_MASK_DATA[jy][jx] |= value;
+        }
+    }
+    return true;
+}
+
+
+void psastroMinMaxForShape (float *min, float *max, float y, psEllipseShape shape) {
+
+	// XXX optimize this
+	float A = 1.0;
+	float B = shape.sxy*y*PS_SQR(shape.sx);
+	float C = PS_SQR(y*shape.sx/shape.sy) - PS_SQR(shape.sx);
+
+	float T1 = PS_SQR(B);
+	float T2 = 4.0*A*C;
+	float R = T1 - T2;
+	if (R < 0) R = 0;
+
+	*min =  (-B - sqrt (R)) / (2.0*A);
+	*max =  (-B + sqrt (R)) / (2.0*A);
+}
+
+bool psastroMaskEllipse (psImage *mask, psImageMaskType value, float x0, float y0, psEllipseAxes axes) {
+
+    psEllipseShape shape = psEllipseAxesToShape (axes);
+
+    // phi is the coordinate along the elliptical path
+    // phiMin, phiMax are the points on the path for Ymin and Ymax
+    float phiMin = atan2 (-axes.minor*cos(axes.theta), axes.major*sin(axes.theta));
+    float phiMax = phiMin + M_PI;
+
+    float Ymin = -axes.major*cos(phiMin)*sin(axes.theta) + axes.minor*sin(phiMin)*cos(axes.theta);
+    float Ymax = -axes.major*cos(phiMax)*sin(axes.theta) + axes.minor*sin(phiMax)*cos(axes.theta);
+    if (Ymin > Ymax) PS_SWAP (Ymin, Ymax);
+  
+    for (int iy = Ymin; iy <= Ymax; iy++) {
+	int jy = iy + y0;
+	if (jy < 0) continue;
+	if (jy >= mask->numRows) continue;
+
+	float Xmin, Xmax;
+	psastroMinMaxForShape (&Xmin, &Xmax, iy, shape);
+	fprintf (stderr, "mask %d : %f -> %f == %f -> %f (%x)\n", jy, Xmin, Xmax, Xmin + x0, Xmax + x0, value);
+
+	for (int ix = Xmin; ix <= Xmax; ix++) {
+	    int jx = ix + x0;
+	    if (jx < 0) continue;
+	    if (jx >= mask->numCols) continue;
+	    mask->data.PS_TYPE_IMAGE_MASK_DATA[jy][jx] |= value;
+	}
+    }
+    return true;
+}
+
+bool psastroMaskEllipticalAnnulus (psImage *mask, psImageMaskType value, float x0, float y0, psEllipseAxes eInner, psEllipseAxes eOuter) {
+
+    // skip the masking if the outer ellipse is nonsensical
+    psEllipseShape sOuter = psEllipseAxesToShape (eOuter);
+    if (isnan(sOuter.sx) || isnan(sOuter.sy) || isnan(sOuter.sxy)) return false;
+
+    psEllipseShape sInner = psEllipseAxesToShape (eInner);
+    if (isnan(sInner.sx) || isnan(sInner.sy) || isnan(sInner.sxy)) {
+	// use a solid ellipse if the inner ellipse is nonsensical
+	sInner.sx = 0.1;
+	sInner.sy = 0.1;
+	sInner.sxy = 0.0;
+    }
+
+    // phi is the coordinate along the elliptical path
+    // phiMin, phiMax are the points on the path for Ymin and Ymax
+    float phiMinInner = atan2 (-eInner.minor*cos(eInner.theta), eInner.major*sin(eInner.theta));
+    float phiMaxInner = phiMinInner + M_PI;
+    float phiMinOuter = atan2 (-eOuter.minor*cos(eOuter.theta), eOuter.major*sin(eOuter.theta));
+    float phiMaxOuter = phiMinOuter + M_PI;
+
+    float YminInner = -eInner.major*cos(phiMinInner)*sin(eInner.theta) + eInner.minor*sin(phiMinInner)*cos(eInner.theta);
+    float YmaxInner = -eInner.major*cos(phiMaxInner)*sin(eInner.theta) + eInner.minor*sin(phiMaxInner)*cos(eInner.theta);
+    if (YminInner > YmaxInner) PS_SWAP (YminInner, YmaxInner);
+
+    float YminOuter = -eOuter.major*cos(phiMinOuter)*sin(eOuter.theta) + eOuter.minor*sin(phiMinOuter)*cos(eOuter.theta);
+    float YmaxOuter = -eOuter.major*cos(phiMaxOuter)*sin(eOuter.theta) + eOuter.minor*sin(phiMaxOuter)*cos(eOuter.theta);
+    if (YminOuter > YmaxOuter) PS_SWAP (YminOuter, YmaxOuter);
+  
+    for (int iy = YminOuter; iy <= YmaxOuter; iy++) {
+	int jy = iy + y0;
+	if (jy < 0) continue;
+	if (jy >= mask->numRows) continue;
+
+	float XminOuter, XmaxOuter;
+	psastroMinMaxForShape (&XminOuter, &XmaxOuter, iy, sOuter);
+
+	if ((iy > YmaxInner) || (iy < YminInner)) {
+	    for (int ix = XminOuter; ix <= XmaxOuter; ix++) {
+		int jx = ix + x0;
+		if (jx < 0) continue;
+		if (jx >= mask->numCols) continue;
+		mask->data.PS_TYPE_IMAGE_MASK_DATA[jy][jx] |= value;
+	    }
+	} else {
+	    float XminInner, XmaxInner;
+	    psastroMinMaxForShape (&XminInner, &XmaxInner, iy, sInner);
+
+	    for (int ix = XminOuter; ix <= XminInner; ix++) {
+		int jx = ix + x0;
+		if (jx < 0) continue;
+		if (jx >= mask->numCols) continue;
+		mask->data.PS_TYPE_IMAGE_MASK_DATA[jy][jx] |= value;
+	    }
+	    for (int ix = XmaxInner; ix <= XmaxOuter; ix++) {
+		int jx = ix + x0;
+		if (jx < 0) continue;
+		if (jx >= mask->numCols) continue;
+		mask->data.PS_TYPE_IMAGE_MASK_DATA[jy][jx] |= value;
+	    }
+	}
+    }
+    return true;
+}
+
+bool psastroMaskBox (psImage *mask, psImageMaskType value, float x0, float y0, float dL, float dW, float theta) {
+
+    // draw a series of lines (from -0.5*dW to +0.5*dW) of length dL, starting at x0, y0, angle theta
+
+    float xs = x0;
+    float ys = y0;
+
+    float xe = xs + dL*cos(theta);
+    float ye = ys + dL*sin(theta);
+
+    psastroMaskLine (mask, value, xs, ys, xe, ye, (int) dW);
+    return true;
+}
+
+/**
+ * identify the quadrant and draw the correct line
+ */
+void psastroMaskLine (psImage *mask, psImageMaskType value, double x1, double y1, double x2, double y2, int dW) {
+
+  int FlipDirect, FlipCoords;
+  int X1, Y1, X2, Y2, dX, dY;
+
+  /* rather than draw the line from float positions, we find the closest
+     integer end-points and draw the line between those pixels */
+
+  X1 = ROUND(x1);
+  Y1 = ROUND(y1);
+  X2 = ROUND(x2);
+  Y2 = ROUND(y2);
+
+  dX = X2 - X1;
+  dY = Y2 - Y1;
+
+  FlipCoords = (abs(dX) < abs(dY));
+  FlipDirect = FlipCoords ? (y1 > y2) : (x1 > x2);
+
+  if (!FlipDirect && !FlipCoords) psastroMaskLineBresen (mask, value, X1, Y1, X2, Y2, dW, FALSE);
+  if ( FlipDirect && !FlipCoords) psastroMaskLineBresen (mask, value, X2, Y2, X1, Y1, dW, FALSE);
+  if (!FlipDirect &&  FlipCoords) psastroMaskLineBresen (mask, value, Y1, X1, Y2, X2, dW, TRUE);
+  if ( FlipDirect &&  FlipCoords) psastroMaskLineBresen (mask, value, Y2, X2, Y1, X1, dW, TRUE);
+
+  return;
+}
+
+/**
+ * use the Bresenham line drawing technique
+ * integer-only Bresenham line-draw version which is fast
+ */
+void psastroMaskLineBresen (psImage *mask, psImageMaskType value, int X1, int Y1, int X2, int Y2, int dW, int swapcoords) {
+
+    int X, Y, dX, dY;
+    int e, e2;
+
+    dX = X2 - X1;
+    dY = Y2 - Y1;
+
+    Y = Y1;
+    e = 0;
+    for (X = X1; X <= X2; X++) {
+        if (X > 0) {
+            if (swapcoords) {
+                if (X >= mask->numRows) continue;
+                for (int y = Y - dW; y <= Y + dW; y++) {
+                    if (y < 0) continue;
+                    if (y >= mask->numCols) continue;
+                    mask->data.PS_TYPE_IMAGE_MASK_DATA[X][y] |= value;
+                }
+            } else {
+                if (X >= mask->numCols) continue;
+                for (int y = Y - dW; y <= Y + dW; y++) {
+                    if (y < 0) continue;
+                    if (y >= mask->numRows) continue;
+                    mask->data.PS_TYPE_IMAGE_MASK_DATA[y][X] |= value;
+                }
+            }
+        }
+        e += dY;
+        e2 = 2 * e;
+        if (e2 > dX) {
+            Y++;
+            e -= dX;
+        }
+        if (e2 < -dX) {
+            Y--;
+            e += dX;
+        }
+    }
+    return;
+}
+
+void psastroMaskRectangle (psImage *mask, psImageMaskType value, int x0, int y0, int x1, int y1) {
+
+    int xs = PS_MAX (0, PS_MIN (mask->numCols, PS_MIN (x0, x1)));
+    int xe = PS_MAX (0, PS_MIN (mask->numCols, PS_MAX (x0, x1)));
+    int ys = PS_MAX (0, PS_MIN (mask->numRows, PS_MIN (y0, y1)));
+    int ye = PS_MAX (0, PS_MIN (mask->numRows, PS_MAX (y0, y1)));
+
+    for (int iy = ys; iy < ye; iy++) {
+        for (int ix = xs; ix < xe; ix++) {
+            mask->data.PS_TYPE_IMAGE_MASK_DATA[iy][ix] |= value;
+        }
+    }
+}
+
+
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMetadataStats.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMetadataStats.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMetadataStats.c	(revision 24244)
@@ -13,5 +13,5 @@
 # include "psastroInternal.h"
 
-bool psastroMetadataStats (pmConfig *config) {
+bool psastroMetadataStats (pmConfig *config, psMetadata *stats) {
 
     bool status;
@@ -24,27 +24,22 @@
 
     if (!output) {
-	psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find output file (PSASTRO.OUTPUT).");
-	return false;
+        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find output file (PSASTRO.OUTPUT).");
+        return false;
     }
 
-    // create output stats metadata
-    psMetadata *stats = psMetadataAlloc ();
-
-    // extract stats for the complete fpa 
+    // extract stats for the complete fpa
     pmFPAview *view = pmFPAviewAlloc(0);
 
     if (!ppStatsMetadata(stats, output->fpa, view, 0, config)) {
-	psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate stats for image.");
-	psFree(view);
-	psFree(stats);
-	return false;
+        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate stats for image.");
+        psFree(view);
+        return false;
     }
 
     // if we did not request any specific stats, the structure is empty
     if (stats && stats->list->n == 0) {
-	psWarning ("stats output specified, but no requested stats entries in headers");
-	psFree(view);
-	psFree(stats);
-	return true;
+        psWarning ("stats output specified, but no requested stats entries in headers");
+        psFree(view);
+        return true;
     }
 
@@ -52,7 +47,7 @@
     char *statsMDC = psMetadataConfigFormat(stats);
     if (!statsMDC || strlen(statsMDC) == 0) {
-	psError(PS_ERR_IO, false, "Unable to serialize stats metadata.\n");
-	return false;
-    } 
+        psError(PS_ERR_IO, false, "Unable to serialize stats metadata.\n");
+        return false;
+    }
 
     // convert to a real UNIX filename
@@ -60,21 +55,20 @@
     FILE *statsFile = fopen (resolved, "w");
     if (!statsFile) {
-	psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved);
-	psFree(stats);
-	psFree(view);
-	psFree(statsMDC);
-	psFree(resolved);
-	return false;
+        psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved);
+        psFree(view);
+        psFree(statsMDC);
+        psFree(resolved);
+        return false;
     }
 
     // write the stats MDC to a file
-    // XXX why does this not call psMetadataConfigPrint?
     fprintf(statsFile, "%s", statsMDC);
     fclose(statsFile);
+
+    pmConfigRunFilenameAddWrite(config, "STATS", filename);
 
     psFree(resolved);
     psFree(statsMDC);
     psFree(view);
-    psFree(stats);
     return true;
 }
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroModelAnalysis.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroModelAnalysis.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroModelAnalysis.c	(revision 24244)
@@ -153,5 +153,8 @@
 
     psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV);
-    psVectorStats (stats, posZero, NULL, NULL, 0);
+    if (!psVectorStats (stats, posZero, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
 
     fprintf (outfile, "# pos zero %f +/- %f\n", stats->sampleMedian, stats->sampleStdev);
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroModelFitBoresite.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroModelFitBoresite.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroModelFitBoresite.c	(revision 24244)
@@ -62,10 +62,16 @@
 
     // center (Xo) = mean(Xo), RX = range / 2
-    psVectorStats (stats, Xo, NULL, NULL, 0);
+    if (!psVectorStats (stats, Xo, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return NULL;
+    }
     params->data.F32[PAR_X0] = stats->sampleMean;
     params->data.F32[PAR_RX] = (stats->max - stats->min) / 2.0;
 
     // center (Yo) = mean(Yo), RY = range / 2
-    psVectorStats (stats, Yo, NULL, NULL, 0);
+    if (!psVectorStats (stats, Yo, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return NULL;
+    }
     params->data.F32[PAR_Y0] = stats->sampleMean;
     params->data.F32[PAR_RY] = (stats->max - stats->min) / 2.0;
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMosaicOneChip.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMosaicOneChip.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroMosaicOneChip.c	(revision 24244)
@@ -58,8 +58,13 @@
 
     // modify the order to correspond to the actual number of matched stars:
-    if ((match->n < 17) && (order >= 3)) order = 2;
-    if ((match->n < 13) && (order >= 2)) order = 1;
-    if ((match->n <  9) && (order >= 1)) order = 0;
-    if ((match->n <  3) || (order < 0) || (order > 3)) {
+    int Ndof_min = 3;
+    int order_max = 0.5*(sqrt(4*match->n - 4*Ndof_min + 1) - 3);
+    order = PS_MIN (order, order_max);
+
+    // if ((match->n < 17) && (order >= 3)) order = 2;
+    // if ((match->n < 13) && (order >= 2)) order = 1;
+    // if ((match->n <  9) && (order >= 1)) order = 0;
+
+    if (order < 0) {
         psLogMsg ("psastro", 3, "insufficient stars (%ld) or invalid order (%d)", match->n, order);
         return false;
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroOneChipFit.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroOneChipFit.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroOneChipFit.c	(revision 24244)
@@ -64,6 +64,18 @@
         // modify the order to correspond to the actual number of matched stars:
         int Ndof_min = 3;
-        int order_max = 0.5*(3 + sqrt(4*match->n - 4*Ndof_min + 1));
+        int order_max = 0.5*(sqrt(4*match->n - 4*Ndof_min + 1) - 3);
         order = PS_MIN (order, order_max);
+
+	// order 0 : Ro -> nterms = 1 * 2;
+	// order 1 : Ro, Rx, Ry -> nterms = 3 * 2;
+	// order 2 : Ro, Rx, Ry, Rxx, Rxy, Ryy -> nterms = 6 * 2;
+	// order 3 : Ro, Rx, Ry, Rxx, Rxy, Ryy, Rxxx, Rxxy, Rxyy, Ryyy -> nterms = 10 * 2
+	// 2*(N+1)*(N+2)/2 = (N+1)*(N+2) = nterms;
+	// (order+1)(order+2) + ndof = nvalues
+	// order^2 + 3*order + 2 + ndof = nvalue;
+	// order^2 + 3*order + 2 + ndof - nvalue = 0;
+	// 2*order = -3 +/- sqrt (9 - 4*(2 - nvalue + ndof));
+	// 2*order = -3 +/- sqrt (9 - 8 + 4*nvalue - 4*ndof);
+	// 2*order = (sqrt (1 + 4*nvalue - 4*ndof) - 3);
 
         // if ((match->n < 11) && (order >= 3)) order = 2;
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroParseCamera.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroParseCamera.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroParseCamera.c	(revision 24244)
@@ -45,13 +45,4 @@
     psFree (chips);
 
-    psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
-    if (dump_file) {
-        pmConfigCamerasCull(config, NULL);
-        pmConfigRecipesCull(config, "PPIMAGE,PPSTATS,PSPHOT,MASKS,PSASTRO");
-
-        pmConfigDump(config, input->fpa, dump_file);
-    }
-
-
     psTrace("psastro", 1, "Done with psastroParseCamera...\n");
     return true;
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroVersion.c	(revision 24244)
@@ -9,4 +9,5 @@
 
 #include "psastroInternal.h"
+#include "psastroVersionDefinitions.h"
 
 #ifndef PSASTRO_VERSION
@@ -20,11 +21,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString psastroVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PSASTRO_BRANCH), xstr(PSASTRO_VERSION));
+    psStringAppend(&value, "%s@%s", PSASTRO_BRANCH, PSASTRO_VERSION);
     return value;
 }
@@ -32,5 +30,5 @@
 psString psastroSource(void)
 {
-  return psStringCopy(xstr(PSASTRO_SOURCE));
+  return psStringCopy(PSASTRO_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PSASTRO_VERSION_DEFINITIONS_H
+#define PSASTRO_VERSION_DEFINITIONS_H
+
+#define PSASTRO_VERSION @PSASTRO_VERSION@ // SVN version
+#define PSASTRO_BRANCH  @PSASTRO_BRANCH@  // SVN branch
+#define PSASTRO_SOURCE  @PSASTRO_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroZeroPoint.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroZeroPoint.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroZeroPoint.c	(revision 24244)
@@ -126,5 +126,8 @@
     // this analysis has too few data points to use the robust median method
     psStats *stats = psStatsAlloc (PS_STAT_CLIPPED_MEAN | PS_STAT_CLIPPED_STDEV);
-    psVectorStats (stats, dMag, NULL, NULL, 0);
+    if (!psVectorStats (stats, dMag, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
+    }
     fprintf (stderr, "zero point %f +/- %f using %d stars; transparency %f\n", stats->clippedMean, stats->clippedStdev, Npts, zeropt - stats->clippedMean);
 
Index: /branches/cnb_branches/cnb_branch_20090301/psconfig/psbuild
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psconfig/psbuild	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psconfig/psbuild	(revision 24244)
@@ -13,6 +13,6 @@
 $stop = "";
 $verbose = 0;
-$use_svn = 1;
 $svn_trunk = "";
+$offline = 0;
 
 $extlibs = "none";
@@ -90,7 +90,7 @@
         &bootstrap ();
     }
-    if ($ARGV[0] eq "-skip-svn") {
-	$use_svn = 0;
-	shift; next;
+    if ($ARGV[0] eq "-offline") {
+        $offline = 1;
+        shift; next;
     }
     if ($ARGV[0] eq "-env") {
@@ -167,60 +167,4 @@
 sub build_distribution {
 
-    # set environment variables used to supply SVN info to the compilation
-
-    if ($use_svn) {
-	# example dump from svn info:
-	# pikake: svn info
-	# Path: .
-	# URL: https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/branches/eam_branches/eam_branch_20090303/ppImage
-	# Repository Root: https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp
-	# Repository UUID: 60eb6cdc-a59c-4636-a4e0-dba66a9721fd
-	# Revision: 23158
-	# Node Kind: directory
-	# Schedule: normal
-	# Last Changed Author: price
-	# Last Changed Rev: 23125
-	# Last Changed Date: 2009-03-03 15:41:16 -1000 (Tue, 03 Mar 2009)
-	
-	$svn_version = `svnversion`; chomp $svn_version;
-	@svn_info = `svn info`;
-
-	# get the svn_root first:
-	foreach $line (@svn_info) {
-	    if ($line =~ m|^Repository Root:|) {
-		($svn_root) = $line =~ m|^Repository Root:\s*(\S*)|;
-		last;
-	    }
-	}
-
-	# now get the branch and UUID values
-	foreach $line (@svn_info) {
-	    if ($line =~ m|^URL:|) {
-		($svn_branch) = $line =~ m|^URL:\s*$svn_root/*(\S*)|;
-	    }
-	    if ($line =~ m|^Repository UUID:|) {
-		($svn_source) = $line =~ m|^Repository UUID:\s*(\S*)|;
-	    }
-	}
-	
-	$ENV{SVN_VERSION} = $svn_version;
-	$ENV{SVN_BRANCH}  = $svn_branch;
-	$ENV{SVN_SOURCE}  = $svn_source;
-    } else {
-	# alternatively, grab these from the following files:
-	if (! -e "SVNINFO") { 
-	    print "missing SVNINFO file for repository info, skipping\n";
-	} else {
-	    @svn_info = `cat SVNINFO`;
-	    foreach $line (@svn_info) {
-		($name, $value) = split (" ", $line);
-		$ENV{$name} = $value;
-	    }
-	}
-    }
-    print "SVN_VERSION $ENV{SVN_VERSION}\n";
-    print "SVN_BRANCH  $ENV{SVN_BRANCH}\n";
-    print "SVN_SOURCE  $ENV{SVN_SOURCE}\n";
-
     # use psconfig.csh to set needed build aliases
     if ($extlibs eq "check") {
@@ -279,4 +223,5 @@
     if ($optimize) { $psopts = "$psopts --enable-optimize"; }
     if ($profile)  { $psopts = "$psopts --enable-profile --disable-shared --enable-static"; }
+    $psopts .= " --disable-version" if $offline;
 
     $homedir = `pwd`; chomp $homedir;
@@ -299,7 +244,7 @@
 
         if ($svn_trunk && (!-d $workdir || !-r $workdir || !-x $workdir)) {
-	    # try trunk instead
+            # try trunk instead
             print STDERR "$module[$i] missing from local path, trying trunk path\n";
-	    $workdir = "$svn_trunk/$module[$i]";
+            $workdir = "$svn_trunk/$module[$i]";
         }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psconfig/pschecklibs
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psconfig/pschecklibs	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psconfig/pschecklibs	(revision 24244)
@@ -25,5 +25,12 @@
     if ($ARGV[0] eq "-force") {
         if (@ARGV < 2) { die "-force must be coupled to a library name\n"; }
-        $force{lc($ARGV[1])} = 1;
+        if (lc($ARGV[1]) eq 'build') {
+            $force{'autoconf'} = 1;
+            $force{'automake'} = 1;
+            $force{'libtool'} = 1;
+            $force{'pkg-config'} = 1;
+        } else {
+            $force{lc($ARGV[1])} = 1;
+        }
         shift; shift; next;
     }
Index: /branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.7.perl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.7.perl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.7.perl	(revision 24244)
@@ -81,3 +81,4 @@
   77    Filesys::Df                     Filesys-Df-0.92.tar.gz                  0.92
   78    SQL::Interp                     SQL-Interp-1.06.tar.gz
+  79    Log::Dispatch::Email::MailSend  Log-Dispatch-2.22.tar.gz
 
Index: /branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.8.libs
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.8.libs	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.8.libs	(revision 24244)
@@ -18,4 +18,11 @@
 #   make install options
 
+
+# Build tools
+bin autoconf             NONE           NONE   autoconf-2.63.tar.gz     autoconf-2.63    N NONE NONE NONE
+bin automake             NONE           NONE   automake-1.10.tar.gz     automake-1.10    N NONE NONE NONE
+bin libtool              NONE           NONE   libtool-2.2.6a.tar.gz    libtool-2.2.6    N NONE NONE NONE
+bin pkg-config           NONE           NONE   pkg-config-0.23.tar.gz   pkg-config-0.23  N NONE NONE NONE
+
 lib libm                 NONE           NONE   NONE                     NONE             N NONE NONE NONE
 lib libX11               NONE           NONE   NONE                     NONE             N NONE NONE NONE
@@ -27,12 +34,8 @@
 lib libjpeg              NONE           jpeg   jpegsrc.v6b-p1.tar.gz    jpeg-6b          N --enable-shared NONE install-lib
 lib libcfitsio           NONE           NONE   cfitsio3100-pap.tar.gz   cfitsio3100-pap  N --enable-shared shared NONE
-#lib libmysqlclient      NONE           mysql  mysql-5.0.27.tar.gz      mysql-5.0.27     N NONE NONE NONE
 lib libmysqlclient       NONE           mysql  mysql-5.0.51a.tar.gz     mysql-5.0.51a    N NONE NONE NONE
 lib libgsl               NONE           NONE   gsl-1.11.tar.gz          gsl-1.11         N NONE NONE NONE
 lib libfftw3f            NONE           NONE   fftw-3.0.1.tar.gz        fftw-3.0.1       N --enable-float,--enable-shared,--disable-fortran NONE NONE
-#lib libfftw3            NONE           NONE   fftw-3.0.1.tar.gz        fftw-3.0.1       N --enable-shared,--disable-fortran NONE NONE
-# paul claims we are not currently using double-point FFTs anywhere
 
-bin pkg-config           NONE           NONE   pkg-config-0.22.tar.gz   pkg-config-0.22  N NONE NONE NONE
 
 inc X11/Xatom.h          NONE           NONE   NONE                     NONE             N NONE NONE NONE 
@@ -55,5 +58,5 @@
 inc gsl/gsl_rng.h        NONE           NONE   gsl-1.11.tar.gz          gsl-1.11         N NONE NONE NONE 
 inc inttypes.h           NONE           NONE   NONE                     NONE             N NONE NONE NONE 
-inc jpeglib.h            NONE           jpeg   jpegsrc.v6b.tar.gz       jpeg-6b          N --enable-shared NONE install-lib
+inc jpeglib.h            NONE           jpeg   jpegsrc.v6b-p1.tar.gz    jpeg-6b          N --enable-shared NONE install-lib
 inc limits.h             NONE           NONE   NONE                     NONE             N NONE NONE NONE 
 inc malloc.h             NONE           NONE   NONE                     NONE             N NONE NONE NONE 
Index: /branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.8.perl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.8.perl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.8.perl	(revision 24244)
@@ -81,3 +81,3 @@
   77    Filesys::Df                     Filesys-Df-0.92.tar.gz                  0.92
   78    SQL::Interp                     SQL-Interp-1.06.tar.gz
-
+  79    Log::Dispatch::Email::MailSend  Log-Dispatch-2.22.tar.gz
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/configure.ac	(revision 24244)
@@ -204,4 +204,6 @@
 echo "PSPHOT_LIBS: $PSPHOT_LIBS"
 
+IPP_VERSION
+
 AC_SUBST([PSPHOT_CFLAGS])
 AC_SUBST([PSPHOT_LIBS])
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/Makefile.am	(revision 24244)
@@ -1,14 +1,26 @@
 lib_LTLIBRARIES = libpsphot.la
 
-# PSPHOT_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-# PSPHOT_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-# PSPHOT_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
-# 
-# # Force recompilation of psphotVersion.c, since it gets the version information
-# psphotVersion.c: FORCE
-# 	touch psphotVersion.c
-# FORCE: ;
+if HAVE_SVNVERSION
+PSPHOT_VERSION=`$(SVNVERSION) ..`
+else
+PSPHOT_VERSION="UNKNOWN"
+endif
 
-libpsphot_la_CFLAGS = $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPSPHOT_VERSION=$(PSPHOT_VERSION) -DPSPHOT_BRANCH=$(PSPHOT_BRANCH) -DPSPHOT_SOURCE=$(SVN_SOURCE)
+if HAVE_SVN
+PSPHOT_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PSPHOT_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PSPHOT_BRANCH="UNKNOWN"
+PSPHOT_SOURCE="UNKNOWN"
+endif
+
+# Force recompilation of psphotVersion.c, since it gets the version information
+psphotVersion.c: psphotVersionDefinitions.h
+psphotVersionDefinitions.h: psphotVersionDefinitions.h.in FORCE
+	-$(RM) psphotVersionDefinitions.h
+	$(SED) -e "s|@PSPHOT_VERSION@|\"$(PSPHOT_VERSION)\"|" -e "s|@PSPHOT_BRANCH@|\"$(PSPHOT_BRANCH)\"|" -e "s|@PSPHOT_SOURCE@|\"$(PSPHOT_SOURCE)\"|" psphotVersionDefinitions.h.in > psphotVersionDefinitions.h
+FORCE: ;
+
+libpsphot_la_CFLAGS = $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
 libpsphot_la_LDFLAGS = $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 
@@ -129,5 +141,5 @@
 
 # Error codes.
-BUILT_SOURCES = psphotErrorCodes.h psphotErrorCodes.c
+BUILT_SOURCES = psphotErrorCodes.h psphotErrorCodes.c psphotVersionDefinitions.h
 CLEANFILES = psphotErrorCodes.h psphotErrorCodes.c
 EXTRA_DIST = psphotErrorCodes.dat psphotErrorCodes.c.in psphotErrorCodes.h.in \
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphot.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphot.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphot.c	(revision 24244)
@@ -1,8 +1,3 @@
 # include "psphotStandAlone.h"
-
-static void usage (void) {
-    fprintf (stderr, "USAGE: psphot [-file image(s)] [-list imagelist] (output)\n");
-    exit (PS_EXIT_CONFIG_ERROR);
-}
 
 int main (int argc, char **argv) {
@@ -14,8 +9,5 @@
     // load command-line arguments, options, and system config data
     pmConfig *config = psphotArguments (argc, argv);
-    if (!config) {
-        psErrorStackPrint(stderr, "Error reading arguments\n");
-        usage ();
-    }
+    assert(config);
 
     psphotVersionPrint();
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphot.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphot.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphot.h	(revision 24244)
@@ -31,4 +31,6 @@
 bool            psphotReadoutCleanup (pmConfig *config, pmReadout *readout, psMetadata *recipe, pmDetections *detections, pmPSF *psf, psArray *sources);
 bool            psphotDefineFiles (pmConfig *config, pmFPAfile *input);
+void            psphotFilesActivate(pmConfig *config, bool state);
+
 bool            psphotSetMaskBits (pmConfig *config);
 bool            psphotSetMaskRecipe (pmConfig *config, psImageMaskType maskValue, psImageMaskType markValue);
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotApResid.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotApResid.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotApResid.c	(revision 24244)
@@ -355,12 +355,15 @@
 
         if (j > 2) {
-            bool status = true;
-            status &= psVectorStats (statsS, dASubset, NULL, mkSubset, 0xff);
-            status &= psVectorStats (statsM, dMSubset, NULL, mkSubset, 0xff);
-            if (!status) { psErrorClear (); }
+            if (!psVectorStats (statsS, dASubset, NULL, mkSubset, 0xff)) {
+		psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		return false;
+	    }
+            if (!psVectorStats (statsM, dMSubset, NULL, mkSubset, 0xff)) {
+		psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+		return false;
+	    }
             dSo->data.F32[i] = statsS->robustStdev;
             dMo->data.F32[i] = statsM->sampleMean;
             dRo->data.F32[i] = statsS->robustStdev / statsM->sampleMean;
-            //      fprintf (stderr, "%d (%d) : sys: %f, phot: %f, rat: %f\n", i, j, dSo->data.F32[i], dMo->data.F32[i], dRo->data.F32[i]);
         } else {
             dSo->data.F32[i] = NAN;
@@ -375,6 +378,6 @@
     psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN);
     if (!psVectorStats (stats, dRo, NULL, NULL, 0)) {
-        // XXX better testing of raised error
-        psErrorClear();
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats");
+	return false;
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotArguments.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotArguments.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotArguments.c	(revision 24244)
@@ -1,3 +1,99 @@
 # include "psphotStandAlone.h"
+
+static void writeHelpInfo(const char* program, pmConfig* config, FILE* ofile)
+{
+  fprintf(ofile,
+	  "Usage: one of the following\n"
+	  "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n"
+	  "     -variance varfile1[,varfile2,...] OutFileBaseName\n"
+	  "\n"
+	  "%s -list FileNameList [-masklist MaskFileNameList] \n"
+	  "     -variancelist VarFileNameList OutFileBaseName\n"
+	  "\n"
+	  "%s -help\n"
+	  "\n"
+	  "%s -version\n"
+	  "\n"
+	  "where:\n"
+	  "  FileNameList is a text file containing filenames, one per line\n"
+	  "  MaskFileNameList is a text file of mask filenames, one per line\n"
+	  "  VarFileNameList is a text file of variance filenames, one per line\n"
+	  "  OutFileBaseName is the 'root name' for output files\n"
+	  "\n"
+	  "additional options:\n"
+	  "  -modeltest xObj yObj [-model DEFAULT|ModelName] \n"
+	  "        [-fitmode DEFAULT|PSF|CONV] [-fitset FitFileName]\n"
+	  "     Test fit for object at the given coordinates.  ModelName\n"
+	  "     is one of PS_MODEL_GAUSS, PS_MODEL_PGAUSS, PS_MODEL_QGAUSS,\n"
+	  "     PS_MODEL_RGAUSS, PS_MODEL_PS1_V1, or PS_MODEL_SERSIC.\n"
+	  "     FitFileName is a file of x,y,Io triples.\n"
+	  "  -psf PsfFile1[,PsfFile2,...] or -psflist PsfFileNameList\n"
+	  "     specify PSF rather than letting %s estimate it\n"
+	  "  -src SrcFile1[,SrcFile2,...] or -srclist SrcFileNameList\n"
+	  "     specify additional sources for PSF generation\n"
+	  "  -chip nn[,nn,...]\n"
+	  "     select detector chips to process; default is all.\n"
+	  "     Indices correspond to zero-based offset in the FPA metadata table.\n"
+	  "  -photcode PhotoCodeName\n"
+	  "     specify photocode\n"
+	  "  -region RegionString\n"
+	  "     specify analysis region.  String is of form '[x0:x1,y0:y1]'\n"
+	  "     To use this option you must define a default in psphot.config\n"
+	  "  -visual\n"
+	  "     turns on interactive display mode\n"
+	  "  -dumpconfig CfgFileName\n"
+          "     causes config info to be dumped to the named file.\n"
+	  "  -break NOTHING|BACKMDL|PEAKS|MOMENTS|PSFMODEL|ENSEMBLE|PASS1\n"
+	  "     choose a point at which to exit processing early\n"
+	  "  -nthreads n\n"
+	  "     set number of parallel threads of execution\n"
+	  "  -F OldFileRule ReplacementFileRule\n"
+	  "     change file naming rule; e.g. '-F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF'\n"
+	  "  -D name stringval\n"
+	  "     set a string-valued config parameter\n"
+	  "  -Di name intval\n"
+	  "     set an integer-valued config parameter\n"
+	  "  -Df name fval\n"
+	  "     set a float-valued config parameter\n"
+	  "  -Db name boolval\n"
+	  "     set a boolean-valued config parameter\n"
+	  "  -v, -vv, -vvv\n"
+	  "     set increasing levels of verbosity\n"
+	  "  -logfmt FormatString\n"
+	  "     set format string used for log messages\n"
+	  "  -trace Fac Lvl\n"
+	  "     set tracing for facility Fac to integer Lvl, e.g. '-trace err 10'\n"
+	  "  -trace-levels\n"
+	  "     print current trace levels\n",
+	  program,program,program,program,program);
+    psFree(config);
+    pmConfigDone();
+    psLibFinalize();
+    exit(PS_EXIT_SUCCESS);
+}
+
+static void usage(const char *program,  // Name of the program
+                  psMetadata *arguments, // Command-line arguments
+                  pmConfig *config,      // Configuration
+		  int exitCode
+		  ) 
+{
+  fprintf(stderr,
+	  "Usage: one of the following\n"
+	  "%s -file fname1[,fname2,...] -mask maskfile1[,maskfile2,...]\n"
+	  "     -variance varfile1[,varfile2,...] OutFileBaseName\n"
+	  "\n"
+	  "%s -list FileNameList [-masklist MaskFileNameList] \n"
+	  "     -variancelist VarFileNameList OutFileBaseName\n"
+	  "\n"
+	  "Try '%s -help' for more options and explanation\n",
+	  program,program,program);
+    if (exitCode != PS_EXIT_SUCCESS)
+      psErrorStackPrint(stderr, "Error reading arguments\n");
+    psFree(config);
+    pmConfigDone();
+    psLibFinalize();
+    exit(exitCode);
+}
 
 pmConfig *psphotArguments(int argc, char **argv) {
@@ -5,17 +101,4 @@
     int N;
     bool status;
-
-    if (argc == 1) {
-        psError(PSPHOT_ERR_ARGUMENTS, true, "Too few arguments: %d", argc);
-        return NULL;
-    }
-
-    if ((N = psArgumentGet (argc, argv, "-version"))) {
-        psString version;
-        version = psphotVersionLong();    fprintf (stdout, "%s\n", version); psFree (version);
-        version = psModulesVersionLong(); fprintf (stdout, "%s\n", version); psFree (version);
-        version = psLibVersionLong();     fprintf (stdout, "%s\n", version); psFree (version);
-        exit (0);
-    }
 
     // load config data from default locations
@@ -23,6 +106,8 @@
     if (config == NULL) {
         psError(PSPHOT_ERR_CONFIG, false, "Can't read site configuration");
-        return NULL;
-    }
+	exit(PS_EXIT_CONFIG_ERROR);
+    }
+
+    PSARGUMENTS_INSTANTIATE_GENERICS( psphot, config, argc, argv );
 
     // save the following additional recipe values based on command-line options
@@ -30,18 +115,15 @@
     psMetadata *options = pmConfigRecipeOptions (config, PSPHOT_RECIPE);
 
-    // Number of threads
-    if ((N = psArgumentGet(argc, argv, "-threads"))) {
-        psArgumentRemove(N, &argc, argv);
-        int nThreads = atoi(argv[N]);
-        psMetadataAddS32(config->arguments, PS_LIST_TAIL, "NTHREADS", 0, "number of psphot threads", nThreads);
-        psArgumentRemove(N, &argc, argv);
-
-        // create the thread pool with number of desired threads, supplying our thread launcher function
-        psThreadPoolInit (nThreads);
-    }
-    psphotSetThreads();
+    // Number of threads is handled
+    PSARGUMENTS_INSTANTIATE_THREADSARG( psphot, config, argc, argv )
 
     // run the test model (requires X,Y coordinate)
     if ((N = psArgumentGet (argc, argv, "-modeltest"))) {
+        if (argc<=N+2) {
+          psError(PSPHOT_ERR_ARGUMENTS, true, 
+		  "Expected to see 2 more arguments; saw %d", argc - 1);
+	  exit(PS_EXIT_CONFIG_ERROR);
+	}
+
         psMetadataAddBool (options, PS_LIST_TAIL, "TEST_FIT",   0, "", true);
         psMetadataAddF32  (options, PS_LIST_TAIL, "TEST_FIT_X", 0, "", atof(argv[N+1]));
@@ -54,4 +136,9 @@
         // specify the modeltest model
         if ((N = psArgumentGet (argc, argv, "-model"))) {
+	    if (argc<=N+1) {
+	      psError(PSPHOT_ERR_ARGUMENTS, true, 
+		      "Expected to see 1 more argument; saw %d", argc - 1);
+	      exit(PS_EXIT_CONFIG_ERROR);
+	    }
             psArgumentRemove (N, &argc, argv);
             psMetadataAddStr (options, PS_LIST_TAIL, "TEST_FIT_MODEL", 0, "", argv[N]);
@@ -61,4 +148,9 @@
         // specify the test fit mode
         if ((N = psArgumentGet (argc, argv, "-fitmode"))) {
+	    if (argc<=N+1) {
+	      psError(PSPHOT_ERR_ARGUMENTS, true, 
+		      "Expected to see 1 more argument; saw %d", argc - 1);
+	      exit(PS_EXIT_CONFIG_ERROR);
+	    }
             psArgumentRemove (N, &argc, argv);
             psMetadataAddStr (options, PS_LIST_TAIL, "TEST_FIT_MODE", 0, "", argv[N]);
@@ -66,4 +158,9 @@
         }
         if ((N = psArgumentGet (argc, argv, "-fitset"))) {
+	    if (argc<=N+1) {
+	      psError(PSPHOT_ERR_ARGUMENTS, true, 
+		      "Expected to see 1 more argument; saw %d", argc - 1);
+	      exit(PS_EXIT_CONFIG_ERROR);
+	    }
             psArgumentRemove (N, &argc, argv);
             psMetadataAddStr (options, PS_LIST_TAIL, "TEST_FIT_SET", 0, "", argv[N]);
@@ -74,4 +171,9 @@
     // photcode : used in output to supplement header data (argument or recipe?)
     if ((N = psArgumentGet (argc, argv, "-photcode"))) {
+        if (argc<=N+1) {
+	  psError(PSPHOT_ERR_ARGUMENTS, true, 
+		  "Expected to see 1 more argument; saw %d", argc - 1);
+	  exit(PS_EXIT_CONFIG_ERROR);
+	}
         psArgumentRemove (N, &argc, argv);
         psMetadataAddStr (options, PS_LIST_TAIL, "PHOTCODE", PS_META_REPLACE, "", argv[N]);
@@ -87,13 +189,11 @@
     // break : used from recipe throughout psphotReadout
     if ((N = psArgumentGet (argc, argv, "-break"))) {
+	if (argc<=N+1) {
+	  psError(PSPHOT_ERR_ARGUMENTS, true, 
+		  "Expected to see 1 more argument; saw %d", argc - 1);
+	  exit(PS_EXIT_CONFIG_ERROR);
+	}
         psArgumentRemove (N, &argc, argv);
         psMetadataAddStr (options, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, "", argv[N]);
-        psArgumentRemove (N, &argc, argv);
-    }
-
-    // fitmode : used from recipe throughout psphotReadout
-    if ((N = psArgumentGet (argc, argv, "-fitmode"))) {
-        psArgumentRemove (N, &argc, argv);
-        psMetadataAddStr (options, PS_LIST_TAIL, "FITMODE", PS_META_REPLACE, "", argv[N]);
         psArgumentRemove (N, &argc, argv);
     }
@@ -101,4 +201,9 @@
     // analysis region : overrides recipe value, used in psphotReadout/psphotEnsemblePSF
     if ((N = psArgumentGet (argc, argv, "-region"))) {
+	if (argc<=N+1) {
+	  psError(PSPHOT_ERR_ARGUMENTS, true, 
+		  "Expected to see 1 more argument; saw %d", argc - 1);
+	  exit(PS_EXIT_CONFIG_ERROR);
+	}
         psArgumentRemove (N, &argc, argv);
         psMetadataAddStr (options, PS_LIST_TAIL, "ANALYSIS_REGION", 0, "", argv[N]);
@@ -108,4 +213,9 @@
     // chip selection is used to limit chips to be processed
     if ((N = psArgumentGet (argc, argv, "-chip"))) {
+	if (argc<=N+1) {
+	  psError(PSPHOT_ERR_ARGUMENTS, true, 
+		  "Expected to see 1 more argument; saw %d", argc - 1);
+	  exit(PS_EXIT_CONFIG_ERROR);
+	}
         psArgumentRemove (N, &argc, argv);
         psMetadataAddStr (config->arguments, PS_LIST_TAIL, "CHIP_SELECTIONS", PS_DATA_STRING, "", argv[N]);
@@ -121,14 +231,23 @@
     pmConfigFileSetsMD (config->arguments, &argc, argv, "SRC",    "-src",    "-srclist");
 
+    if (argc == 1) {
+        psError(PSPHOT_ERR_ARGUMENTS, true, "Too few arguments: %d", argc);
+	usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR);
+    }
+
+    if (psArgumentGet(argc, argv, "-help")
+	|| psArgumentGet(argc, argv, "-h"))
+      writeHelpInfo(argv[0], config, stdout);
+      
     // the input file is a required argument; if not found, we will exit
     status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list");
     if (!status) {
         psError(PSPHOT_ERR_ARGUMENTS, false, "pmConfigFileSetsMD failed to parse arguments");
-        return NULL;
+	usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR);
     }
 
     if (argc != 2) {
         psError(PSPHOT_ERR_ARGUMENTS, true, "Expected to see one more argument; saw %d", argc - 1);
-        return NULL;
+	usage(argv[0], config->arguments, config, PS_EXIT_CONFIG_ERROR);
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotBlendFit.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotBlendFit.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotBlendFit.c	(revision 24244)
@@ -197,5 +197,5 @@
         if (source->type == PM_SOURCE_TYPE_SATURATED) continue;
 
-        // skip DBL second sources (ie, added by psphotFitBlob)
+        // skip DBL second sources (ie, added by psphotFitBlob
         if (source->mode &  PM_SOURCE_MODE_PAIR) continue;
 
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotChoosePSF.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotChoosePSF.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotChoosePSF.c	(revision 24244)
@@ -361,5 +361,9 @@
 
     psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_SAMPLE_QUARTILE);
-    psVectorStats (stats, fwhmMajor, NULL, NULL, 0);
+    if (!psVectorStats (stats, fwhmMajor, NULL, NULL, 0)) {
+        psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MAJOR");
+        return false;
+    }
+
     psMetadataAddF32 (recipe, PS_LIST_TAIL, "FWHM_MAJ",   PS_META_REPLACE, "PSF FWHM Major axis (mean)", stats->sampleMean);
     psMetadataAddF32 (recipe, PS_LIST_TAIL, "FW_MJ_SG",   PS_META_REPLACE, "PSF FWHM Major axis (sigma)", stats->sampleStdev);
@@ -367,5 +371,8 @@
     psMetadataAddF32 (recipe, PS_LIST_TAIL, "FW_MJ_UQ",   PS_META_REPLACE, "PSF FWHM Major axis (upper quartile)", stats->sampleUQ);
 
-    psVectorStats (stats, fwhmMinor, NULL, NULL, 0);
+    if (!psVectorStats (stats, fwhmMinor, NULL, NULL, 0)) {
+        psError(PS_ERR_UNKNOWN, false, "failure to measure stats for FWHM MINOR");
+        return false;
+    }
     psMetadataAddF32 (recipe, PS_LIST_TAIL, "FWHM_MIN",   PS_META_REPLACE, "PSF FWHM Minor axis (mean)", stats->sampleMean);
     psMetadataAddF32 (recipe, PS_LIST_TAIL, "FW_MN_SG",   PS_META_REPLACE, "PSF FWHM Minor axis (sigma)", stats->sampleStdev);
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotCleanup.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotCleanup.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotCleanup.c	(revision 24244)
@@ -2,4 +2,13 @@
 
 void psphotCleanup (pmConfig *config) {
+
+    // Dump configuration if requested
+    bool status;
+    psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
+    if (dump_file) {
+        (void)pmConfigCamerasCull(config,NULL);
+	(void)pmConfigRecipesCull(config,NULL);
+        pmConfigDump(config, dump_file);
+    }
 
     psFree (config);
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotDefineFiles.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotDefineFiles.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotDefineFiles.c	(revision 24244)
@@ -1,3 +1,9 @@
 # include "psphotInternal.h"
+
+// List of output files
+static const char *outputFiles[] = { "PSPHOT.OUTPUT", "PSPHOT.RESID", "PSPHOT.BACKMDL",
+                                     "PSPHOT.BACKMDL.STDEV", "PSPHOT.BACKGND", "PSPHOT.BACKSUB",
+                                     "PSPHOT.PSF.SAVE", "SOURCE.PLOT.MOMENTS", "SOURCE.PLOT.PSFMODEL",
+                                     "SOURCE.PLOT.APRESID", NULL };
 
 // XXX we need to be able to distinguish several cases:
@@ -133,2 +139,13 @@
     return true;
 }
+
+void psphotFilesActivate(pmConfig *config, bool state)
+{
+    for (int i = 0; outputFiles[i]; i++) {
+        if (!pmFPAfileActivate(config->files, state, outputFiles[i])) {
+            psErrorClear();
+        }
+    }
+
+    return;
+}
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotDiagnosticPlots.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotDiagnosticPlots.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotDiagnosticPlots.c	(revision 24244)
@@ -40,5 +40,8 @@
 
     psStats *stats = psStatsAlloc (PS_STAT_MAX | PS_STAT_MIN);
-    psVectorStats (stats, values, NULL, NULL, 0);
+    if (!psVectorStats (stats, values, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats for histogram");
+        return false;
+    }
 
     psHistogram *histogram = psHistogramAlloc (stats->min, stats->max, 1000);
@@ -63,5 +66,8 @@
     psStatsInit (stats);
     stats->options = PS_STAT_MAX | PS_STAT_MIN;
-    psVectorStats (stats, histogram->nums, NULL, NULL, 0);
+    if (!psVectorStats (stats, histogram->nums, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats for histogram");
+        return false;
+    }
 
     // scale the plot to hold the histogram
@@ -100,5 +106,8 @@
     psStatsInit (stats);
     stats->options = PS_STAT_MAX | PS_STAT_MIN;
-    psVectorStats (stats, histogram->nums, NULL, NULL, 0);
+    if (!psVectorStats (stats, histogram->nums, NULL, NULL, 0)) {
+	psError(PS_ERR_UNKNOWN, false, "failure to measure stats for histogram");
+        return false;
+    }
 
     // scale the plot to hold the histogram
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotErrorCodes.h.in	(revision 24244)
@@ -9,5 +9,5 @@
  */
 typedef enum {
-    PSPHOT_ERR_BASE = 1300,
+    PSPHOT_ERR_BASE = 3000,
     PSPHOT_ERR_${ErrorCode},
     PSPHOT_ERR_NERROR
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotImageLoop.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotImageLoop.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotImageLoop.c	(revision 24244)
@@ -75,4 +75,11 @@
                 }
 
+                psImageMaskType maskSat = pmConfigMaskGet("SAT", config); // Mask value for saturated pixels
+                if (!pmReadoutMaskNonfinite(readout, maskSat)) {
+                    psError(psErrorCodeLast(), false, "Unable to mask non-finite pixels.");
+                    psFree(view);
+                    return false;
+                }
+
                 // run the actual photometry analysis on this chip/cell/readout
                 if (!psphotReadout (config, view)) {
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotImageQuality.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotImageQuality.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotImageQuality.c	(revision 24244)
@@ -88,4 +88,5 @@
                      "Number of stars used for IQ measurements", M2->n);
 
+// XXX make this a recipe option
 #if (USE_SAMPLE)
     psStats *stats = psStatsAlloc (PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV | PS_STAT_SAMPLE_QUARTILE);
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotMakeResiduals.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotMakeResiduals.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotMakeResiduals.c	(revision 24244)
@@ -200,6 +200,15 @@
             // measure the robust median to determine a baseline reference value
             *fluxClip = *fluxClipDef;
-            psVectorStats (fluxClip, fluxes, NULL, fmasks, fmaskVal);
-            psErrorClear();             // clear (ignore) any outstanding errors
+            if (!psVectorStats (fluxClip, fluxes, NULL, fmasks, fmaskVal)) {
+		psError(PSPHOT_ERR_CONFIG, false, "Error calculating residual stats");
+		return false;
+	    }
+	    if (isnan(fluxClip->robustMedian)) {
+                resid->Ro->data.F32[oy][ox] = 0.0;
+                resid->Rx->data.F32[oy][ox] = 0.0;
+                resid->Ry->data.F32[oy][ox] = 0.0;
+                resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = badMask;
+                continue;
+	    }
 
             // mark input pixels which are more than N sigma from the median
@@ -220,10 +229,19 @@
                 // measure the desired statistic on the unclipped pixels
                 *fluxStats = *fluxStatsDef;
-                psVectorStats (fluxStats, fluxes, NULL, fmasks, fmaskVal);
-                psErrorClear();         // clear (ignore) any outstanding errors
+                if (!psVectorStats (fluxStats, fluxes, NULL, fmasks, fmaskVal)) {
+		    psError(PSPHOT_ERR_CONFIG, false, "Error calculating residual stats");
+		    return false;
+		}
 
                 resid->Ro->data.F32[oy][ox] = psStatsGetValue(fluxStats, statOption);
                 resid->Rx->data.F32[oy][ox] = resid->Ry->data.F32[oy][ox] = 0.0;
-                //resid->variance->data.F32[oy][ox] = fluxStats->sampleStdev;
+
+		if (isnan(resid->Ro->data.F32[oy][ox])) {
+		    resid->Ro->data.F32[oy][ox] = 0.0;
+		    resid->Rx->data.F32[oy][ox] = 0.0;
+		    resid->Ry->data.F32[oy][ox] = 0.0;
+		    resid->mask->data.PM_TYPE_RESID_MASK_DATA[oy][ox] = badMask;
+		    continue;
+		}
 
                 if (fabs(resid->Ro->data.F32[oy][ox]) < pixelSN*fluxStats->sampleStdev/sqrt(nKeep)) {
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotMaskReadout.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotMaskReadout.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotMaskReadout.c	(revision 24244)
@@ -39,8 +39,8 @@
       psImage *mk = readout->mask;
       for (int j = 0; j < im->numRows; j++) {
-	for (int i = 0; i < im->numCols; i++) {
-	  if (isfinite(im->data.F32[j][i]) && isfinite(wt->data.F32[j][i])) continue;
-	  mk->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskBad;
-	}
+        for (int i = 0; i < im->numCols; i++) {
+          if (isfinite(im->data.F32[j][i]) && isfinite(wt->data.F32[j][i])) continue;
+          mk->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskBad;
+        }
       }
     }
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotModelBackground.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotModelBackground.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotModelBackground.c	(revision 24244)
@@ -189,9 +189,9 @@
 
             if (gotX && gotY) {
-                psTraceSetLevel ("psLib.math.vectorFittedStats_v4", 6);
-                psTraceSetLevel ("psLib.math.vectorRobustStats", 6);
+                (void) psTraceSetLevel ("psLib.math.vectorFittedStats_v4", 6);
+                (void) psTraceSetLevel ("psLib.math.vectorRobustStats", 6);
             } else {
-                psTraceSetLevel ("psLib.math.vectorFittedStats_v4", 0);
-                psTraceSetLevel ("psLib.math.vectorRobustStats", 0);
+                (void) psTraceSetLevel ("psLib.math.vectorFittedStats_v4", 0);
+                (void) psTraceSetLevel ("psLib.math.vectorRobustStats", 0);
             }
             # endif
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadout.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadout.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadout.c	(revision 24244)
@@ -61,5 +61,5 @@
     // display the backsub and backgnd images
     psphotVisualShowBackground (config, view, readout);
-
+    
     // run a single-model test if desired (exits from here if test is run)
     psphotModelTest (config, view, recipe);
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadoutCleanup.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadoutCleanup.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadoutCleanup.c	(revision 24244)
@@ -9,11 +9,11 @@
     if (psErrorCodeLast() == PSPHOT_ERR_DATA) {
         psErrorStackPrint(stderr, "Error in the psphot readout analysis");
-	psErrorClear();
-    } 
-    if (psErrorCodeLast() != PS_ERR_NONE) { 
-	psFree (psf);
-	psFree (sources);
-	psFree (detections);
-	return false;
+        psErrorClear();
+    }
+    if (psErrorCodeLast() != PS_ERR_NONE) {
+        psFree (psf);
+        psFree (sources);
+        psFree (detections);
+        return false;
     }
 
@@ -22,7 +22,7 @@
         if (!psphotPSFstats (readout, recipe, psf)) {
             psError(PSPHOT_ERR_PROG, false, "Failed to measure PSF shape parameters");
-	    psFree (psf);
-	    psFree (sources);
-	    psFree (detections);
+            psFree (psf);
+            psFree (sources);
+            psFree (detections);
             return false;
         }
@@ -32,10 +32,24 @@
         if (!psphotMomentsStats (readout, recipe, sources)) {
             psError(PSPHOT_ERR_PROG, false, "Failed to measure Moment shape parameters");
-	    psFree (psf);
-	    psFree (sources);
-	    psFree (detections);
+            psFree (psf);
+            psFree (sources);
+            psFree (detections);
             return false;
         }
     }
+
+    // Check to see if the image quality was measured
+    if (!psf) {
+        bool mdok;                      // Status of MD lookup
+        int nIQ = psMetadataLookupS32(&mdok, recipe, "IQ_NSTAR"); // Number of stars for IQ measurement
+        if (!mdok || nIQ <= 0) {
+            psError(PSPHOT_ERR_DATA, false, "Unable to measure image quality");
+            psFree (psf);
+            psFree (sources);
+            psFree (detections);
+            return false;
+        }
+    }
+
 
     // write NSTARS to the image header
@@ -48,19 +62,19 @@
     psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSPHOT.HEADER",  PS_DATA_METADATA, "header stats", header);
     if (sources) {
-	psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSPHOT.SOURCES", PS_DATA_ARRAY, "psphot sources", sources);
+        psMetadataAdd (readout->analysis, PS_LIST_TAIL, "PSPHOT.SOURCES", PS_DATA_ARRAY, "psphot sources", sources);
     }
     if (psf) {
-	// save the psf for possible output.  if there was already an entry, it was loaded from external sources
-	// the new one may have been updated or modified, so replace the existing entry.  We
-	// are required to save it on the chip, but this will cause problems if we ever want to
-	// run psphot on an unmosaiced image
-	pmCell *cell = readout->parent;
-	pmChip *chip = cell->parent;
-	psMetadataAdd (chip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN | PS_META_REPLACE,  "psphot psf", psf);
+        // save the psf for possible output.  if there was already an entry, it was loaded from external sources
+        // the new one may have been updated or modified, so replace the existing entry.  We
+        // are required to save it on the chip, but this will cause problems if we ever want to
+        // run psphot on an unmosaiced image
+        pmCell *cell = readout->parent;
+        pmChip *chip = cell->parent;
+        psMetadataAdd (chip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN | PS_META_REPLACE,  "psphot psf", psf);
     }
 
     if (psErrorCodeLast() != PS_ERR_NONE) {
         psErrorStackPrint(stderr, "unexpected remaining errors");
-	abort();
+        abort();
     }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotRoughClass.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotRoughClass.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotRoughClass.c	(revision 24244)
@@ -91,5 +91,5 @@
 	return false;
     }
-    if (!psfClump.X || !psfClump.Y) {
+    if (!psfClump.X || !psfClump.Y || isnan(psfClump.X) || isnan(psfClump.Y)) {
 	psLogMsg ("psphot", 4, "Failed to find a valid PSF clump for region %f,%f - %f,%f\n", region->x0, region->y0, region->x1, region->y1);
 	return false;
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotSourceFits.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotSourceFits.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotSourceFits.c	(revision 24244)
@@ -234,8 +234,10 @@
     pmSource *tmpSrc = pmSourceAlloc ();
 
+    // XXX need to handle failures better here
     pmModel *EXT = psphotFitEXT (readout, source, modelTypeEXT, maskVal, markVal);
     okEXT = psphotEvalEXT (tmpSrc, EXT);
-    chiEXT = EXT->chisq / EXT->nDOF;
-
+    chiEXT = EXT ? EXT->chisq / EXT->nDOF : NAN;
+
+    // DBL will always be defined, but DBL->data[n] might not
     psArray *DBL = psphotFitDBL (readout, source, maskVal, markVal);
     okDBL  = psphotEvalDBL (tmpSrc, DBL->data[0]);
@@ -244,19 +246,22 @@
 
     // correct first model chisqs for flux trend
+    chiDBL = NAN;
     ONE = DBL->data[0];
-    chiTrend = psPolynomial1DEval (psf->ChiTrend, ONE->params->data.F32[1]);
-    ONE->chisqNorm = ONE->chisq / chiTrend;
-
-    // save chisq for double-star/galaxy comparison
-    chiDBL = ONE->chisq / ONE->nDOF;
+    if (ONE) {
+      chiTrend = psPolynomial1DEval (psf->ChiTrend, ONE->params->data.F32[1]);
+      ONE->chisqNorm = ONE->chisq / chiTrend;
+      chiDBL = ONE->chisq / ONE->nDOF; // save chisq for double-star/galaxy comparison
+    }
 
     // correct second model chisqs for flux trend
     ONE = DBL->data[1];
-    chiTrend = psPolynomial1DEval (psf->ChiTrend, ONE->params->data.F32[1]);
-    ONE->chisqNorm = ONE->chisq / chiTrend;
+    if (ONE) {
+      chiTrend = psPolynomial1DEval (psf->ChiTrend, ONE->params->data.F32[1]);
+      ONE->chisqNorm = ONE->chisq / chiTrend;
+    }
 
     psFree (tmpSrc);
 
-    // psTraceSetLevel("psModules.objects.pmSourceFitSet", 0);
+    // (void) psTraceSetLevel("psModules.objects.pmSourceFitSet", 0);
 
     if (okEXT && okDBL) {
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotTest.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotTest.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotTest.c	(revision 24244)
@@ -50,5 +50,5 @@
     }
 
-    psTraceSetLevel ("psLib.sys.mutex", 3);
+    (void) psTraceSetLevel ("psLib.sys.mutex", 3);
 
     int nThreads = atoi (argv[2]);
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotVersion.c	(revision 24244)
@@ -1,3 +1,4 @@
 #include "psphotInternal.h"
+#include "psphotVersionDefinitions.h"
 
 #ifdef HAVE_KAPA
@@ -15,11 +16,8 @@
 #endif
 
-#define xstr(s) str(s)
-#define str(s) #s
-
 psString psphotVersion(void)
 {
     char *value = NULL;
-    psStringAppend(&value, "%s@%s", xstr(PSPHOT_BRANCH), xstr(PSPHOT_VERSION));
+    psStringAppend(&value, "%s@%s", PSPHOT_BRANCH, PSPHOT_VERSION);
     return value;
 }
@@ -27,5 +25,5 @@
 psString psphotSource(void)
 {
-    return psStringCopy(xstr(PSPHOT_SOURCE));
+    return psStringCopy(PSPHOT_SOURCE);
 }
 
Index: /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PSPHOT_VERSION_DEFINITIONS_H
+#define PSPHOT_VERSION_DEFINITIONS_H
+
+#define PSPHOT_VERSION @PSPHOT_VERSION@ // SVN version
+#define PSPHOT_BRANCH  @PSPHOT_BRANCH@  // SVN branch
+#define PSPHOT_SOURCE  @PSPHOT_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/pstamp/src/pstampErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pstamp/src/pstampErrorCodes.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pstamp/src/pstampErrorCodes.dat	(revision 24244)
@@ -2,6 +2,6 @@
 # This file is used to generate pstampErrorCodes.h
 #
-BASE = 800		First value we use; lower values belong to psLib
-UNKNOWN			Unknown PM error code
+BASE = 10000		First value we use; lower values belong to psLib
+UNKNOWN			Unknown pstamp error code
 NOT_IMPLEMENTED		Desired feature is not yet implemented
 ARGUMENTS		Incorrect arguments
Index: /branches/cnb_branches/cnb_branch_20090301/pstamp/src/pstampErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pstamp/src/pstampErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pstamp/src/pstampErrorCodes.h.in	(revision 24244)
@@ -9,5 +9,5 @@
  */
 typedef enum {
-    PSTAMP_ERR_BASE = 650,
+    PSTAMP_ERR_BASE = 10000,
     PSTAMP_ERR_${ErrorCode},
     PSTAMP_ERR_NERROR
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/configure.ac
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/configure.ac	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/configure.ac	(revision 24244)
@@ -37,4 +37,6 @@
 echo "PSWARP_LIBS: $PSWARP_LIBS"
 
+IPP_VERSION
+
 AC_SUBST([PSWARP_CFLAGS])
 AC_SUBST([PSWARP_LIBS])
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/src/Makefile.am
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/Makefile.am	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/src/Makefile.am	(revision 24244)
@@ -1,14 +1,26 @@
 bin_PROGRAMS = pswarp
 
-PSWARP_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
-PSWARP_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
-PSWARP_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+if HAVE_SVNVERSION
+PSWARP_VERSION=`$(SVNVERSION) ..`
+else
+PSWARP_VERSION="UNKNOWN"
+endif
+
+if HAVE_SVN
+PSWARP_BRANCH=`$(SVN) info .. | $(SED) -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'`
+PSWARP_SOURCE=`$(SVN) info | $(SED) -n -e 's/Repository UUID: // p'`
+else
+PSWARP_BRANCH="UNKNOWN"
+PSWARP_SOURCE="UNKNOWN"
+endif
 
 # Force recompilation of pswarpVersion.c, since it gets the version information
-pswarpVersion.c: FORCE
-	touch pswarpVersion.c
+pswarpVersion.c: pswarpVersionDefinitions.h
+pswarpVersionDefinitions.h: pswarpVersionDefinitions.h.in FORCE
+	-$(RM) pswarpVersionDefinitions.h
+	$(SED) -e "s|@PSWARP_VERSION@|\"$(PSWARP_VERSION)\"|" -e "s|@PSWARP_BRANCH@|\"$(PSWARP_BRANCH)\"|" -e "s|@PSWARP_SOURCE@|\"$(PSWARP_SOURCE)\"|" pswarpVersionDefinitions.h.in > pswarpVersionDefinitions.h
 FORCE: ;
 
-pswarp_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PSWARP_CFLAGS) -DPSWARP_VERSION=\"$(PSWARP_VERSION)\" -DPSWARP_BRANCH=\"$(PSWARP_BRANCH)\" -DPSWARP_SOURCE=\"$(PSWARP_SOURCE)\"
+pswarp_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PSWARP_CFLAGS)
 pswarp_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PPSTATS_LIBS) $(PSPHOT_LIBS) $(PSWARP_LIBS)
 
@@ -24,5 +36,5 @@
 	pswarpMatchRange.c		\
 	pswarpParseCamera.c		\
-	pswarpPixelFraction.c		\
+	pswarpPixelsLit.c		\
 	pswarpSetMaskBits.c		\
 	pswarpSetThreads.c	        \
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarp.h
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarp.h	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarp.h	(revision 24244)
@@ -102,8 +102,8 @@
                           const char *filename, const char *argname);
 
-/// Check to see if the readout has a minimum fraction of "lit" pixels
-bool pswarpPixelFraction(const pmReadout *readout, ///< Readout to inspect
-                         psMetadata *stats, ///< Statistics to update with the result
-                         const pmConfig *config ///< Configuration
+/// Get the range of lit pixels
+bool pswarpPixelsLit(const pmReadout *readout, ///< Readout to inspect
+                     psMetadata *stats, ///< Statistics to update with the result
+                     const pmConfig *config ///< Configuration
     );
 
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpArguments.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpArguments.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpArguments.c	(revision 24244)
@@ -163,10 +163,4 @@
     }
 
-    float acceptFrac = psMetadataLookupF32(&status, recipe, "ACCEPT.FRAC"); ///< Min fraction of good pixels
-    if (!status) {
-        acceptFrac = 0.0;
-        psWarning("ACCEPT.FRAC is not set in the %s recipe --- defaulting to %f.", PSWARP_RECIPE, poorFrac);
-    }
-
     // Set recipe values in the recipe (since we've possibly altered some)
     psMetadataAddS32(recipe, PS_LIST_TAIL, "GRID.NX", PS_META_REPLACE,
@@ -180,6 +174,4 @@
     psMetadataAddF32(recipe, PS_LIST_TAIL, "POOR.FRAC", PS_META_REPLACE,
                      "Fraction of bad flux for a pixel to be marked as poor", poorFrac);
-    psMetadataAddF32(recipe, PS_LIST_TAIL, "ACCEPT.FRAC", PS_META_REPLACE,
-                     "Minimum fraction of good pixels for result to be accepted", acceptFrac);
 
     // Set recipe values in the arguments
@@ -194,6 +186,4 @@
     psMetadataAddF32(config->arguments, PS_LIST_TAIL, "POOR.FRAC", 0,
                      "Fraction of bad flux for a pixel to be marked as poor", poorFrac);
-    psMetadataAddF32(config->arguments, PS_LIST_TAIL, "ACCEPT.FRAC", 0,
-                     "Minimum fraction of good pixels for result to be accepted", acceptFrac);
 
     psTrace("pswarp", 1, "Done with pswarpArguments...\n");
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpErrorCodes.dat
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpErrorCodes.dat	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpErrorCodes.dat	(revision 24244)
@@ -2,6 +2,6 @@
 # This file is used to generate pswarpErrorClasses.h
 #
-BASE = 800		First value we use; lower values belong to psLib
-UNKNOWN			Unknown PM error code
+BASE = 8000		First value we use; lower values belong to psLib
+UNKNOWN			Unknown PSWARP error code
 NOT_IMPLEMENTED		Desired feature is not yet implemented
 ARGUMENTS		Incorrect arguments
@@ -9,2 +9,3 @@
 IO			Problem in FITS I/O
 DATA                    Problem in data values
+NO_OVERLAP		No overlap between input and skycell
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpErrorCodes.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpErrorCodes.h.in	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpErrorCodes.h.in	(revision 24244)
@@ -21,5 +21,5 @@
  */
 typedef enum {
-    PSWARP_ERR_BASE = 600,
+    PSWARP_ERR_BASE = 8000,
     PSWARP_ERR_${ErrorCode},
     PSWARP_ERR_NERROR
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpLoop.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpLoop.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpLoop.c	(revision 24244)
@@ -142,4 +142,5 @@
         psFree(resolved);
         stats = psMetadataAlloc();
+        psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", 0, "No problems", 0);
     }
 
@@ -273,22 +274,32 @@
     }
 
+    if (!output->data_exists) {
+        psWarning("No overlap between input and skycell.");
+        if (stats) {
+            psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE,
+                             "No overlap between input and skycell", PSWARP_ERR_NO_OVERLAP);
+        }
+        psphotFilesActivate(config, false);
+        psFree(cells);
+        psFree(view);
+        goto DONE;
+    }
+
     pmCell *outCell = output->parent;   ///< Output cell
     pmChip *outChip = outCell->parent;  ///< Output chip
     pmFPA *outFPA = outChip->parent;    ///< Output FP
 
-    if (!pswarpPixelFraction(output, stats, config)) {
-        // Don't write output images, and don't bother about anything else
-        output->data_exists = outCell->data_exists = outChip->data_exists = false;
+    if (!pswarpPixelsLit(output, stats, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to calculate pixel regions.");
         psFree(cells);
         psFree(view);
-        goto COMPLETED;
+        return false;
     }
 
     // Set variance factor
     {
-        float varFactor = psMetadataLookupF32(NULL, output->analysis, PSWARP_ANALYSIS_VARFACTOR);
-        psAssert(isfinite(varFactor), "Should be something here.");
-        long goodPix = psMetadataLookupS64(NULL, output->analysis, PSWARP_ANALYSIS_GOODPIX);
-        psAssert(goodPix > 0, "Should be something here.");
+        bool mdok;                      // Status of MD lookup
+        float varFactor = psMetadataLookupF32(&mdok, output->analysis, PSWARP_ANALYSIS_VARFACTOR);
+        long goodPix = psMetadataLookupS64(&mdok, output->analysis, PSWARP_ANALYSIS_GOODPIX);
         varFactor /= goodPix;
 
@@ -364,6 +375,6 @@
     fileActivation(config, independentFiles, false);
 
-    // We need a new PSF model for the warped frame.  It would be good to generate this analytically, but that's going to be tricky.
-    // We have a list of sources, so we use those to redetermine the PSF model.
+    // We need a new PSF model for the warped frame.  It would be good to generate this analytically, but
+    // that's going to be tricky.  We have a list of sources, so we use those to redetermine the PSF model.
 
     if (psMetadataLookupBool(&mdok, config->arguments, "PSF")) {
@@ -394,6 +405,14 @@
         // measure the PSF using these sources
         if (!psphotReadoutFindPSF(config, view, sources)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to determine PSF for warped image.");
-            return false;
+            // This is likely a data quality issue
+            // XXX Split into multiple cases using error codes?
+            psErrorStackPrint(stderr, "Unable to determine PSF");
+            psWarning("Unable to determine PSF --- suspect bad data quality.");
+            if (stats && psMetadataLookupS32(NULL, stats, "QUALITY") == 0) {
+                psMetadataAddS32(stats, PS_LIST_TAIL, "QUALITY", PS_META_REPLACE,
+                                 "Unable to determine PSF", psErrorCodeLast());
+            }
+            psErrorClear();
+            psphotFilesActivate(config, false);
         }
 
@@ -448,5 +467,6 @@
     // Now done with the skycell side of things
 
-    COMPLETED:
+ DONE:
+
     // Write out summary statistics
     if (stats) {
@@ -464,4 +484,5 @@
         psFree((void*)statsMDC);
         fclose(statsFile);
+        pmConfigRunFilenameAddWrite(config, "STATS", statsName);
 
         psFree(stats);
@@ -471,6 +492,5 @@
     psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
     if (dump_file) {
-        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PSWARP.INPUT"); // Input file
-        pmConfigDump(config, input->fpa, dump_file);
+        pmConfigDump(config, dump_file);
     }
 
Index: anches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpPixelFraction.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpPixelFraction.c	(revision 24243)
+++ 	(revision )
@@ -1,102 +1,0 @@
-/** @file pswarpPixelFraction.c
- *
- *  @brief
- *
- *  @ingroup pswarp
- *
- *  @author IfA
- *  @version $Revision: 1.6 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2009-02-05 20:44:04 $
- *  Copyright 2009 Institute for Astronomy, University of Hawaii
- */
-
-#ifdef HAVE_CONFIG_H
-#include <config.h>
-#endif
-
-#include <stdio.h>
-#include <pslib.h>
-#include <psmodules.h>
-
-#include "pswarp.h"
-
-bool pswarpPixelFraction(const pmReadout *readout, psMetadata *stats, const pmConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(readout, false);
-    PS_ASSERT_IMAGE_NON_NULL(readout->image, false);
-    PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, false);
-    if (!readout->mask) {
-        // Can't do anything
-        return true;
-    }
-    PS_ASSERT_IMAGE_NON_NULL(readout->mask, false);
-    PS_ASSERT_IMAGES_SIZE_EQUAL(readout->mask, readout->image, false);
-    PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_IMAGE_MASK, false);
-
-    if (stats) {
-        PS_ASSERT_METADATA_NON_NULL(stats, false);
-    }
-    PS_ASSERT_PTR_NON_NULL(config, false);
-    PS_ASSERT_METADATA_NON_NULL(config->arguments, false);
-
-    bool status; 
-
-    float minFrac = psMetadataLookupF32(NULL, config->arguments, "ACCEPT.FRAC"); ///< Minimum fraction
-
-    // load the recipe
-    psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSWARP_RECIPE);
-    if (!recipe) {
-        psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSWARP_RECIPE);
-        return false;
-    }
-
-    // output mask bits
-    psImageMaskType maskValue = psMetadataLookupImageMask(&status, recipe, "MASK.OUTPUT"); 
-    psAssert (status, "MASK.OUTPUT was not defined");
-
-    psImage *image = readout->image;    ///< Image of interest
-    psImage *mask = readout->mask;      ///< Mask image
-
-    int numCols = image->numCols, numRows = image->numRows; ///< Size of image
-    long numPix = numCols * numRows;
-
-    // Range of valid pixels
-    int xMin = INT_MAX, xMax = -INT_MAX, yMin = INT_MAX, yMax = -INT_MAX;
-
-    long numBad = 0;                     ///< Number of bad pixels
-    for (int y = 0; y < numRows; y++) {
-        for (int x = 0; x < numCols; x++) {
-            if (mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskValue) {
-                numBad++;
-            } else {
-                if (y > yMax) {
-                    yMax = y;
-                }
-                if (y < yMin) {
-                    yMin = y;
-                }
-                if (x > xMax) {
-                    xMax = x;
-                }
-                if (x < xMin) {
-                    xMin = x;
-                }
-            }
-        }
-    }
-
-    float goodFrac = (numPix - numBad) / (float)numPix; ///< Fraction of good pixels
-    bool accept = (goodFrac >= minFrac ? true : false); ///< Accept this readout?
-
-    if (stats) {
-        psMetadataAddBool(stats, PS_LIST_HEAD, "ACCEPT", 0, "Accept this readout?", accept);
-
-        psMetadataAddS32(stats, PS_LIST_TAIL, "RANGE.XMIN", 0, "Minimum valid x value", xMin);
-        psMetadataAddS32(stats, PS_LIST_TAIL, "RANGE.XMAX", 0, "Maximum valid x value", xMax);
-        psMetadataAddS32(stats, PS_LIST_TAIL, "RANGE.YMIN", 0, "Minimum valid y value", yMin);
-        psMetadataAddS32(stats, PS_LIST_TAIL, "RANGE.YMAX", 0, "Maximum valid y value", yMax);
-    }
-
-    return accept;
-}
-
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpPixelsLit.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpPixelsLit.c	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpPixelsLit.c	(revision 24244)
@@ -0,0 +1,92 @@
+/** @file pswarpPixelFraction.c
+ *
+ *  @brief
+ *
+ *  @ingroup pswarp
+ *
+ *  @author IfA
+ *  @version $Revision: 1.6 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-05 20:44:04 $
+ *  Copyright 2009 Institute for Astronomy, University of Hawaii
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <pslib.h>
+#include <psmodules.h>
+
+#include "pswarp.h"
+
+bool pswarpPixelsLit(const pmReadout *readout, psMetadata *stats, const pmConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(readout, false);
+    PS_ASSERT_IMAGE_NON_NULL(readout->image, false);
+    PS_ASSERT_IMAGE_TYPE(readout->image, PS_TYPE_F32, false);
+    if (!readout->mask) {
+        // Can't do anything
+        return true;
+    }
+    PS_ASSERT_IMAGE_NON_NULL(readout->mask, false);
+    PS_ASSERT_IMAGES_SIZE_EQUAL(readout->mask, readout->image, false);
+    PS_ASSERT_IMAGE_TYPE(readout->mask, PS_TYPE_IMAGE_MASK, false);
+
+    if (!stats) {
+        // No point in continuing --- we record results to the statistics
+        return true;
+    }
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    PS_ASSERT_METADATA_NON_NULL(config->arguments, false);
+
+    bool status;
+
+    // load the recipe
+    psMetadata *recipe = psMetadataLookupPtr (NULL, config->recipes, PSWARP_RECIPE);
+    if (!recipe) {
+        psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSWARP_RECIPE);
+        return false;
+    }
+
+    // output mask bits
+    psImageMaskType maskValue = psMetadataLookupImageMask(&status, recipe, "MASK.OUTPUT");
+    psAssert(status, "MASK.OUTPUT was not defined");
+
+    psImage *image = readout->image;    ///< Image of interest
+    psImage *mask = readout->mask;      ///< Mask image
+
+    int numCols = image->numCols, numRows = image->numRows; ///< Size of image
+
+    // Range of valid pixels
+    int xMin = INT_MAX, xMax = -INT_MAX, yMin = INT_MAX, yMax = -INT_MAX;
+
+    for (int y = 0; y < numRows; y++) {
+        for (int x = 0; x < numCols; x++) {
+            if (!(mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] & maskValue)) {
+                if (y > yMax) {
+                    yMax = y;
+                }
+                if (y < yMin) {
+                    yMin = y;
+                }
+                if (x > xMax) {
+                    xMax = x;
+                }
+                if (x < xMin) {
+                    xMin = x;
+                }
+            }
+        }
+    }
+
+    if (stats) {
+        psMetadataAddS32(stats, PS_LIST_TAIL, "RANGE.XMIN", 0, "Minimum valid x value", xMin);
+        psMetadataAddS32(stats, PS_LIST_TAIL, "RANGE.XMAX", 0, "Maximum valid x value", xMax);
+        psMetadataAddS32(stats, PS_LIST_TAIL, "RANGE.YMIN", 0, "Minimum valid y value", yMin);
+        psMetadataAddS32(stats, PS_LIST_TAIL, "RANGE.YMAX", 0, "Maximum valid y value", yMax);
+    }
+
+    return true;
+}
+
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpVersion.c
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpVersion.c	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpVersion.c	(revision 24244)
@@ -20,6 +20,7 @@
 #include <ppStats.h>
 
-psString pswarpVersion(void)
-{
+#include "pswarp.h"
+#include "pswarpVersionDefinitions.h"
+
 #ifndef PSWARP_VERSION
 #error "PSWARP_VERSION is not set"
@@ -28,12 +29,17 @@
 #error "PSWARP_BRANCH is not set"
 #endif
-    return psStringCopy(PSWARP_BRANCH "@" PSWARP_VERSION);
+#ifndef PSWARP_SOURCE
+#error "PSWARP_SOURCE is not set"
+#endif
+
+psString pswarpVersion(void)
+{
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", PSWARP_BRANCH, PSWARP_VERSION);
+    return value;
 }
 
 psString pswarpSource(void)
 {
-#ifndef PSWARP_SOURCE
-#error "PSWARP_SOURCE is not set"
-#endif
     return psStringCopy(PSWARP_SOURCE);
 }
Index: /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpVersionDefinitions.h.in
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpVersionDefinitions.h.in	(revision 24244)
+++ /branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpVersionDefinitions.h.in	(revision 24244)
@@ -0,0 +1,8 @@
+#ifndef PSWARP_VERSION_DEFINITIONS_H
+#define PSWARP_VERSION_DEFINITIONS_H
+
+#define PSWARP_VERSION @PSWARP_VERSION@ // SVN version
+#define PSWARP_BRANCH  @PSWARP_BRANCH@  // SVN branch
+#define PSWARP_SOURCE  @PSWARP_SOURCE@  // SVN source
+
+#endif
Index: /branches/cnb_branches/cnb_branch_20090301/tools/diff_outputs.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/tools/diff_outputs.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/tools/diff_outputs.pl	(revision 24244)
@@ -12,4 +12,19 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 
+# List of products
+use constant PRODUCTS => { 'REGULAR' => [ 'IMAGE', 'MASK', 'VARIANCE',
+                                          'SOURCES', 'JPEG1', 'JPEG2',
+                                          'KERNEL' ],
+                           'ALL' => [ 'IMAGE', 'MASK', 'VARIANCE',
+                                      'SOURCES', 'JPEG1', 'JPEG2',
+                                      'KERNEL', 'INVERSE.IMAGE',
+                                      'INVERSE.MASK', 'INVERSE.VARIANCE',
+                                      'INVERSE.SOURCES' ],
+                           'INVERSE' => [ 'INVERSE.IMAGE', 'INVERSE.MASK',
+                                          'INVERSE.VARIANCE',
+                                          'INVERSE.SOURCES' ],
+                       };
+
+# Extensions to add to path_base
 use constant EXTENSIONS => { 'IMAGE' => '.fits', # Image
                              'MASK' => '.mask.fits', # Mask
@@ -19,4 +34,8 @@
                              'JPEG2' => '.b2.jpg', # Binned JPEG
                              'KERNEL' => '.subkernel', # Convolution kernel
+                             'INVERSE.IMAGE' => '.inv.fits', # Image
+                             'INVERSE.MASK' => '.inv.mask.fits', # Mask
+                             'INVERSE.VARIANCE' => '.inv.wt.fits', # Variance
+                             'INVERSE.SOURCES' => '.inv.cmf', # Sources
                          };
 
@@ -50,23 +69,32 @@
                        ) or die "Unable to connect to database: $DBI::errstr";
 
+
 # Query to run
-my $sql = "SELECT path_base FROM diffSkyfile WHERE diff_id = $diff_id";
-$sql .= " AND skycell_id = $skycell_id" if defined $skycell_id;
+my $sql = "SELECT path_base, bothways FROM diffSkyfile JOIN diffRun USING(diff_id) WHERE diff_id = $diff_id AND fault = 0 AND quality = 0";
+$sql .= " AND skycell_id = '$skycell_id'" if defined $skycell_id;
 
 # List of diffs
-my $diffs = $db->selectcol_arrayref( $sql,
-                                     { Columns => [1] }
+my $diffs = $db->selectall_arrayref( $sql, { Slice => {} }
                                      ) or die "Unable to execute SQL: $DBI::errstr";
 
 my @products;                   # Array of products
 if (defined $products) {
-    @products = split /,/, $products;
+  PRODUCT_SEARCH: foreach my $product ( split(/,/, $products) ) {
+      foreach my $key ( keys %{PRODUCTS()} ) {
+          if ($product eq $key) {
+              push @products, @{${PRODUCTS()}{$key}};
+              next PRODUCT_SEARCH;
+          }
+      }
+      push @products, $product;
+  }
 } else {
-    @products = keys %{EXTENSIONS()};
+    @products = @{${PRODUCTS()}{'ALL'}};
 }
 
 foreach my $diff ( @$diffs ) {
     foreach my $product ( @products ) {
-        copy_extension( $diff, ${EXTENSIONS()}{$product} );
+        next if (not defined $diff->{bothways} or $diff->{bothways} eq 'NULL' or not $diff->{bothways}) and $product =~ /INVERSE/;
+        copy_extension( $diff->{path_base}, ${EXTENSIONS()}{$product} );
     }
 }
Index: /branches/cnb_branches/cnb_branch_20090301/tools/ipp_apply_burntool.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/tools/ipp_apply_burntool.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/tools/ipp_apply_burntool.pl	(revision 24244)
@@ -91,4 +91,20 @@
 
 my $REALRUN = 1;
+my $RAWTABLES = 1;
+# XXX if we want to avoid using the raw tables in this script, we need to delay deletion of the tempfile until the next pass.
+
+# first pass: mark the db entries to catch failures
+# these will not be identified as burned by chip processing (user_1 > 0.5)
+if (! $skip_burned)  {
+    foreach my $file (@files) {
+	my $exp_id = $file->{exp_id};
+
+	my $status;
+	$status = vsystem ("$regtool -dbname $dbname -updateprocessedimfile -exp_id $exp_id -class_id $class_id -user_1 0.1", 1);
+	if ($status) {
+	    &my_die("failed to update imfile");
+	}
+    }
+}
 
 my $prevFileOpt = "";
@@ -120,9 +136,14 @@
     # print "outImfile: $outImfile -> $outImfileReal\n";
 
-    # destination for the burntool artifacts
-    my $artImfile = $rawImfile;
-    $artImfile =~ s/fits$/burn.tbl/;
-    my $artImfileReal = $ipprc->file_resolve($artImfile, 1);
-    # print "artImfile: $artImfile -> $artImfileReal\n";
+    # burntool now can write the artifacts to the fits file.
+    # destination for the burntool artifacts.  use this if RAWTABLES is set.
+    my $artImfile;
+    my $artImfileReal;
+    if ($RAWTABLES) {
+	$artImfile = $rawImfile;
+	$artImfile =~ s/fits$/burn.tbl/;
+	$artImfileReal = $ipprc->file_resolve($artImfile, 1);
+	# print "artImfile: $artImfile -> $artImfileReal\n";
+    }
 
     # uncompress the image (do we need to check if it is compressed?)
@@ -132,5 +153,9 @@
     }
 
-    $status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt", $REALRUN);
+    if ($RAWTABLES) {
+	$status = vsystem ("$burntool $tmpImfileReal out=$artImfileReal $prevFileOpt", $REALRUN);
+    } else {
+	$status = vsystem ("$burntool $tmpImfileReal $prevFileOpt", $REALRUN);
+    }
     if ($status) {
 	&my_die("failed on burntool");
@@ -149,5 +174,9 @@
 
     # save the artifact file for the next image
-    $prevFileOpt = "in=$artImfileReal";
+    if ($RAWTABLES) {
+	$prevFileOpt = "in=$artImfileReal";
+    } else {
+	$prevFileOpt = "infits=$artImfileReal";
+    }
     print "\n";
 }
Index: /branches/cnb_branches/cnb_branch_20090301/tools/warp_outputs.pl
===================================================================
--- /branches/cnb_branches/cnb_branch_20090301/tools/warp_outputs.pl	(revision 24243)
+++ /branches/cnb_branches/cnb_branch_20090301/tools/warp_outputs.pl	(revision 24244)
@@ -49,5 +49,5 @@
 
 # Query to run
-my $sql = "SELECT path_base FROM warpSkyfile WHERE warp_id = $warp_id AND ignored = 0";
+my $sql = "SELECT path_base FROM warpSkyfile WHERE warp_id = $warp_id AND fault = 0 AND quality = 0";
 $sql .= " AND skycell_id = \'$skycell_id\'" if defined $skycell_id;