Index: /branches/eam_branches/ipp-20100621/DataStoreServer/scripts/dsreg
===================================================================
--- /branches/eam_branches/ipp-20100621/DataStoreServer/scripts/dsreg	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/DataStoreServer/scripts/dsreg	(revision 28794)
@@ -270,4 +270,7 @@
             die("failed trying to create fileset directory $fileset_dir");
         }
+	if (! -d $fileset_dir ) {
+            die("cannot access just created fileset directory $fileset_dir");
+	}
     }
 
@@ -328,7 +331,12 @@
                     my $dest = "$fileset_dir/$filename";
 
+		    if ( !-e $src or !-r $src) {
+		    	print STDERR "file $src does not exist or is not accessible\n";
+			exit $PS_EXIT_SYS_ERROR;
+		    }
                     if ($linkfiles) {
                         if (! symlink $src, $dest) {
-                            die("failed trying to link $src to $dest");
+                            print STDERR "failed trying to link $src to $dest\n";
+			    exit $PS_EXIT_SYS_ERROR;
                         }
                     } else {
Index: /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/neb-host
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/neb-host	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/neb-host	(revision 28794)
@@ -16,5 +16,5 @@
 use Pod::Usage qw( pod2usage );
 
-my ($host, $state,
+my ($host, $state, $note,
     $db, $dbhost, $dbpass, $dbuser,
     $allocate, $available, $debug);
@@ -51,9 +51,10 @@
     'host=s'         => \$host,
     'state=s'        => \$state,
-    
+    'note=s'         => \$note,
     'db=s'           => \$db,
     'dbhost=s'       => \$dbhost,
     'dbuser=s'       => \$dbuser,
     'dbpass=s'       => \$dbpass,
+
     
     ) || pod2usage( 2 );
@@ -101,5 +102,6 @@
     $set{'v.allocate'} = $allocate if defined $allocate;
     $set{'v.available'} = $available if defined $available;
-    
+    $set{'v.note'} = $note if defined $note;
+
     eval {
 	my ($q, @bind) = sql_interp("UPDATE volume AS v SET", \%set, "WHERE", \%constraint);
@@ -134,9 +136,13 @@
 $dbh->commit;
 
-my $format  = "%-15s %-15s %-10s %-10s %-10s %-10s\n";
-my @columns = qw(host name mounted allocate available xattr);
+my $format  = "%-15s %-15s %-7s %-8s %-9s %-5s %s\n";
+my @columns = qw(host name mounted allocate available xattr note);
 printf($format, @columns);
 
 while (my $row = $query->fetchrow_hashref) {
+    unless ((exists($$row{ 'note' })) && (defined($$row{ 'note' }))) {
+	$$row{ 'note' } = '-';
+    }
+
     printf($format, @$row{@columns});
 }
@@ -153,4 +159,5 @@
 
     neb-host [<nebulous host> <nebulous state>]
+    [--note '<status note>']
     [--host <nebulous host>] [--state <up|down|repair>]
     [--dbhost <database host>] [--db <database name>]
@@ -168,4 +175,8 @@
 =over 4
 
+=item * --note <status note>
+
+Set a status note for the specified volume.
+
 =item * --host <nebulous host>
 
Index: /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/neb-voladd
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/neb-voladd	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/neb-voladd	(revision 28794)
@@ -19,5 +19,5 @@
 use Pod::Usage qw( pod2usage );
 
-my ($db, $dbhost, $dbuser, $dbpass, $mountpoint, $vname, $vhost, $uri);
+my ($db, $dbhost, $dbuser, $dbpass, $mountpoint, $vname, $vhost, $uri, $note);
 
 $db     = $ENV{'NEB_DB'} unless $db;
@@ -35,4 +35,5 @@
     'vhost=s'           => \$vhost,
     'vname|n=s'         => \$vname,
+    'note=s'            => \$note,
 ) || pod2usage( 2 );
 
@@ -69,5 +70,5 @@
 
 my $query = $dbh->prepare( $sql->new_volume );
-$query->execute( $vname, $vhost, $path, $mountpoint);
+$query->execute( $vname, $vhost, $path, $mountpoint, $note);
 
 print " OK\n";
@@ -131,4 +132,8 @@
 
 Optional.  Defaults to C<localhost>.
+
+=item * --note <note>
+
+Add a note for this volume.
 
 =item * --mountpoint <path>
Index: /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/neb-voladm
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/neb-voladm	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/neb-voladm	(revision 28794)
@@ -33,4 +33,5 @@
     $xattr,
     $cab_id,
+    $note,
 );
 
@@ -53,4 +54,5 @@
     'cab_id|c=i'        => \$cab_id,
     'xattr=i'           => \$xattr,
+    'note=s'            => \$note,
 ) || pod2usage( 2 );
 
@@ -89,4 +91,5 @@
 $set{'v.xattr'} = $xattr if defined $xattr;
 $set{'v.cab_id'} = $cab_id if defined $cab_id;
+$set{'v.note'} = $note if defined $note;
 
 if (%set) {
@@ -150,5 +153,5 @@
     neb-voladm [--vname <volume name>] [--vhost <volume host>]
     [--allocate <0|1>] [--available <0|1>] [--xattr <0|1>]
-    [--cab_id <cab_id>]
+    [--cab_id <cab_id>] [--note <note>]
     [--host <database host> ] [--db <database name>]
     [--user <database username>] [--pass <database password>]
@@ -189,4 +192,10 @@
 
 Sets the cabinet id for this storage volume.
+
+Requires either the C<--vname> or c<--vhost> options.
+
+=item * --note <note>
+
+Set a note for this volume.
 
 Requires either the C<--vname> or c<--vhost> options.
Index: /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/nebdiskd
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/nebdiskd	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous-Server/bin/nebdiskd	(revision 28794)
@@ -53,5 +53,9 @@
 # set the process name for easy pgrep-ability
 $0 = 'nebdiskd';
-my $rcfile = "$ENV{HOME}/.nebdiskrc";
+#my $rcfile = "$ENV{HOME}/.nebdiskrc";
+my $rcfile = $ENV{NEBDISKDRC};
+unless (defined($rcfile)) {
+    $rcfile = '/etc/nebdiskd.rc';
+}
 $rcfile = File::Spec->canonpath($rcfile);
 
@@ -71,6 +75,4 @@
 my %host_removed = ();
 my $failure_limit = 5;
-
-
 
 #my $mounts = $c->get_mounts;
@@ -143,4 +145,6 @@
 $SIG{HUP}  = sub { $c = read_rcfile($rcfile) }; 
 
+my $date = localtime();
+$log->warn("BEGIN: $date with RCFILE: $rcfile");
 
 while (1) {
@@ -187,5 +191,5 @@
 
     eval {
-        my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, ?, ? FROM volume WHERE mountpoint = ?");
+        my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, ?, ?,note FROM volume WHERE mountpoint = ?");
 #	my $d_query = $dbh->prepare_cached("UPDATE mountedvol SET allocate = ?, available = ? WHERE mountpoint = ?");
         my $d_query = $dbh->prepare_cached("DELETE FROM mountedvol WHERE mountpoint = ?");
@@ -224,4 +228,13 @@
                 }
             };
+
+            # fetch stats on the mounted device.  this has to be done AFTER
+            # we determine if it's a valid mountpoint incase
+            # is_mountpoint() invokes the automounter
+            my $dev_info = df($mountpoint, 1024);
+            unless (defined $dev_info) {
+		$valid_mountpoint = 0;
+            }
+
             if (!$valid_mountpoint) {
                 # try is_mountpoint() again if $retry > 1
@@ -254,5 +267,5 @@
             # we determine if it's a valid mountpoint incase
             # is_mountpoint() invokes the automounter
-            my $dev_info = df($mountpoint, 1024);
+            $dev_info = df($mountpoint, 1024);
             unless (defined $dev_info) {
                 $log->error("can't find device info for $mountpoint");
Index: /branches/eam_branches/ipp-20100621/Nebulous-Server/docs/changes.txt
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous-Server/docs/changes.txt	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/Nebulous-Server/docs/changes.txt	(revision 28794)
@@ -0,0 +1,5 @@
+-- 2010-06-24 edit to add notes.
+
+ALTER TABLE volume ADD COLUMN note VARCHAR(255) AFTER cab_id;
+ALTER TABLE mountedvol ADD COLUMN note VARCHAR(255) AFTER used;
+
Index: /branches/eam_branches/ipp-20100621/Nebulous-Server/lib/Nebulous/Server.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous-Server/lib/Nebulous/Server.pm	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous-Server/lib/Nebulous/Server.pm	(revision 28794)
@@ -9,5 +9,5 @@
 no warnings qw( uninitialized );
 
-our $VERSION = '0.17';
+our $VERSION = '0.18';
 
 use base qw( Class::Accessor::Fast );
@@ -1510,4 +1510,104 @@
 }
 
+sub find_objects_wildcard
+{
+    my $self = shift;
+
+    my $log = $self->log;
+    $log->debug( "entered - @_" );
+
+    my ($pattern) = validate_pos( @_,
+        {
+            type        => SCALAR,
+            optional    => 1,
+        },
+    );
+
+    my $sql = $self->sql;
+    my $db  = $self->_db_for_index(0);
+    
+    # validate that we have a key to deal with
+    eval {
+        $pattern = parse_neb_key($pattern) if defined $pattern;
+    };
+    $log->logdie("$@") if $@;
+
+    unless (defined $pattern) {
+        $log->debug( "leaving" );
+        $log->logdie("no keys found");
+    }
+
+    # parse out the directory we're working in, and decide if we are a directory
+    my $dir_id = $self->_resolve_dir_parent_id(key => $pattern, dir_id => 1);
+    my $work_dir;
+    my $file_pattern;
+    if (defined $dir_id) {
+	$work_dir = $pattern;
+	$file_pattern = '%';
+    }
+    else {
+	my $dir_plain = dirname($pattern);
+	if ($dir_plain eq 'neb:') {
+	    $dir_plain = 'neb:///';
+	}
+	if ($dir_plain) {
+	    $work_dir = parse_neb_key($dir_plain);
+	}
+	else {
+	    $work_dir = parse_neb_key('/');
+	}
+	$file_pattern = basename $pattern;
+	unless ($file_pattern) {
+	    $file_pattern = '%';
+	}
+	
+	$dir_id = $self->_resolve_dir_parent_id(key => $work_dir, dir_id => 1);
+	unless (defined $dir_id) {
+	    $log->logdie("pattern $work_dir does not match any key or directory");
+	}
+    }
+    
+    # find dirs under dir
+    my @dir_keys;
+
+    eval {
+        $log->debug("looking for directories under dir: $dir_id");
+        my $query = $db->prepare_cached( $sql->find_dir_by_parent_id . " AND dirname LIKE ? ");
+        $query->execute( $dir_id, $file_pattern );
+
+        while ( my $row = $query->fetchrow_hashref ) {
+            next if $row->{'dir_id'} == 1;
+            my $dir = $row->{'dirname'};
+            if ($dir_id == 1) {
+                push @dir_keys, $dir . '/' if $dir;
+            } else {
+                push @dir_keys, $work_dir->path . '/' . $dir . '/' if $dir;
+            }
+            $log->debug( "matched $dir" ) if $dir;
+        }
+    };
+    $log->logdie("database error: $@") if $@;
+
+    # find files under dir
+    my @keys;
+    eval {
+        $log->debug("looking for objects under dir: $dir_id");
+        my $query = $db->prepare_cached( $sql->find_object_by_dir_id . " AND ext_id_basename LIKE ? ");
+        $query->execute( $dir_id , $file_pattern);
+
+        while ( my $row = $query->fetchrow_hashref ) {
+            my $key = $row->{ 'ext_id' };
+            push @keys, $key if $key;
+            $log->debug( "matched $key" ) if $key;
+        }
+    };
+    $log->logdie("database error: $@") if $@;
+
+    $log->debug( "leaving" );
+    
+    return [sort(@dir_keys), sort(@keys)];
+
+}
+
 # find matching objects from the given server
 sub _find_objects_for_index
@@ -1598,4 +1698,58 @@
 }
 
+sub find_ext_id_by_volume
+{
+    my $self = shift;
+    my $log = $self->log;
+    $log->debug("entered - @_");
+    my ($vol_name,$limit) = validate_pos(@_,
+					{
+					    type      => SCALAR,
+# # 					    callbacks => {
+# # 						'is valid volume name' => sub {
+# # 						    return 1 if not defined $_[0];
+# # 						    $self->_is_valid_volume_name($_[0])
+# # 						},
+# 					    },
+					},
+					{
+					    type      => SCALAR|UNDEF,
+					    optional => 1,
+					},
+	);
+    unless (defined($limit)) {
+	$limit = 50000;
+    }
+
+    my $sql = $self->sql;
+    my @ext_ids;
+    my $db = $self->_db_for_index(0);
+    eval {
+	my $query;
+	$query = $db->prepare_cached( $sql->get_ext_id_by_vol_name );
+	my $rows = $query->execute($vol_name, 1,$limit);
+	unless ($rows > 0) {
+	    $query->finish;
+	    $log->logdie("no instances on storage volume or volume is not avaiable for volume: $vol_name");
+	}
+        while (my $row = $query->fetchrow_hashref) {
+            my $ext_id = $row->{ 'ext_id' };
+            push @ext_ids, $ext_id if $ext_id;
+        }
+    };
+    if ($@) {
+        $db->rollback;
+        $log->logdie("database error: $@");
+    }
+
+    # XXX remove this?
+    $log->logdie("no ext_ids found") unless (scalar @ext_ids);
+
+    $log->debug("found: \@ext_ids");
+
+    $log->debug("leaving");
+
+    return \@ext_ids;
+}
 
 sub find_instances
Index: /branches/eam_branches/ipp-20100621/Nebulous-Server/lib/Nebulous/Server/SQL.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous-Server/lib/Nebulous/Server/SQL.pm	(revision 28794)
@@ -266,4 +266,18 @@
             AND mountedvol.name = ?
             AND mountedvol.available = ?
+    },
+    get_ext_id_by_vol_name => qq{
+        SELECT
+            ext_id
+        FROM instance
+        JOIN storage_object
+            USING (so_id)
+        JOIN mountedvol
+            USING(vol_id)
+        JOIN volume
+            USING(vol_id)
+        WHERE volume.name = ?
+            AND mountedvol.available = ?
+        LIMIT ?
     },
     # volume handler
@@ -353,6 +367,6 @@
     },
     new_volume          => qq{
-        INSERT INTO volume (name, host, path, allocate, available, xattr, mountpoint, cab_id)
-        VALUES (?, ?, ?, TRUE, TRUE, FALSE, ?, NULL)
+        INSERT INTO volume (name, host, path, allocate, available, xattr, mountpoint, cab_id, note)
+        VALUES (?, ?, ?, TRUE, TRUE, FALSE, ?, NULL, ?)
     },
     get_volume_by_name => qq{
@@ -371,5 +385,6 @@
             v.xattr,
             mountedvol.vol_id IS NOT NULL as mounted,
-            v.cab_id
+            v.cab_id,
+            v.note
         FROM volume AS v
         LEFT JOIN mountedvol
@@ -675,4 +690,5 @@
     mountpoint VARCHAR(255) NOT NULL,
     cab_id INT,
+    note VARCHAR(255),
     PRIMARY KEY(vol_id),
     UNIQUE KEY(name),
@@ -701,4 +717,5 @@
     total BIGINT NOT NULL,
     used BIGINT NOT NULL,
+    note VARCHAR(255),
     PRIMARY KEY(vol_id),
     KEY(name),
Index: /branches/eam_branches/ipp-20100621/Nebulous/Build.PL
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous/Build.PL	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous/Build.PL	(revision 28794)
@@ -113,7 +113,9 @@
         bin/neb-locate
         bin/neb-ls
+        bin/neb-migrate
         bin/neb-mv
         bin/neb-replicate
         bin/neb-rm
+        bin/neb-shift
         bin/neb-stat
         bin/neb-swap
Index: /branches/eam_branches/ipp-20100621/Nebulous/MANIFEST
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous/MANIFEST	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous/MANIFEST	(revision 28794)
@@ -14,7 +14,9 @@
 bin/neb-insert
 bin/neb-ls
+bin/neb-migrate
 bin/neb-mv
 bin/neb-replicate
 bin/neb-rm
+bin/neb-shift
 bin/neb-stat
 bin/neb-swap
Index: /branches/eam_branches/ipp-20100621/Nebulous/bin/neb-ls
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous/bin/neb-ls	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous/bin/neb-ls	(revision 28794)
@@ -1,5 +1,5 @@
 #!/usr/bin/env perl
 
-# Copyright (C) 2007-2009  Joshua Hoblitt
+# Copyright (C) 2007-2009  Joshua Hoblitt, 2010 Chris Waters
 
 use strict;
@@ -16,18 +16,22 @@
 my (
     $server,
-    $long,
-#    $recursive,
+    $path,
+    $ls,
+    $column,
 );
 
 $server = $ENV{'NEB_SERVER'} unless $server;
 
-# make --long the default
-$long = 1;
 
 GetOptions(
     'server|s=s'    => \$server,
-#    'recursive|r'   => \$recursive,
-    'l|1'           => \$long,
+    'path|p'        => \$path,
+    'l'             => \$ls,
+    'column|c'      => \$column,
 ) || pod2usage( 2 );
+
+# twiddle column so that it does it by default.
+
+$column = not $column;
 
 my $pattern = shift;
@@ -47,19 +51,38 @@
 $pattern ||= "/";
 
-#if ($recursive) {
-#    $pattern = "^" . $pattern . ".*";
-#} else {
-#    $pattern = "^" . $pattern . "\$";
-#}
+my $keys = $neb->find_objects_wildcard($pattern);
 
-my $keys = $neb->find_objects($pattern);
-
+if ($path) {
+    my @files;
+    foreach my $key (@{ $keys }) {
+	my $uris = $neb->find_instances($key);
+	if (defined $uris) {
+	    push @files, URI->new(@$uris[0])->file;
+	}
+    }
+    @{ $keys } = @files;
+} 
+if ($ls) {
+    my @files;
+    foreach my $key (@{ $keys }) {
+	my $uris = `ls -al $key`;
+	chomp($uris);
+	if (defined $uris) {
+	    push @files, $uris;
+	}
+    }
+    @{ $keys } = @files;
+}    
 if ($keys) {
-    if ($long) {
-        print join("\n", @{ $keys }), "\n";
-    } else {
-        print join(" ", @{ $keys }), "\n";
+    if ($column) {
+	open(COLUMN,"|column") || die "Cannot open column command.";
+	print COLUMN join("\n", @{ $keys }), "\n";
+	close(COLUMN);
+    }
+    else {
+	print join("\n", @{ $keys }), "\n";
     }
 }
+
 
 __END__
@@ -73,11 +96,11 @@
 =head1 SYNOPSIS
 
-    neb-ls [--server <URL>] [-l|-1] [--recursive] <pattern>
+    neb-ls [--server <URL>] [-c] [-p] <pattern>
 
 =head1 DESCRIPTION
 
 This program list Nebulous keys matched by C<<pattern>>.  Call it with no
-arguments is equivalanet to searching with the pattern C<.*>.  Where
-C<<pattern>> is a POSIX 1003.2 compatable regular repression.
+arguments is equivalent to searching with the pattern C<neb://>.  SQL like
+wildcards are supported to select out certain files.
 
 =head1 OPTIONS
@@ -85,19 +108,14 @@
 =over 4
 
-=item * -l|-1
+=item * --path|-p
 
-Use a long listing format.
+Find the disk files corresponding to the keys found.
+
+=item * --column|-c
+
+Disable column formatting, and output results one to a line.
 
 Optional
 
-=cut
-#=item * --recursive|-r
-#
-#By default C<neb-ls> will only try to match the exact string provided to it.
-#With this option set all keys which match C<<pattern>> as a REGEX or substring
-#will be returned.
-#
-#Optional
-#
 =item * --server|-s <URL>
 
@@ -122,8 +140,4 @@
 =back
 
-=head1 CREDITS
-
-Just me, myself, and I.
-
 =head1 SUPPORT
 
@@ -132,9 +146,9 @@
 =head1 AUTHOR
 
-Joshua Hoblitt <jhoblitt@cpan.org>
+Joshua Hoblitt <jhoblitt@cpan.org>, Chris Waters
 
 =head1 COPYRIGHT
 
-Copyright (C) 2007-2009  Joshua Hoblitt.  All rights reserved.
+Copyright (C) 2007-2010  Joshua Hoblitt / Chris Waters.  All rights reserved.
 
 This program is free software; you can redistribute it and/or modify it under
Index: /branches/eam_branches/ipp-20100621/Nebulous/bin/neb-migrate
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous/bin/neb-migrate	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/Nebulous/bin/neb-migrate	(revision 28794)
@@ -0,0 +1,180 @@
+#!/usr/bin/env perl
+
+# Copyright (C) 2007-2010  Joshua Hoblitt/Chris Waters
+
+use strict;
+use warnings FATAL => qw( all );
+
+use vars qw( $VERSION );
+$VERSION = '0.03';
+
+use Nebulous::Client;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version );
+use Pod::Usage qw( pod2usage );
+
+my (
+    $server,
+    $limit,
+    $dest_volume,
+    $no_pretend,
+#    $recursive,
+);
+
+$server = $ENV{'NEB_SERVER'} unless $server;
+
+GetOptions(
+    'server|s=s'    => \$server,
+    'limit|l=s'     => \$limit,
+    'destination|d=s' => \$dest_volume,
+    'no_pretend'    => \$no_pretend,
+) || pod2usage( 2 );
+
+if ($limit && $limit <= 0) {
+    die "Limit must be positive.";
+}
+
+
+my $host = shift;
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --server", -exitval => 2 )
+    unless $server;
+
+my $neb = Nebulous::Client->new(
+    proxy => "$server",
+);
+
+die "can't connected to Nebulous Server: $server"
+    unless defined $neb;
+
+# Find the keys on this volume.
+my $keys;
+if ($limit) {
+    $keys = $neb->find_ext_id_by_volume($host,$limit);
+}
+else {
+    $keys = $neb->find_ext_id_by_volume($host);
+}
+
+# Print the keys on this volume, and if we're not pretending, do the neb-shift code to move them off it.
+if ($keys) {
+    foreach my $src (@{ $keys }) {
+	print "$src";
+	if ($no_pretend) {
+	    print "\t";
+	    my $uris = $neb->find_instances($src, "~$host");
+	    unless (defined $uris) {
+		die "No copy of $src exists on $host";
+	    }
+	    
+	    my $status;
+	    if ($dest_volume) {
+		$status = $neb->replicate( $src, $dest_volume);
+	    }
+	    else {
+		$status = $neb->replicate($src);
+	    }
+	    die "Replicate phase failed for $src ($host) ($dest_volume)" unless $status;
+	    
+	    $status = $neb->cull($src,$host);
+	    
+	    warn "Cull phase failed for $src from $host" unless $status;
+	    
+	    print "Success!";
+	}
+	print "\n";
+    }
+}
+
+__END__
+
+=pod
+
+=head1 NAME
+
+neb-migrate - list Nebulous keys on a specific volume and move them off it
+
+=head1 SYNOPSIS
+
+    neb-migrate [--server <URL>] [--limit <N>] [--no_pretend] <volume>
+
+=head1 DESCRIPTION
+
+This program list Nebulous keys found on volume <<volume>>, and will move them 
+to another volume if requested to do so.  
+
+=head1 OPTIONS
+
+=over 4
+
+Optional
+
+=item * --limit
+
+Limit the number of results.
+
+=item * --no_pretend
+
+Actually move the files off of the specified host.
+
+=item * --destination|-d <dest_volume>
+
+Specify the destination volume for the keys moved.
+
+=item * --server|-s <URL>
+
+URL of the Nebulous server to connect to.
+
+Optional if the appropriate environment variable is set.
+
+=back
+
+=head1 ENVIRONMENT
+
+These environment variables may be used in place of the specified command line
+options.  All command line option will override the corresponding environment
+value.
+
+=over 4
+
+=item * C<NEB_SERVER>
+
+Equivalent to --server|-s
+
+=back
+
+=head1 SUPPORT
+
+Please contact the author directly via e-mail.
+
+=head1 AUTHOR
+
+Chris Waters <watersc1@ifa.hawaii.edu>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2007-2010 Josh Hoblitt / Chris Waters.  All rights reserved.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place - Suite 330, Boston, MA  02111-1307, USA.
+
+The full text of the license can be found in the L<perlgpl> Pod as supplied
+with Perl 5.8.1 and later.
+
+=head1 SEE ALSO
+
+L<Nebulous::Server>, L<neb-initdb>, L<neb-addvol>, L<nebdiskd>, L<neb-df>,
+L<neb-touch>, C<regex(7)>
+
+=cut
Index: /branches/eam_branches/ipp-20100621/Nebulous/bin/neb-shift
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous/bin/neb-shift	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/Nebulous/bin/neb-shift	(revision 28794)
@@ -0,0 +1,146 @@
+#!/usr/bin/env perl
+
+# Copyright (C) 2007-2008  Joshua Hoblitt
+#
+# $Id: neb-shift,v 1.5 2008-01-22 21:40:39 jhoblitt Exp $
+
+use strict;
+use warnings FATAL => qw( all );
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use Nebulous::Client;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version );
+use Pod::Usage qw( pod2usage );
+
+my ($volume, $server);
+
+$server = $ENV{'NEB_SERVER'} unless $server;
+
+GetOptions(
+    'volume=s'      => \$volume,
+    'server|s=s'    => \$server,
+) || pod2usage( 2 );
+
+my $src = shift;
+my $abandon_vol = shift;
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --server <source key> <source volume>",
+        -exitval => 2 )
+        unless defined $server and defined $src and defined $abandon_vol;
+
+my $neb = Nebulous::Client->new(
+    proxy => "$server",
+);
+
+die "can't connected to Nebulous Server: $server"
+    unless defined $neb;
+
+my $uris = $neb->find_instances($src, "~$abandon_vol");
+unless (defined $uris) {
+    die "No copy of $src exists on $abandon_vol";
+}
+
+my $status;
+if ($volume) {
+    $status = $neb->replicate( $src, $volume);
+}
+else {
+    $status = $neb->replicate($src);
+}
+die "Replicate phase failed for $src ($abandon_vol) ($volume)" unless $status;
+
+$status = $neb->cull($src,$abandon_vol);
+
+warn "Cull phase failed for $src from $abandon_vol" unless $status;
+
+__END__
+
+=pod
+
+=head1 NAME
+
+neb-shift - list Nebulous keys
+
+=head1 SYNOPSIS
+
+    neb-shift [--server <URL>] [--volume <destination volume name>] <source key> <source volume>
+
+=head1 DESCRIPTION
+
+Shift a Nebulous storage key from a specific source volume, optionally assigning it
+to a specific destination volume.
+
+=head1 OPTIONS
+
+=over 4
+
+=item * --server|-s <URL>
+
+URL of the Nebulous server to connect to.
+
+Optional if the appropriate environment variable is set.
+
+=item * --volume <volume name>
+
+The volume to place the key's new storage instance on.
+
+Optional.
+
+=back
+
+=head1 ENVIRONMENT
+
+These environment variables may be used in place of the specified command line
+options.  All command line option will override the corresponding environment
+value.
+
+=over 4
+
+=item * C<NEB_SERVER>
+
+Equivalent to --server|-s
+
+=back
+
+=head1 CREDITS
+
+Just me, myself, and I.
+
+=head1 SUPPORT
+
+Please contact the author directly via e-mail.
+
+=head1 AUTHOR
+
+Joshua Hoblitt <jhoblitt@cpan.org>
+
+=head1 COPYRIGHT
+
+Copyright (C) 2007-2008  Joshua Hoblitt.  All rights reserved.
+
+This program is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This program is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+this program; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place - Suite 330, Boston, MA  02111-1307, USA.
+
+The full text of the license can be found in the L<perlgpl> Pod as supplied
+with Perl 5.8.1 and later.
+
+=head1 SEE ALSO
+
+L<Nebulous::Server>, L<neb-initdb>, L<neb-addvol>, L<nebdiskd>, L<neb-df>,
+L<neb-touch>, C<regex(7)>
+
+=cut
Index: /branches/eam_branches/ipp-20100621/Nebulous/lib/Nebulous/Client.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/Nebulous/lib/Nebulous/Client.pm	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/Nebulous/lib/Nebulous/Client.pm	(revision 28794)
@@ -9,5 +9,5 @@
 no warnings qw( uninitialized );
 
-our $VERSION = '0.17';
+our $VERSION = '0.18';
 
 use Digest::MD5;
@@ -741,4 +741,46 @@
 }
 
+sub find_objects_wildcard
+{
+    my $self = shift;
+
+    my @args = validate_pos( @_,
+        {
+            type        => SCALAR,
+            optional    => 1,
+        },
+    );
+
+    $log->debug( "entered - @_" );
+
+    my $response = $self->{ 'server' }->find_objects_wildcard( @args );
+    if ( $response->fault ) {
+        $self->set_err($response->faultstring);
+
+        if ($response->faultstring =~ /no keys found/) {
+            $log->debug( "leaving" );
+            return;
+        }
+        if ($response->faultstring =~ /does not match any key or directory/) {
+            $log->debug( "leaving" );
+            return;
+        }
+
+        $log->logdie("unhandled fault - ", $self->err);
+    }
+
+    my $keys = $response->result;
+
+    $log->debug( "server found: @$keys" );
+
+#    foreach my $path ( @{ $uris } ) {
+#        $path = _get_file_path( $path );
+#    }
+
+    $log->debug( "leaving" );
+
+    return $keys;
+}
+
 
 sub find_instances
@@ -788,4 +830,38 @@
 
     return $uris;
+}
+
+sub find_ext_id_by_volume
+{
+    my $self = shift;
+    my ($vol_name, $limit) = validate_pos( @_,
+					   { 
+					       type => SCALAR,
+					   },
+					   {
+					       type => SCALAR|UNDEF,
+					       optional => 1,
+					   },
+	);
+    
+    $log->debug( "entered - @_" );
+
+    my $response = $self->{ 'server' }->find_ext_id_by_volume( $vol_name, $limit);
+    if ( $response->fault ) {
+	$self->set_err($response->faultstring);
+	if ($response->faultstring =~ /no instances on storage volume/) {
+	    $log->debug( "leaving" );
+	    return;
+	}
+
+	$log->logdie("unhandled fault - ", $self->err);
+    }
+
+    my $ext_ids = $response->result;
+    
+    $log->debug( "server found @$ext_ids" );
+    $log->debug( "leaving" );
+    
+    return($ext_ids);
 }
 
Index: /branches/eam_branches/ipp-20100621/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm	(revision 28794)
@@ -514,4 +514,8 @@
         if ($image->{fault}) {
             print STDERR "selected difference image $id $image->{diff_id} $skycell_id has fault: $image->{fault}\n";
+            next;
+        }
+        if ($image->{quality}) {
+            print STDERR "selected difference image $id $image->{diff_id} $skycell_id has poor quality: $image->{quality}\n";
             next;
         }
Index: /branches/eam_branches/ipp-20100621/archive/noise_model/plot.gp
===================================================================
--- /branches/eam_branches/ipp-20100621/archive/noise_model/plot.gp	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/archive/noise_model/plot.gp	(revision 28794)
@@ -7,59 +7,83 @@
 set yrange [0:0.045]
 set border linewidth 3
-set term postscript color eps font "Times-Roman,14"
+set term postscript color eps linewidth 2.5 font "Times-Roman,30
 
 # Input
-set output "input.ps"
+set output "input.eps"
 plot \
     "hist_0.dat" notitle with lines lt 1 lw 3 lc 1 , \
-    "hist_18.dat" notitle with lines lt 1 lw 3 lc 3 , \
-    "hist_36.dat" notitle with lines lt 1 lw 3 lc 11 , \
+    "hist_24.dat" notitle with lines lt 1 lw 3 lc 3 , \
+    "hist_42.dat" notitle with lines lt 1 lw 3 lc 11 , \
+    "gauss.dat" notitle with lines lt 1 lw 3 lc 7
+
+# Input phot
+set output "input_phot.eps"
+plot \
+    "hist_1.dat" notitle with lines lt 1 lw 3 lc 1 , \
+    "hist_25.dat" notitle with lines lt 1 lw 3 lc 3 , \
+    "hist_43.dat" notitle with lines lt 1 lw 3 lc 11 , \
+    "gauss.dat" notitle with lines lt 1 lw 3 lc 7
+
+# Warp
+set output "warp.eps"
+plot \
+    "hist_2.dat" notitle with lines lt 1 lw 3 lc 1 , \
+    "hist_26.dat" notitle with lines lt 1 lw 3 lc 3 , \
+    "hist_44.dat" notitle with lines lt 1 lw 3 lc 11 , \
     "gauss.dat" notitle with lines lt 1 lw 3 lc 7
 
 # Warp phot
-set output "warp_phot.ps"
+set output "warp_phot.eps"
 plot \
     "hist_3.dat" notitle with lines lt 1 lw 3 lc 1 , \
-    "hist_21.dat" notitle with lines lt 1 lw 3 lc 3 , \
-    "hist_39.dat" notitle with lines lt 1 lw 3 lc 11 , \
+    "hist_27.dat" notitle with lines lt 1 lw 3 lc 3 , \
+    "hist_45.dat" notitle with lines lt 1 lw 3 lc 11 , \
+    "gauss.dat" notitle with lines lt 1 lw 3 lc 7
+
+# Warp conv
+set output "warp_conv.eps"
+plot \
+    "hist_4.dat" notitle with lines lt 1 lw 3 lc 1 , \
+    "hist_28.dat" notitle with lines lt 1 lw 3 lc 3 , \
+    "hist_46.dat" notitle with lines lt 1 lw 3 lc 11 , \
     "gauss.dat" notitle with lines lt 1 lw 3 lc 7
 
 # Warp conv phot
-set output "warp_conv_phot.ps"
+set output "warp_conv_phot.eps"
 plot \
     "hist_5.dat" notitle with lines lt 1 lw 3 lc 1 , \
-    "hist_23.dat" notitle with lines lt 1 lw 3 lc 3 , \
-    "hist_41.dat" notitle with lines lt 1 lw 3 lc 11 , \
+    "hist_29.dat" notitle with lines lt 1 lw 3 lc 3 , \
+    "hist_47.dat" notitle with lines lt 1 lw 3 lc 11 , \
     "gauss.dat" notitle with lines lt 1 lw 3 lc 7
 
 # Stack
-set output "stack.ps"
+set output "stack.eps"
 plot \
     "hist_50.dat" notitle with lines lt 1 lw 3 lc 1 , \
     "hist_51.dat" notitle with lines lt 1 lw 3 lc 3 , \
+    "hist_58.dat" notitle with lines lt 1 lw 3 lc 11 , \
+    "gauss.dat" notitle with lines lt 1 lw 3 lc 7
+
+# Stack phot
+set output "stack_phot.eps"
+plot \
+    "hist_52.dat" notitle with lines lt 1 lw 3 lc 1 , \
+    "hist_53.dat" notitle with lines lt 1 lw 3 lc 3 , \
     "hist_59.dat" notitle with lines lt 1 lw 3 lc 11 , \
     "gauss.dat" notitle with lines lt 1 lw 3 lc 7
 
-# Stack phot
-set output "stack_phot.ps"
-plot \
-    "hist_51.dat" notitle with lines lt 1 lw 3 lc 1 , \
-    "hist_52.dat" notitle with lines lt 1 lw 3 lc 3 , \
-    "hist_60.dat" notitle with lines lt 1 lw 3 lc 11 , \
-    "gauss.dat" notitle with lines lt 1 lw 3 lc 7
-
 # Diff
-set output "diff.ps"
+set output "diff.eps"
 plot \
     "hist_48.dat" notitle with lines lt 1 lw 3 lc 1 , \
     "hist_56.dat" notitle with lines lt 1 lw 3 lc 3 , \
-    "hist_63.dat" notitle with lines lt 1 lw 3 lc 11 , \
+    "hist_62.dat" notitle with lines lt 1 lw 3 lc 11 , \
     "gauss.dat" notitle with lines lt 1 lw 3 lc 7
 
 # Diff phot
-set output "diff_phot.ps"
+set output "diff_phot.eps"
 plot \
     "hist_49.dat" notitle with lines lt 1 lw 3 lc 1 , \
     "hist_57.dat" notitle with lines lt 1 lw 3 lc 3 , \
-    "hist_64.dat" notitle with lines lt 1 lw 3 lc 11 , \
+    "hist_63.dat" notitle with lines lt 1 lw 3 lc 11 , \
     "gauss.dat" notitle with lines lt 1 lw 3 lc 7
Index: /branches/eam_branches/ipp-20100621/archive/noise_model/simulate.c
===================================================================
--- /branches/eam_branches/ipp-20100621/archive/noise_model/simulate.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/archive/noise_model/simulate.c	(revision 28794)
@@ -6,9 +6,8 @@
 #define DET_SIZE 2000
 #define SKY_SIZE 1000
-#define SIZE 1024
 #define OUTPUT_ROOT "test"
-#define SCALE 0.654321
+#define SCALE 0.7654321
 #define ROT M_PI_2
-#define INTERPOLATION PS_INTERPOLATE_LANCZOS4
+#define INTERPOLATION PS_INTERPOLATE_LANCZOS3
 #define OFFSET 16
 #define SMOOTH_SIGMA 6.54321
@@ -16,4 +15,5 @@
 #define DUAL_KERNEL "sub.subkernel"
 #define WARP_NUM 10000
+#define CONV_NUM 1000
 
 static const float variances[] = { 3.0, 10.0, 30.0, 100.0, 300.0, 1000.0, 3000.0, 10000.0 };
@@ -21,8 +21,13 @@
 static const char *rootNames[] = { "o5298g0209o.fake.warp", "o5298g0210o.fake.warp", "o5298g0211o.fake.warp",
                                    "o5298g0212o.fake.warp", "o5298g0213o.fake.warp", "o5298g0214o.fake.warp",
+
                                    "o5298g0215o.fake.warp", "o5298g0216o.fake.warp" };
+#if 1
 static const char *kernels[] = { "test.5.kernel", "test.11.kernel", "test.17.kernel", "test.23.kernel",
                                  "test.29.kernel", "test.35.kernel", "test.41.kernel", "test.47.kernel" };
-
+#else
+static const char *kernels[] = { "test.11.kernel", "test.11.kernel", "test.11.kernel", "test.11.kernel",
+                                 "test.11.kernel", "test.11.kernel", "test.11.kernel", "test.11.kernel" };
+#endif
 
 void writeImage(const psImage *image, const char *suffix)
@@ -76,10 +81,8 @@
 float meanVar(const psImage *var, const psImage *mask, const psKernel *covar)
 {
-    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
     psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN);
-    psImageBackground(stats, NULL, var, mask, 0xFF, rng);
+    psImageStats(stats, var, mask, 0xFF);
     float variance = stats->sampleMedian * psImageCovarianceFactor(covar);
     psFree(stats);
-    psFree(rng);
     return variance;
 }
@@ -93,4 +96,7 @@
         for (int x = 0; x < image->numCols; x++) {
             sn->data.F32[y][x] = image->data.F32[y][x] / sqrtf(var->data.F32[y][x] * varFactor);
+            if (!isfinite(sn->data.F32[y][x])) {
+                mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] = 0xFF;
+            }
         }
     }
@@ -100,4 +106,11 @@
     float offset = stats->sampleMean;
     psFree(stats);
+
+    static int number = 0;
+
+    psString snName = NULL;
+    psStringAppend(&snName, "sn.%d.fits", number);
+    writeImage(sn, snName);
+    fprintf(stderr, "Writing S/N image %d\n", number);
 
     psHistogram *hist = psHistogramAlloc(-5, +5, 101);
@@ -121,8 +134,7 @@
     psFree(sn);
 
-    static int number = 0;
     psString name = NULL;
     fprintf(stderr, "Writing histogram %d\n", number);
-    psStringAppend(&name, "hist_%d.dat", number++);
+    psStringAppend(&name, "hist_%d.dat", number);
     FILE *file = fopen(name, "w");
     psFree(name);
@@ -136,4 +148,6 @@
     psFree(hist);
 
+    number++;
+
     return noise;
 }
@@ -144,14 +158,7 @@
 {
     psKernel *kernel2 = psKernelAlloc(kernel->xMin, kernel->xMax, kernel->yMin, kernel->yMax);
-    double sum = 0.0, sum2 = 0.0;
     for (int y = kernel->yMin; y <= kernel->yMax; y++) {
         for (int x = kernel->xMin; x <= kernel->xMax; x++) {
-            sum += kernel->kernel[y][x];
-            sum2 += kernel2->kernel[y][x] = PS_SQR(kernel->kernel[y][x]);
-        }
-    }
-    for (int y = kernel->yMin; y <= kernel->yMax; y++) {
-        for (int x = kernel->xMin; x <= kernel->xMax; x++) {
-            kernel2->kernel[y][x] /= sum2;
+            kernel2->kernel[y][x] = PS_SQR(kernel->kernel[y][x]);
         }
     }
@@ -176,21 +183,23 @@
 
     psKernel *kernel = psImageSmoothKernel(SMOOTH_SIGMA, SMOOTH_N_SIGMA); // Kernel used for smoothing
+    double sum2 = 0.0;                                               // Sum of kernel squared
+    for (int y = kernel->yMin; y <= kernel->yMax; y++) {
+        for (int x = kernel->xMin; x <= kernel->xMax; x++) {
+            sum2 += PS_SQR(kernel->kernel[y][x]);
+        }
+    }
+    float factor = 1.0 / sum2;
     psKernel *smoothCovar = psImageCovarianceCalculate(kernel, covar);
     psFree(kernel);
-    psImageCovarianceTransfer(smoothVariance, smoothCovar);
+
+    // Apply square root of significance image scaling factor to image
+    psBinaryOp(smoothImage, smoothImage, "*", psScalarAlloc(sqrtf(factor), PS_TYPE_F32));
+
 #else
     psKernel *kernel = psImageSmoothKernel(SMOOTH_SIGMA, SMOOTH_N_SIGMA); // Kernel used for smoothing
     psKernel *kernel2 = psKernelAlloc(kernel->xMin, kernel->xMax, kernel->yMin, kernel->yMax);
-    double sum = 0.0, sum2 = 0.0;
     for (int y = kernel->yMin; y <= kernel->yMax; y++) {
         for (int x = kernel->xMin; x <= kernel->xMax; x++) {
-            sum += kernel->kernel[y][x];
-            sum2 += kernel2->kernel[y][x] = PS_SQR(kernel->kernel[y][x]);
-        }
-    }
-    fprintf(stderr, "Kernel sum: %f %f\n", sum, sum2);
-    for (int y = kernel->yMin; y <= kernel->yMax; y++) {
-        for (int x = kernel->xMin; x <= kernel->xMax; x++) {
-            kernel2->kernel[y][x] /= sum2;
+            kernel2->kernel[y][x] = PS_SQR(kernel->kernel[y][x]);
         }
     }
@@ -332,4 +341,19 @@
 }
 
+float covarianceSum(const psKernel *covar)
+{
+    if (!covar) {
+        return 1.0;
+    }
+    double sum = 0.0;
+    for (int y = covar->yMin; y <= covar->yMax; y++) {
+        for (int x = covar->xMin; x <= covar->xMax; x++) {
+            sum += covar->kernel[y][x];
+        }
+    }
+    return sum;
+}
+
+
 int main(int argc, char *argv[])
 {
@@ -383,9 +407,10 @@
         }
 
-        fprintf(stderr, "Input image %d: S/N: %f Covar: %f Var: %f\n",
+        fprintf(stderr, "Input image %d: S/N: %f Covar: %f Var: %f CovarSum: %f\n",
                 i,
                 signoise(inImage, inMask, inVariance, inCovar),
                 psImageCovarianceFactor(inCovar),
-                meanVar(inVariance, inMask, inCovar));
+                meanVar(inVariance, inMask, inCovar),
+                covarianceSum(inCovar));
 
         phot(inImage, inMask, inVariance, inCovar);
@@ -417,9 +442,10 @@
         }
 
-        fprintf(stderr, "Warp image %d: S/N: %f Covar: %f Var: %f\n",
+        fprintf(stderr, "Warp image %d: S/N: %f Covar: %f Var: %f CovarSum: %f\n",
                 i,
                 signoise(warpImage, warpMask, warpVariance, warpCovar),
                 psImageCovarianceFactor(warpCovar),
-                meanVar(warpVariance, warpMask, warpCovar));
+                meanVar(warpVariance, warpMask, warpCovar),
+                covarianceSum(warpCovar));
 
         phot(warpImage, warpMask, warpVariance, warpCovar);
@@ -434,4 +460,11 @@
         pmReadoutReadSubtractionKernels(ro, fits);
         pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, ro->analysis, PM_SUBTRACTION_ANALYSIS_KERNEL);
+        int normIndex = PM_SUBTRACTION_INDEX_NORM(kernels);
+        int bgIndex = PM_SUBTRACTION_INDEX_BG(kernels);
+#if 1
+//        kernels->solution1->data.F64[normIndex] += 1.0;
+        kernels->solution1->data.F64[bgIndex] = 0.0;
+#endif
+        fprintf(stderr, "Norm: %f BG: %f\n", kernels->solution1->data.F64[normIndex], kernels->solution1->data.F64[bgIndex]);
 #if 0
         for (int i = 0; i < kernels->num; i++) {
@@ -455,4 +488,36 @@
         psFitsClose(fits);
 
+#if 0
+        {
+            psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
+            psArray *covariances = psArrayAlloc(CONV_NUM);
+            psVector *factors = psVectorAlloc(CONV_NUM, PS_TYPE_F32);
+            double mean = 0.0;
+            for (int i = 0; i < CONV_NUM; i++) {
+                float x, y;
+                p_pmSubtractionPolynomialNormCoords(&x, &y, psRandomUniform(rng) * SKY_SIZE,
+                                                    psRandomUniform(rng) * SKY_SIZE,
+                                                    kernels->xMin, kernels->xMax,
+                                                    kernels->yMin, kernels->yMax);
+                psKernel *kernel = pmSubtractionKernel(kernels, x, y, false);
+                psKernel *covar = covariances->data[i] = psImageCovarianceCalculate(kernel, inCovar);
+                psFree(kernel);
+                mean += factors->data.F32[i] = psImageCovarianceFactor(covar);
+            }
+            psFree(rng);
+            psFree(covariances);
+
+            mean /= WARP_NUM;
+
+            double stdev = 0.0;
+            for (int i = 0; i < CONV_NUM; i++) {
+                stdev += PS_SQR(factors->data.F32[i] - mean);
+            }
+            stdev = sqrt(stdev/(CONV_NUM-1));
+            fprintf(stderr, "Conv covariance mean: %f stdev: %f\n", mean, stdev);
+            psFree(factors);
+        }
+#endif
+
         pmReadout *conv = pmReadoutAlloc(NULL);
 #if 1
@@ -460,5 +525,5 @@
         conv->mask = psImageAlloc(SKY_SIZE, SKY_SIZE, PS_TYPE_IMAGE_MASK);
         conv->variance = psImageAlloc(SKY_SIZE, SKY_SIZE, PS_TYPE_F32);
-        if (!pmSubtractionMatchPrecalc(NULL, conv, NULL, ro, ro->analysis, 32, 0.0, 0.01,
+        if (!pmSubtractionMatchPrecalc(NULL, conv, NULL, ro, ro->analysis, 2 * kernels->size + 1, 0.0, 0.01,
                                        0xFF, 0xF0, 0x0F, 0.1, 1.0)) {
             psErrorStackPrint(stderr, "Error:");
@@ -494,13 +559,15 @@
         }
 
-        fprintf(stderr, "Conv Image %d: S/N: %f Covar: %f Var: %f\n",
+        fprintf(stderr, "Conv Image %d: S/N: %f Covar: %f Var: %f CovarSum: %f\n",
                 i,
                 signoise(conv->image, conv->mask, conv->variance, conv->covariance),
                 psImageCovarianceFactor(conv->covariance),
-                meanVar(conv->variance, conv->mask, conv->covariance));
+                meanVar(conv->variance, conv->mask, conv->covariance),
+                covarianceSum(conv->covariance));
 
         phot(conv->image, conv->mask, conv->variance, conv->covariance);
         readouts->data[i] = conv;
 
+        //        exit(1);
     }
 
@@ -562,4 +629,14 @@
                 psImageCovarianceFactor(stackCovar2),
                 meanVar(stackVariance2, stackMask2, stackCovar2));
+
+        writeImage(stackImage1, "stack1.image.fits");
+        writeImage(stackMask1, "stack1.mask.fits");
+        writeImage(stackVariance1, "stack1.variance.fits");
+        writeImage(stackCovar1->image, "stack1.covar.fits");
+        writeImage(stackImage2, "stack2.image.fits");
+        writeImage(stackMask2, "stack2.mask.fits");
+        writeImage(stackVariance2, "stack2.variance.fits");
+        writeImage(stackCovar2->image, "stack2.covar.fits");
+
         phot(stackImage1, stackMask1, stackVariance1, stackCovar1);
         phot(stackImage2, stackMask2, stackVariance2, stackCovar2);
@@ -584,4 +661,12 @@
         kernel->yMax = SKY_SIZE;
 
+        int normIndex = PM_SUBTRACTION_INDEX_NORM(kernel);
+        int bgIndex = PM_SUBTRACTION_INDEX_BG(kernel);
+#if 1
+//        kernel->solution1->data.F64[normIndex] += 1.0;
+        kernel->solution1->data.F64[bgIndex] = 0.0;
+#endif
+        fprintf(stderr, "Norm: %f BG: %f\n", kernel->solution1->data.F64[normIndex], kernel->solution1->data.F64[bgIndex]);
+
         pmReadout *convStack1 = pmReadoutAlloc(NULL);
         pmReadout *convStack2 = pmReadoutAlloc(NULL);
@@ -594,5 +679,5 @@
 
         if (!pmSubtractionMatchPrecalc(convStack1, convStack2, stack1, stack2, stack1->analysis,
-                                       32, 0.0, 0.001, 0xFF, 0xF0, 0x0F, 0.1, 1.0)) {
+                                       2 * kernel->size + 1, 0.0, 0.001, 0xFF, 0xF0, 0x0F, 0.1, 1.0)) {
             psErrorStackPrint(stderr, "Error:");
             exit(1);
@@ -600,4 +685,13 @@
         psFree(stack1);
         psFree(stack2);
+
+        writeImage(convStack1->image, "stack1.conv.image.fits");
+        writeImage(convStack1->mask, "stack1.conv.mask.fits");
+        writeImage(convStack1->variance, "stack1.conv.variance.fits");
+        writeImage(convStack1->covariance->image, "stack1.conv.covar.fits");
+        writeImage(convStack2->image, "stack2.conv.image.fits");
+        writeImage(convStack2->mask, "stack2.conv.mask.fits");
+        writeImage(convStack2->variance, "stack2.conv.variance.fits");
+        writeImage(convStack2->covariance->image, "stack2.conv.covar.fits");
 
         psImageCovarianceTransfer(convStack1->variance, convStack1->covariance);
@@ -624,6 +718,4 @@
                 meanVar(diffVariance, diffMask, diffCovar));
 
-        phot(diffImage, diffMask, diffVariance, diffCovar);
-
         writeImage(diffImage, "ssdiff.image.fits");
         writeImage(diffMask, "ssdiff.mask.fits");
@@ -674,4 +766,12 @@
         kernel->xMax = SKY_SIZE;
         kernel->yMax = SKY_SIZE;
+
+        int normIndex = PM_SUBTRACTION_INDEX_NORM(kernel);
+        int bgIndex = PM_SUBTRACTION_INDEX_BG(kernel);
+#if 1
+//        kernel->solution1->data.F64[normIndex] += 1.0;
+        kernel->solution1->data.F64[bgIndex] = 0.0;
+#endif
+        fprintf(stderr, "Norm: %f BG: %f\n", kernel->solution1->data.F64[normIndex], kernel->solution1->data.F64[bgIndex]);
 
         pmReadout *warp = readouts->data[readouts->n - 1];
Index: /branches/eam_branches/ipp-20100621/dbconfig/background.md
===================================================================
--- /branches/eam_branches/ipp-20100621/dbconfig/background.md	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/dbconfig/background.md	(revision 28794)
@@ -0,0 +1,72 @@
+# Tables to support background replacement
+
+# Background replacement on a chipRun
+chipBackgroundRun METADATA
+    chip_bg_id          S64     0
+    chip_id             S64     0
+    state               STR     64
+    workdir             STR     255
+    label               STR     64
+    data_group          STR     64
+    dist_group          STR     64
+    reduction	        STR	64
+    note                STR     255
+    registered          TAI     NULL
+    magicked            S64     0
+END
+
+# Results of background replacement from chipBackgroundRun
+chipBackgroundImfile METADATA
+    chip_bg_id          S64     0
+    class_id            STR     64
+    path_base           STR     255
+    magicked            S64     0
+    dtime_script        F32     0.0
+    hostname            STR     64
+    quality             S16     0
+    fault               S16     0
+    software_ver        STR     16
+    bg                  F64     0.0
+    bg_stdev            F64     0.0
+    maskfrac_npix       F32     0.0
+    maskfrac_static     F32	0.0
+    maskfrac_dynamic    F32     0.0
+    maskfrac_magic      F32     0.0
+    maskfrac_advisory   F32     0.0
+END
+
+# Background replacement on a warpRun (utilising chipBackgroundRun)
+warpBackgroundRun METADATA
+    warp_bg_id          S64     0
+    warp_id             S64     0
+    chip_bg_id          S64     0
+    state               STR     64
+    workdir             STR     255
+    label               STR     64
+    data_group          STR     64
+    dist_group          STR     64
+    reduction	        STR	64
+    note                STR     255
+    registered          TAI     NULL
+    magicked            S64     0
+END
+
+# Results of background replacement from warpBackgroundRun
+warpBackgroundSkyfile METADATA
+    warp_bg_id          S64     0
+    skycell_id          STR     64
+    path_base           STR     255
+    magicked            S64     0
+    dtime_script        F32     0.0
+    hostname            STR     64
+    quality             S16     0
+    fault               S16     0
+    software_ver        STR     16
+    bg                  F64     0.0
+    bg_stdev            F64     0.0
+    maskfrac_npix       F32     0.0
+    maskfrac_static     F32	0.0
+    maskfrac_dynamic    F32     0.0
+    maskfrac_magic      F32     0.0
+    maskfrac_advisory   F32     0.0
+END
Index: /branches/eam_branches/ipp-20100621/dbconfig/changes.txt
===================================================================
--- /branches/eam_branches/ipp-20100621/dbconfig/changes.txt	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/dbconfig/changes.txt	(revision 28794)
@@ -1718,4 +1718,102 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+
+-- Tables to support background restoration
+-- Background replacement on a chipRun
+CREATE TABLE chipBackgroundRun (
+    chip_bg_id BIGINT AUTO_INCREMENT, -- unique identifier
+    chip_id BIGINT NOT NULL,          -- link to chipRun
+    state VARCHAR(64) NOT NULL,       -- state of run (new, full, etc.)
+    workdir VARCHAR(255) NOT NULL,    -- working directory
+    label VARCHAR(64),                -- processing label
+    data_group VARCHAR(64),           -- group for data
+    dist_group VARCHAR(64),           -- group for distribution
+    reduction VARCHAR(64),    -- reduction class (for altering recipe)
+    note VARCHAR(255),        -- note
+    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
+    magicked BIGINT DEFAULT 0 NOT NULL, -- magic status
+    PRIMARY KEY(chip_bg_id),
+    KEY(chip_id),
+    KEY(state),
+    KEY(label),
+    KEY(data_group),
+    KEY(dist_group),
+    FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Results of background replacement from chipBackgroundRun
+CREATE TABLE chipBackgroundImfile (
+    chip_bg_id BIGINT NOT NULL,        -- unique identifier
+    class_id VARCHAR(64) NOT NULL,     -- class (component) identifier
+    path_base VARCHAR(255) NOT NULL,   -- root name for outputs
+    magicked BIGINT,                   -- magic_id if magicked
+    dtime_script FLOAT,                -- elapsed time for script
+    hostname VARCHAR(64) NOT NULL,     -- host that executed script
+    quality SMALLINT NOT NULL,         -- bad quality flag
+    fault SMALLINT NOT NULL,           -- fault code
+    software_ver VARCHAR(16),          -- software version
+    bg FLOAT,                          -- background level
+    bg_stdev FLOAT,                    -- stdev of background
+    maskfrac_npix FLOAT,               -- Number of pixels masked
+    maskfrac_static FLOAT,             -- Fraction masked static
+    maskfrac_dynamic FLOAT,            -- Fraction masked dynamic
+    maskfrac_magic FLOAT,              -- Fraction masked magic
+    maskfrac_advisory FLOAT,           -- Fraction masked advisory
+    PRIMARY KEY(chip_bg_id,class_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Background replacement on a warpRun (utilising chipBackgroundRun)
+CREATE TABLE warpBackgroundRun (
+    warp_bg_id BIGINT AUTO_INCREMENT, -- unique identifier
+    warp_id BIGINT NOT NULL,          -- link to warpRun
+    chip_bg_id BIGINT NOT NULL,       -- link to chipBackgroundRun
+    state VARCHAR(64) NOT NULL,       -- state of run (new, full, etc.)
+    workdir VARCHAR(255) NOT NULL,    -- working directory
+    label VARCHAR(64),                -- processing label
+    data_group VARCHAR(64),           -- group for data
+    dist_group VARCHAR(64),           -- group for distribution
+    reduction VARCHAR(64),    -- reduction class (for altering recipe)
+    note VARCHAR(255),        -- note
+    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
+    magicked BIGINT DEFAULT 0 NOT NULL, -- magic status
+    PRIMARY KEY(warp_bg_id),
+    KEY(warp_id),
+    KEY(chip_bg_id),
+    KEY(state),
+    KEY(label),
+    KEY(data_group),
+    KEY(dist_group),
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id),
+    FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Results of background replacement from warpBackgroundRun
+CREATE TABLE warpBackgroundSkyfile (
+    warp_bg_id BIGINT NOT NULL,        -- unique identifier
+    skycell_id VARCHAR(64) NOT NULL,   -- skycell identifier
+    path_base VARCHAR(255) NOT NULL,   -- root name for outputs
+    magicked BIGINT,                   -- magic_id if magicked
+    dtime_script FLOAT,                -- elapsed time for script
+    hostname VARCHAR(64) NOT NULL,     -- host that executed script
+    quality SMALLINT NOT NULL,         -- bad quality flag
+    fault SMALLINT NOT NULL,           -- fault code
+    software_ver VARCHAR(16),          -- software version
+    bg FLOAT,                          -- background level
+    bg_stdev FLOAT,                    -- stdev of background
+    maskfrac_npix FLOAT,               -- Number of pixels masked
+    maskfrac_static FLOAT,             -- Fraction masked static
+    maskfrac_dynamic FLOAT,            -- Fraction masked dynamic
+    maskfrac_magic FLOAT,              -- Fraction masked magic
+    maskfrac_advisory FLOAT,           -- Fraction masked advisory
+    PRIMARY KEY(warp_bg_id,skycell_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(warp_bg_id) REFERENCES warpBackgroundRun(warp_bg_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+
 -- Version 1.1.68 addition of minidvodb tables and changes to addTables 
 
@@ -1768,4 +1866,5 @@
     registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
     note VARCHAR(255),             -- note
+    magicked BIGINT NOT NULL DEFAULT 0, -- magic mask applied
     PRIMARY KEY(diff_phot_id),
     KEY(diff_id),
Index: /branches/eam_branches/ipp-20100621/dbconfig/diffphot.md
===================================================================
--- /branches/eam_branches/ipp-20100621/dbconfig/diffphot.md	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/dbconfig/diffphot.md	(revision 28794)
@@ -10,4 +10,5 @@
     registered      TAI         NULL
     note            STR         255
+    magicked        S64         0
 END
 
Index: /branches/eam_branches/ipp-20100621/dbconfig/ipp.m4
===================================================================
--- /branches/eam_branches/ipp-20100621/dbconfig/ipp.m4	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/dbconfig/ipp.m4	(revision 28794)
@@ -33,3 +33,4 @@
 include(receive.md)
 include(publish.md)
+include(background.md)
 include(diffphot.md)
Index: /branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/Makefile.in	(revision 28794)
@@ -14,9 +14,9 @@
 PROGRAMS = \
 $(DESTBIN)/skyplot.dvo \
-$(DESTBIN)/czartool_getLabels.pl \
-$(DESTBIN)/czartool_checkServer.pl \
 $(DESTBIN)/czartool_getServerStatus.pl \
+$(DESTBIN)/czartool_revert.pl \
 $(DESTBIN)/build_histogram.dvo \
-
+$(DESTBIN)/helpers.dvo \
+$(DESTBIN)/plot_x_vs_y.dvo
 
 RAWSRC = \
@@ -36,4 +36,6 @@
 $(DESTWWW)/ipp.load.dat \
 $(DESTWWW)/ipp.load.php \
+$(DESTWWW)/ipp.plots.dat \
+$(DESTWWW)/ipp.plots.php \
 $(DESTWWW)/ipp.science.dat \
 $(DESTWWW)/ipp.science.php \
@@ -51,8 +53,13 @@
 $(DESTWWW)/ipp.czartool.dat \
 $(DESTWWW)/czartool_labels.php \
-$(DESTWWW)/czartool_servers.php \
+$(DESTWWW)/czartool_getplot.php \
 $(DESTWWW)/histogram.php \
 $(DESTWWW)/show_and_delete_image.php \
-
+$(DESTWWW)/scatterPlot.php \
+$(DESTWWW)/plotHistogram.php \
+$(DESTWWW)/columns_in_db.php \
+$(DESTWWW)/cleanTmpDirectory.php \
+$(DESTWWW)/warpProcessedExp_Images.php \
+$(DESTWWW)/warpProcessedExp.php
 
 DEFSRC = \
@@ -125,4 +132,5 @@
 $(DESTWWW)/rawScienceExp.php \
 $(DESTWWW)/rawExp.php \
+$(DESTWWW)/rawSummary.php \
 $(DESTWWW)/rawExpStats.php \
 $(DESTWWW)/summitExp.php \
@@ -135,4 +143,5 @@
 $(DESTWWW)/warpFailedSkyfiles.php \
 $(DESTWWW)/diffSummary.php \
+$(DESTWWW)/diffSummary_Images.php \
 $(DESTWWW)/diffRun.php \
 $(DESTWWW)/diffInputSkyfile.php \
@@ -142,4 +151,5 @@
 $(DESTWWW)/stackRun.php \
 $(DESTWWW)/stackSummary.php \
+$(DESTWWW)/stackSummary_Images.php \
 $(DESTWWW)/stackInputSkyfile.php \
 $(DESTWWW)/stackProcessedSkyfile.php \
@@ -175,5 +185,16 @@
 $(DESTWWW)/flatcorrRun.php \
 $(DESTWWW)/flatcorrChip.php \
-$(DESTWWW)/flatcorrCamera.php
+$(DESTWWW)/flatcorrCamera.php \
+$(DESTWWW)/maskStats.php \
+$(DESTWWW)/simplePlotRawImage.php \
+$(DESTWWW)/simplePlotChipImage.php \
+$(DESTWWW)/simplePlotCamImage.php \
+$(DESTWWW)/histogramBackgroundImage.php \
+$(DESTWWW)/histogramCamProcessedExpImage.php \
+$(DESTWWW)/histogramZptObsImage.php \
+$(DESTWWW)/scatterPlotAirMassFwhmImage.php \
+$(DESTWWW)/scatterCpiBgReMaImage.php \
+$(DESTWWW)/scatterCpiBgReMpImage.php \
+$(DESTWWW)/scatterCpiBgReSaImage.php
 
 PICTURES = \
@@ -189,5 +210,6 @@
 $(DESTWWW)/up.png \
 $(DESTWWW)/up_sel.png \
-$(DESTWWW)/missing.png
+$(DESTWWW)/missing.png \
+$(DESTWWW)/noimage.png
 
 DESTLINK = $(DESTWWW)/index.php
@@ -203,4 +225,7 @@
 	ln -s $(DESTWWW)/Login.php $@
 
+$(SRC)/%Image.php: $(DEF)/%Image.d $(DEF)/autocodeImage.php $(GENERATE)
+	@if [ ! -d $(SRC) ]; then mkdir -p $(SRC); fi
+	$(GENERATE) $< $(DEF)/autocodeImage.php $@
 $(SRC)/%.php: $(DEF)/%.d $(DEF)/autocode.php $(GENERATE)
 	@if [ ! -d $(SRC) ]; then mkdir -p $(SRC); fi
@@ -208,4 +233,8 @@
 
 # generated php code is built into SRC first
+$(DESTWWW)/%Image.php: $(SRC)/%Image.php
+	@if [ ! -d $(DESTWWW) ]; then mkdir -p $(DESTWWW) || exit; fi
+	rm -f $(DESTWWW)/$*Image.php || exit
+	cp $(SRC)/$*Image.php $(DESTWWW)/$*Image.php || exit
 $(DESTWWW)/%.php: $(SRC)/%.php
 	@if [ ! -d $(DESTWWW) ]; then mkdir -p $(DESTWWW) || exit; fi
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/README
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/README	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/README	(revision 28794)
@@ -98,2 +98,84 @@
 output column in the HTML table.
 
+Drawing Figures
+===============
+- Definition files (def/*.d) MUST be appended with Image.d to allow specific
+  processing (the template is not def/autocode.php but autocodeImage.php)
+     e.g.:   	  plotMyStuffImage.php
+- No changes to the previous file content except for these 4 items:
+
+     1) The 'restrict' keyword should be used in the 'show' column of
+     	FIELD entry if displaying the column values is wanted.
+	Such FIELDs are used in the WHERE restriction clause, not in
+	the query.
+
+	Note for developpers: actually any word different from the
+	existing keywords '', 'value', 'none', 'image=xxx',
+	'value=xxx' are ok.  No code was developped to support this
+	keyword in the scritps/generate script.
+
+     2) 'TOPLOT <list of db columns>': defines the list of columns to
+     	query.
+	e.g.: 
+	      TOPLOT camProcessedExp.fwhm_major
+	      TOPLOT chipProcessedImfile.bg, rawExp.sun_angle
+	All values (no LIMIT) are written to a temporary (text) file
+	of the /tmp directory.
+	In a row, the different values are separated by a single space
+	(i.e. ' ').
+
+     3) 'PLOTTER <php_script_name>': defines the name of the PHP
+        script that will call an external tool (e.g. a DVO script).
+	
+	It is expected that the PHP script named 'php_script_name'
+	supports the following parameters:
+	- input: the name of an input text filename;
+	- output: the name of an output image filename;
+	- title: a string (see item 4) PLOTTITLE).
+
+	e.g.:
+		PLOTTER scatterPlot.php	
+		PLOTTER plotHistogram.php
+
+	The list of existing plotters are detailed further.
+
+     4) 'PLOTTITLE <A title>': defines the title of a figure. Could be
+     	used in the external program which is effectively used for
+     	rendering the figure.
+
+	e.g.:
+		PLOTTITLE A title
+
+Existing plotting scripts
+=========================
+raw/skyplot.php (calls scripts/skyplot.dvo)
+	Text input file needs to contain 3 columns
+
+raw/plotHistogram.php (calls scripts/build_histogram.dvo)
+	Text input file needs to contain 1 column
+	Computes the histogram of the input file data for 100 bins.
+
+raw/scatterPlot.php (calls scripts/pot_x_vs_y.dvo)
+	Text input file needs to contain 2 columns
+	Scatter plot of the first column values vs the second ones
+	Regression line is also plotted (correlation coefficient is
+	shown in title).
+
+Plotting with PHP and temporary files
+=====================================
+
+Plots in PHP use temporary files (to provide data to dvo scripts and
+to store result images). Unfortunately, if the files are deleted at
+the end of main php script, the browser does not have the time to
+render the image. If the file is not deleted, they remain in the /tmp
+directory leading to disk space exhaustion.
+
+To avoid this problem, the function delete_old_tmp_files() in ipp.php
+was created. It checks the /tmp directory and removes all files
+belonging to the $DELETION_USER user and that are older than
+$DELETION_DELAY minutes. delete_old_tmp_files() is called at the
+beginning of the autocodeImage.php template script (line 4) and is
+therefore present in all generated plotting scripts.
+
+$DELETION_USER and $DELETION_DELAY are defined in site.php.in (values
+are 'apache' and '15').
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/autocodeImage.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/autocodeImage.php	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/autocodeImage.php	(revision 28794)
@@ -0,0 +1,162 @@
+<?php 
+
+include 'ipp.php';
+delete_old_tmp_files();
+
+$ID = checkID ();
+
+// require an explicit project
+if (! $ID['proj']) { projectform ($ID); }
+
+$db = dbconnect($ID['proj']);
+
+if ($ID['menu']) {
+  $myMenu = $ID['menu'];
+} else {
+  $myMenu = "$MENU";
+}
+
+menu($myMenu, '$TITLE', '$STYLE', $ID['link'], $ID['proj']);
+
+echo "<p> $TITLE </p>";
+
+// set up the form
+echo "<form action=\"$FILE\" method=\"POST\">\n";
+
+$restricted = 0;
+
+// define restrictiosn to the queries
+// ** TABLE RESTRICTIONS **
+
+if ($restricted == 0) {//Don't select anything
+  if ($WHERE == "") {
+    $WHERE = "WHERE (0 > 1)";
+  } else {
+    $WHERE = "WHERE (0 > 1)";
+  }
+}
+
+// get the result table count
+$sql = "SELECT count(*) FROM $TABLE $WHERE";
+
+$qry = $db->query($sql);
+if (dberror($qry)) {
+  echo "<b>error reading $TABLE table count</b><br>\n";
+  echo "<br><small><b> count query : $sql </b></small><br>\n";
+  menu_end();
+}
+if (!$qry->fetchInto($row)) {
+  echo "<b>error reading $TABLE table count</b><br>\n";
+  echo "<br><small><b> count query : $sql </b></small><br>\n";
+  menu_end();
+}
+$nValues = $row[0];
+echo "<b>$nValues match selection</b><br>\n";
+
+$sql = "SELECT $TOPLOT FROM $TABLE $WHERE";
+$qry = $db->query($sql);
+
+if (dberror($qry)) {
+  echo "<b>error reading $TABLE table</b><br>\n";
+  echo "<br><small><b> table query : $sql </b></small><br>\n";
+  menu_end();
+}
+
+// ** HEAD CODE **
+
+// ** BUTTON RESTRICTIONS **
+navigate_buttons ($rowStart, $rowLast, $dTABLE, $rowTotal, $buttonLink, $ID, '$FILE');
+
+echo "&nbsp; : &nbsp; or enter start row: <input type=\"text\" name=\"from\" size=\"5\" value=\"$rowStart\">\n";
+
+// set up the table
+echo "<table class=list>\n";
+
+// echo "<tr><td></td>\n"; // first field is a label set below for the query rows only
+// ** TABLE HEADER **
+// echo "</tr>\n";
+
+// query restriction form
+// echo "<tr>\n";
+// echo "<td class=list> <input type=\"text\" name=\"expID\"></td>\n";
+// ** TABLE QUERY **
+// echo "</tr>\n";
+
+// query restriction form
+// echo "<tr>\n";
+// echo "<td class=list> <input type=\"text\" name=\"expID\"></td>\n";
+// TABLE QUERY 
+// echo "</tr>\n";
+
+// close the table and form
+echo "</table>\n";
+echo "<input type=\"submit\" name=\"constraints\" value=\"refine search\">\n";
+$pass = $ID['pass'];
+$proj = $ID['proj'];
+$menu = $ID['menu'];
+echo "<input type=\"hidden\" name=\"pass\" value=\"$pass\">\n";
+echo "<input type=\"hidden\" name=\"proj\" value=\"$proj\">\n";
+echo "<input type=\"hidden\" name=\"menu\" value=\"$menu\">\n";
+echo "</form>\n";
+
+if ($row[0] == 0) {
+  menu_end();
+}
+
+$datFilename = tempnam("/tmp", "dvodat.");
+$pngFilename = $datFilename.".png";
+
+$datfile = fopen($datFilename, "w");
+chmod($datFilename, 0777);
+
+// output the results to tmpfile
+while ($qry->fetchInto($row)) {
+  $line = "";
+  foreach ($row as $i => $value) {
+    switch ($value) {
+    case 'g.00000':
+      $filter = 1;
+      break;
+    case 'r.00000':
+      $filter = 2;
+      break;
+    case 'i.00000':
+      $filter = 3;
+      break;
+    case 'z.00000':
+      $filter = 4;
+      break;
+    case 'y.00000':
+      $filter = 5;
+      break;
+    case 'w.00000':
+      $filter = 6;
+      break;
+    case 'OPEN':
+      $filter = 0;
+      break;
+    default:
+      $filter = $value;
+      break;
+    }
+    $line .= "$filter ";
+  }
+  if (preg_match('/^[ \t]*$/', $line) == 0) {//Don't write empty lines
+    $line = preg_replace('/[ \t]*$/', '', $line);
+    fwrite ($datfile, $line."\n");
+  }
+}
+fclose($datfile);
+
+echo "<img src=\"$PLOTTER?input=$datFilename&output=$pngFilename&title='$PLOTTITLE'\"><br>\n";
+
+// ** TAIL CODE **
+
+echo "<small> WHERE: $WHERE<br><br></small>\n";
+echo "<small> SQL: $sql<br><br></small>\n";
+
+echo "Temporary files are [$datFilename] and [$pngFilename]\n";
+
+menu_end();
+
+?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/camSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/camSummary.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/camSummary.d	(revision 28794)
@@ -16,2 +16,4 @@
 GROUP label
 GROUP workdir
+GROUP data_group
+GROUP tess_id
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/chipSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/chipSummary.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/chipSummary.d	(revision 28794)
@@ -10,5 +10,5 @@
 FIELD tess_id,    	 7, %s,     tess_id
 FIELD state,    	 7, %s,     state
-FIELD count(state) as n, 7, %d,     count
+FIELD count(state),      7, %d,     count
 
 MODE  summary
@@ -16,2 +16,4 @@
 GROUP label
 GROUP workdir
+GROUP data_group
+GROUP tess_id
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/destreakSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/destreakSummary.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/destreakSummary.d	(revision 28794)
@@ -15,2 +15,3 @@
 GROUP stage
 GROUP label
+GROUP data_group
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/diffSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/diffSummary.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/diffSummary.d	(revision 28794)
@@ -16,2 +16,4 @@
 GROUP label
 GROUP workdir
+GROUP data_group
+GROUP tess_id
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/diffSummary_Images.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/diffSummary_Images.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/diffSummary_Images.d	(revision 28794)
@@ -0,0 +1,52 @@
+TABLE diffRun, diffSkyfile, diffSummary
+TITLE Diff Summary w/ Images
+FILE  diffSummary_Images.php
+MENU  ipp.stack.dat
+
+WHERE diffRun.diff_id             = diffSkyfile.diff_id
+WHERE diffSummary.diff_id = diffRun.diff_id
+# WHERE diffRun.state = 'full'
+# WHERE diffSkyfile.fault = 0
+
+# define image names to be used below
+# IMAGE VAR basename filerule camera class_id
+IMAGE JPEG2 OP3 PPSKYCELL.JPEG2 GPC1 NONE
+OP   OP3  $diffSummary.path_base . $diffSummary.projection_cell
+ARGS ARG7 name=$diffSummary.path_base.$diffSummary.projection_cell
+ARGS ARG7 rule=PPSKYCELL.JPEG1
+ARGS ARG7 camera=GPC1
+ARGS ARG7 class_id=NONE
+
+# XXX need to get camera from lookup
+ARGS  ARG7 diffSkyfile.diff_id=$diffSkyfile.diff_id
+ARGS  ARG7 diffSkyfile.skycell_id=$diffSkyfile.skycell_id
+ARGS  ARG7 camera=GPC1
+ARGS  ARG7 basename=$diffSkyfile.path_base
+
+#     field                         size  format  name           show     link to         extras
+FIELD diffSkyfile.diff_id,             5, %d,     Diff ID
+FIELD diffSkyfile.skycell_id,          5, %s,     Skycell ID
+FIELD diffRun.label,    	       5, %s,     Label
+FIELD diffRun.data_group,    	       5, %s,     data grp
+FIELD diffRun.dist_group,    	       5, %s,     dist grp
+FIELD *,    	                       8, %s,     image, image=JPEG2, getimage.php, ARG7
+# FIELD diffRun.filter,    	       5, %s,     Filter
+FIELD diffRun.tess_id,    	       5, %s,     Tess ID
+FIELD diffRun.state,    	       7, %s,     State,         value,   diffProcessedSkyfile.php, ARG7
+FIELD diffSkyfile.bg,                  5, %.2f,   backgnd
+FIELD diffSkyfile.bg_stdev,            5, %.2f,   stdev    
+FIELD diffSkyfile.dtime_diff,          5, %.2f,   t(diff)
+FIELD diffSkyfile.dtime_match,         5, %.2f,   t(match)
+FIELD diffSkyfile.dtime_phot,          5, %.2f,   t(phot)
+FIELD diffSkyfile.dtime_script,        5, %.2f,   t(script)
+FIELD diffSkyfile.stamps_num,          5, %d,     n(stamps)
+FIELD diffSkyfile.stamps_mean,         5, %.2f,   mean(stamps)
+FIELD diffSkyfile.stamps_rms,          5, %.2f,   rms(stamps)
+FIELD diffSkyfile.sources,             5, %d,     n(sources)
+FIELD diffSkyfile.path_base,           5, %s,     path_base,     none
+
+FIELD diffSummary.path_base,           5, %s,     path_base,     none
+FIELD diffSummary.projection_cell,     5, %s,     projection_cell,     none
+
+# TAIL PHP insert_image ('PPSUB.OUTPUT.JPEG1', 'width=90%');
+TAIL PHP insert_log ('LOG.EXP');
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/distRun.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/distRun.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/distRun.d	(revision 28794)
@@ -9,5 +9,5 @@
 
 #     field                   size  format  name         show    link to                  extras
-FIELD distRun.dist_id,        5, %d,     Dist ID
+FIELD distinct(distRun.dist_id),        5, %d,     Dist ID
 FIELD distRun.target_id,      5, %d,     Target ID
 FIELD distRun.stage,          5, %s,     Stage
@@ -25,2 +25,4 @@
 FIELD distComponent.fault,    5, %d,     Comp. Fault
 FIELD distRun.note,           5, %s,     Note
+
+GROUP dist_id
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/distSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/distSummary.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/distSummary.d	(revision 28794)
@@ -6,5 +6,5 @@
 #     field                   size  format  name         show    link to                  extras
 FIELD distRun.label,          15, %s,     Label
-FIELD distRun.data_group,          15, %s,     data grp
+#FIELD distRun.data_group,          15, %s,     data grp
 FIELD distRun.stage,          5, %s,     Stage
 FIELD distRun.state,          5, %s,     State
@@ -16,2 +16,3 @@
 GROUP stage
 GROUP state
+GROUP data_group
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/fakeSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/fakeSummary.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/fakeSummary.d	(revision 28794)
@@ -16,2 +16,4 @@
 GROUP label
 GROUP workdir
+GROUP data_group
+GROUP tess_id
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramBackgroundImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramBackgroundImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramBackgroundImage.d	(revision 28794)
@@ -0,0 +1,31 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp
+TITLE New histogram background
+FILE  histogramBackgroundImage.php
+MENU  ipp.plots.dat
+
+WHERE camRun.state = 'full'
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT rawExp.bg
+PLOTTER plotHistogram.php
+PLOTTITLE Background values
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramCamProcessedExpImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramCamProcessedExpImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramCamProcessedExpImage.d	(revision 28794)
@@ -0,0 +1,31 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp
+TITLE Histogram fwhm major (camProcessedExp)
+FILE  histogramCamProcessedExpImage.php
+MENU  ipp.plots.dat
+
+WHERE camRun.state = 'full'
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT camProcessedExp.fwhm_major
+PLOTTER plotHistogram.php
+PLOTTITLE Fwhm_major from camProcessedExp
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramChipProcessedImfileFwhmMajorImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramChipProcessedImfileFwhmMajorImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramChipProcessedImfileFwhmMajorImage.d	(revision 28794)
@@ -0,0 +1,32 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp, chipProcessedImfile
+TITLE Histogram fwhm major (chipProcessedImfile)
+FILE  histogramChipProcessedImfileFwhmMajorImage.php
+MENU  ipp.plots.dat
+
+WHERE camRun.state = 'full'
+WHERE chipProcessedImfile.exp_id = rawExp.exp_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT chipProcessedImfile.fwhm_major
+PLOTTER plotHistogram.php
+PLOTTITLE Fwhm_major from ChipProcessedImfile
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramChipRunFwhmMajorImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramChipRunFwhmMajorImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramChipRunFwhmMajorImage.d	(revision 28794)
@@ -0,0 +1,31 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp
+TITLE Histogram fwhm major (camProcessedExp)
+FILE  histogramCamProcessedExpImage.php
+MENU  ipp.imfiles.dat
+
+WHERE camRun.state = 'full'
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT camProcessedExp.fwhm_major
+PLOTTER plotHistogram.php
+PLOTTITLE Fwhm_major values
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramFwhmMajorImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramFwhmMajorImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramFwhmMajorImage.d	(revision 28794)
@@ -0,0 +1,31 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp
+TITLE New histogram fwhm major
+FILE  histogramFwhmMajorImage.php
+MENU  ipp.imfiles.dat
+
+WHERE camRun.state = 'full'
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT camProcessedExp.fwhm_major
+PLOTTER plotHistogram.php
+PLOTTITLE Fwhm_major values
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramFwhmMinorImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramFwhmMinorImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramFwhmMinorImage.d	(revision 28794)
@@ -0,0 +1,31 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp
+TITLE New histogram fwhm minor
+FILE  histogramFwhmMinorImage.php
+MENU  ipp.imfiles.dat
+
+WHERE camRun.state = 'full'
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT camProcessedExp.fwhm_minor
+PLOTTER plotHistogram.php
+PLOTTITLE Fwhm_minor values
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramZptObsImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramZptObsImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/histogramZptObsImage.d	(revision 28794)
@@ -0,0 +1,31 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp
+TITLE Histogram of ZptObs
+FILE  histogramZptObsImage.php
+MENU  ipp.plots.dat
+
+WHERE camRun.state = 'full'
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT camProcessedExp.zpt_obs
+PLOTTER plotHistogram.php
+PLOTTITLE ZptObs
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/magicSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/magicSummary.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/magicSummary.d	(revision 28794)
@@ -14,2 +14,4 @@
 GROUP state
 GROUP label
+GROUP data_group
+GROUP workdir
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/maskStats.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/maskStats.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/maskStats.d	(revision 28794)
@@ -0,0 +1,50 @@
+TABLE rawExp, chipRun, camRun, fakeRun, warpRun
+TITLE Mask Stats
+FILE  maskStats.php
+MENU  ipp.imfiles.dat
+
+WHERE rawExp.exp_id = chipRun.exp_id
+WHERE chipRun.chip_id = camRun.chip_id
+WHERE camRun.cam_id = fakeRun.cam_id
+WHERE fakeRun.fake_id = warpRun.fake_id
+
+OP   OP1  $ra * 57.295783
+OP   OP2  $decl * 57.295783
+
+#        field              size format  name           show          link to   extras
+FIELD    rawExp.exp_name,    10,  %s,     Exp Name
+FIELD    rawExp.exp_id,       5,  %d,     Exp ID,        value,   rawImfile.php,  ARG1
+FIELD    rawExp.exp_type,     8,  %s,     Type    
+FIELD    rawExp.state,        5,  %s,     State
+FIELD    rawExp.obs_mode,     8,  %s,     obs mode
+FIELD    rawExp.dateobs,     19,  %T,     Date/Time
+FIELD    rawExp.ra,           8,  %C,     RA,           op=OP1      
+FIELD    rawExp.decl,         8,  %C,     DEC,          op=OP2
+FIELD    rawExp.filter,      10,  %s,     FILTER
+FIELD    rawExp.airmass,      5,  %.4f,   airmass     
+FIELD    rawExp.exp_time,     5,  %.2f,   exp_time    
+
+FIELD    chipRun.maskfrac_npix,         8,  %.3e,     chip npix   
+FIELD    camRun.maskfrac_ref_npix,     	8,  %.3e,     cam ref npix   
+FIELD    camRun.maskfrac_max_npix,     	8,  %.3e,     cam max npix   
+FIELD    warpRun.maskfrac_npix,     	8,  %.3e,     warp npix   
+
+FIELD    chipRun.maskfrac_static,   	8,  %.3f,     chip static 
+FIELD    camRun.maskfrac_ref_static,   	8,  %.3f,     cam ref static 
+FIELD    camRun.maskfrac_max_static,   	8,  %.3f,     cam max static 
+FIELD    warpRun.maskfrac_static,   	8,  %.3f,     warp static 
+
+FIELD    chipRun.maskfrac_dynamic,  	8,  %.3f,     chip dynamic
+FIELD    camRun.maskfrac_ref_dynamic,  	8,  %.3f,     cam ref dynamic
+FIELD    camRun.maskfrac_max_dynamic,  	8,  %.3f,     cam max dynamic
+FIELD    warpRun.maskfrac_dynamic,  	8,  %.3f,     warp dynamic
+
+FIELD    chipRun.maskfrac_magic,    	8,  %.3f,     chip magic  
+FIELD    camRun.maskfrac_ref_magic,    	8,  %.3f,     cam ref magic  
+FIELD    camRun.maskfrac_max_magic,    	8,  %.3f,     cam max magic  
+FIELD    warpRun.maskfrac_magic,    	8,  %.3f,     warp magic  
+
+FIELD    chipRun.maskfrac_advisory, 	8,  %.3f,     chip advisory
+FIELD    camRun.maskfrac_ref_advisory, 	8,  %.3f,     cam ref advisory
+FIELD    camRun.maskfrac_max_advisory, 	8,  %.3f,     cam max advisory
+FIELD    warpRun.maskfrac_advisory,     8,  %.3f,     warp advisory 
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/rawDetrendExp_detrend.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/rawDetrendExp_detrend.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/rawDetrendExp_detrend.d	(revision 28794)
@@ -7,9 +7,9 @@
 WHERE rawExp.exp_type != 'OBJECT'
 
-ARGS ARG1 exp_id=$exp_id
+ARGS ARG1 rawImfile.exp_id=$exp_id
 
 #        field        	size  format  name           show     link to         extras
-FIELD    exp_id,      	 5,   %d,     Exp ID
-FIELD    exp_name,    	10,   %s,     Exp Name,     value,   rawImfile.php,  ARG1
+FIELD    exp_id,      	 5,   %d,     Exp ID,     value,   rawImfile.php,  ARG1
+FIELD    exp_name,    	10,   %s,     Exp Name
 FIELD    telescope,   	10,   %s,     Telescope
 FIELD    camera,      	10,   %s,     Camera
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/rawExp.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/rawExp.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/rawExp.d	(revision 28794)
@@ -15,4 +15,5 @@
 FIELD    camera,      10,  %s,     Camera
 FIELD    exp_type,     8,  %s,     Type    
+FIELD    obs_mode,     8,  %s,     obs mode
 FIELD    dateobs,     19,  %T,     Date/Time
 FIELD    ra,           8,  %C, RA,           op=OP1      
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/rawSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/rawSummary.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/rawSummary.d	(revision 28794)
@@ -0,0 +1,17 @@
+TABLE rawExp
+TITLE Raw Summary
+FILE  rawSummary.php
+MENU  ipp.load.dat
+
+#     field            size  format  name         show    link to         extras
+FIELD exp_type,    	 15, %s,     OBSTYPE
+FIELD obs_mode,    	 15, %s,     OBS_MODE
+FIELD state,    	  7, %s,     state
+FIELD MIN(dateobs),    	  19, %T,    min date
+FIELD MAX(dateobs),    	  19, %T,    max date
+FIELD count(exp_id) as n, 7, %d,     count
+
+MODE  summary
+GROUP state
+GROUP exp_type
+GROUP obs_mode
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterCpiBgReMaImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterCpiBgReMaImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterCpiBgReMaImage.d	(revision 28794)
@@ -0,0 +1,33 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp, chipProcessedImfile
+TITLE Scatter plot Background (chipProcessedImfile) vs Moon angle (rawExp)
+FILE  scatterCpiBgReMaImage.php
+MENU  ipp.plots.dat
+
+WHERE camRun.state = 'full'
+WHERE chipProcessedImfile.exp_id = rawExp.exp_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.moon_phase,          5, %.2f,   Moon Phase,      restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT chipProcessedImfile.bg, rawExp.moon_angle
+PLOTTER scatterPlot.php
+PLOTTITLE Airmass vs Fwhm_major (ChipProcessedImfile)
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterCpiBgReMpImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterCpiBgReMpImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterCpiBgReMpImage.d	(revision 28794)
@@ -0,0 +1,33 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp, chipProcessedImfile
+TITLE Scatter plot Background (chipProcessedImfile) vs Moon phase (rawExp)
+FILE  scatterCpiBgReMpImage.php
+MENU  ipp.plots.dat
+
+WHERE camRun.state = 'full'
+WHERE chipProcessedImfile.exp_id = rawExp.exp_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.moon_angle,          5, %.2f,   Moon Angle,      restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT chipProcessedImfile.bg, rawExp.moon_phase
+PLOTTER scatterPlot.php
+PLOTTITLE Background (chipProcessedImfile) vs Moon Phase (rawExp)
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterCpiBgReSaImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterCpiBgReSaImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterCpiBgReSaImage.d	(revision 28794)
@@ -0,0 +1,33 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp, chipProcessedImfile
+TITLE Scatter plot Background (chipProcessedImfile) vs Sun Angle (rawExp)
+FILE  scatterCpiBgReSaImage.php
+MENU  ipp.plots.dat
+
+WHERE camRun.state = 'full'
+WHERE chipProcessedImfile.exp_id = rawExp.exp_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.moon_angle,          5, %.2f,   Moon Angle,      restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT chipProcessedImfile.bg, rawExp.sun_angle
+PLOTTER scatterPlot.php
+PLOTTITLE Background (chipProcessedImfile) vs Sun Angle (rawExp)
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterPlotAirMassFwhmImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterPlotAirMassFwhmImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/scatterPlotAirMassFwhmImage.d	(revision 28794)
@@ -0,0 +1,32 @@
+TABLE camRun, chipRun, rawExp, camProcessedExp, chipProcessedImfile
+TITLE Scatter plot airmass vs fwhm major (chipProcessedImfile)
+FILE  scatterPlotAirMassFwhmImage.php
+MENU  ipp.plots.dat
+
+WHERE camRun.state = 'full'
+WHERE chipProcessedImfile.exp_id = rawExp.exp_id
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camRun.chip_id = chipRun.chip_id
+WHERE camProcessedExp.cam_id = camRun.cam_id
+WHERE camProcessedExp.fault = 0
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT rawExp.airmass, chipProcessedImfile.fwhm_major
+PLOTTER scatterPlot.php
+PLOTTITLE Airmass vs Fwhm_major (ChipProcessedImfile)
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/simplePlotCamImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/simplePlotCamImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/simplePlotCamImage.d	(revision 28794)
@@ -0,0 +1,29 @@
+TABLE camRun, chipRun, rawExp
+TITLE New sky plot - cam
+FILE  simplePlotCamImage.php
+MENU  ipp.plots.dat
+
+WHERE camRun.state != 'new' 
+WHERE chipRun.exp_id = rawExp.exp_id
+WHERE camRun.chip_id = chipRun.chip_id
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT rawExp.ra,rawExp.decl,rawExp.filter
+PLOTTER skyplot.php
+PLOTTITLE Sky Plot Cam
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/simplePlotChipImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/simplePlotChipImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/simplePlotChipImage.d	(revision 28794)
@@ -0,0 +1,28 @@
+TABLE chipRun, rawExp
+TITLE New sky plot - chip
+FILE  simplePlotChipImage.php
+MENU  ipp.plots.dat
+
+WHERE chipRun.state != 'new' 
+WHERE chipRun.exp_id = rawExp.exp_id
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT rawExp.ra,rawExp.decl,rawExp.filter
+PLOTTER skyplot.php
+PLOTTITLE Sky Plot Chip
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/simplePlotRawImage.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/simplePlotRawImage.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/simplePlotRawImage.d	(revision 28794)
@@ -0,0 +1,25 @@
+TABLE rawExp
+TITLE New sky plot - raw
+FILE  simplePlotRawImage.php
+MENU  ipp.plots.dat
+
+#    field                size  format  name           show     link to         extras
+FIELD rawExp.exp_name,            5, %s,     Exp Name, 	      restrict
+FIELD rawExp.exp_id,             5, %d,      Exp ID, 	      restrict
+FIELD rawExp.telescope,          10, %s,     Telescope,       restrict
+FIELD rawExp.camera,             10, %s,     Camera, 	      restrict
+FIELD rawExp.dateobs,            19, %T,     Date/Time,       restrict
+FIELD rawExp.ra,                  8, %C,     RA, 	      restrict
+FIELD rawExp.decl,                8, %C,     DEC, 	      restrict
+FIELD rawExp.comment,            65, %s,     Comment, 	      restrict
+FIELD rawExp.object,              8, %s,     Object, 	      restrict
+FIELD rawExp.filter,             10, %s,     FILTER, 	      restrict
+FIELD rawExp.exp_time,            5, %.2f,   exp_time, 	      restrict
+FIELD rawExp.airmass,             5, %.4f,   airmass, 	      restrict
+FIELD rawExp.bg,                  5, %.2f,   backgnd, 	      restrict
+FIELD rawExp.bg_stdev,            5, %.2f,   stdev, 	      restrict
+
+# What to plot
+TOPLOT rawExp.ra,rawExp.decl,rawExp.filter
+PLOTTER skyplot.php
+PLOTTITLE Sky Plot Raw
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/stackSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/stackSummary.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/stackSummary.d	(revision 28794)
@@ -16,2 +16,4 @@
 GROUP label
 GROUP workdir
+GROUP data_group
+GROUP tess_id
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/stackSummary_Images.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/stackSummary_Images.d	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/stackSummary_Images.d	(revision 28794)
@@ -0,0 +1,29 @@
+TABLE stackAssociation, stackSummary
+TITLE Stack Summary w/ Images
+FILE  stackSummary_Images.php
+MENU  ipp.stack.dat
+
+# TODO
+WHERE stackSummary.sass_id = stackAssociation.sass_id
+
+# define image names to be used below
+# IMAGE VAR basename filerule camera class_id
+IMAGE JPEG2 OP1 PPSKYCELL.JPEG2 GPC1 NONE
+OP   OP1  $stackSummary.path_base . $stackSummary.projection_cell
+ARGS ARG7 name=$stackSummary.path_base.$stackSummary.projection_cell
+ARGS ARG7 rule=PPSKYCELL.JPEG1
+ARGS ARG7 camera=GPC1
+ARGS ARG7 class_id=NONE
+
+#        field                   size  format  name           show     link to         extras
+FIELD stackSummary.sass_id,         5, %s,     SASS ID
+FIELD stackAssociation.data_group,  5, %s,     data grp
+FIELD stackAssociation.tess_id,     5, %s,     Tess ID
+FIELD stackAssociation.filter,      5, %s,     Filter
+FIELD *,    	                    8, %s,     image,	       image=JPEG2, getimage.php, ARG7
+FIELD stackSummary.path_base,       5, %s,     path_base,      none
+FIELD stackSummary.projection_cell, 5, %s,     projection_cell, none
+FIELD stackAssociation.sass_id,     5, %s,     sAMsai,         none
+
+# TAIL PHP insert_image ('PPSTACK.OUTPUT.JPEG1', 'width=90%');
+TAIL PHP insert_log ('LOG.EXP');
Index: /branches/eam_branches/ipp-20100621/ippMonitor/def/warpSummary.d
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/def/warpSummary.d	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/def/warpSummary.d	(revision 28794)
@@ -18,2 +18,4 @@
 GROUP label
 GROUP workdir
+GROUP data_group
+GROUP tess_id
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/cleanTmpDirectory.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/cleanTmpDirectory.php	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/cleanTmpDirectory.php	(revision 28794)
@@ -0,0 +1,32 @@
+<?php
+
+
+include 'ipp.php';
+
+$ID = checkID ();
+
+// require an explicit project
+if (! $ID['proj']) { projectform ($ID); }
+
+menu('ipp.imfiles.dat', 'Imfile Tables', 'ipp.css', $ID['link'], $ID['proj']);
+
+// document body
+
+
+exec("find /tmp -user apache", $output, $status);
+
+echo "<html><body>\n";
+echo "Cleaning /tmp file (removing all files belonging to user apache)<br><br>\n";
+$count = 0;
+foreach ($output as $key => $value) {
+  echo "Deleting [$value]<br>\n";
+  exec("rm -f $value");
+  $count++;
+}
+echo "<br>/tmp cleaned ($count files were deleted)<br>\n";
+echo "</body></html>\n";
+
+
+menu_end();
+
+?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/columns_in_db.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/columns_in_db.php	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/columns_in_db.php	(revision 28794)
@@ -0,0 +1,155 @@
+<?php
+
+$pageTitle = 'Columns in DB Tables';
+
+include 'ipp.php';
+
+$ID = checkID ();
+
+if (! $ID['proj']) { projectform ($ID); }
+
+$db = dbconnect($ID['proj']);
+
+if ($ID['menu']) {
+  $myMenu = $ID['menu'];
+} else {
+  $myMenu = "ipp.menu.dat";
+}
+
+menu($myMenu, $pageTitle, 'ipp.css', $ID['link'], $ID['proj']);
+
+echo "<p> $pageTitle </p>";
+
+// Table names query
+$tableNames = array (
+		     "Label", 
+		     "addMask", 
+		     "addProcessedExp", 
+		     "addRun", 
+		     "calDB", 
+		     "calRun", 
+		     "camMask", 
+		     "camProcessedExp", 
+		     "camRun", 
+		     "chipBackgroundImfile", 
+		     "chipBackgroundRun", 
+		     "chipImfile", 
+		     "chipMask", 
+		     "chipProcessedImfile", 
+		     "chipRun", 
+		     "dbversion", 
+		     "detInputExp", 
+		     "detNormalizedExp", 
+		     "detNormalizedImfile", 
+		     "detNormalizedStatImfile", 
+		     "detProcessedExp", 
+		     "detProcessedImfile", 
+		     "detRegisteredImfile", 
+		     "detResidExp", 
+		     "detResidImfile", 
+		     "detRun", 
+		     "detRunSummary", 
+		     "detStackedImfile", 
+		     "diffInputSkyfile", 
+		     "diffPhotRun", 
+		     "diffPhotSkyfile", 
+		     "diffRun", 
+		     "diffSkyfile", 
+		     "diffSummary", 
+		     "distComponent", 
+		     "distRun", 
+		     "distTarget", 
+		     "dqstatsContent", 
+		     "dqstatsRun", 
+		     "fakeMask", 
+		     "fakeProcessedImfile", 
+		     "fakeRun", 
+		     "flatcorrAddstarLink", 
+		     "flatcorrCamLink", 
+		     "flatcorrChipLink", 
+		     "flatcorrRun", 
+		     "magicDSFile", 
+		     "magicDSRun", 
+		     "magicInputSkyfile", 
+		     "magicMask", 
+		     "magicNodeResult", 
+		     "magicRun", 
+		     "magicTree", 
+		     "minidvodbProcessed", 
+		     "minidvodbRun", 
+		     "newExp", 
+		     "newImfile", 
+		     "publishClient", 
+		     "publishDone", 
+		     "publishRun", 
+		     "pzDataStore", 
+		     "pzDownloadExp", 
+		     "pzDownloadImfile", 
+		     "rawExp", 
+		     "rawImfile", 
+		     "rcDSFileset", 
+		     "rcDestination", 
+		     "rcInterest", 
+		     "rcRun", 
+		     "receiveFile", 
+		     "receiveFileset", 
+		     "receiveResult", 
+		     "receiveSource", 
+		     "stackAssociation", 
+		     "stackAssociationMap", 
+		     "stackInputSkyfile", 
+		     "stackRun", 
+		     "stackSumSkyfile", 
+		     "stackSummary", 
+		     "staticskyInput", 
+		     "staticskyResult", 
+		     "staticskyRun", 
+		     "summitExp", 
+		     "summitImfile", 
+		     "warpBackgroundRun", 
+		     "warpBackgroundSkyfile", 
+		     "warpImfile", 
+		     "warpMask", 
+		     "warpRun", 
+		     "warpSkyCellMap", 
+		     "warpSkyfile", 
+		     "warpSummary", 
+		     );
+
+echo "<b>This script does not do anything but showing the current columns in the various db tables</b><br/><br/>";
+
+foreach ($tableNames as $tIndex => $tableName) {
+  $sql = "show columns from $tableName";
+  $qry = $db->query($sql);
+  if (dberror($qry)) {
+    echo "<b>error reading querying table</b><br>\n";
+    echo "<br><small><b> table query : $sql </b></small><br>\n";
+    menu_end();
+  }
+  $count = 0;
+  $line = "";
+  $values = array();
+  while ($qry->fetchInto($row)) {
+    // print "[".$row[0]."]<br>\n";
+    foreach ($row as $index => $value) {
+      if ($index == 0) {
+	$values[] = $value;
+      }
+    }
+  }
+  // print count($values)."<br>\n";
+  asort($values);
+  // print count($values)."<br>\n";
+  foreach ($values as $index => $value) {
+    // echo "&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;".$count." => ".$value."<br>";
+    // $line .= "<option value=\"$value\">$count => $value</option>\n";
+    $line .= "<option value=\"$value\">$value</option>\n";
+    $count++;
+  }
+  echo "<b>Table name: $tableName</b><br>\n";
+  echo "Count = $count columns<br>\n";
+  echo "<select name=\"blah\">\n".$line."\n</select><br>\n";
+}
+
+menu_end();
+?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_getplot.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_getplot.php	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_getplot.php	(revision 28794)
@@ -0,0 +1,31 @@
+<?php
+
+$debug = 0;
+include 'site.php';
+
+### these need to be set to the correct locations!!
+$MISSING = "missing.png";
+
+### we must have been past arguments with GET:
+if ($_SERVER[REQUEST_METHOD] != 'GET') {
+    exit ();
+}
+
+$type = $_GET[type];
+$label = $_GET[label];
+$stage = $_GET[stage];
+
+if ($type=="t")
+$filePath = "/tmp/czarplot_".$label."_".$stage."_".$type.".png";
+else if ($type=="h")
+$filePath = "/tmp/czarplot_".$label."_all_stages_".$type.".png";
+$file = fopen ($filePath, "r");
+if ($file && !$debug) {
+    header ('Content-Type: image/png');
+    fpassthru ($file);
+}
+
+exit();
+
+?>
+~                                 
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_labels.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_labels.php	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/czartool_labels.php	(revision 28794)
@@ -3,13 +3,22 @@
 include 'site.php';
 
-$ID = checkID ();
+$ID = checkID();
 
 // require an explicit project
 if (! $ID['proj']) { projectform ($ID); }
 
-$db = dbconnect($ID['proj']);
+//$db = dbconnect($ID['proj']);
+$czardb = dbconnect("czardb"); // HACK to connect to czarDb 
+$gpc1db = dbconnect($ID['proj']);
+
+$PATH = getenv("PATH");
+putenv("PATH=$BINDIR:$PATH");
+
+$LD_LIBRARY_PATH = getenv("LD_LIBRARY_PATH");
+putenv("LD_LIBRARY_PATH=$LIBDIR:$LD_LIBRARY_PATH");
+
 
 if ($ID['menu']) {$myMenu = $ID['menu'];}
-else {$myMenu = "ipp.czartool.dat";}
+else {$myMenu = "ipp.imfiles.dat";}
 
 menu($myMenu, 'Czartool', 'ipp.css', $ID['link'], $ID['proj']);
@@ -19,19 +28,27 @@
 $menu = $ID['menu'];
 
-$userSelection = $_GET[selection];
-
-if ($userSelection == "") {$userSelection = $_POST['state'];}
-if ($userSelection == "") {$userSelection = "new";}
-
+$selectedStage = $_GET[stage];
+$selectedLabel = $_GET[label];
+$selectedServer = $_GET[server];
+$selectedRevertServer = $_GET[revertserver];
+$selectedRevertStage = $_GET[revertstage];
+$selectedRevertMode = $_GET[revertmode];
+
+if ($selectedLabel == "") $selectedLabel = "all_labels";
+if ($selectedStage == "") $selectedStage = "all_stages";
+
+// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
+$currentRevertMode;
+if ($selectedRevertStage != "" && $selectedRevertMode != "" && $selectedRevertServer != "" ) {
+
+    exec("czartool_revert.pl -s $selectedRevertServer -t $selectedRevertStage -o $selectedRevertMode", $response, $status);
+    if ($response[0] == "off") $currentRevertMode = 0;
+    else if ($response[0] == "on") $currentRevertMode = 1;
+}
 $debug = 0;
 
-$PATH = getenv("PATH");
-putenv("PATH=$BINDIR:$PATH");
-
-$LD_LIBRARY_PATH = getenv("LD_LIBRARY_PATH");
-putenv("LD_LIBRARY_PATH=$LIBDIR:$LD_LIBRARY_PATH");
-
-exec("czartool_getLabels.pl -s stdscience", $stdsLabels, $status);
-exec("czartool_getLabels.pl -s distribution", $distLabels, $status);
+$stdsLabels = getLabels($czardb, "stdscience");
+$distLabels = getLabels($czardb, "distribution");
+$pubLabels = getLabels($czardb, "publishing");
 
 if ($debug) {
@@ -43,5 +60,17 @@
 
 $states=array("full","new","drop","wait");
-$stages=array("chip","cam","fake","warp","stack","diff","magic","destreak","dist");
+$stages=array("chip","cam","fake","warp","stack","diff","magic","magicDS","dist");
+$servers=array(
+        "addstar",
+        "cleanup",
+        "detrend",
+        "distribution",
+        "pstamp",
+        "update",
+        "publishing",
+        "registration",
+        "replication",
+        "stdscience",
+        "summitcopy");
 
 // set up the form
@@ -52,15 +81,41 @@
 echo "<input type=\"hidden\" name=\"menu\" value=\"$menu\">\n";
 echo "</form>\n";
-$stateChosen = 0;
-foreach ($states as &$state) {
-
-    if ($userSelection == $state) {
-        showAllLabels($pass, $proj, $db, $stdsLabels, $distLabels, $stages, $states, $state);
-        $stateChosen=1;
-        break;
-    }
-}
-
-if (!$stateChosen) showOneLabel($pass, $proj, $db, $userSelection, $stages, $states);
+$lastUpdateTime = getLastUpdateTime($czardb);
+echo "<p  align=\"center\"> Current status of IPP (any faults are shown in parentheses). NOTE: This data is good as of: $lastUpdateTime </p>";
+
+echo "<table>\n";
+  echo "<tr>\n";
+
+    echo "<td> \n";
+      echo "<img src=\"czartool_getplot.php?type=t&label=$selectedLabel&stage=$selectedStage\"><br>\n";
+    echo "</td>\n";
+
+    echo "<td> \n";
+      createLabelsTable($pass, $proj, $czardb, $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $selectedRevertStage, $currentRevertMode);
+    echo "</td>\n";
+
+  echo "</tr>\n";
+  echo "<tr>\n";
+
+echo "<table>\n";
+  echo "<tr valign=top>\n";
+    echo "<td> \n";
+      echo "<img src=\"czartool_getplot.php?type=h&label=$selectedLabel&stage=$selectedStage\"><br>\n";
+    echo "</td>\n";
+    echo "<td> \n";
+      createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage);
+    echo "</td>\n";
+
+    echo "<td> \n";
+      $today = date("Y-m-d");                         // 03.10.01
+      showSummitData($gpc1db, $today);
+    echo "</td>\n";
+  echo "</tr>\n";
+echo "</table>\n";
+if ($selectedServer) showServerStatus($selectedServer);
+
+  echo "</tr>\n";
+echo "</table>\n";
+
 
 menu_end();
@@ -69,10 +124,300 @@
 ###########################################################################
 #
-# Checks one label and prints results in a table
-#
-###########################################################################
-function showOneLabel($pass, $proj, $db, $label, $stages, $states) {
-
-    echo "<p> Current status for label '$label' (any faults are shown in parentheses) </p>";
+# Checks summitExp table agains rawExp table 
+#
+###########################################################################
+function showSummitData($gpc1db, $date) {
+
+    $sql = "SELECT DISTINCT exp_type FROM summitExp WHERE dateobs > '$date'";
+
+    $qry = $gpc1db->query($sql);
+    if (dberror($qry)) {
+        echo "<b>error reading newExp table</b><br>\n";
+        echo "<br><small><b> table query : $sql </b></small><br>\n";
+        menu_end();
+    }
+
+    // set up the table
+    echo "<table class=list>\n";
+    echo "<tr><td></td>\n";
+    write_header_cell ("list", "Exposure type");
+    write_header_cell ("list", "At summit");
+    write_header_cell ("list", "Registered at MHPCC");
+
+    $msg = "No science images taken since $date";
+
+    // list the results
+    while ($qry->fetchInto($expType)) {
+
+        $sql = "SELECT COUNT(*) FROM summitExp WHERE dateobs > '$date' AND exp_type = '$expType[0]'";
+        $qry2 = $gpc1db->query($sql);
+        $qry2->fetchInto($summit);
+        $sql = "SELECT COUNT(*) FROM summitExp JOIN rawExp ON summitExp.exp_name = rawExp.exp_name WHERE summitExp.dateobs > '$date' AND summitExp.exp_type = '$expType[0]'";
+        $qry2 = $gpc1db->query($sql);
+        $qry2->fetchInto($mhpcc);
+
+        $class = "list";
+        echo "<tr><td></td>\n";
+
+        if ($expType[0] == "OBJECT") {
+            if ($summit[0] == $mhpcc[0]) $msg = "All science exposures taken since $date<br>have been registered at MHPCC";
+            else $msg = "Warning: Not all science exposures taken since $date<br>have been registered at MHPCC";
+        }
+
+        write_table_cell ($class, '%s', "", $expType[0]);
+        write_table_cell ($class, '%d', "", $summit[0]);
+        write_table_cell ($class, '%d', $link, $mhpcc[0]);
+        echo "</tr>\n";
+    }
+
+    echo "</table>\n";
+
+    echo "<p> $msg </p>";
+
+}
+
+###########################################################################
+#
+# Gets labels for this server
+#
+###########################################################################
+function getLabels($db, $server) {
+
+    # order by descending priority as set in gpc1 database
+    $sql = "SELECT label FROM current_labels WHERE server LIKE '$server' ORDER BY priority DESC";
+    if ($debug) {echo "$sql<br>";}
+
+    $qry = $db->query($sql);
+    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
+    while ($qry->fetchInto($row)) {
+
+        $labels[] = $row[0];
+    }
+
+    return $labels;
+}
+
+###########################################################################
+#
+# Creates table for all labels showing all stages for given 'state'
+#
+###########################################################################
+function createLabelsTable($pass, $proj, $db, $stdsLabels, $distLabels, $pubLabels, $stages, $states, $selectedState, $selectedLabel, $selectedStage, $selectedRevertStage, $currentRevertMode) {
+
+    // set up table columns
+    $class = "list";
+    echo "<table class=$class >\n";
+    echo "<tr><td></td>\n";
+
+    write_header_cell($class, "");
+    write_header_cell($class, "");
+    write_header_cell($class, "Reverts:");
+    foreach ($stages as &$stage) {
+
+        $revertOnOff=array();
+        if ($stage == "destreak" or $stage == "dist") $server = "distribution";
+        else $server = "stdscience";
+        if ($stage == $selectedRevertStage) $reverting = $currentRevertMode;
+        else $reverting = getRevertStatus($db, $stage);
+        $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&revertserver=" . $server . "&revertstage=" . $stage . "&revertmode=";
+        if(!$reverting) {$label =  "Start";$link = $link . "on";}
+        if($reverting) {$label = "Stop";$link = $link . "off";}
+        unset($reverting);
+        write_table_cell($class, '%s', $link, $label);
+    }
+    write_header_cell($class, "");
+
+    echo "</tr>\n";
+    echo "<tr><td></td>\n";
+    write_header_cell($class, "Label (in order of priority)");
+    write_header_cell($class, "Distributing?");
+    write_header_cell($class, "Publishing?");
+    foreach ($stages as &$stage) {
+        
+        if ($stage == $selectedStage) $link = "";
+        else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $stage;
+        write_table_cell($class, '%s', $link, $stage);
+    }
+
+    if ($selectedStage=="all_stages") $link = "";
+    else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=all_stages";
+    write_table_cell($class, '%s', $link, "All stages");
+
+    echo "</tr>\n";
+    echo "<tr><td></td>\n";
+
+    $defaultlink = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj;
+
+    // write rows
+    foreach ($stdsLabels as &$stdsLabel) {
+
+        $distributing = false;
+        $publishing = false;
+        foreach ($distLabels as &$distLabel) {
+
+            if ($stdsLabel == $distLabel) { $distributing = true; break;}
+        }
+        foreach ($pubLabels as &$pubLabel) {
+
+            if ($stdsLabel == $pubLabel) { $publishing = true; break;}
+        }
+
+        // create link to label summary page for each label
+        if ($stdsLabel == $selectedLabel) $link = "";
+        else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel . "&stage=" . $selectedStage;
+
+        echo "<tr><td></td>\n";
+        write_table_cell($class, '%s', $link, $stdsLabel);
+        write_table_cell($class, '%s', "", $distributing ? "yes" : "NO");
+        write_table_cell($class, '%s', "", $publishing ? "yes" : "NO");
+
+        $str = "";
+        $anyFaults = false; 
+
+        $link = "chipProcessedImfile_failure.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.label=" . $stdsLabel . "&chipRun.state=new";
+        getStateAndFaults($db, $stdsLabel, $selectedState, "chip", $str, $anyFaults);
+        write_table_cell($class, '%s', $anyFaults ? $link : "", $str);
+
+        $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&camRun.label=" . $stdsLabel . "&camRun.state=new";
+        getStateAndFaults($db, $stdsLabel, $selectedState, "cam", $str, $anyFaults);
+        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+        $link = $defaultlink;
+        getStateAndFaults($db, $stdsLabel, $selectedState, "fake", $str, $anyFaults);
+        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+        $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.label=" . $stdsLabel . "&warpRun.state=new";
+        getStateAndFaults($db, $stdsLabel, $selectedState, "warp", $str, $anyFaults);
+        write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
+
+        $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.label=" . $stdsLabel . "&stackRun.state=new";
+        getStateAndFaults($db, $stdsLabel, $selectedState, "stack", $str, $anyFaults);
+        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+        $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.label=" . $stdsLabel . "&diffRun.state=new";
+        getStateAndFaults($db, $stdsLabel, $selectedState, "diff", $str, $anyFaults);
+        write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
+
+        $link = $defaultlink;
+        getStateAndFaults($db, $stdsLabel, $selectedState, "magic", $str, $anyFaults);
+        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+        $link = $defaultlink;
+        getStateAndFaults($db, $stdsLabel, $selectedState, "magicDS", $str, $anyFaults);
+        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+        $link = $defaultlink;
+        getStateAndFaults($db, $stdsLabel, $selectedState, "dist", $str, $anyFaults);
+        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+        echo "</tr>\n";
+    }
+
+    if ($selectedLabel == "all_labels") $link = "";
+    else  $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=all_labels&stage=".$selectedStage;
+
+    echo "<tr><td></td>\n";
+    write_table_cell($class, '%s', $link, "All labels:");
+    write_table_cell($class, '%s', "", "-");
+    write_table_cell($class, '%s', "", "-");
+
+    $str = "";
+    $anyFaults = false; 
+
+    $link = "chipProcessedImfile_failure.php?pass=" . $pass . "&proj=" . $proj . "&chipRun.state=new";
+    getStateAndFaults($db, "all_labels", $selectedState, "chip", $str, $anyFaults);
+    write_table_cell($class, '%s', $anyFaults ? $link : "", $str);
+
+    $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&camRun.state=new";
+    getStateAndFaults($db, "all_labels", $selectedState, "cam", $str, $anyFaults);
+    write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+    $link = $defaultlink;
+    getStateAndFaults($db, "all_labels", $selectedState, "fake", $str, $anyFaults);
+    write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+    $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj . "&warpRun.state=new";
+    getStateAndFaults($db, "all_labels", $selectedState, "warp", $str, $anyFaults);
+    write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
+
+    $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&stackRun.state=new";
+    getStateAndFaults($db, "all_labels", $selectedState, "stack", $str, $anyFaults);
+    write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+    $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj . "&diffRun.state=new";
+    getStateAndFaults($db, "all_labels", $selectedState, "diff", $str, $anyFaults);
+    write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
+
+    $link = $defaultlink;
+    getStateAndFaults($db, "all_labels", $selectedState, "magic", $str, $anyFaults);
+    write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+    $link = $defaultlink;
+    getStateAndFaults($db, "all_labels", $selectedState, "magicDS", $str, $anyFaults);
+    write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+    $link = $defaultlink;
+    getStateAndFaults($db, "all_labels", $selectedState, "dist", $str, $anyFaults);
+    write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
+
+
+    echo "</tr>\n";
+    echo "</table>\n";
+}
+
+###########################################################################
+#
+# Returns whether this stage is reverting or not
+#
+###########################################################################
+function getRevertStatus($db, $stage) {
+
+    $anyFaults = false;
+    $sql = "SELECT reverting FROM $stage ORDER BY timestamp DESC LIMIT 1";
+    if ($debug) {echo "$sql<br>";}
+
+    $qry = $db->query($sql);
+    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
+    $qry->fetchInto($row);
+
+    return $row[0];
+}
+
+###########################################################################
+#
+# Returns state and fault-count (if new) as a string
+#
+###########################################################################
+function getStateAndFaults($db, $label, $state, $stage, &$str, &$anyFaults) {
+
+    $anyFaults = false;
+    $sql = "SELECT pending, faults FROM $stage WHERE label LIKE '$label' ORDER BY timestamp DESC LIMIT 1";
+    if ($debug) {echo "$sql<br>";}
+
+    $qry = $db->query($sql);
+    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
+    $qry->fetchInto($row);
+
+    $pending = $row[0];
+    $faults = $row[1];
+
+    $str = "$pending";
+
+    if ($state == "new") {
+
+        if ($faults > 0) {
+
+            $str = $str."(".$faults.")";
+            $anyFaults = true;
+        }
+    }
+}
+
+###########################################################################
+#
+# Checks the status of all the pantasks servers 
+#
+###########################################################################
+function createServersTable($pass, $proj, $db, $servers, $selectedLabel, $selectedStage) {
 
     // set up table columns
@@ -80,156 +425,24 @@
     echo "<table class=$class>\n";
     echo "<tr><td></td>\n";
-    write_header_cell($class, "state");
-    foreach ($stages as &$stage) {write_header_cell($class, $stage);}
+    write_header_cell($class, "Server");
+    write_header_cell($class, "Alive?");
+    write_header_cell($class, "Scheduler running?");
     echo "</tr>\n";
 
-    // write rows
-    foreach ($states as &$state) {
-
-        $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&selection=" . $state;
-        $defaultlink = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj;
+    foreach ($servers as &$server) {
+
+        $link = "czartool_labels.php?pass=".$pass."&proj=".$proj."&server=".$server."&label=".$selectedLabel."&stage=".$selectedStage;
+        //      $link = "";
+
+        getServerStatus($db, $server, $alive, $running);
 
         echo "<tr><td></td>\n";
-        write_table_cell($class, '%s', $link, $state);
-
-        $str = "";
-        $anyFaults = false; 
-        $link = "chipProcessedImfile_failure.php?menu=ipp.science.dat&pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel;
-        getStateAndFaults($db, $label,"chipRun", $state, "chip", $str, $anyFaults);
-        write_table_cell($class, '%s', $anyFaults ? $link : "", $str);
-
-        $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel;
-        getStateAndFaults($db, $label,"camRun", $state, "cam", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
-        $link = $defaultlink;
-        getStateAndFaults($db, $label,"fakeRun", $state, "fake", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
-        $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj;
-        getStateAndFaults($db, $label,"warpRun", $state, "warp", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
-        
-        $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj;
-        getStateAndFaults($db, $label,"stackRun", $state, "stack", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
-        $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj;
-        getStateAndFaults($db, $label,"diffRun", $state, "diff", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
-
-        $link = $defaultlink;
-        getStateAndFaults($db, $label,"magicRun", $state, "magic", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
-        $link = $defaultlink;
-        getStateAndFaults($db, $label,"magicDSRun", $state, "magicDS", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-       
-        $link = $defaultlink;
-        getStateAndFaults($db, $label,"distRun", $state, "dist", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
+        write_table_cell($class, '%s', $link, $server);
+        write_table_cell($class, '%s', "", $alive ? "yes" : "NO");
+        write_table_cell($class, '%s', "", $running ? "yes" : "NO");
         echo "</tr>\n";
     }
 
     echo "</table>\n";
-
-    echo "<p> - Click on a state to see all labels for that state <br>";
-    echo "- Click on a fault to find logfile </p>";
-}
-
-###########################################################################
-#
-# Creates table for all labels showing all stages for given 'state'
-#
-###########################################################################
-function showAllLabels($pass, $proj, $db, $stdsLabels, $distLabels, $stages, $states, $selectedState) {
-
-    echo "<p> Current stdscience labels for '$selectedState' (any faults are shown in parentheses)</p>";
-
-    // set up table columns
-    $class = "list";
-    //echo "<table class=$class BORDER=\"0\" CELLSPACING=\"0\" CELLPADDING=\"5\">\n";
-    echo "<table class=$class >\n";
-    echo "<tr><td></td>\n";
-    write_header_cell($class, "label");
-    write_header_cell($class, "distributing?");
-    //    echo "<td class=\"$class\"><a href=\"$link\"> $myValue </a></td>\n";
-    //    echo "<th class=\"$class\"><a href=\"czartool.php\">label</th>\n";
-
-    foreach ($stages as &$stage) {write_header_cell($class, $stage);}
-    //foreach ($stages as &$stage) {
-
-    //  $link = "czartool.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stage;
-    // echo "<th class=\"$class\"><a href=\"$link\">$stage</th>\n";
-    // }
-
-    echo "</tr>\n";
-
-    // write rows
-    foreach ($stdsLabels as &$stdsLabel) {
-
-        $distributing = false;
-        foreach ($distLabels as &$distLabel) {
-
-            if ($stdsLabel == $distLabel) { $distributing = true; break;}
-        }
-
-
-        // create link to label summary page for each label
-        $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&selection=" . $stdsLabel;
-        $defaultlink = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj;
-
-        echo "<tr><td></td>\n";
-        write_table_cell($class, '%s', $link, $stdsLabel);
-        write_table_cell($class, '%s', "", $distributing ? "yes" : "NO");
-
-        $str = "";
-        $anyFaults = false; 
-
-        $link = "chipProcessedImfile_failure.php?menu=ipp.science.dat&pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel;
-        getStateAndFaults($db, $stdsLabel,"chipRun", $selectedState, "chip", $str, $anyFaults);
-        write_table_cell($class, '%s', $anyFaults ? $link : "", $str);
-
-        $link = "camProcessedExp_failure.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $stdsLabel;
-        getStateAndFaults($db, $stdsLabel,"camRun", $selectedState, "cam", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
-        $link = $defaultlink;
-        getStateAndFaults($db, $stdsLabel,"fakeRun", $selectedState, "fake", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
-        $link = "warpFailedSkyfiles.php?pass=" . $pass . "&proj=" . $proj;
-        getStateAndFaults($db, $stdsLabel,"warpRun", $selectedState, "warp", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
-        
-        $link = "stackFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj;
-        getStateAndFaults($db, $stdsLabel,"stackRun", $selectedState, "stack", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
-        $link = "diffFailedSkyfile.php?pass=" . $pass . "&proj=" . $proj;
-        getStateAndFaults($db, $stdsLabel,"diffRun", $selectedState, "diff", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "",  $str);
-
-        $link = $defaultlink;
-        getStateAndFaults($db, $stdsLabel,"magicRun", $selectedState, "magic", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
-        $link = $defaultlink;
-        getStateAndFaults($db, $stdsLabel,"magicDSRun", $selectedState, "magicDS", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-       
-        $link = $defaultlink;
-        getStateAndFaults($db, $stdsLabel,"distRun", $selectedState, "dist", $str, $anyFaults);
-        write_table_cell($class, '%s',  $anyFaults ? $link : "", $str);
-
-        echo "</tr>\n";
-    }
-
-    echo "</table>\n";
-
-    echo "<p> - Click on a label to see all states for that label (full, new, drop etc)<br>";
-    echo "- Click on a fault to find logfile </p>";
 }
 
@@ -239,69 +452,57 @@
 #
 ###########################################################################
-function getStateAndFaults($db, $label, $table, $state, $stage, &$str, &$anyFaults) {
-
-    $anyFaults = false;
-
-    $str = checkLabel($db, $label, $table, $state, $stage);
-
-    if ($state == "new") {
-
-        $faults = countFaults($db, $label,$table,$stage);
-
-        if ($faults > 0) {
-
-            $str = $str."(".$faults.")";
-            $anyFaults = true;
-        }
-    }
-}
-
-###########################################################################
-#
-# Returns count of exposures with this state for this label
-#
-###########################################################################
-function checkLabel($db, $label, $table, $state, $stage) {
-
-    $sql = "SELECT COUNT(state) FROM $table WHERE label LIKE '$label' AND state = '$state'";
-    if ($debug) {echo "$sql<br>";}
+function getServerStatus($db, $server, &$alive, &$running) {
+
+    $sql = "SELECT alive, running FROM servers WHERE server LIKE '$server' ORDER BY timestamp DESC LIMIT 1";
+    if($debug){echo "$sql<br>";}
 
     $qry = $db->query($sql);
     if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
-    if (!$qry->fetchInto($row)) {echo "<b>error with $sql </b><br>\n";}
-    return $row[0];
-}
-
-###########################################################################
-#
-# Returns count of faults for this stage and label
-#
-###########################################################################
-function countFaults($db, $label, $table, $stage) {
-
-    $joinTable = 0;
-    $id = $stage."_id";
-
-    if ($stage == "chip") {$joinTable="chipProcessedImfile";}
-    elseif ($stage == "cam") {$joinTable="camProcessedExp";}
-    elseif ($stage == "fake") {$joinTable="fakeProcessedImfile";}
-    elseif ($stage == "warp") {$joinTable="warpSkyfile";}
-    elseif ($stage == "stack") {$joinTable="stackSumSkyfile";}
-    elseif ($stage == "diff") {$joinTable="diffSkyfile";}
-    elseif ($stage == "magic") {$joinTable="magicNodeResult";}
-    elseif ($stage == "magicDS") {$id = "magic_ds_id"; $joinTable="magicDSFile";}
-    elseif ($stage == "dist") {$joinTable="distComponent";}
-    else {return -1;}
-
-    $faultCol =  $joinTable.".fault";
-
-    $sql = "SELECT COUNT(DISTINCT $id) FROM $table JOIN $joinTable USING ($id) WHERE label LIKE '$label' AND $faultCol != 0 AND $table.state = 'new'";
+    $qry->fetchInto($row);
+
+    $alive = $row[0];
+    $running = $row[1];
+}
+
+###########################################################################
+#
+# Returns time of last czarDb update 
+#
+###########################################################################
+function getLastUpdateTime($db) {
+
+    $anyFaults = false;
+    $sql = "SELECT timestamp FROM chip ORDER BY timestamp DESC LIMIT 1";
     if ($debug) {echo "$sql<br>";}
 
     $qry = $db->query($sql);
     if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
-    if (!$qry->fetchInto($row)) {echo "<b>error with $sql </b><br>\n";}
+    $qry->fetchInto($row);
+
     return $row[0];
 }
+
+###########################################################################
+#
+# Shows the status of the provided pantasks server 
+#
+###########################################################################
+function showServerStatus($server) {
+
+    echo "<p> Status for $server server </p>";
+
+    $results=array();
+    exec("czartool_getServerStatus.pl -s $server", $results, $status);
+
+    foreach ($results as &$line) {
+        echo "<pre>\n";
+        echo "$line\n";
+        echo "</pre>\n";
+
+    }
+
+}
+
+
 ?>
 
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/getimage.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/getimage.php	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/getimage.php	(revision 28794)
@@ -11,4 +11,5 @@
 // global $PSCONFIG
 
+
 ### these need to be set to the correct locations!!
 $MISSING = "missing.png";
@@ -20,5 +21,4 @@
 }
 
-
 putenv("PSCONFDIR=$PSCONFDIR");
 putenv("PSCONFIG=$PSCONFIG");
@@ -28,12 +28,23 @@
 
 if ($debug) {
+  echo "DEBUG1<br>";
   echo "args: $args<br>";
   echo "path: $PATH<br>";
   echo "perl: $PERLLIB<br>";
   echo "bindir: $BINDIR<br>";
+  echo "site=$SITE<br>\n";
 }
 
-# $basename may contain filename@filerule
+// $basename may contain filename@filerule
 $basename = $_GET[name];
+if ($basename == "noimage.noimage") {
+  $file = fopen ("noimage.png", "r");//$MISSING?
+  if ($file && !$debug) {
+    # do I need to use image/jpg? can I modify this to image/png, etc, based on the type?
+	   header ('Content-Type: image/jpg');
+    fpassthru ($file);
+  }
+  exit ();
+}
 $filerule = $_GET[rule];
 $camera   = $_GET[camera];
@@ -43,4 +54,5 @@
 
 if ($debug) {
+  echo "DEBUG2<br>";
   echo "basename: $basename<br>";
   echo "filerule: $filerule<br>";
@@ -64,6 +76,8 @@
 
 if ($debug) {
-  exec ("which ipp_filename.pl", $output, $status);
-  echo "which ipp_filename.pl output:<br>";
+  echo "DEBUG3<br>";
+  $output = array();
+  exec ("which $BINDIR/ipp_filename.pl", $output, $status);
+  echo "which $BINDIR/ipp_filename.pl output:<br>";
   for ($i = 0; $i < count($output); $i++) {
     echo "output $i: $output[$i]<br>";
@@ -71,4 +85,5 @@
   echo "status:   $status<br>";
 
+  $output = array();
   exec ("whoami", $output, $status);
   echo "which whoami output:<br>";
@@ -79,5 +94,7 @@
 }
 
-exec ("ipp_filename.pl --site=$SITE --basename $basename --filerule $filerule --camera $camera --class_id $class_id", $output, $status);
+/* --site=$SITE */
+$output = array();
+exec ("ipp_filename.pl --site=$SITE --basename $basename --filerule $filerule --camera $camera --class_id $class_id 2> /tmp/errors", $output, $status);
 
 # use these to check the environment
@@ -91,9 +108,10 @@
 
 if ($debug) {
+  echo "DEBUG4<br>";
+  echo "Command = ipp_filename.pl --site=$SITE --basename $basename --filerule $filerule --camera $camera --class_id $class_id<br>\n";
   echo "basename: $basename<br>";
   for ($i = 0; $i < count($output); $i++) {
     echo "output $i: $output[$i]<br>";
   }
-  echo "output:   $output[0]<br>";
   echo "status:   $status<br>";
   echo "filename: $filename<br>";
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.css
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.css	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.css	(revision 28794)
@@ -5,33 +5,144 @@
  */
 
-body  { bgcolor: #a0d0e0; background-color: #a0d0e0; padding: 5px; margin: 5px }
-
-a:link, a:visited, a:active { text-decoration: underline; font-weight: bold; color: #ff0000 }
-
-a.button      { text-decoration: none; font-weight: bold;   color: #000000; background: #0080c0}
-a.menutop     { text-decoration: none; font-weight: bold;   color: #ffffff}
-a.menutops    { text-decoration: none; font-weight: bold;   color: #80f0ff}
-a.menuext     { text-decoration: none; font-weight: bold;   color: #ffffff}
-a.menulink    { text-decoration: none; font-weight: normal; color: #000000}
-a.menuselect  { text-decoration: none; font-weight: normal; color: #0000ff}
-a.piclink     { text-decoration: none; font-weight: normal; color: #0080c0}
-
-td.menutop    { text-align: left; background: #0080c0; font-size: small; color: #ffffff; padding: 2px; }
-td.menutops   { text-align: left; background: #0080c0; font-size: small; color: #000000; padding: 2px; }
-td.menuext    { text-align: left; background: #303070; font-size: small; color: #ffffff; padding: 2px; }
-td.menulink   { text-align: left; background: #d0d0ff; font-size: small; color: #000000; padding: 2px; font-weight: normal; text-indent: 0px; }
-td.menuselect { text-align: left; background: #d0d0ff; font-size: small; color: #ff0000; padding: 2px; font-weight: normal; text-indent: 0px; }
-td.menuline   { text-align: left; background: #d0d0ff; font-size: small; color: #000000; padding: 2px; font-weight: normal; text-indent: 0px; }
-
-td.picture    { text-align: left; background: #00ffff; font-size: small; color: #ff0000; padding: 0px; text-indent: 2px; font-weight: normal; }
-td.piclink    { text-align: left; background: #0080c0; font-size: small; color: #ff0000; padding: 0px; text-indent: 0px; font-weight: normal; }
-
-table.page { border: 0px solid #0080c0; background: #a0d0e0; padding: 0px}
-
-td.title { background-color: #ff0000; padding: 5px; font-size: x-large; text-align: left; vertical-align: top}
+body  { 
+    background-color: #a0d0e0; 
+    padding: 5px; 
+    margin: 5px
+}
+
+div.debug {
+    height: 15%;
+    background-color: #ffffaa;
+    overflow: auto;
+}
+
+a:link, a:visited, a:active { 
+    text-decoration: underline; 
+    font-weight: bold; 
+    color: #ff0000
+}
+
+a.button      { 
+    text-decoration: none; 
+    font-weight: bold;   
+    color: #000000; 
+    background: #0080c0
+}
+a.menutop     { 
+    text-decoration: none; 
+    font-weight: bold;   
+    color: #ffffff
+}
+a.menutops    { 
+    text-decoration: none; 
+    font-weight: bold;   
+    color: #80f0ff
+}
+a.menuext     { 
+    text-decoration: none; 
+    font-weight: bold;   
+    color: #ffffff
+}
+a.menulink    { 
+    text-decoration: none; 
+    font-weight: normal; 
+    color: #000000
+}
+a.menuselect  { 
+    text-decoration: none; 
+    font-weight: normal; 
+    color: #0000ff
+}
+a.piclink     { 
+    text-decoration: none; 
+    font-weight: normal; 
+    color: #0080c0
+}
+
+td.menutop    { 
+    text-align: left; 
+    background: #0080c0; 
+    font-size: small; 
+    color: #ffffff; 
+    padding: 2px; 
+}
+td.menutops   { 
+    text-align: left; 
+    background: #0080c0; 
+    font-size: small; 
+    color: #000000; 
+    padding: 2px; 
+}
+td.menuext    { 
+    text-align: left; 
+    background: #303070; 
+    font-size: small; 
+    color: #ffffff; 
+    padding: 2px; 
+}
+td.menulink   { 
+    text-align: left; 
+    background: #d0d0ff; 
+    font-size: small; 
+    color: #000000; 
+    padding: 2px; 
+    font-weight: normal; 
+    text-indent: 0px; 
+}
+td.menuselect { 
+    text-align: left; 
+    background: #d0d0ff; 
+    font-size: small; 
+    color: #ff0000; 
+    padding: 2px; 
+    font-weight: normal; 
+    text-indent: 0px; 
+}
+td.menuline   { 
+    text-align: left; 
+    background: #d0d0ff; 
+    font-size: small; 
+    color: #000000; 
+    padding: 2px; 
+    font-weight: normal; 
+    text-indent: 0px; 
+}
+
+td.picture    { 
+    text-align: left; 
+    background: #00ffff; 
+    font-size: small; 
+    color: #ff0000; 
+    padding: 0px; 
+    text-indent: 2px; 
+    font-weight: normal; 
+}
+td.piclink    { 
+    text-align: left; 
+    background: #0080c0; 
+    font-size: small; 
+    color: #ff0000; 
+    padding: 0px; 
+    text-indent: 0px; 
+    font-weight: normal; 
+}
+
+table.page { 
+    border: 0px solid #0080c0; 
+    background: #a0d0e0; 
+    padding: 0px
+}
+
+td.title { 
+    background-color: #ff0000; 
+    padding: 5px; 
+    font-size: x-large; 
+    text-align: left; 
+    vertical-align: top
+}
 
 td.body  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: normal;  
            vertical-align: top;
@@ -42,35 +153,37 @@
 }
 
-table.menu { text-align: left; 
-             font-size: small; 
-	     font-weight: normal; 
-	     background: #ff0000; 
-	     border: 3px solid #0080c0; 
-	     padding: 0px; 
-}
-
-table.select { text-align: left; 
-             font-size: small; 
-	     font-weight: normal; 
-	     background: #0080c0; 
-	     border: 1px solid #0080c0; 
-	     padding: 0px; 
+table.menu { 
+    text-align: left; 
+    font-size: small; 
+    font-weight: normal; 
+    background: #ff0000; 
+    border: 3px solid #0080c0; 
+    padding: 0px; 
+}
+
+table.select { 
+    text-align: left; 
+    font-size: small; 
+    font-weight: normal; 
+    background: #0080c0; 
+    border: 1px solid #0080c0; 
+    padding: 0px; 
 }
 
 td.select  { 
-           text-align: left; 
-           font-size: normal;  
-           font-weight: bold;  
-           vertical-align: top;
-	   color: #ffffff; 
-	   background: #0080c0; 
-	   background-color: #0080c0; 
-	   border: 2px solid #0080c0; 
-	   padding: 2px; 
+    text-align: left; 
+    font-size: small;  
+    font-weight: bold;  
+    vertical-align: top;
+    color: #ffffff; 
+    background: #0080c0; 
+    background-color: #0080c0; 
+    border: 2px solid #0080c0; 
+    padding: 2px; 
 }
 
 th.select  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: bold;  
            vertical-align: top;
@@ -92,5 +205,5 @@
 td.list  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: normal;  
            vertical-align: top;
@@ -103,5 +216,5 @@
 td.list_off  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: normal;  
            vertical-align: top;
@@ -114,5 +227,5 @@
 td.list_drop  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: normal;  
            vertical-align: top;
@@ -125,5 +238,5 @@
 td.list_run  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: normal;  
            vertical-align: top;
@@ -136,5 +249,5 @@
 td.det_off  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: normal;  
            vertical-align: top;
@@ -147,5 +260,5 @@
 td.det_drop  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: normal;  
            vertical-align: top;
@@ -158,5 +271,5 @@
 td.det_add  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: normal;  
            vertical-align: top;
@@ -169,5 +282,5 @@
 th.list  { 
            text-align: left; 
-           font-size: normal;  
+           font-size: small;  
            font-weight: bold;  
            vertical-align: top;
@@ -178,2 +291,38 @@
 }
 
+td.ul3x2    { 
+    text-align: right; 
+    vertical-align: bottom;
+    padding: 0px; 
+    text-indent: 0px; 
+}
+td.uc3x2    { 
+    text-align: center; 
+    vertical-align: bottom;
+    padding: 0px; 
+    text-indent: 0px; 
+}
+td.ur3x2    { 
+    text-align: left; 
+    vertical-align: bottom;
+    padding: 0px; 
+    text-indent: 0px; 
+}
+td.ll3x2    { 
+    text-align: right; 
+    vertical-align: top;
+    padding: 0px; 
+    text-indent: 0px; 
+}
+td.lc3x2    { 
+    text-align: center; 
+    vertical-align: top;
+    padding: 0px; 
+    text-indent: 0px; 
+}
+td.lr3x2    { 
+    text-align: left; 
+    vertical-align: top;
+    padding: 0px; 
+    text-indent: 0px; 
+}
Index: anches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.czartool.dat
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.czartool.dat	(revision 28793)
+++ 	(revision )
@@ -1,5 +1,0 @@
-input     | ipp.imfiles.dat 
-
-menutop   | menutop      | plain   | &nbsp;                       | 
-menulink  | menuselect 	 | link    | Check labels             	  | czartool_labels.php  
-menulink  | menuselect 	 | link    | Check servers             	  | czartool_servers.php                         
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.imfiles.dat	(revision 28794)
@@ -18,8 +18,11 @@
 menutop   | menutop      | link    | test page                    | phptest.php     
 menutop   | menutop      | link    | night summary                | nightSummary.php
-menutop   | menutop      | link    | simple plot - raw            | simplePlotraw.php
-menutop   | menutop      | link    | simple plot - chip           | simplePlot.php
-menutop   | menutop      | link    | simple plot - cam            | simplePlotcam.php
+menutop   | menutop      | link    | simple plot - raw (to remove?) | simplePlotraw.php
+menutop   | menutop      | link    | simple plot - chip (to remove?) | simplePlot.php
+menutop   | menutop      | link    | simple plot - cam (to remove?) | simplePlotcam.php
 menutop   | menutop      | link    | czartool                     | czartool_labels.php
-menutop   | menutop      | link    | histogram                    | histogram.php
+menutop   | menutop      | link    | mask stats                   | maskStats.php
 
+menutop   | menutop      | plain   | &nbsp;                       | 
+menulink  | menuselect 	 | link    | Tables columns               | columns_in_db.php
+menulink  | menuselect 	 | link    | Clean /tmp directory         | cleanTmpDirectory.php
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.load.dat
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.load.dat	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.load.dat	(revision 28794)
@@ -2,4 +2,5 @@
 
 menutop   | menutop      | plain   | &nbsp;                       | 
+menulink  | menuselect 	 | link    | Raw Exp Summary              | rawSummary.php             
 menulink  | menuselect 	 | link    | Raw Exposures                | rawExp.php             
 menulink  | menuselect 	 | link    | Raw Detrend Exp              | rawDetrendExp.php             
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.menu.dat
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.menu.dat	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.menu.dat	(revision 28794)
@@ -24,4 +24,5 @@
 menutop   | menutop      | link    | Distribution                 | ipp.dist.php
 menutop   | menutop      | link    | Calibration                  | ipp.cal.php
+menutop   | menutop      | link    | Plots              	  | ipp.plots.php
 menutop   | menutop      | link    | Admin and Debug              | ipp.imfiles.php
 
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.php	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.php	(revision 28794)
@@ -273,7 +273,20 @@
 }
 
+function head ($title) {
+  echo "<head>\n";
+  echo "  <title> $title </title>\n";
+  /* JavaScript code */
+  echo "  <script type=\"text/javascript\">\n";
+  echo "  function changeCell(cellId, newContent){\n";
+  echo "    document.getElementById(cellId).innerHTML=newContent;\n";
+  echo "  }\n";
+  echo "  </script>\n";
+  echo "</head>\n\n";
+}
+
 function menu ($source, $title, $sheet, $append, $project) {
 
-  echo "<html><head><title> $title </title></head>\n\n";
+  echo "<html>\n";
+  head($title);
   echo "<link rel=\"STYLESHEET\" HREF=\"$sheet\">\n";
   echo "<body>\n";
@@ -871,3 +884,20 @@
 }
 
+//////////////////////////////////////////////////////////////////////////
+// Extra filtering parameters array
+$filteringParameters = array();
+
+//
+// Delete temporary files if they are older than $expire_time minutes
+//
+function delete_old_tmp_files() {
+  global $DELETION_USER;    // defined in raw/site.php.in
+  global $DELETION_DELAY;   // defined in raw/site.php.in
+  exec("find /tmp -user $DELETION_USER -amin +$DELETION_DELAY", $output, $status);
+  foreach ($output as $key => $value) {
+    exec("rm -f $value");
+    $count++;
+  }
+}
+
 ?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.plots.dat
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.plots.dat	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.plots.dat	(revision 28794)
@@ -0,0 +1,17 @@
+input     | ipp.menu.dat 
+
+menutop   | menutop      | plain   | &nbsp;                                 | 
+menulink  | menuselect      | link    | Simple plot - raw            	    | simplePlotRawImage.php
+menulink  | menuselect      | link    | Simple plot - chip           	    | simplePlotChipImage.php
+menulink  | menuselect      | link    | Simple plot - cam            	    | simplePlotCamImage.php
+
+menutop   | menutop      | plain   | &nbsp;                                 | 
+menulink  | menuselect      | link    | Histogram background                   | histogramBackgroundImage.php
+menulink  | menuselect      | link    | Histogram Fwhm Major (camProcessedExp) | histogramCamProcessedExpImage.php
+menulink  | menuselect      | link    | Histogram Zpt Obs (camProcessedExp)  | histogramZptObsImage.php
+
+menutop   | menutop      | plain   | &nbsp;                                 | 
+menulink  | menuselect      | link    | Plot airmass vs fwhm_major (chipProcessedImfile) | scatterPlotAirMassFwhmImage.php
+menulink  | menuselect      | link    | Plot Background (chipProcessedImfile) vs Moon Angle (rawExp) | scatterCpiBgReMaImage.php
+menulink  | menuselect      | link    | Plot Background (chipProcessedImfile) vs Moon Phase (rawExp) | scatterCpiBgReMpImage.php
+menulink  | menuselect      | link    | Plot Background (chipProcessedImfile) vs Sun Angle (rawExp) | scatterCpiBgReSaImage.php
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.plots.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.plots.php	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.plots.php	(revision 28794)
@@ -0,0 +1,17 @@
+<?php 
+
+include 'ipp.php';
+
+$ID = checkID ();
+
+// require an explicit project
+if (! $ID['proj']) { projectform ($ID); }
+
+menu('ipp.plots.dat', 'Plots', 'ipp.css', $ID['link'], $ID['proj']);
+
+// document body
+echo "Plots scripts<br>\n";
+
+menu_end();
+
+?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.science.dat
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.science.dat	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.science.dat	(revision 28794)
@@ -33,8 +33,9 @@
 menulink  | menuselect   | link    | Warp Summary                 | warpSummary.php
 menulink  | menuselect   | link    | Warp Stage Exp               | warpStageExp.php
+menulink  | menuselect   | link    | Warp Processed Exp           | warpProcessedExp.php
+menulink  | menuselect   | link    | Warp Processed Exp w/ Skycell Images | warpProcessedExp_Images.php
 menulink  | menuselect   | link    | Warp Stage Skyfiles          | warpStageSkyfiles.php
 menulink  | menuselect   | link    | Warp Skyfile Inputs          | warpStageSkyfileInputs.php
 menulink  | menuselect   | link    | Warp Processed Skyfiles      | warpProcessedSkyfiles.php
-# menulink  | menuselect   | link    | Warp Processed Skyfiles      | warpProcessedSkyfiles_Images.php
 menulink  | menuselect   | link    | Warp Failed Skyfiles         | warpFailedSkyfiles.php
 
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.stack.dat
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.stack.dat	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/ipp.stack.dat	(revision 28794)
@@ -3,4 +3,5 @@
 menutop   | menutop      | plain   | &nbsp;                       | 
 menulink  | menuselect   | link    | Stack Summary                | stackSummary.php
+menulink  | menuselect   | link    | Stack Summary w/ Images        | stackSummary_Images.php
 menulink  | menuselect   | link    | Stack Run                    | stackRun.php
 menulink  | menuselect   | link    | Stack Input Skyfile          | stackInputSkyfile.php
@@ -11,4 +12,5 @@
 menutop   | menutop      | plain   | &nbsp;                       | 
 menulink  | menuselect   | link    | Diff Summary                 | diffSummary.php
+menulink  | menuselect   | link    | Diff Summary w/ Images       | diffSummary_Images.php
 menulink  | menuselect   | link    | Diff Run                     | diffRun.php
 menulink  | menuselect   | link    | Diff Input Skyfile           | diffInputSkyfile.php
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/plotHistogram.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/plotHistogram.php	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/plotHistogram.php	(revision 28794)
@@ -0,0 +1,72 @@
+<?php
+
+# this program takes two arguments: 
+# an input filename of a file containing floating point values
+# an output png filename
+
+$debug = 0;
+include 'site.php';
+
+### these need to be set to the correct locations!!
+$MISSING = "missing.png";
+
+### we must have been past arguments with GET:
+if ($_SERVER[REQUEST_METHOD] != 'GET') { 
+  exit ();
+}
+
+$PATH = getenv("PATH");
+putenv("PATH=$BINDIR:$PATH");
+
+$LD_LIBRARY_PATH = getenv("LD_LIBRARY_PATH");
+putenv("LD_LIBRARY_PATH=$LIBDIR:$LD_LIBRARY_PATH");
+
+if ($debug) {
+  echo "path: $PATH<br>";
+
+  $newpath = getenv("PATH");
+  echo "new path: $newpath<br>";
+
+  $newpath = getenv("LD_LIBRARY_PATH");
+  echo "new path: $newpath<br>";
+
+  echo "bindir: $BINDIR<br>";
+}
+
+if ($debug) {
+  exec ("which build_histogram.dvo", $output, $status);
+  echo "which output:<br>";
+  for ($i = 0; $i < count($output); $i++) {
+    echo "output $i: $output[$i]<br>";
+  }
+  echo "status:   $status<br>";
+}
+
+$infile  = $_GET[input];
+$outfile = $_GET[output];
+$title = $_GET[title];
+
+if ($debug) {
+  echo "Calling: build_histogram.dvo $infile $outfile 0.1 100 $title\n";
+}
+exec ("build_histogram.dvo $infile $outfile 0.1 100 $title", $output, $status);
+
+if ($debug) {
+  for ($i = 0; $i < count($output); $i++) {
+    echo "output $i: $output[$i]<br>";
+  }
+  echo "status:   $status<br>";
+  exit ();
+}
+
+# use these to check the environment
+# passthru ("env");
+
+$file = fopen ($outfile, "r");
+if ($file && !$debug) {
+  header ('Content-Type: image/png');
+  fpassthru ($file);
+}
+exit ();
+?>
+
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/scatterPlot.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/scatterPlot.php	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/scatterPlot.php	(revision 28794)
@@ -0,0 +1,80 @@
+<?php
+
+# this program takes two arguments: 
+# an input filename of a file containing floating point values
+# an output png filename
+
+$debug = 0;
+include 'site.php';
+
+### these need to be set to the correct locations!!
+$MISSING = "missing.png";
+
+### we must have been past arguments with GET:
+if ($_SERVER[REQUEST_METHOD] != 'GET') { 
+  exit ();
+}
+
+$PATH = getenv("PATH");
+putenv("PATH=$BINDIR:$PATH");
+
+$LD_LIBRARY_PATH = getenv("LD_LIBRARY_PATH");
+putenv("LD_LIBRARY_PATH=$LIBDIR:$LD_LIBRARY_PATH");
+
+if ($debug) {
+  echo "path: $PATH<br>";
+
+  $newpath = getenv("PATH");
+  echo "new path: $newpath<br>";
+
+  $newpath = getenv("LD_LIBRARY_PATH");
+  echo "new path: $newpath<br>";
+
+  echo "bindir: $BINDIR<br>";
+}
+
+if ($debug) {
+  exec ("which plot_x_vs_y.dvo", $output, $status);
+  echo "which output:<br>";
+  for ($i = 0; $i < count($output); $i++) {
+    echo "output $i: $output[$i]<br>";
+  }
+  echo "status:   $status<br>";
+}
+
+$infile  = $_GET[input];
+$outfile = $_GET[output];
+$title = $_GET[title];
+
+$title = preg_replace("/'/", "", $title);
+$title = preg_replace("/'/", "", $title);
+$title = preg_replace("/\\\\/", "", $title);
+$title = preg_replace("/\\\\/", "", $title);
+$xtitle = preg_replace("/_vs_.*$/", "", $title);
+$ytitle = preg_replace("/^.*_vs_/", "", $title);
+
+$debug = 0;
+if ($debug) {
+  echo "Calling: plot_x_vs_y.dvo $infile $outfile [$xtitle] [$ytitle]\n";
+}
+$debug = 0;
+exec ("plot_x_vs_y.dvo $infile $outfile '$xtitle' '$ytitle'", $output, $status);
+
+if ($debug) {
+  for ($i = 0; $i < count($output); $i++) {
+    echo "output $i: $output[$i]<br>";
+  }
+  echo "status:   $status<br>";
+  exit ();
+}
+
+# use these to check the environment
+# passthru ("env");
+
+$file = fopen ($outfile, "r");
+if ($file && !$debug) {
+  header ('Content-Type: image/png');
+  fpassthru ($file);
+}
+exit ();
+?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/site.php.in
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/site.php.in	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/site.php.in	(revision 28794)
@@ -15,3 +15,9 @@
 
 $DBI     = "@DBI@";
+
+// Temporary files belonging to DELETION_USER and older than 
+// DELETION_DELAY minutes will be delayed by the
+// delete_old_tmp_files() function (see raw/ipp.php)
+$DELETION_USER  = "apache";
+$DELETION_DELAY = 15;
 ?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/skyplot.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/skyplot.php	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/skyplot.php	(revision 28794)
@@ -47,5 +47,5 @@
 $outfile = $_GET[output];
 
-exec ("skyplot.dvo $infile $outfile", $output, $status);
+$output = shell_exec("skyplot.dvo $infile $outfile");//, $output, $status);
 
 if ($debug) {
@@ -65,4 +65,6 @@
   fpassthru ($file);
 }
-exit ();
+
+exit();
+
 ?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/warpProcessedExp.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/warpProcessedExp.php	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/warpProcessedExp.php	(revision 28794)
@@ -0,0 +1,367 @@
+<?php 
+
+include 'ipp.php';
+
+$ID = checkID ();
+
+// require an explicit project
+if (! $ID['proj']) { projectform ($ID); }
+
+$db = dbconnect($ID['proj']);
+
+if ($ID['menu']) {
+  $myMenu = $ID['menu'];
+} else {
+  $myMenu = "ipp.science.dat";
+}
+
+menu($myMenu, 'Warp Processed Exposures', 'ipp.css', $ID['link'], $ID['proj']);
+
+echo "<p> Warp Processed Exposures </p>";
+
+// set up the form
+echo "<form action=\"warpProcessedExp.php\" method=\"POST\">\n";
+
+$restricted = 0;
+
+// define restrictiosn to the queries
+// ** TABLE RESTRICTIONS **
+$WHERE = "WHERE warpRun.fake_id = fakeRun.fake_id AND fakeRun.cam_id = camRun.cam_id AND camRun.chip_id = chipRun.chip_id  AND chipRun.exp_id = rawExp.exp_id";
+$WHERE = check_restrict ('rawExp.exp_name', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.exp_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.exp_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.exp_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('chipRun.chip_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('chipRun.chip_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('chipRun.chip_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('camRun.cam_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('camRun.cam_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('camRun.cam_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('fakeRun.fake_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('fakeRun.fake_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('fakeRun.fake_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('warpRun.warp_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpRun.warp_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('warpRun.warp_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('warpRun.state', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpRun.label', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpRun.data_group', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpRun.dist_group', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.telescope', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.camera', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.ra', $WHERE, 'min', 0.017453);
+$WHERE = check_restrict ('rawExp.ra', $WHERE, 'max', 0.017453);
+$WHERE = check_restrict ('rawExp.decl', $WHERE, 'min', 0.017453);
+$WHERE = check_restrict ('rawExp.decl', $WHERE, 'max', 0.017453);
+$WHERE = check_restrict ('rawExp.object', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.filter', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.bg', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.bg', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.bg', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.bg_stdev', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.bg_stdev', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.bg_stdev', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.comment', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpSummary.path_base', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpSummary.projection_cell', $WHERE, 'string', 1.0);
+$WHERE = check_ordering ('', $WHERE);
+
+if ($restricted == 0) {
+  if ("none" != "none") {
+    $WHERE = "$WHERE none";
+  }
+}
+
+// get the result table count
+if (basic == "basic") {
+  $sql = "SELECT count(*) FROM warpRun, fakeRun, camRun, chipRun, rawExp, warpSummary $WHERE";
+}
+if (basic == "summary") {
+  $sql = "SELECT count(*) FROM (SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,warpSummary.path_base,warpSummary.projection_cell FROM warpRun, fakeRun, camRun, chipRun, rawExp, warpSummary $WHERE) as TEMP";
+}
+
+$qry = $db->query($sql);
+if (dberror($qry)) {
+  echo "<b>error reading warpRun, fakeRun, camRun, chipRun, rawExp, warpSummary table count</b><br>\n";
+  echo "<br><small><b> count query : $sql </b></small><br>\n";
+  menu_end();
+}
+if (!$qry->fetchInto($row)) {
+  echo "<b>error reading warpRun, fakeRun, camRun, chipRun, rawExp, warpSummary table count</b><br>\n";
+  echo "<br><small><b> count query : $sql </b></small><br>\n";
+  menu_end();
+}
+// set up the row counter variables
+if ($ID['from']) {
+  $rowStart = $ID['from'];
+} else {
+  $rowStart = 0;
+}
+$rowLast = $rowStart + $dTABLE;
+$rowTotal = $row[0];
+if ($rowLast > $rowTotal) { $rowLast = $rowTotal; }
+echo "<b> $rowStart to $rowLast of $rowTotal items</b><br>\n";
+
+// query the database
+if (basic == "basic") {
+  $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage') FROM fakeRun,camRun,chipRun,rawExp,warpRun LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE LIMIT $dTABLE OFFSET $rowStart";
+}
+
+$qry = $db->query($sql);
+if (dberror($qry)) {
+  echo "<b>error reading warpRun, fakeRun, camRun, chipRun, rawExp, warpSummary table</b><br>\n";
+  echo "<br><small><b> table query : $sql </b></small><br>\n";
+  menu_end();
+}
+
+// ** HEAD CODE **
+
+// ** BUTTON RESTRICTIONS **
+$buttonLink = button_restrict_string ('rawExp.exp_name', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.exp_id', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.exp_id', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.exp_id', $buttonLink);
+$buttonLink = button_restrict_string ('chipRun.chip_id', $buttonLink);
+$buttonLink = button_restrict_min ('chipRun.chip_id', $buttonLink);
+$buttonLink = button_restrict_max ('chipRun.chip_id', $buttonLink);
+$buttonLink = button_restrict_string ('camRun.cam_id', $buttonLink);
+$buttonLink = button_restrict_min ('camRun.cam_id', $buttonLink);
+$buttonLink = button_restrict_max ('camRun.cam_id', $buttonLink);
+$buttonLink = button_restrict_string ('fakeRun.fake_id', $buttonLink);
+$buttonLink = button_restrict_min ('fakeRun.fake_id', $buttonLink);
+$buttonLink = button_restrict_max ('fakeRun.fake_id', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.warp_id', $buttonLink);
+$buttonLink = button_restrict_min ('warpRun.warp_id', $buttonLink);
+$buttonLink = button_restrict_max ('warpRun.warp_id', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.state', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.label', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.data_group', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.dist_group', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.telescope', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.camera', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.dateobs', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.dateobs', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.dateobs', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.ra', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.ra', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.ra', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.decl', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.decl', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.decl', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.object', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.filter', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.exp_time', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.exp_time', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.exp_time', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.airmass', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.airmass', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.airmass', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.bg', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.bg', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.bg', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.bg_stdev', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.bg_stdev', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.bg_stdev', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.comment', $buttonLink);
+$buttonLink = button_restrict_string ('warpSummary.path_base', $buttonLink);
+$buttonLink = button_restrict_string ('warpSummary.projection_cell', $buttonLink);
+navigate_buttons ($rowStart, $rowLast, $dTABLE, $rowTotal, $buttonLink, $ID, 'warpProcessedExp.php');
+
+echo "&nbsp; : &nbsp; or enter start row: <input type=\"text\" name=\"from\" size=\"5\" value=\"$rowStart\">\n";
+
+// set up the table
+echo "<table class=list>\n";
+
+// echo "<tr><td></td>\n"; // first field is a label set below for the query rows only
+// ** TABLE HEADER **
+echo "<tr><td></td>\n";
+write_header_cell ("list", "Exp Name");
+write_header_cell ("list", "Exp ID");
+write_header_cell ("list", "Chip ID");
+write_header_cell ("list", "Cam ID");
+write_header_cell ("list", "Fake ID");
+write_header_cell ("list", "Warp ID");
+write_header_cell ("list", "image");
+write_header_cell ("list", "state");
+write_header_cell ("list", "label");
+write_header_cell ("list", "data grp");
+write_header_cell ("list", "dist grp");
+write_header_cell ("list", "Telescope");
+write_header_cell ("list", "Camera");
+write_header_cell ("list", "Date/Time");
+write_header_cell ("list", "RA");
+write_header_cell ("list", "DEC");
+write_header_cell ("list", "Object");
+write_header_cell ("list", "FILTER");
+write_header_cell ("list", "exp_time    ");
+write_header_cell ("list", "airmass     ");
+write_header_cell ("list", "backgnd");
+write_header_cell ("list", "stdev    ");
+write_header_cell ("list", "Comment");
+echo "</tr>\n";
+echo "<tr><td></td>\n";
+write_sort_cell ("list", "rawExp.exp_name", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.exp_id", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "chipRun.chip_id", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "camRun.cam_id", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "fakeRun.fake_id", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "warpRun.warp_id", $buttonLink, $ID, 'warpProcessedExp.php');
+echo "<td></td>\n";
+write_sort_cell ("list", "warpRun.state", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "warpRun.label", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "warpRun.data_group", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "warpRun.dist_group", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.telescope", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.camera", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.dateobs", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.ra * 57.295783", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.decl * 57.295783", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.object", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.filter", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.exp_time", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.airmass", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.bg", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.bg_stdev", $buttonLink, $ID, 'warpProcessedExp.php');
+write_sort_cell ("list", "rawExp.comment", $buttonLink, $ID, 'warpProcessedExp.php');
+echo "</tr>\n";
+// echo "</tr>\n";
+
+// query restriction form
+// echo "<tr>\n";
+// echo "<td class=list> <input type=\"text\" name=\"expID\"></td>\n";
+// ** TABLE QUERY **
+echo "<tr><td>&ge;</td>\n";
+write_query_row ('rawExp.exp_name', 5, 'string');
+write_query_row ('rawExp.exp_id', 5, 'min');
+write_query_row ('chipRun.chip_id', 7, 'min');
+write_query_row ('camRun.cam_id', 7, 'min');
+write_query_row ('fakeRun.fake_id', 7, 'min');
+write_query_row ('warpRun.warp_id', 7, 'min');
+echo "<td> &nbsp; </td>\n";
+write_query_row ('warpRun.state', 7, 'string');
+write_query_row ('warpRun.label', 7, 'string');
+write_query_row ('warpRun.data_group', 7, 'string');
+write_query_row ('warpRun.dist_group', 7, 'string');
+write_query_row ('rawExp.telescope', 10, 'string');
+write_query_row ('rawExp.camera', 10, 'string');
+write_query_row ('rawExp.dateobs', 19, 'min');
+write_query_row ('rawExp.ra', 8, 'min');
+write_query_row ('rawExp.decl', 8, 'min');
+write_query_row ('rawExp.object', 8, 'string');
+write_query_row ('rawExp.filter', 10, 'string');
+write_query_row ('rawExp.exp_time', 5, 'min');
+write_query_row ('rawExp.airmass', 5, 'min');
+write_query_row ('rawExp.bg', 5, 'min');
+write_query_row ('rawExp.bg_stdev', 5, 'min');
+write_query_row ('rawExp.comment', 65, 'string');
+echo "</tr><tr><td>&le;</td>\n";
+echo "<td> &nbsp; </td>\n";
+write_query_row ('rawExp.exp_id', 5, 'max');
+write_query_row ('chipRun.chip_id', 7, 'max');
+write_query_row ('camRun.cam_id', 7, 'max');
+write_query_row ('fakeRun.fake_id', 7, 'max');
+write_query_row ('warpRun.warp_id', 7, 'max');
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+write_query_row ('rawExp.dateobs', 19, 'max');
+write_query_row ('rawExp.ra', 8, 'max');
+write_query_row ('rawExp.decl', 8, 'max');
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+write_query_row ('rawExp.exp_time', 5, 'max');
+write_query_row ('rawExp.airmass', 5, 'max');
+write_query_row ('rawExp.bg', 5, 'max');
+write_query_row ('rawExp.bg_stdev', 5, 'max');
+echo "<td> &nbsp; </td>\n";
+echo "</tr>\n";
+// echo "</tr>\n";
+
+// list the results
+while ($qry->fetchInto($row)) {
+  // $link = "warpProcessedExp.php" . "?expID=" . $row[0] . "&" . $ID['link'];
+
+  $class = "list";
+  // ** TD CLASS **
+
+  echo "<tr><td></td>\n";
+  // ** TABLE DATA **
+  write_table_cell ($class, '%s', "", $row[0]);
+  $link = "rawImfile.php" . "?menu=ipp.science.dat&" . $ID['link'];
+  $link = $link . "&rawImfile.exp_id=$row[1]";
+  write_table_cell ($class, '%d', $link, $row[1]);
+  $link = "chipProcessedImfile.php" . "?menu=ipp.science.dat&" . $ID['link'];
+  $link = $link . "&chipRun.chip_id=$row[2]";
+  write_table_cell ($class, '%d', $link, $row[2]);
+  $link = "camProcessedExp.php" . "?menu=ipp.science.dat&" . $ID['link'];
+  $link = $link . "&camRun.cam_id=$row[3]";
+  write_table_cell ($class, '%d', $link, $row[3]);
+  $link = "fakeProcessedImfile.php" . "?menu=ipp.science.dat&" . $ID['link'];
+  $link = $link . "&fakeRun.fake_id=$row[4]";
+  write_table_cell ($class, '%d', $link, $row[4]);
+  $link = "warpStageExp.php" . "?menu=ipp.science.dat&" . $ID['link'];
+  $link = $link . "&warpRun.warp_id=$row[5]";
+  write_table_cell ($class, '%d', $link, $row[5]);
+  $link = "getimage.php" . "?menu=ipp.science.dat&" . $ID['link'];
+  $link = $link . "&name=$row[22].$row[23]";
+  $link = $link . "&rule=PPSKYCELL.JPEG1";
+  $link = $link . "&camera=GPC1";
+  $link = $link . "&class_id=NONE";
+  echo "<td class=\"$class\"><a href=\"$link\"> <img src=\"getimage.php?name=$row[22].$row[23]&rule=PPSKYCELL.JPEG2&camera=$row[11]&class_id=NONE\"> </a></td>\n";
+  write_table_cell ($class, '%s', "", $row[6]);
+  write_table_cell ($class, '%s', "", $row[7]);
+  write_table_cell ($class, '%s', "", $row[8]);
+  write_table_cell ($class, '%s', "", $row[9]);
+  write_table_cell ($class, '%s', "", $row[10]);
+  write_table_cell ($class, '%s', "", $row[11]);
+  write_table_cell ($class, '%T', "", $row[12]);
+  write_table_cell ($class, '%C', "", $row[13] * 57.295783);
+  write_table_cell ($class, '%C', "", $row[14] * 57.295783);
+  write_table_cell ($class, '%s', "", $row[15]);
+  write_table_cell ($class, '%s', "", $row[16]);
+  write_table_cell ($class, '%.2f', "", $row[17]);
+  write_table_cell ($class, '%.4f', "", $row[18]);
+  write_table_cell ($class, '%.2f', "", $row[19]);
+  write_table_cell ($class, '%.2f', "", $row[20]);
+  write_table_cell ($class, '%s', "", $row[21]);
+  echo "</tr>\n";
+}
+
+// query restriction form
+// echo "<tr>\n";
+// echo "<td class=list> <input type=\"text\" name=\"expID\"></td>\n";
+// TABLE QUERY 
+// echo "</tr>\n";
+
+// close the table and form
+echo "</table>\n";
+echo "<input type=\"submit\" name=\"constraints\" value=\"refine search\">\n";
+$pass = $ID['pass'];
+$proj = $ID['proj'];
+$menu = $ID['menu'];
+echo "<input type=\"hidden\" name=\"pass\" value=\"$pass\">\n";
+echo "<input type=\"hidden\" name=\"proj\" value=\"$proj\">\n";
+echo "<input type=\"hidden\" name=\"menu\" value=\"$menu\">\n";
+echo "</form>\n";
+
+// ** TAIL CODE **
+
+echo "<small> WHERE: $WHERE<br><br></small>\n";
+echo "<small> SQL: $sql<br><br></small>\n";
+
+menu_end();
+
+?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/raw/warpProcessedExp_Images.php
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/raw/warpProcessedExp_Images.php	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/raw/warpProcessedExp_Images.php	(revision 28794)
@@ -0,0 +1,478 @@
+<?php 
+
+include 'ipp.php';
+
+$ID = checkID ();
+
+// require an explicit project
+if (! $ID['proj']) { projectform ($ID); }
+
+$db = dbconnect($ID['proj']);
+
+if ($ID['menu']) {
+  $myMenu = $ID['menu'];
+} else {
+  $myMenu = "ipp.science.dat";
+}
+
+menu($myMenu, 'Warp Stage Exposures w/ Images', 'ipp.css', $ID['link'], $ID['proj']);
+
+echo "<p> Warp Stage Exposures w/ Images</p>";
+
+// set up the form
+echo "<form action=\"warpProcessedExp_Images.php\" method=\"POST\">\n";
+
+$restricted = 0;
+
+// define restrictiosn to the queries
+// ** TABLE RESTRICTIONS **
+$WHERE = "WHERE warpRun.fake_id = fakeRun.fake_id AND fakeRun.cam_id = camRun.cam_id AND camRun.chip_id = chipRun.chip_id AND chipRun.exp_id = rawExp.exp_id";
+$WHERE = check_restrict ('rawExp.exp_name', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.exp_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.exp_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.exp_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('chipRun.chip_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('chipRun.chip_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('chipRun.chip_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('camRun.cam_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('camRun.cam_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('camRun.cam_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('fakeRun.fake_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('fakeRun.fake_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('fakeRun.fake_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('warpRun.warp_id', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpRun.warp_id', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('warpRun.warp_id', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('warpRun.state', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpRun.label', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpRun.data_group', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpRun.dist_group', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.telescope', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.camera', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.dateobs', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.ra', $WHERE, 'min', 0.017453);
+$WHERE = check_restrict ('rawExp.ra', $WHERE, 'max', 0.017453);
+$WHERE = check_restrict ('rawExp.decl', $WHERE, 'min', 0.017453);
+$WHERE = check_restrict ('rawExp.decl', $WHERE, 'max', 0.017453);
+$WHERE = check_restrict ('rawExp.object', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.filter', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.exp_time', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.airmass', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.bg', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.bg', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.bg', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.bg_stdev', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('rawExp.bg_stdev', $WHERE, 'min', 1.0);
+$WHERE = check_restrict ('rawExp.bg_stdev', $WHERE, 'max', 1.0);
+$WHERE = check_restrict ('rawExp.comment', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpSummary.path_base', $WHERE, 'string', 1.0);
+$WHERE = check_restrict ('warpSummary.projection_cell', $WHERE, 'string', 1.0);
+$WHERE = check_ordering ('', $WHERE);
+
+if ($restricted == 0) {
+  if ("none" != "none") {
+    $WHERE = "$WHERE none";
+  }
+}
+
+// get the result table count
+if (basic == "basic") {
+  $sql = "SELECT count(*) FROM fakeRun,camRun,chipRun,rawExp,warpRun LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE";
+}
+
+$qry = $db->query($sql);
+if (dberror($qry)) {
+  echo "<b>error reading warpRun, fakeRun, camRun, chipRun, rawExp, warpSummary table count</b><br>\n";
+  echo "<br><small><b> count query : $sql </b></small><br>\n";
+  menu_end();
+}
+if (!$qry->fetchInto($row)) {
+  echo "<b>error reading warpRun, fakeRun, camRun, chipRun, rawExp, warpSummary table count</b><br>\n";
+  echo "<br><small><b> count query : $sql </b></small><br>\n";
+  menu_end();
+}
+// set up the row counter variables
+if ($ID['from']) {
+  $rowStart = $ID['from'];
+} else {
+  $rowStart = 0;
+}
+$rowLast = $rowStart + $dTABLE;
+$rowTotal = $row[0];
+if ($rowLast > $rowTotal) { $rowLast = $rowTotal; }
+echo "<b> $rowStart to $rowLast of $rowTotal items</b><br>\n";
+
+// query the database
+if (basic == "basic") {
+  $sql = "SELECT rawExp.exp_name,rawExp.exp_id,chipRun.chip_id,camRun.cam_id,fakeRun.fake_id,warpRun.warp_id,warpRun.state,warpRun.label,warpRun.data_group,warpRun.dist_group,rawExp.telescope,rawExp.camera,rawExp.dateobs,rawExp.ra,rawExp.decl,rawExp.object,rawExp.filter,rawExp.exp_time,rawExp.airmass,rawExp.bg,rawExp.bg_stdev,rawExp.comment,ifnull(warpSummary.path_base,'noimage'),ifnull(warpSummary.projection_cell,'noimage') FROM fakeRun,camRun,chipRun,rawExp,warpRun LEFT JOIN warpSummary ON warpSummary.warp_id = warpRun.warp_id $WHERE LIMIT $dTABLE OFFSET $rowStart";
+}
+
+/* echo "<tr><td>SQL = $sql</td></tr>\n"; */
+/* menu_end(); */
+
+$qry = $db->query($sql);
+if (dberror($qry)) {
+  echo "<b>error reading warpRun, fakeRun, camRun, chipRun, rawExp, warpSummary table</b><br>\n";
+  echo "<br><small><b> table query : $sql </b></small><br>\n";
+  menu_end();
+}
+
+// ** HEAD CODE **
+
+// ** BUTTON RESTRICTIONS **
+$buttonLink = button_restrict_string ('rawExp.exp_name', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.exp_id', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.exp_id', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.exp_id', $buttonLink);
+$buttonLink = button_restrict_string ('chipRun.chip_id', $buttonLink);
+$buttonLink = button_restrict_min ('chipRun.chip_id', $buttonLink);
+$buttonLink = button_restrict_max ('chipRun.chip_id', $buttonLink);
+$buttonLink = button_restrict_string ('camRun.cam_id', $buttonLink);
+$buttonLink = button_restrict_min ('camRun.cam_id', $buttonLink);
+$buttonLink = button_restrict_max ('camRun.cam_id', $buttonLink);
+$buttonLink = button_restrict_string ('fakeRun.fake_id', $buttonLink);
+$buttonLink = button_restrict_min ('fakeRun.fake_id', $buttonLink);
+$buttonLink = button_restrict_max ('fakeRun.fake_id', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.warp_id', $buttonLink);
+$buttonLink = button_restrict_min ('warpRun.warp_id', $buttonLink);
+$buttonLink = button_restrict_max ('warpRun.warp_id', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.state', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.label', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.data_group', $buttonLink);
+$buttonLink = button_restrict_string ('warpRun.dist_group', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.telescope', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.camera', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.dateobs', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.dateobs', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.dateobs', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.ra', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.ra', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.ra', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.decl', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.decl', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.decl', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.object', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.filter', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.exp_time', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.exp_time', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.exp_time', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.airmass', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.airmass', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.airmass', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.bg', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.bg', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.bg', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.bg_stdev', $buttonLink);
+$buttonLink = button_restrict_min ('rawExp.bg_stdev', $buttonLink);
+$buttonLink = button_restrict_max ('rawExp.bg_stdev', $buttonLink);
+$buttonLink = button_restrict_string ('rawExp.comment', $buttonLink);
+$buttonLink = button_restrict_string ('warpSummary.path_base', $buttonLink);
+$buttonLink = button_restrict_string ('warpSummary.projection_cell', $buttonLink);
+navigate_buttons ($rowStart, $rowLast, $dTABLE, 
+		  $rowTotal, $buttonLink, $ID, 'warpProcessedExp_Images.php');
+
+echo "&nbsp; : &nbsp; or enter start row: <input type=\"text\" name=\"from\" size=\"5\" value=\"$rowStart\">\n";
+
+// set up the table
+echo "<table class=list>\n";
+
+// echo "<tr><td></td>\n"; // first field is a label set below for the query rows only
+// ** TABLE HEADER **
+echo "<tr><td></td>\n";
+write_header_cell ("list", "Exp Name");
+write_header_cell ("list", "Exp ID");
+write_header_cell ("list", "Chip ID");
+write_header_cell ("list", "Cam ID");
+write_header_cell ("list", "Fake ID");
+write_header_cell ("list", "Warp ID");
+write_header_cell ("list", "image");
+write_header_cell ("list", "image");
+write_header_cell ("list", "state");
+write_header_cell ("list", "label");
+write_header_cell ("list", "data grp");
+write_header_cell ("list", "dist grp");
+write_header_cell ("list", "Telescope");
+write_header_cell ("list", "Camera");
+write_header_cell ("list", "Date/Time");
+write_header_cell ("list", "RA");
+write_header_cell ("list", "DEC");
+write_header_cell ("list", "Object");
+write_header_cell ("list", "FILTER");
+write_header_cell ("list", "exp_time    ");
+write_header_cell ("list", "airmass     ");
+write_header_cell ("list", "backgnd");
+write_header_cell ("list", "stdev    ");
+write_header_cell ("list", "Comment");
+echo "</tr>\n";
+echo "<tr><td></td>\n";
+write_sort_cell ("list", "rawExp.exp_name", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.exp_id", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "chipRun.chip_id", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "camRun.cam_id", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "fakeRun.fake_id", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "warpRun.warp_id", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+echo "<td></td>\n";
+echo "<td></td>\n";
+write_sort_cell ("list", "warpRun.state", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "warpRun.label", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "warpRun.data_group", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "warpRun.dist_group", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.telescope", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.camera", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.dateobs", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.ra * 57.295783", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.decl * 57.295783", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.object", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.filter", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.exp_time", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.airmass", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.bg", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.bg_stdev", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+write_sort_cell ("list", "rawExp.comment", $buttonLink, $ID, 'warpProcessedExp_Images.php');
+echo "</tr>\n";
+// echo "</tr>\n";
+
+// query restriction form
+// echo "<tr>\n";
+// echo "<td class=list> <input type=\"text\" name=\"expID\"></td>\n";
+// ** TABLE QUERY **
+echo "<tr><td>&ge;</td>\n";
+write_query_row ('rawExp.exp_name', 5, 'string');
+write_query_row ('rawExp.exp_id', 5, 'min');
+write_query_row ('chipRun.chip_id', 7, 'min');
+write_query_row ('camRun.cam_id', 7, 'min');
+write_query_row ('fakeRun.fake_id', 7, 'min');
+write_query_row ('warpRun.warp_id', 7, 'min');
+echo "<td> &nbsp; </td>\n";
+echo "<td></td>\n";
+write_query_row ('warpRun.state', 7, 'string');
+write_query_row ('warpRun.label', 7, 'string');
+write_query_row ('warpRun.data_group', 7, 'string');
+write_query_row ('warpRun.dist_group', 7, 'string');
+write_query_row ('rawExp.telescope', 10, 'string');
+write_query_row ('rawExp.camera', 10, 'string');
+write_query_row ('rawExp.dateobs', 19, 'min');
+write_query_row ('rawExp.ra', 8, 'min');
+write_query_row ('rawExp.decl', 8, 'min');
+write_query_row ('rawExp.object', 8, 'string');
+write_query_row ('rawExp.filter', 10, 'string');
+write_query_row ('rawExp.exp_time', 5, 'min');
+write_query_row ('rawExp.airmass', 5, 'min');
+write_query_row ('rawExp.bg', 5, 'min');
+write_query_row ('rawExp.bg_stdev', 5, 'min');
+write_query_row ('rawExp.comment', 65, 'string');
+echo "</tr><tr><td>&le;</td>\n";
+echo "<td> &nbsp; </td>\n";
+write_query_row ('rawExp.exp_id', 5, 'max');
+write_query_row ('chipRun.chip_id', 7, 'max');
+write_query_row ('camRun.cam_id', 7, 'max');
+write_query_row ('fakeRun.fake_id', 7, 'max');
+write_query_row ('warpRun.warp_id', 7, 'max');
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+write_query_row ('rawExp.dateobs', 19, 'max');
+write_query_row ('rawExp.ra', 8, 'max');
+write_query_row ('rawExp.decl', 8, 'max');
+echo "<td> &nbsp; </td>\n";
+echo "<td> &nbsp; </td>\n";
+write_query_row ('rawExp.exp_time', 5, 'max');
+write_query_row ('rawExp.airmass', 5, 'max');
+write_query_row ('rawExp.bg', 5, 'max');
+write_query_row ('rawExp.bg_stdev', 5, 'max');
+echo "<td> &nbsp; </td>\n";
+echo "</tr>\n";
+// echo "</tr>\n";
+
+/* $warpIdCollection[$warpId]["00"] = "ul"; */
+/* $warpIdCollection[$warpId]["01"] = "ur"; */
+/* $warpIdCollection[$warpId]["10"] = "ll"; */
+/* $warpIdCollection[$warpId]["11"] = "lr"; */
+$warpIdCollection = array();
+
+$noImageValue = "<a href=\"noimage.png\"><img src=\"noimage.png\"></a>";
+
+// list the results
+while ($qry->fetchInto($row)) {
+  // $link = "warpProcessedExp_Images.php" . "?expID=" . $row[0] . "&" . $ID['link'];
+
+  $class = "list";
+  // ** TD CLASS **
+
+  //Identify current warpId
+  $warpId = $row[5];
+/*   echo "<tr><td>Number of elements in warpIdCollection: ".count($warpIdCollection)."</td></tr>\n"; */
+/*   echo "<tr><td>(warpId=$warpId)</td></tr>\n"; */
+
+  if (!isSet($warpIdCollection[$warpId])) {
+/*     echo "<tr><td>$warpId not in warpIdCollection</td></tr>"; */
+    $warpIdCollection[$warpId] = array();
+    $skyCellValue = preg_replace("|skycell[.]|","",$row[23]);
+    $warpIdCollection[$warpId]["12"] = $skyCellValue;
+/*     echo "<tr><td>Skycell is [$skyCellValue]</td></tr>"; */
+
+    echo "<tr>\n";
+    // ** TABLE DATA **
+    echo "<td></td>\n";
+    write_table_cell ($class, '%s', "", $row[0]);
+    $link = "rawImfile.php" . "?menu=ipp.science.dat&" . $ID['link'];
+    $link = $link . "&rawImfile.exp_id=$row[1]";
+    write_table_cell ($class, '%d', $link, $row[1]);
+    $link = "chipProcessedImfile.php" . "?menu=ipp.science.dat&" . $ID['link'];
+    $link = $link . "&chipRun.chip_id=$row[2]";
+    write_table_cell ($class, '%d', $link, $row[2]);
+    $link = "camProcessedExp.php" . "?menu=ipp.science.dat&" . $ID['link'];
+    $link = $link . "&camRun.cam_id=$row[3]";
+    write_table_cell ($class, '%d', $link, $row[3]);
+    $link = "fakeProcessedImfile.php" . "?menu=ipp.science.dat&" . $ID['link'];
+    $link = $link . "&fakeRun.fake_id=$row[4]";
+    write_table_cell ($class, '%d', $link, $row[4]);
+    $link = "warpStageExp.php" . "?menu=ipp.science.dat&" . $ID['link'];
+    $link = $link . "&warpRun.warp_id=$row[5]";
+    write_table_cell ($class, '%d', $link, $row[5]);
+    /* Image */
+    echo "<td class=\"$class\">\n";
+    echo "<table border=\"1\"><tr>\n";
+/*     echo "  <td class=\"ul3x2\" id=\"$warpId.00\">$noImageValue</td>\n"; */
+    echo "  <td class=\"ul3x2\" id=\"$warpId.00\"></td>\n";
+    echo "  <td class=\"uc3x2\" id=\"$warpId.01\">$noImageValue</td>\n";
+    echo "  <td class=\"ur3x2\" id=\"$warpId.02\">$noImageValue</td>\n";
+    echo "</tr><tr>\n";
+/*     echo "  <td class=\"ll3x2\" id=\"$warpId.10\">$noImageValue</td>\n"; */
+    echo "  <td class=\"ll3x2\" id=\"$warpId.10\"></td>\n";
+    echo "  <td class=\"lc3x2\" id=\"$warpId.11\">$noImageValue</td>\n";
+    echo "  <td class=\"lr3x2\" id=\"$warpId.12\">$noImageValue</td>\n";
+    echo "</tr></table>\n";
+    echo "</td>\n";
+    $link = "getimage.php" . "?menu=ipp.science.dat&" . $ID['link'];
+    $link = $link . "&name=$row[22].$row[23]";
+    $link = $link . "&rule=PPSKYCELL.JPEG1";
+    $link = $link . "&camera=GPC1";
+    $link = $link . "&class_id=NONE";
+
+    /* Image as Text */
+    echo "<td class=\"$class\">\n";
+    echo "<table border=\"1\"><tr>\n";
+/*     echo "  <td class=\"ul3x2\" id=\"t$warpId.00\">No image</td>\n"; */
+    echo "  <td class=\"ul3x2\" id=\"t$warpId.00\"></td>\n";
+    echo "  <td class=\"uc3x2\" id=\"t$warpId.01\">No image</td>\n";
+    echo "  <td class=\"ur3x2\" id=\"t$warpId.02\">No image</td>\n";
+    echo "</tr><tr>\n";
+/*     echo "  <td class=\"ll3x2\" id=\"t$warpId.10\">No image</td>\n"; */
+    echo "  <td class=\"ll3x2\" id=\"t$warpId.10\"></td>\n";
+    echo "  <td class=\"lc3x2\" id=\"t$warpId.11\">No image</td>\n";
+    echo "  <td class=\"lr3x2\" id=\"t$warpId.12\">No image</td>\n";
+    echo "</tr></table>\n";
+    echo "</td>\n";
+
+    $cellContent = "<a href=\\\"$link\\\"> <img src=\\\"getimage.php?name=$row[22].$row[23]&rule=PPSKYCELL.JPEG2&camera=$row[11]&class_id=NONE\\\"> </a>";
+    echo "<script type=\"text/javascript\">changeCell(\"$warpId.12\", \"$cellContent\");</script>";
+    $imageNameValue = ($row[23]=="noimage" ?
+		       "No image" : $row[23]);
+    echo "<script type=\"text/javascript\">changeCell(\"t$warpId.12\", \"$imageNameValue\");</script>";
+
+    //resume with "regular" columns
+    write_table_cell ($class, '%s', "", $row[6]);
+    write_table_cell ($class, '%s', "", $row[7]);
+    write_table_cell ($class, '%s', "", $row[8]);
+    write_table_cell ($class, '%s', "", $row[9]);
+    write_table_cell ($class, '%s', "", $row[10]);
+    write_table_cell ($class, '%s', "", $row[11]);
+    write_table_cell ($class, '%T', "", $row[12]);
+    write_table_cell ($class, '%C', "", $row[13] * 57.295783);
+    write_table_cell ($class, '%C', "", $row[14] * 57.295783);
+    write_table_cell ($class, '%s', "", $row[15]);
+    write_table_cell ($class, '%s', "", $row[16]);
+    write_table_cell ($class, '%.2f', "", $row[17]);
+    write_table_cell ($class, '%.4f', "", $row[18]);
+    write_table_cell ($class, '%.2f', "", $row[19]);
+    write_table_cell ($class, '%.2f', "", $row[20]);
+    write_table_cell ($class, '%s', "", $row[21]);
+    echo "</tr>\n";
+
+  } else {
+    // Update table cell content
+/*     echo "<tr><td>$warpId in warpIdCollection</td></tr>"; */
+    $lrValue =  $warpIdCollection[$warpId]["12"];
+    $skyCellValue = preg_replace("|skycell[.]|","",$row[23]);
+/*     echo "<tr><td>Current skycell is [$skyCellValue] / LowerRight is [$lrValue]</td></tr>"; */
+
+    $link = "getimage.php" . "?menu=ipp.science.dat&" . $ID['link'];
+    $link = $link . "&name=$row[22].$row[23]";
+    $link = $link . "&rule=PPSKYCELL.JPEG1";
+    $link = $link . "&camera=GPC1";
+    $link = $link . "&class_id=NONE";
+    $cellContent = "<a href=\\\"$link\\\"> <img src=\\\"getimage.php?name=$row[22].$row[23]&rule=PPSKYCELL.JPEG2&camera=$row[11]&class_id=NONE\\\"> </a>";
+
+    if ($skyCellValue == $lrValue + 1) {
+      $warpIdCollection[$warpId]["11"] = $skyCellValue;
+      echo "<script type=\"text/javascript\">changeCell(\"$warpId.11\", \"$cellContent\");</script>";
+      echo "<script type=\"text/javascript\">changeCell(\"t$warpId.11\", \"$row[23]\");</script>";
+    } else {
+      if ($skyCellValue == $lrValue + 2) {
+	$warpIdCollection[$warpId]["10"] = $skyCellValue;
+	echo "<script type=\"text/javascript\">changeCell(\"$warpId.10\", \"$cellContent\");</script>";
+	echo "<script type=\"text/javascript\">changeCell(\"t$warpId.10\", \"$row[23]\");</script>";
+      } else {
+	if (!isSet($warpIdCollection[$warpId]["02"])) {
+	  /* 	echo "<tr><td>Update UpperRight</td></tr>"; */
+	  $warpIdCollection[$warpId]["02"] = $skyCellValue;
+	  echo "<script type=\"text/javascript\">changeCell(\"$warpId.02\", \"$cellContent\");</script>";
+	  echo "<script type=\"text/javascript\">changeCell(\"t$warpId.02\", \"$row[23]\");</script>";
+	} else {
+	  $urValue = $warpIdCollection[$warpId]["02"];
+	  if ($urValue + 1 == $skyCellValue) {
+	    echo "<script type=\"text/javascript\">changeCell(\"$warpId.01\", \"$cellContent\");</script>";
+	    echo "<script type=\"text/javascript\">changeCell(\"t$warpId.01\", \"$row[23]\");</script>";
+	  } else {
+	    if ($urValue + 2 == $skyCellValue) {
+	      echo "<script type=\"text/javascript\">changeCell(\"$warpId.00\", \"$cellContent\");</script>";
+	      echo "<script type=\"text/javascript\">changeCell(\"t$warpId.00\", \"$row[23]\");</script>";
+	    } else {
+	      $expected = $urValue + 1;
+	      $actual = $skyCellValue;
+	      echo "<script type=\"text/javascript\">changeCell(\"$warpId.00\", \"Problem in UpperLeft corner: expected $expected but was $actual\");</script>";
+	      echo "<tr><td>Problem in Upper part!</td></tr>";
+	    }
+	  } 
+	}
+      }
+    }
+  }
+}
+
+// query restriction form
+// echo "<tr>\n";
+// echo "<td class=list> <input type=\"text\" name=\"expID\"></td>\n";
+// TABLE QUERY 
+// echo "</tr>\n";
+
+// close the table and form
+echo "</table>\n";
+echo "<input type=\"submit\" name=\"constraints\" value=\"refine search\">\n";
+$pass = $ID['pass'];
+$proj = $ID['proj'];
+$menu = $ID['menu'];
+echo "<input type=\"hidden\" name=\"pass\" value=\"$pass\">\n";
+echo "<input type=\"hidden\" name=\"proj\" value=\"$proj\">\n";
+echo "<input type=\"hidden\" name=\"menu\" value=\"$menu\">\n";
+echo "</form>\n";
+
+// ** TAIL CODE **
+
+echo "<small> WHERE: $WHERE<br><br></small>\n";
+echo "<small> SQL: $sql<br><br></small>\n";
+
+menu_end();
+
+?>
Index: /branches/eam_branches/ipp-20100621/ippMonitor/scripts/build_histogram.dvo
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/build_histogram.dvo	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/scripts/build_histogram.dvo	(revision 28794)
@@ -37,14 +37,15 @@
   plot dx vhist -x 1 -c blue
   $binsCount = dx[]
-  myecho "  Labeling"
-  myecho "    -x $3"
+  strsub $3 "_" "\\\\_" -var xLabel
+  myecho "  Labelling"
+  myecho "    -x $xLabel"
   myecho "    -y Occurrences "
-  myecho "    +x 'Histogram of $3 "
+  myecho "    +x 'Histogram of $xLabel "
   myecho "           interval = $minvalue : $maxvalue "
   myecho "	     / $binsCount bins"
   myecho "           / bin width: $delta)'"
-  labels -x $3
+  labels -x $xLabel
   labels -y Occurrences
-  labels +x "Histogram of $3 / interval = $minvalue : $maxvalue / $binsCount bins / bin width: $delta"
+  labels +x "Histogram of $xLabel / interval = $minvalue : $maxvalue / $binsCount bins / bin width: $delta"
 
   myecho "Saving graphics to [$2]"
Index: anches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_checkServer.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_checkServer.pl	(revision 28793)
+++ 	(revision )
@@ -1,25 +1,0 @@
-#!/usr/bin/perl -w
-
-use warnings;
-use strict;
-use Getopt::Long;
-
-my $server = undef;
-GetOptions ("server|s=s" => \$server);
-
-my @cmdOut = `echo "status ;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
-my $line;
-my $foundStatus = 0;
-my $isRunning = 0;
-foreach $line (@cmdOut) {
-
-    chomp($line);
-    if ($line =~ m/Scheduler is stopped/) {$isRunning = 0; $foundStatus=1;last;}
-    if ($line =~ m/Scheduler is running/) {$isRunning = 1; $foundStatus=1;last;}
-    if ($line =~ m/Task Staus/) {last;}
-
-}
-
-($foundStatus) ? print "yes\n" : print "no\n";
-($isRunning) ? print "yes\n" : print "no\n";
-
Index: anches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_getLabels.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_getLabels.pl	(revision 28793)
+++ 	(revision )
@@ -1,25 +1,0 @@
-#!/usr/bin/perl -w
-
-use warnings;
-use strict;
-use Getopt::Long;
-
-my $server = undef;
-GetOptions ("server|s=s" => \$server);
-
-my @cmdOut = `echo "show.labels;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
-my $line;
-my $passedHeader=0;
-foreach $line (@cmdOut) {
-
-    chomp($line);
-    if ($line =~ m/pantasks:\s+(.*)/) {
-  
-        # HACK to get around 'dummy' label in stdscience
-        if ($1 !~ m/.*dummy.*/) {print "$1\n"};
-        $passedHeader=1; 
-        next;
-    }
-
-    if ($passedHeader){print "$line\n";}
-}
Index: /branches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_revert.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_revert.pl	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/scripts/czartool_revert.pl	(revision 28794)
@@ -0,0 +1,38 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use Getopt::Long;
+
+my $server = undef;
+my $task = undef;
+my $onoff = undef;
+
+GetOptions (
+        "server|s=s" => \$server,
+        "task|t=s" => \$task,
+        "onoff|o=s" => \$onoff);
+
+my $prob = 0;
+if (!$server) {print "ERROR: need to define a server (-s)\n"; $prob=1;}
+if (!$task) {print "ERROR: need to define a task (-t)\n"; $prob=1;}
+if ($prob) {exit;}
+
+if ($task eq "cam") {$task = "camera";}
+
+my $revert = undef;
+if ($onoff) {$revert = "$task.revert.$onoff;";}
+else {$revert="";}
+
+my @cmdOut = `echo "$revert status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+
+my $line;
+foreach $line (@cmdOut) {
+
+    chomp($line);
+    if ($line =~ m/\s+([+-])[+-]\s+$task\.revert.*/) {
+
+        if ($1 =~ m/-/) {print "off\n";}
+        elsif ($1 =~ m/\+/) {print "on\n";}
+    }
+}
Index: /branches/eam_branches/ipp-20100621/ippMonitor/scripts/generate
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/generate	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/scripts/generate	(revision 28794)
@@ -35,4 +35,7 @@
     &init_key ("FIELDS");
     &init_key ("GROUPS");
+    &init_key ("TOPLOT");
+    &init_key ("PLOTTER");
+    &init_key ("PLOTTITLE");
 
     foreach $line (@list) {
@@ -43,4 +46,10 @@
         if ($key eq "TABLE") { ($value) = $value =~ m|^\s*(.+)\s*$|; }
         if ($key eq "TYPE")  { ($value) = $value =~ m|^\s*(.+)\s*$|; }
+	if ($key eq "TOPLOT") { ($value) = $value =~ m|^\s*(.+)\s*$|; }
+	if ($key eq "PLOTTER") { ($value) = $value =~ m|^\s*(.+)\s*$|; }
+	if ($key eq "PLOTTITLE") { 
+	    ($value) = $value =~ m|^\s*(.+)\s*$|;
+	    $value =~ s/\s/_/g;
+	}
 
         &set_keypair ($key, $value);
@@ -153,10 +162,10 @@
     }
 
+    foreach $opword (@opwords) {
+        &parse_opwords ($opword);
+    }
+
     foreach $imagedef (@imagedefs) {
         &parse_imagedef ($imagedef);
-    }
-
-    foreach $opword (@opwords) {
-        &parse_opwords ($opword);
     }
 
@@ -253,5 +262,4 @@
 # generate a table header cell (<th> </th>) for each field (show != none)
 sub write_table_header {
-
     # print the table header (field labels)
     print FILE "echo \"<tr><td></td>\\n\";\n";
@@ -438,4 +446,5 @@
             next;
         }
+
     }
 }
@@ -617,4 +626,9 @@
     ($var, $name, $rule, $camera, $class) = split (" ", $value);
     $name   = &parse_fieldname ($name);
+    if (exists $ops{$name}) {
+	#if OPx was defined, replace it by its value
+	$name = $ops{$name};
+	$name =~ s/\s+//g;
+    }
     $camera = &parse_fieldname ($camera);
     $class  = &parse_fieldname ($class);
@@ -682,5 +696,5 @@
         if ($key eq $key[$i]) {
             if ($VERBOSE) { print "found $key: $key[$i]  -- $value[$i] (def: $default)\n"; }
-            if (($default eq "") && ($value[$i] eq "")) { die "missing value for required key $key[$i]\n"; }
+            if (($default eq "") && ($value[$i] eq "")) { die "check_key: missing value for required key $key[$i]\n"; }
             if ($value[$i] eq "") { $value[$i] = $default; }
             return;
@@ -695,5 +709,5 @@
         if ($VERBOSE) { print "$key[$i]  -- $value[$i]\n"; }
         if ($line =~ m|\$$key[$i]|) {
-            if ($value[$i] eq "") { die "missing value for required key $key[$i]\n"; }
+            if ($value[$i] eq "") { die "check_keypairs: missing value for required key $key[$i]\n"; }
             $line =~ s|\$$key[$i]|$value[$i]|g;
         }
@@ -708,3 +722,2 @@
 # against the expectation at runtime.  for the moment,
 # calculate by hand and add to def.d file
-
Index: /branches/eam_branches/ipp-20100621/ippMonitor/scripts/helpers.dvo
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/helpers.dvo	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/scripts/helpers.dvo	(revision 28794)
@@ -0,0 +1,29 @@
+#!/usr/bin/env dvo
+
+macro minimum
+  set values = $1
+  set minvalue = values[0]
+  for i 0 values[]
+    if ($minvalue>values[$i])
+      $minvalue = values[$i]
+    end
+  end
+  return $minvalue
+end
+
+macro maximum
+  set values = $1
+  set maxvalue = values[0]
+  for i 0 values[]
+    if ($maxvalue<values[$i])
+      $maxvalue = values[$i]
+    end
+  end
+  return $minvalue
+end
+
+macro myecho
+  if ($debug == 1)
+     echo $1
+  end
+end
Index: /branches/eam_branches/ipp-20100621/ippMonitor/scripts/plot_x_vs_y.dvo
===================================================================
--- /branches/eam_branches/ipp-20100621/ippMonitor/scripts/plot_x_vs_y.dvo	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippMonitor/scripts/plot_x_vs_y.dvo	(revision 28794)
@@ -0,0 +1,60 @@
+#!/usr/bin/env dvo
+
+$debug=0
+
+macro build_x_vs_y
+  myecho "Loading data from = $1<br>"
+  data $1
+  read v 1 w 2
+  sort v w
+  limits v w
+  box
+  plot v w -x 0 -c blue
+  strsub $3 "_" "\\\\_" -var xlabel
+  strsub $4 "_" "\\\\_" -var ylabel
+  labels -x $xlabel -y $ylabel +x "$xlabel vs $ylabel"
+
+  fit v w 1
+  $y0 = $C0 + $C1 * $XMIN
+  $y1 = $C0 + $C1 * $XMAX
+  create xfit 0 2
+  xfit[0] = $XMIN
+  xfit[1] = $XMAX
+  create yfit 0 2
+  yfit[0] = $y0
+  yfit[1] = $y1
+  plot xfit yfit -c red -x 0
+
+  vstat v
+  set vUnbiased = v - $MEAN
+  $vsigma = $SIGMA
+  vstat w
+  set wUnbiased = w - $MEAN
+  $wsigma = $SIGMA
+  set vwUnbiased = vUnbiased * wUnbiased
+  vstat vwUnbiased
+  $correlation = $MEAN / ($vsigma * $wsigma)
+  label -ul "Correlation = $correlation"
+
+  myecho "Saving graphics to [$2]"
+  png -name $2
+end
+
+if ($SCRIPT)
+  $KAPA = kapa -noX
+  resize 1000 1000
+  if ($argv:n != 4)
+    echo "USAGE: build_x_vs_y (input) (output) (x-label) (y-label)"
+    echo "  TODO"
+    exit 1
+  end
+
+  echo "Loading dependencies"
+  $helpersFilename = `which helpers.dvo`
+  input $helpersFilename
+  echo "Dependencies loaded"
+
+  echo $argv:3
+  build_x_vs_y $argv:0 $argv:1 $argv:2 $argv:3
+  exit 0
+end
Index: /branches/eam_branches/ipp-20100621/ippScripts/Build.PL
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/Build.PL	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/Build.PL	(revision 28794)
@@ -109,4 +109,6 @@
         scripts/bundle_detrends.pl
         scripts/ipp_cluster_load_monitor.pl
+        scripts/background_chip.pl
+        scripts/background_warp.pl
         scripts/skycell_jpeg.pl
         scripts/diffphot.pl
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl	(revision 28794)
@@ -21,4 +21,5 @@
 my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
 my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
@@ -40,5 +41,5 @@
 # Grab options
 my ( $date, $camera, $dbname, $logfile, $verbose, $manual);
-my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $check_mode);
+my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode);
 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips);
 my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old);
@@ -59,4 +60,5 @@
     'this_target_only=s'   => \$this_target_only,
     'this_filter_only=s'   => \$this_filter_only,
+    'this_mode_only=s'     => \$this_mode_only,
     'check_registration'   => \$check_registration,
     'define_burntool'      => \$define_burntool,
@@ -86,4 +88,5 @@
            --this_target_only     Process only a single target.
            --this_filter_only     Process only a single filter.
+           --this_mode_only       Process only a single clean mode.
         Modes:
            --check_registration   Confirm the data downloaded correctly.
@@ -129,4 +132,5 @@
 my %detfilter_list = ();
 my %detmax_list = ();
+my @mode_list = ();
 my %clean_commands = ();
 my %clean_retention = ();
@@ -151,4 +155,5 @@
 	    if (${ $mentry }{name} eq 'MODE') {
 		$this_mode = ${ $mentry }{value};
+		push @mode_list, $this_mode;
 	    }
 	    elsif (${ $mentry }{name} eq 'COMMAND') {
@@ -263,4 +268,14 @@
     }
     die("$this_filter_only is invalid.") if ($#filter_list != 0);
+}
+
+if (defined($this_mode_only)) {
+    foreach my $t (@mode_list) {
+        if ($t eq $this_mode_only) {
+            @mode_list = ($this_mode_only);
+            last;
+        }
+    }
+    die("$this_mode_only is invalid.") if ($#mode_list != 0);
 }
 
@@ -717,5 +732,5 @@
     my $date = shift;
 
-    my $command = construct_dqstatstool_cmd($date);
+    my $command = construct_dqstats_cmd($date);
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run ( command => $command, verbose => $verbose );
@@ -754,4 +769,5 @@
     
     my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 ";
+    my $use_limits = " -use_begin ${date}T00:00:00 -use_end ${date}T23:59:59 ";
     $date =~ s/-//g;
 
@@ -766,4 +782,5 @@
     $cmd .= " -workdir $workdir ";
     $cmd .= " -label $label ";
+    $cmd .= " $use_limits ";
     if ($maxN > 0) {
 	$cmd .= " -random_subset -random_limit $maxN ";
@@ -839,5 +856,5 @@
     }
     if (($metadata_out{nsState} eq 'CHECKDETRENDS') && ($exposures > 0)) {
-	$metadata_out{nsState} eq 'QUEUE_DETRENDS';
+	$metadata_out{nsState} eq 'QUEUEDETRENDS';
     }
 }
@@ -1021,6 +1038,9 @@
     my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($cleaning_date,$target);
     my $args = $command;
-    if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
+    if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) {
 	$args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -time_stamp_end $cleaning_date ";
+    }
+    elsif ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'B')) {
+	$args .= " -dbname $dbname -updaterun -set_state goto_cleaned -state full -set_label goto_cleaned -label $label ";
     }
     else {
@@ -1037,6 +1057,6 @@
     my $pretend = shift;
 
-    foreach my $mode (sort (keys (%clean_commands))) {
-	if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode})) {
+    foreach my $mode (@mode_list) {
+	if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) {
 	    my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode);
 	    if ($cleaning_date eq 'no clean') {
@@ -1050,5 +1070,5 @@
 		unless ($success) {
 		    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-		    &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+		    &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
 		}
 	    }
@@ -1070,5 +1090,5 @@
 		    unless ($success) {
 			$error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-			&my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
+			&my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);
 		    }
 		}
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/background_chip.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/background_chip.pl	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/background_chip.pl	(revision 28794)
@@ -0,0 +1,259 @@
+#!/usr/bin/env perl
+
+use Carp;
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+my $date = `date`;
+print "\n\n";
+print "Starting script $0 on $host at $date\n\n";
+
+use DateTime;
+my $mjd_start = DateTime->now->mjd;   # MJD of starting script
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+use File::Temp qw( tempfile );
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+
+# Look for programs we need
+my $missing_tools;
+my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $ppBackground = can_run('ppBackground') or (warn "Can't find ppBackground" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my @ARGS = @ARGV;
+
+# Parse the command-line arguments
+my ( $chip_bg_id, $class_id, $camera, $outroot, $dbname, $reduction, $verbose,
+     $threads, $no_update, $save_temps, $no_op, $redirect, $deburned );
+GetOptions(
+    'chip_bg_id=s'      => \$chip_bg_id,    # chipBackgroundRun identifier
+    'class_id=s'        => \$class_id,  # Class identifier
+    'camera|c=s'        => \$camera,    # Camera
+    'outroot|w=s'       => \$outroot,   # output file base name
+    'dbname|d=s'        => \$dbname,    # Database name
+    'reduction=s'       => \$reduction, # Reduction class
+    'threads=s'         => \$threads,   # Number of threads to use
+    'verbose'           => \$verbose,   # Print to stdout
+    'no-update'         => \$no_update, # Don't update the database?
+    'no-op'             => \$no_op,     # Don't do any operations?
+    'redirect-output'   => \$redirect,
+    'save-temps'        => \$save_temps, # Save temporary files?
+    ) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage( -msg => "Required options: --chip_bg_id --class_id --camera --outroot",
+           -exitval => 3) unless
+    defined $chip_bg_id and
+    defined $class_id and
+    defined $camera and
+    defined $outroot;
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $chip_bg_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+
+my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $chip_bg_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+
+if ($redirect) {
+    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR );
+    print STDOUT "\n\n";
+    print STDOUT "Starting script $0 on $host\n\n";
+    print STDOUT "FULL COMMAND: $0 @ARGS\n\n";
+}
+
+# Recipes to use based on reduction class
+$reduction = 'DEFAULT' unless defined $reduction;
+my $recipe_ppBackground = $ipprc->reduction($reduction, 'BACKGROUND_PPBACKGROUND'); # ppBackground recipe
+unless ($recipe_ppBackground) {
+    &my_die("Couldn't find selected reduction for BACKGROUND_PPBACKGROUND: $reduction\n", $chip_bg_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# Get inputs
+my $in_path;                    # Input path
+my $magicked;                   # Input is magicked?
+{
+    my $command = "bgtool -chipinputs -chip_bg_id $chip_bg_id -class_id $class_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("Unable to get inputs: $error_code\n");
+        exit($error_code);
+    }
+
+    my $inputs = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $chip_bg_id, $class_id, $PS_EXIT_PROG_ERROR);
+    &my_die("Input list does not contain exactly one entry", $chip_bg_id, $class_id, $PS_EXIT_PROG_ERROR) unless scalar @$inputs == 1;
+    my $input = $$inputs[0];    # Input of interest
+    $in_path = $input->{path_base};
+    $magicked = $input->{magicked};
+}
+
+my $in_image = $ipprc->filename("PPIMAGE.CHIP", $in_path, $class_id);
+my $in_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $in_path, $class_id);
+my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id);
+my $in_pattern = $ipprc->filename("PPIMAGE.PATTERN", $in_path, $class_id);
+my $in_config = $ipprc->filename("PPIMAGE.CONFIG", $in_path, $class_id);
+
+# Determine what to apply
+my ($apply_bg, $apply_pattern); # Apply the background and pattern?
+{
+    &my_die("Cannot find input file: $in_config", $chip_bg_id, $class_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($in_config);
+
+    my $command = "$ppConfigDump -ipprc $in_config -dump-recipe PPIMAGE -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 0);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+    my $recipe = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR);
+
+    $apply_bg = metadataLookupBool($recipe, "BACKGROUND");
+    my $row = metadataLookupBool($recipe, "PATTERN.ROW");
+    my $cell = metadataLookupBool($recipe, "PATTERN.CELL");
+    $apply_pattern = ($row or $cell);
+}
+
+# Set up files
+&my_die("Couldn't find input file: $in_image\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_image);
+&my_die("Couldn't find input file: $in_mask\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_mask);
+&my_die("Couldn't find input file: $in_bg\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_bg) or !$apply_bg;
+&my_die("Couldn't find input file: $in_pattern\n", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($in_pattern) or !$apply_pattern;
+
+$ipprc->outroot_prepare($outroot);
+my $out_image = $ipprc->filename("PPBACKGROUND.OUTPUT", $outroot, $class_id);
+my $out_mask = $ipprc->filename("PPBACKGROUND.OUTPUT.MASK", $outroot, $class_id);
+my $out_stats = $ipprc->filename("PPBACKGROUND.STATS", $outroot, $class_id);
+my $out_config = $ipprc->filename("PPBACKGROUND.CONFIG", $outroot, $class_id);
+my $traceDest = $ipprc->filename("TRACE.IMFILE", $outroot, $class_id);
+
+# Run ppImage
+unless ($no_op) {
+    my $command  = "$ppBackground $outroot";
+    $command .= " -image $in_image";
+    $command .= " -mask $in_mask";
+    $command .= " -stats $out_stats";
+    $command .= " -background $in_bg" if $apply_bg;
+    $command .= " -pattern $in_pattern" if $apply_pattern;
+    $command .= " -recipe PPBACKGROUND $recipe_ppBackground";
+    $command .= " -recipe PPSTATS CHIPSTATS";
+    $command .= " -dbname $dbname" if defined $dbname;
+    $command .= " -dumpconfig $out_config";
+    $command .= " -tracedest $traceDest -log $logDest";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppBackground: $error_code", $chip_bg_id, $class_id, $error_code);
+    }
+}
+
+# Gather command-line arguments from statistics
+my $cmdflags;                   # Command-line flags to add
+my $quality;                    # Quality flag
+{
+    &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats);
+
+    my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_CHIP";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppStatsFromMetadata: $error_code", $chip_bg_id, $class_id, $error_code);
+    }
+    foreach my $line (@$stdout_buf) {
+        $cmdflags .= " $line";
+    }
+    chomp $cmdflags;
+    ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
+}
+
+if (!$quality and !$no_op) {
+    &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image);
+    &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_mask);
+    &my_die("Couldn't find expected output file: $out_stats", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_config);
+
+}
+
+# Update database
+{
+    my $command = "$bgtool -addchip";
+    $command .= " -chip_bg_id $chip_bg_id";
+    $command .= " -class_id $class_id";
+    $command .= " -path_base $outroot";
+    $command .= " -set_magicked $magicked" if $magicked;
+    $command .= " -hostname $host" if defined $host;
+    $command .= " $cmdflags";
+    $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            warn("Unable to perform chiptool -addprocessedimfile: $error_code\n");
+            exit($error_code);
+        }
+    } else {
+        print "skipping command: $command\n";
+    }
+}
+
+
+### Pau.
+
+
+sub my_die
+{
+    my $msg = shift; # Warning message on die
+    my $chip_bg_id = shift; # Chiptool identifier
+    my $class_id = shift; # Class identifier
+    my $exit_code = shift; # Exit code to add
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    carp($msg);
+    if (defined $chip_bg_id and defined $class_id and not $no_update) {
+        my $command = "$bgtool -addchip";
+        $command .= " -chip_bg_id $chip_bg_id";
+        $command .= " -class_id $class_id";
+        $command .= " -fault $exit_code";
+        $command .= " -path_base $outroot";
+        $command .= " -hostname $host" if defined $host;
+        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
+        $command .= " -dbname $dbname" if defined $dbname;
+        system($command);
+    }
+    exit $exit_code;
+}
+
+END {
+    my $status = $?;
+    system("sync") == 0
+        or die "failed to execute sync: $!" ;
+    $? = $status;
+}
+
+__END__
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/background_warp.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/background_warp.pl	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/background_warp.pl	(revision 28794)
@@ -0,0 +1,253 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+## report the program and machine
+use Sys::Hostname;
+my $host = hostname();
+my $date = `date`;
+print "\n\n";
+print "Starting script $0 on $host at $date\n\n";
+
+use DateTime;
+my $mjd_start = DateTime->now->mjd;   # MJD of starting script
+
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Spec;
+use File::Temp qw( tempfile );
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+
+# Look for programs we need
+my $missing_tools;
+my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $pswarp = can_run('pswarp') or (warn "Can't find pswarp" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my ($warp_bg_id, $skycell_id, $tess_dir, $reduction, $camera, $dbname, $outroot, $threads, $verbose, $no_update, $no_op, $redirect, $save_temps);
+GetOptions(
+    'warp_bg_id|i=s'      => \$warp_bg_id, # Warp identifier
+    'skycell_id|s=s'      => \$skycell_id, # Skycell identifier
+    'tess_dir|s=s'        => \$tess_dir, # Tesselation identifier
+    'camera|c=s'          => \$camera, # Camera name
+    'dbname|d=s'          => \$dbname, # Database name
+    'reduction=s'         => \$reduction, # Reduction class
+    'outroot=s'           => \$outroot, # Output root name
+    'threads=s'           => \$threads,   # Number of threads to use for pswarp
+    'verbose'             => \$verbose,   # Print to stdout
+    'no-update'           => \$no_update, # Don't update the database?
+    'no-op'               => \$no_op, # Don't do any operations?
+    'redirect-output'     => \$redirect,
+    'save-temps'          => \$save_temps, # Save temporary files?
+) or pod2usage( 2 );
+
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+pod2usage(
+    -msg => "Required options: --warp_bg_id --skycell_id --tess_dir --camera --outroot",
+    -exitval => 3,
+) unless defined $warp_bg_id
+    and defined $skycell_id
+    and defined $tess_dir
+    and defined $camera
+    and defined $outroot;
+
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+
+my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR );
+
+$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
+
+# Recipes to use based on reduction class
+$reduction = 'DEFAULT' unless defined $reduction;
+my $recipe_pswarp = $ipprc->reduction($reduction, 'BACKGROUND_PSWARP'); # Recipe to use
+my $recipe_psastro = $ipprc->reduction($reduction, 'PSASTRO'); # Recipe to use
+unless ($recipe_pswarp and $recipe_psastro) {
+    &my_die("Couldn't find selected reduction: $reduction\n", $warp_bg_id, $skycell_id, $PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# Where do we get the astrometry source from?
+my $astromSource;               # The astrometry source
+{
+    my $command = "$ppConfigDump -camera $camera -recipe PSWARP $recipe_pswarp -dump-recipe PSWARP -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", $warp_bg_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $warp_bg_id, $PS_EXIT_PROG_ERROR);
+    $astromSource = metadataLookupStr($metadata, 'ASTROM.SOURCE');
+}
+
+# Get list of filenames
+my $tempOutRoot = "/tmp/background.warp.$warp_bg_id.$skycell_id";
+my ($imageFile, $imageName) = tempfile( "$tempOutRoot.image.list.XXXX",  UNLINK => !$save_temps);
+my ($maskFile, $maskName) = tempfile( "$tempOutRoot.mask.list.XXXX",   UNLINK => !$save_temps);
+my $astrometry;                 # Astrometry filename
+my $magicked;                   # Magicked status
+{
+    my $command = "$bgtool -warpinputs";
+    $command .= " -warp_bg_id $warp_bg_id";
+    $command .= " -skycell_id $skycell_id";
+    $command .= " -dbname $dbname" if defined $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to get input list: $error_code", $warp_bg_id, $skycell_id, $error_code);
+    }
+
+    my $files = $mdcParser->parse_list(join "", @$stdout_buf) or &my_die("Unable to parse metadata config doc", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR);
+    foreach my $file (@$files) {
+        my $chip_path = $file->{chip_path_base};
+        my $class_id = $file->{class_id};
+        my $image = $ipprc->filename("PPBACKGROUND.OUTPUT", $chip_path, $class_id);
+        my $mask = $ipprc->filename("PPBACKGROUND.OUTPUT.MASK", $chip_path, $class_id );
+        print $imageFile "$image\n";
+        print $maskFile "$mask\n";
+        &my_die("Can't find input image: $image", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($image);
+        &my_die("Can't find input mask: $mask", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($mask);
+
+        &my_die("Magic status don't match: $magicked vs $file->{magicked}", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $magicked and $magicked != $file->{magicked};
+        $magicked = $file->{magicked};
+
+        my $cam_path = $file->{cam_path_base};
+        my $astrom = $ipprc->filename($astromSource, $cam_path);
+        &my_die("Astrometry files don't match: $astrom vs $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) if defined $astrometry and $astrom ne $astrometry;
+        $astrometry = $astrom;
+    }
+}
+close $imageFile;
+close $maskFile;
+
+&my_die("Can't find input astrometry: $astrometry", $warp_bg_id, $skycell_id, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($astrometry);
+
+my $out_image = $ipprc->filename("PSWARP.OUTPUT", $outroot, $skycell_id );
+my $out_mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $outroot, $skycell_id);
+my $out_stats = $ipprc->filename("SKYCELL.STATS", $outroot, $skycell_id );
+my $out_config = $ipprc->filename("PSWARP.CONFIG", $outroot, $skycell_id);
+my $traceDest = $ipprc->filename("TRACE.EXP", $outroot, $skycell_id);
+
+my $skyFile = $ipprc->filename("SKYCELL.TEMPLATE", $outroot, $skycell_id );
+$ipprc->skycell_file( $tess_dir, $skycell_id, $skyFile, $verbose ) or &my_die("Unable to generate template skycell", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR);
+
+# Run pswarp
+unless ($no_op) {
+    my $command = "$pswarp";
+    $command .= " -list $imageName";
+    $command .= " -masklist $maskName";
+    $command .= " -astrom $astrometry";
+    $command .= " $outroot $skyFile";
+    $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";
+    $command .= " -recipe PSWARP $recipe_pswarp";
+    $command .= " -recipe PPSTATS WARPSTATS";
+    $command .= " -stats $out_stats";
+    $command .= " -tracedest $traceDest -log $logDest";
+    $command .= " -threads $threads" if defined $threads;
+    $command .= " -dbname $dbname" if defined $dbname;
+    $command .= " -dumpconfig $out_config";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform pswarp: $error_code", $warp_bg_id, $skycell_id, $error_code);
+    }
+}
+
+# Read the statistics
+my $cmdflags;
+{
+    &my_die("Couldn't find expected output file: $out_stats", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_stats);
+
+    my $command = "$ppStatsFromMetadata $out_stats - BACKGROUND_WARP";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppStatsFromMetadata: $error_code", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR);
+    }
+    foreach my $line (@$stdout_buf) {
+        $cmdflags .= " $line";
+    }
+    chomp $cmdflags;
+
+    my ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
+
+    if (!$quality) {
+        &my_die("Couldn't find expected output file: $out_image", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image);
+        &my_die("Couldn't find expected output file: $out_mask", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_mask);
+        &my_die("Couldn't find expected output file: $out_config", $warp_bg_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_config);
+    }
+}
+
+unless ($no_update) {
+    my $command = "$bgtool -addwarp";
+    $command .= " -warp_bg_id $warp_bg_id";
+    $command .= " -skycell_id $skycell_id";
+    $command .= " -path_base $outroot"; # needed for logfile lookups
+    $command .= " -set_magicked $magicked" if $magicked;
+    $command .= " -hostname $host";
+    $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
+    $command .= " $cmdflags";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("Unable to update database: $error_code\n");
+        exit($error_code);
+    }
+}
+
+### Pau.
+
+
+sub my_die
+{
+    my $msg = shift;            # Warning message on die
+    my $warp_bg_id = shift;     # warpBackgroundRun identifier
+    my $skycell_id = shift;     # Skycell identifier
+    my $exit_code = shift;      # Exit code to add
+
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
+
+    warn($msg);
+    if (defined $warp_bg_id and defined $skycell_id and not $no_update) {
+        my $command = "$bgtool -addwarp";
+        $command .= " -warp_bg_id $warp_bg_id";
+        $command .= " -skycell_id $skycell_id";
+        $command .= " -path_base $outroot";
+        $command .= " -hostname $host" if defined $host;
+        $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
+        $command .= " -fault $exit_code";
+        $command .= " -dbname $dbname" if defined $dbname;
+        run(command => $command, verbose => $verbose);
+    }
+    exit $exit_code;
+}
+
+END {
+    my $status = $?;
+    system("sync") == 0
+        or die "failed to execute sync: $!" ;
+    $? = $status;
+}
+
+__END__
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/diff_skycell.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/diff_skycell.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/diff_skycell.pl	(revision 28794)
@@ -291,6 +291,6 @@
         $command .= " -ipprc $configurationReal";
     }
-    $command .= " -save-inconv" if defined $saveInConv;
-    $command .= " -save-refconv" if defined $saveRefConv;
+    $command .= " -save-inconv" if defined $saveInConv and $run_state eq "new";
+    $command .= " -save-refconv" if defined $saveRefConv and $run_state eq "new";
     $command .= " -recipe PPSUB $recipe_ppSub";
     $command .= " -recipe PSPHOT $recipe_psphot";
@@ -301,4 +301,6 @@
     if ($run_state eq "new") {
         $command .= " -photometry";
+    } else {
+        $command .= " -Db PHOTOMETRY FALSE";
     }
     $command .= " -inverse" if $inverse;
@@ -350,4 +352,6 @@
                 }
             }
+        } elsif ($run_state eq 'update') {
+            &my_die("Update resulted in poor quality image: $quality", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR);
         }
     } else {
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/diffphot.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/diffphot.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/diffphot.pl	(revision 28794)
@@ -31,5 +31,4 @@
 my $diffphottool = can_run('diffphottool') or (warn "Can't find diffphottool" and $missing_tools = 1);
 my $psphot = can_run('psphot') or (warn "Can't find psphot" and $missing_tools = 1);
-my $ppArith = can_run('ppArith') or (warn "Can't find ppArith" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
 if ($missing_tools) {
@@ -177,21 +176,10 @@
 # Do reverse photometry
 if ($bothways) {
-    my ($tempFile, $tempName) = tempfile( "/tmp/diffphot.$diff_phot_id.$skycell_id.XXXX",
-                                          UNLINK => !$save_temps, SUFFIX => '.fits' );
-    {
-        my ($tempRoot) = $tempName =~ /^(.*).fits/;
-        my $command = "$ppArith $tempRoot -file1 $inputImage -op \\* -const2 -1";
-        unless ($no_op) {
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform ppArith: $error_code", $diff_phot_id, $skycell_id, $error_code);
-            }
-            &my_die("Couldn't find expected output file: $tempName", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($tempName);
-        } else {
-            print "Skipping command: $command\n";
-        }
-    }
+    my $inputImage = $ipprc->filename("PPSUB.INVERSE", $inputPath);
+    my $inputMask = $ipprc->filename("PPSUB.INVERSE.MASK", $inputPath);
+    my $inputVariance = $ipprc->filename("PPSUB.INVERSE.VARIANCE", $inputPath);
+    &my_die("Couldn't find input: $inputImage", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputImage);
+    &my_die("Couldn't find input: $inputMask", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask);
+    &my_die("Couldn't find input: $inputVariance", $diff_phot_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance);
 
     # Get the output filenames
@@ -201,5 +189,5 @@
 
     my $command = "$psphot $outroot.neg";
-    $command .= " -file $tempName";
+    $command .= " -file $inputImage";
     $command .= " -mask $inputMask";
     $command .= " -variance $inputVariance";
@@ -252,8 +240,8 @@
 }
 
-unless ($no_update) {
+{
     my $command = "$diffphottool -diff_phot_id $diff_phot_id -skycell_id $skycell_id";
     $command .= " -done -path_base $outroot";
-    $command .= " -magicked $magicked" if $magicked;
+    $command .= " -magicked $magicked" if defined $magicked;
     $command .= " -quality $quality" if defined $quality;
     $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
@@ -261,9 +249,13 @@
     $command .= " -dbname $dbname" if defined $dbname;
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Command failed: $error_code", $diff_phot_id, $skycell_id, $error_code);
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Command failed: $error_code", $diff_phot_id, $skycell_id, $error_code);
+        }
+    } else {
+        print "Skipping update: $command\n";
     }
 }
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_advancerun.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_advancerun.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_advancerun.pl	(revision 28794)
@@ -66,4 +66,5 @@
 my $difftool   = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
 my $stacktool   = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
 if ($missing_tools) {
     &my_die("Can't find required tools.", $dist_id, $PS_EXIT_CONFIG_ERROR);
@@ -82,4 +83,8 @@
     $tool_cmd = "$chiptool -chip_id";
     $list_mode = "-processedimfile";
+    $component_key = "class_id";
+} elsif ($stage eq "chip_bg") {
+    $tool_cmd = "$bgtool -chip_bg_id";
+    $list_mode = "-chip";
     $component_key = "class_id";
 } elsif ($stage eq "camera") {
@@ -95,4 +100,8 @@
     $list_mode = "-warped";
     $component_key = "skycell_id";
+} elsif ($stage eq "warp_bg") {
+    $tool_cmd = "$bgtool -warp_bg_id";
+    $list_mode = "-warp";
+    $component_key = "skycell_id";
 } elsif ($stage eq "stack") {
     $tool_cmd = "$stacktool -stack_id";
@@ -113,9 +122,16 @@
 $tool_cmd .= " $stage_id";
 
+my $exportarg = '-exportrun';
+if ($stage eq 'chip_bg') {
+    $exportarg = '-exportchip';
+} elsif ($stage eq 'warp_bg') {
+    $exportarg = '-exportwarp';
+}
+
 # XXX should we create a file rule for this?
 my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc";
 
 {
-    my $command = "$tool_cmd -exportrun -outfile $dbinfo_file";
+    my $command = "$tool_cmd $exportarg -outfile $dbinfo_file";
     $command .= " -clean" if ((defined $clean) and ($stage ne "raw"));
     $command .= " -dbname $dbname" if defined $dbname;
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_bundle.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_bundle.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_bundle.pl	(revision 28794)
@@ -35,4 +35,6 @@
                      'PPIMAGE.CHIP.MASK' => 'mask',
                      'PPIMAGE.CHIP.VARIANCE' => 'variance' );
+my %chip_bg_cleaned = ( 'PPBACKGROUND.OUTPUT' => 'image',
+                        'PPBACKGROUND.OUTPUT.MASK' => 'mask' );
 my %camera_cleaned = ( 'PSASTRO.OUTPUT.MASK' => 'mask' );
 my %fake_cleaned;
@@ -40,4 +42,6 @@
                      'PSWARP.OUTPUT.MASK' => 'mask',
                      'PSWARP.OUTPUT.VARIANCE' => 'variance' );
+my %warp_bg_cleaned = ( 'PSWARP.OUTPUT' => 'image',
+                        'PSWARP.OUTPUT.MASK' => 'mask' );
 my %diff_cleaned = ( 'PPSUB.OUTPUT' => 'image',
                      'PPSUB.OUTPUT.MASK' => 'mask',
@@ -233,5 +237,7 @@
     if ($stage ne "raw") {
         &my_die("no mask image found in file list", $component, $PS_EXIT_CONFIG_ERROR) if !$mask;
-        &my_die("no variance image found in file list", $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
+        if (($stage ne "chip_bg") and ($stage ne "warp_bg")) {
+            &my_die("no variance image found in file list", $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
+        }
     }
 
@@ -246,9 +252,19 @@
         my $fh = open_with_retries($mask_resolved);
         close $fh;
-    } elsif ($stage eq "chip") {
+    } elsif ($stage eq "chip" or $stage eq "chip_bg") {
         $class_id = $component;
     }
 
-    my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir";
+    my $release_stage;
+    if ($stage eq "chip_bg") {
+        $release_stage = 'chip';
+    } elsif ($stage eq 'warp_bg') {
+        $release_stage = 'warp';
+    } else {
+        $release_stage = $stage;
+    }
+
+
+    my $command = "$streaksrelease -stage $release_stage -image $image -outroot $tmpdir";
     $command .= " -class_id $class_id" if $class_id;
     $command .= " -mask $mask" if $mask;
@@ -262,5 +278,5 @@
     }
     if ($inv_image) {
-        $command = "$streaksrelease -stage $stage -image $inv_image -outroot $tmpdir";
+        $command = "$streaksrelease -stage $release_stage -image $inv_image -outroot $tmpdir";
         $command .= " -mask $inv_mask" if $inv_mask;
         $command .= " -weight $inv_variance" if $inv_variance;
@@ -341,4 +357,6 @@
     } elsif ($stage eq "chip") {
         $type = $chip_cleaned{$rule};
+    } elsif ($stage eq "chip_bg") {
+        $type = $chip_bg_cleaned{$rule};
     } elsif ($stage eq "camera") {
         $type = $camera_cleaned{$rule};
@@ -347,4 +365,6 @@
     } elsif ($stage eq "warp") {
         $type = $warp_cleaned{$rule};
+    } elsif ($stage eq "warp_bg") {
+        $type = $warp_bg_cleaned{$rule};
     } elsif ($stage eq "diff") {
         $type = $diff_cleaned{$rule};
@@ -416,4 +436,6 @@
     if ($stage eq "chip") {
         $config_file_rule = "PPIMAGE.CONFIG";
+    } elsif ($stage eq "chip_bg") {
+        $config_file_rule = "PPBACKGROUND.CONFIG";
     } elsif ($stage eq "camera") {
         $config_file_rule = "PSASTRO.CONFIG";
@@ -422,4 +444,6 @@
         return \@file_list;
     } elsif ($stage eq "warp") {
+        $config_file_rule = "PSWARP.CONFIG";
+    } elsif ($stage eq "warp_bg") {
         $config_file_rule = "PSWARP.CONFIG";
     } elsif ($stage eq "diff") {
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_defineruns.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_defineruns.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_defineruns.pl	(revision 28794)
@@ -83,5 +83,5 @@
     push @stages, $stage;
 } else {
-    @stages = qw( raw chip camera fake warp diff stack SSdiff);
+    @stages = qw( raw chip chip_bg camera fake warp warp_bg diff stack SSdiff);
 }
 
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_make_fileset.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_make_fileset.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/dist_make_fileset.pl	(revision 28794)
@@ -239,4 +239,6 @@
     } elsif ($stage eq 'chip') {
         $query = "SELECT exp_name FROM chipRun JOIN rawExp USING(exp_id) WHERE chip_id = $stage_id";
+    } elsif ($stage eq 'chip_bg') {
+        $query = "SELECT exp_name FROM chipBackgroundRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE chip_bg_id = $stage_id";
     } elsif ($stage eq 'camera') {
         $query = "SELECT exp_name FROM camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"
@@ -248,4 +250,6 @@
         $query = "SELECT exp_name FROM warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id)"
                     . " JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE warp_id = $stage_id";
+    } elsif ($stage eq 'warp_bg') {
+        $query = "SELECT exp_name FROM warpBackgroundRun JOIN warpRun USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) WHERE warp_bg_id = $stage_id";
     } else {
         &my_die("$stage is invalid value for stage\n");
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/dqstats_bundle.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/dqstats_bundle.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/dqstats_bundle.pl	(revision 28794)
@@ -73,5 +73,5 @@
 # Output products
 unless (defined($uri)) {
-    $uri = "/data/${host}.0/tmp/dqstats.${dqstats_id}.fits";
+    $uri = "/tmp/dqstats.${dqstats_id}.fits";
 }
 #$ipprc->outroot_prepare($uri); # hm....need to think more here.
@@ -90,4 +90,5 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	unlink($uri);
         &my_die("Unable to create the bundle.: $error_code", $dqstats_id, $error_code);
     }
@@ -106,4 +107,5 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+	unlink($uri);
         &my_die("Unable to register the bundle.: $error_code", $dqstats_id, $error_code);
     }
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/magic_destreak.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/magic_destreak.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/magic_destreak.pl	(revision 28794)
@@ -97,8 +97,8 @@
 my ($skycell_args, $class_id, $skycell_id);
 
-if (($stage eq "raw") or ($stage eq "chip")) {
+if (($stage eq "raw") or ($stage eq "chip") or $stage eq "chip_bg") {
     $class_id = $component;
     $skycell_args = " -class_id $component";
-} elsif ($stage eq "warp") {
+} elsif ($stage eq "warp" or $stage eq "warp_bg") {
     $skycell_id = $component;
     $skycell_args = " -skycell_id $component";
@@ -286,5 +286,9 @@
         # $sources = $ipprc->filename("PSPHOT.OUT.CMF.SPL",  $path_base);
 
-    } elsif ($stage eq "warp") {
+    } elsif ($stage eq "chip_bg") {
+        $image  = $ipprc->filename("PPBACKGROUND.OUTPUT", $path_base, $class_id);
+        $mask = $ipprc->filename("PPBACKGROUND.OUTPUT.MASK", $path_base, $class_id);
+        $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path_base);
+    } elsif ($stage eq "warp" or $stage eq "warp_bg") {
         $image  = $ipprc->filename("PSWARP.OUTPUT", $path_base);
         $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/make_burntool_pcontrol.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/make_burntool_pcontrol.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/make_burntool_pcontrol.pl	(revision 28794)
@@ -48,5 +48,5 @@
 while(<LAZY>) {
     chomp;
-    if ($_ =~ /BURNTOOL.STATE.GOOD/) {
+    if ($_ =~ /BURNTOOL.STATE.GOOD\s/) {
         @line = split /\s+/;
         $burntoolStateGood = $line[2];
@@ -328,10 +328,10 @@
 	foreach my $target (@target_list) {
 #	    print STDERR "$target $obsmode_list{$target} $exp_type $obs_mode $object $comment\n";
-	    if (($obs_mode =~ /$obsmode_list{$target}/)&&
-		($object   =~ /$object_list{$target}/)&&
-		($comment  =~ /$comment_list{$target}/)) {
+	    if (($obs_mode =~ /$obsmode_list{$target}/i)&&
+		($object   =~ /$object_list{$target}/i)&&
+		($comment  =~ /$comment_list{$target}/i)) {
 		$return_value = 1;
 	    }
-#	    print ">$return_value $exp_type $obs_mode $object $comment $target $obsmode_list{$target} $object_list{$target} $comment_list{$target}\n";
+#	    print ">$return_value $exp_type $obs_mode $object $comment <> $target $obsmode_list{$target} $object_list{$target} $comment_list{$target}\n";
 	    if ($return_value) {
 		return($return_value);
Index: /branches/eam_branches/ipp-20100621/ippScripts/scripts/publish_file.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippScripts/scripts/publish_file.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippScripts/scripts/publish_file.pl	(revision 28794)
@@ -82,5 +82,7 @@
 my $mdcParser = PS::IPP::Metadata::Config->new;
 
+my $mops = ($product =~ /^IPP-MOPS/ ? 1 : 0); # Format for MOPS?
 my ($dsFile, $dsFileName) = tempfile("/tmp/publish.$pub_id.ds.XXXX", UNLINK => !$save_temps );
+my $dsType = $mops ? "IPP-MOPS" : $product; # Type for DataStore
 
 my $comment;                    # Comment for exposure
@@ -155,5 +157,4 @@
         &my_die("Unable to parse metadata config", $pub_id, $PS_EXIT_PROG_ERROR);
 
-    my $mops = ($product =~ /^IPP-MOPS/ ? 1 : 0); # Format for MOPS?
     my ($mopsPositiveFile, $mopsPositiveFileName) = tempfile("/tmp/publish.$pub_id.mops.pos.XXXX", UNLINK => !$save_temps ) if $mops;
     my ($mopsNegativeFile, $mopsNegativeFileName) = tempfile("/tmp/publish.$pub_id.mops.neg.XXXX", UNLINK => !$save_temps ) if $mops;
@@ -230,5 +231,5 @@
                 print $mopsNegativeFile "$filename\n";
             } else {
-                print $dsFile "$filename|||$product|${skycell_id}.neg|\n";
+                print $dsFile "$filename|||$dsType|${skycell_id}.neg|\n";
             }
         }
@@ -241,9 +242,9 @@
         if (scalar keys %positive > 0) {
             my $output = mops_combine(\%positive, "$outroot.pos.mops", $mopsPositiveFileName);
-            print $dsFile "$output|||$product|positive|\n";
+            print $dsFile "$output|||$dsType|positive|\n";
         }
         if (scalar keys %negative > 0) {
             my $output = mops_combine(\%negative, "$outroot.neg.mops", $mopsNegativeFileName);
-            print $dsFile "$output|||$product|negative|\n";
+            print $dsFile "$output|||$dsType|negative|\n";
         }
     }
@@ -253,5 +254,5 @@
 
 unless ($no_update) {
-    my $command = "$dsreg --add pub.$pub_id.$stage.$stage_id --copy --abspath --product $product --type $product --list $dsFileName";
+    my $command = "$dsreg --add pub.$pub_id.$stage.$stage_id --copy --abspath --product $product --type $dsType --list $dsFileName";
     $command .= " --ps0 \"$comment\"" if defined $comment;
 
Index: /branches/eam_branches/ipp-20100621/ippTasks/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/Makefile.am	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/Makefile.am	(revision 28794)
@@ -34,5 +34,7 @@
 	minidvodb.pro \
 	nightly_stacks.pro \
+	burntool.pro \
 	lossy_compress.pro \
+	background.pro \
 	diffphot.pro
 
Index: /branches/eam_branches/ipp-20100621/ippTasks/background.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/background.pro	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTasks/background.pro	(revision 28794)
@@ -0,0 +1,536 @@
+## background.pro : globals and support macros : -*- sh -*-
+
+check.globals
+
+book init chipBackgroundRuns
+book init warpBackgroundRuns
+
+macro bg.status
+  book listbook chipBackgroundRuns
+  book listbook warpBackgroundRuns
+end
+
+macro bg.reset
+  book init chipBackgroundRuns
+  book init warpBackgroundRuns
+end
+
+macro bg.on
+  task bg.chip.load
+    active true
+  end
+  task bg.chip.run
+    active true
+  end
+  task bg.chip.advance
+    active true
+  end
+  task bg.chip.revert
+    active true
+  end
+  task bg.warp.load
+    active true
+  end
+  task bg.warp.run
+    active true
+  end
+  task bg.warp.advance
+    active true
+  end
+  task bg.warp.revert
+    active true
+  end
+end
+
+macro bg.off
+  task bg.chip.load
+    active false
+  end
+  task bg.chip.run
+    active false
+  end
+  task bg.chip.advance
+    active false
+  end
+  task bg.chip.revert
+    active false
+  end
+  task bg.warp.load
+    active false
+  end
+  task bg.warp.run
+    active false
+  end
+  task bg.warp.advance
+    active false
+  end
+  task bg.warp.revert
+    active false
+  end
+end
+
+macro bg.revert.on
+  task bg.chip.revert
+    active true
+  end
+  task bg.warp.revert
+    active true
+  end
+end
+
+macro bg.revert.off
+  task bg.chip.revert
+    active false
+  end
+  task bg.warp.revert
+    active false
+  end
+end
+
+
+# These variables cycle through the known database names
+$bg_chip_load_DB = 0
+$bg_chip_advance_DB = 0
+$bg_chip_revert_DB = 0
+$bg_warp_load_DB = 0
+$bg_warp_advance_DB = 0
+$bg_warp_revert_DB = 0
+
+task	       bg.chip.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/bg.chip.load.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = bgtool -tochip
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$bg_chip_load_DB
+      $run = $run -dbname $DB:$bg_chip_load_DB
+      $bg_chip_load_DB ++
+      if ($bg_chip_load_DB >= $DB:n) set bg_chip_load_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout chipBackgroundRuns -key chip_bg_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook chipBackgroundRuns
+    end
+    process_cleanup chipBackgroundRuns
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       bg.chip.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    periods -exec $RUNEXEC
+
+    book npages chipBackgroundRuns -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    if ($BURNTOOLING == 1) break
+    
+    book getpage chipBackgroundRuns 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword chipBackgroundRuns $pageName pantaskState RUN
+    book getword chipBackgroundRuns $pageName camera -var CAMERA
+    book getword chipBackgroundRuns $pageName chip_bg_id -var CHIP_BG_ID
+    book getword chipBackgroundRuns $pageName class_id -var CLASS_ID
+    book getword chipBackgroundRuns $pageName exp_tag -var EXP_TAG
+    book getword chipBackgroundRuns $pageName workdir -var WORKDIR_TEMPLATE
+    book getword chipBackgroundRuns $pageName dbname -var DBNAME
+    book getword chipBackgroundRuns $pageName reduction -var REDUCTION
+
+    set.host.for.camera $CAMERA $CLASS_ID
+    set.workdir.by.camera $CAMERA $CLASS_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    sprintf outroot "%s/%s/%s.bgc.%s" $WORKDIR $EXP_TAG $EXP_TAG $CHIP_BG_ID
+    stdout $LOGDIR/bg.chip.run.log
+    stderr $LOGDIR/bg.chip.run.log
+
+    $run = background_chip.pl --threads @MAX_THREADS@ --chip_bg_id $CHIP_BG_ID --class_id $CLASS_ID --camera $CAMERA --outroot $outroot --redirect-output
+    if ("$REDUCTION" != "NULL")
+      $run = $run --reduction $REDUCTION
+    end
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit chipBackgroundRuns $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    exec bgtool -addchip -dbname $DBNAME -chip_bg_id $CHIP_BG_ID -class_id $CLASS_ID -fault $EXIT_CRASH_ERR
+    process_exit chipBackgroundRuns $options:0 $EXIT_CRASH_ERR
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword chipBackgroundRuns $options:0 pantaskState TIMEOUT
+  end
+end
+
+# advance exposures for which all imfiles have completed processing
+# sets the exposure state to full and queues warp processing if requested
+task	       bg.chip.advance
+  host         local
+
+  periods      -poll $LOADPOLL
+#  periods      -exec $LOADEXEC
+  periods      -exec 30
+  periods      -timeout 60
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/bg.chip.advance.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = bgtool -advancechip
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$chip_advance_DB
+      $run = $run -dbname $DB:$bg_chip_advance_DB
+      $bg_chip_advance_DB ++
+      if ($bg_chip_advance_DB >= $DB:n) set bg_chip_advance_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task bg.chip.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/bg.chip.revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = bgtool -revertchip
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$bg_chip_revert_DB
+      $run = $run -dbname $DB:$bg_chip_revert_DB
+      $bg_chip_revert_DB ++
+      if ($bg_chip_revert_DB >= $DB:n) set bg_chip_revert_DB = 0
+    end
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       bg.warp.load
+  host         local
+
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/bg.warp.load.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = bgtool -towarp
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$bg_warp_load_DB
+      $run = $run -dbname $DB:$bg_warp_load_DB
+      $bg_warp_load_DB ++
+      if ($bg_warp_load_DB >= $DB:n) set bg_warp_load_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+    # convert 'stdout' to book format
+    ipptool2book stdout warpBackgroundRuns -key warp_bg_id:skycell_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook warpBackgroundRuns
+    end
+    process_cleanup warpBackgroundRuns
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task	       bg.warp.run
+  periods      -poll $RUNPOLL
+  periods      -exec $RUNEXEC
+  periods      -timeout 60
+
+  task.exec
+    periods -exec $RUNEXEC
+
+    book npages warpBackgroundRuns -var N
+    if ($N == 0) break
+    if ($NETWORK == 0) break
+    if ($BURNTOOLING == 1) break
+    
+    book getpage warpBackgroundRuns 0 -var pageName -key pantaskState INIT
+    if ("$pageName" == "NULL") break
+
+    book setword warpBackgroundRuns $pageName pantaskState RUN
+    book getword warpBackgroundRuns $pageName camera -var CAMERA
+    book getword warpBackgroundRuns $pageName warp_bg_id -var WARP_BG_ID
+    book getword warpBackgroundRuns $pageName skycell_id -var SKYCELL_ID
+    book getword warpBackgroundRuns $pageName tess_id -var TESS_DIR
+    book getword warpBackgroundRuns $pageName exp_tag -var EXP_TAG
+    book getword warpBackgroundRuns $pageName workdir -var WORKDIR_TEMPLATE
+    book getword warpBackgroundRuns $pageName dbname -var DBNAME
+    book getword warpBackgroundRuns $pageName reduction -var REDUCTION
+
+    set.host.for.skycell $SKYCELL_ID
+    set.workdir.by.skycell $SKYCELL_ID $WORKDIR_TEMPLATE $default_host WORKDIR
+
+    sprintf outroot "%s/%s/%s.bgw.%s.%s" $WORKDIR $EXP_TAG $EXP_TAG $WARP_BG_ID $SKYCELL_ID
+    stdout $LOGDIR/bg.warp.run.log
+    stderr $LOGDIR/bg.warp.run.log
+
+    $run = background_warp.pl --threads @MAX_THREADS@ --warp_bg_id $WARP_BG_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output
+    if ("$REDUCTION" != "NULL")
+      $run = $run --reduction $REDUCTION
+    end
+    add_standard_args run
+
+    # save the pageName for future reference below
+    options $pageName
+
+    # create the command line
+    if ($VERBOSE > 1)
+      echo command $run
+    end
+    # if we are unable to run the 'exec', use a long retry time
+    periods -exec 0.05
+    command $run
+  end
+
+  # default exit status
+  task.exit    default
+    process_exit warpBackgroundRuns $options:0 $JOB_STATUS
+  end
+
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    exec bgtool -addwarp -dbname $DBNAME -warp_bg_id $WARP_BG_ID -skycell_id $SKYCELL_ID -fault $EXIT_CRASH_ERR
+    process_exit warpBackgroundRuns $options:0 $EXIT_CRASH_ERR
+  end
+
+  # operation timed out?
+  task.exit    timeout
+    showcommand timeout
+    book setword warpBackgroundRuns $options:0 pantaskState TIMEOUT
+  end
+end
+
+# advance exposures for which all imfiles have completed processing
+# sets the exposure state to full and queues warp processing if requested
+task	       bg.warp.advance
+  host         local
+
+  periods      -poll $LOADPOLL
+#  periods      -exec $LOADEXEC
+  periods      -exec 30
+  periods      -timeout 60
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/bg.warp.advance.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = bgtool -advancewarp
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$warp_advance_DB
+      $run = $run -dbname $DB:$bg_warp_advance_DB
+      $bg_warp_advance_DB ++
+      if ($bg_warp_advance_DB >= $DB:n) set bg_warp_advance_DB = 0
+    end
+    add_poll_args run
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task bg.warp.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/bg.warp.revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = bgtool -revertwarp
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$bg_warp_revert_DB
+      $run = $run -dbname $DB:$bg_warp_revert_DB
+      $bg_warp_revert_DB ++
+      if ($bg_warp_revert_DB >= $DB:n) set bg_warp_revert_DB = 0
+    end
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
Index: /branches/eam_branches/ipp-20100621/ippTasks/burntool.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/burntool.pro	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTasks/burntool.pro	(revision 28794)
@@ -0,0 +1,568 @@
+## burntool.pro : -*- sh -*-
+
+check.globals
+
+macro burntool.it.on
+#    bt.initday.on
+    bt.registration.on
+    bt.burntool.on
+    bt.chips.on
+end
+
+macro burntool.it.off
+#    bt.initday.off
+    bt.registration.off
+    bt.burntool.off
+    bt.chips.off
+end
+
+macro bt.on
+    burntool.it.on
+end
+
+macro bt.off
+    burntool.it.off
+end
+
+macro bt.initday.on
+  task bt.initday.load
+    active true
+  end
+end
+
+macro bt.registration.on
+  task bt.registration.load
+    active true
+  end
+end
+
+macro bt.burntool.on
+  task bt.burntool.load
+    active true
+  end
+  task bt.burntool.run
+    active true
+  end
+end
+
+macro bt.chips.on
+  task bt.chips.load
+    active true
+  end
+end
+
+macro bt.initday.off
+  task bt.initday.load
+    active false
+  end
+end
+
+macro bt.registration.off
+  task bt.registration.load
+    active false
+  end
+end
+
+macro bt.burntool.off
+  task bt.burntool.load
+    active false
+  end
+  task bt.burntool.run
+    active false
+  end
+end
+
+macro bt.chips.off
+  task bt.chips.load
+    active false
+  end
+end
+
+$bt_regPAGE = 0
+$bt_burnPAGE = 0
+$bt_RburnPAGE = 0
+$bt_RburnCELL = 0
+$bt_chipPAGE = 0
+$bt_RchipPAGE = 0
+
+book init btData
+book init btBurntool
+#
+# Macros to control the book.
+#
+macro bt.add.date
+   book newpage btData $1
+   book setword btData $1 nsState NEW
+end
+
+macro bt.show.dates
+    book npages btData -var Npages
+    for i 0 $Npages
+       book getpage btData $i -var date
+       book getword btData $date nsState -var state
+       echo $date $state
+    end
+    book npages btBurntool -var Npages
+end
+
+macro bt.del.date
+    book delpage btData $1
+end
+
+macro bt.set.date
+    book setword btData $1 nsState $2
+end
+
+
+
+#
+# Start a new date to work on
+#
+task              bt.initday.load
+  host            local
+  periods         -poll $LOADPOLL
+  periods         -exec $LOADEXEC
+  periods         -timeout 30
+  active          false
+  npending        1
+
+  task.exec
+    stdout $LOGDIR/bt.initday.log
+    stderr $LOGDIR/bt.initday.log
+    $today = `date +%Y-%m-%d`
+    book newpage btData $today
+    book setword btData $today nsState NEW
+
+    command echo $today
+  end
+
+  task.exit       0
+    # nothing to do here?
+  end
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+#
+# Check to see if registration is complete
+#
+task              bt.registration.load
+  host            local
+  periods         -poll $LOADPOLL
+  periods         -exec $LOADEXEC
+  periods         -timeout 30
+  npending        1
+
+  task.exec
+    stdout NULL
+    stderr $LOGDIR/bt.registration.log
+
+    book getpage btData $bt_regPAGE -var date
+    book getword btData $date nsState -var bt_STATE
+    book npages btData -var Npages
+
+    if ($VERBOSE > 5) 
+       echo "bt.registration.load: " $bt_regPAGE $date $bt_STATE $Npages
+    end
+
+    $bt_regPAGE ++
+    if ($bt_regPAGE >= $Npages) set bt_regPAGE = 0
+    option $date
+
+    if ("$bt_STATE" != "NEW") break
+    $run = automate_stacks.pl --check_registration --date $date
+    command $run
+  end
+
+  # success
+  task.exit   0
+#    convert 'stdout' to book format
+    book delpage btData $options:0
+    ipptool2book stdout btData -uniq -key date
+
+    if ($VERBOSE > 2)
+	book listbook btData
+    end
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+#
+# Check to see if the chips have been properly burntooled and are ready for queueing
+#
+task              bt.chips.load
+  host            local
+  periods         -poll $LOADPOLL
+  periods         -exec $LOADEXEC
+  periods         -timeout 300
+  npending        1
+
+  task.exec
+     stdout NULL
+     stderr $LOGDIR/bt.chips.log
+
+     book getpage btData $bt_chipPAGE -var date
+     book getword btData $date nsState -var bt_STATE
+     book npages btData -var Npages
+
+     if ($VERBOSE > 5)
+	echo "bt.chips.load: " $bt_chipPAGE $date $bt_STATE $Npages
+     end
+
+     $bt_chipPAGE ++
+     if ($bt_chipPAGE >= $Npages) set bt_chipPAGE = 0
+     option $date
+
+     if (("$bt_STATE" != "REGISTERED")&&("$bt_STATE" != "BURNING")) break
+     $run = automate_stacks.pl --check_chips --date $date
+
+     if ("$bt_STATE" == "BURNING") 
+	$run = $run --isburning
+     end
+     command $run
+   end
+  # success
+  task.exit   0
+#   convert 'stdout' to book format
+    book delpage btData $options:0
+    ipptool2book stdout btData -uniq -key date
+
+    # remove the burntool page if we're done with it.
+    book getword btData $options:0 nsState -var bt_STATE
+    if ("$bt_STATE" == "QUEUECHIPS") 
+	book delpage btBurntool $options:0
+	book delpage btData $options:0
+    end
+
+    if ($VERBOSE > 2)
+	book listbook btData
+    end
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+#
+# Generate a list of date ranges and chips that need to be processed with burntool
+#
+task              bt.burntool.load
+  host            local
+  periods         -poll $LOADPOLL
+  periods         -exec $LOADEXEC
+  periods         -timeout 30
+  npending        1
+
+  task.exec
+    stdout NULL
+    stderr $LOGDIR/bt.burntool.log
+
+    book getpage btData $bt_burnPAGE -var date
+    book getword btData $date nsState -var bt_STATE
+    book npages btData -var Npages
+
+    if ($VERBOSE > 5) 
+        echo "bt.burntool.load: " $bt_burnPAGE $date $bt_STATE $Npages
+    end
+
+    $bt_burnPAGE ++
+    if ($bt_burnPAGE >= $Npages) set bt_burnPAGE = 0
+    option $date
+    
+    if ("$bt_STATE" != "NEEDSBURNING") break
+
+    $run = automate_stacks.pl --define_burntool --date $date
+    command $run
+  end
+  # success
+  task.exit   0
+#    convert 'stdout' to book format
+#    book delpage btBurntool $options:0
+    ipptool2book stdout btBurntool -uniq -key date
+    book setword btData $options:0 nsState "QUEUEBURNING"
+
+    if ($VERBOSE > 2)
+	book listbook btData
+    end
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+#
+# Magically run burntool on the data, based on the information stored in our book.
+#
+task              bt.burntool.run
+  periods         -poll $RUNPOLL
+  periods         -exec $RUNEXEC
+  periods         -timeout 10800
+  npending        30
+  task.exec
+     periods -exec $RUNEXEC
+     stdout NULL
+     stderr $LOGDIR/bt.burntool.log
+
+     book getpage btData $bt_RburnPAGE -var date
+     book getword btData $date nsState -var bt_STATE
+     book npages btData -var Npages
+
+    if ($VERBOSE > 5) 
+        echo "bt.burntool.run: " $bt_RburnPAGE $date $bt_STATE $Npages
+    end
+
+    $bt_RburnPAGE ++
+    if ($bt_RburnPAGE >= $Npages) set bt_RburnPAGE = 0
+    if (("$bt_STATE" != "QUEUEBURNING")&&("$bt_STATE" != "BURNING")) break
+    $BURNTOOLING = 1
+    # Find out where in the list of jobs we are
+    book getword btBurntool $date btN -var btN
+    book getword btBurntool $date btNCounter -var btNcounter
+
+    if ($VERBOSE > 5) 
+	echo "bt.burntool.run: Status: " $btNcounter $date
+    end
+
+    if ("$bt_STATE" == "QUEUEBURNING")
+	$new_state = "QUEUEBURNING"
+    end
+    if ("$bt_STATE" == "BURNING")
+	$new_state = "BURNING"
+    end
+    if ($btNcounter + 1 > $btN)
+	$new_state = "BURNING"
+    end
+    if ($btNcounter > $btN) 
+    	$btNcounter = 0
+    end
+    if ($VERBOSE > 5) 
+	echo "bt.burntool.run: Status: " $btNcounter $new_state
+    end
+    
+    # Increment the counter in the book for the next job.
+    $counter_update = $btNcounter + 1
+    book setword btBurntool $date btNCounter $counter_update
+#    book setword btData $options:0 nsState $options:1
+    book setword btData $date nsState $new_state
+
+    # Get the current status of this job, and skip if it doesn't need to process.
+    sprintf status_label     "bt%dStatus"  $btNcounter
+    book getword btBurntool $date $status_label   -var status
+
+    if ($VERBOSE > 5) 
+	echo "bt.burntool.run: Status: " $btNcounter $status $date $status_label
+    end
+    if (("$status" == "FINISHED")||("$status" == "RUN")) break	
+    
+    # Continue loading information to process this job
+    sprintf start_date_label "bt%dBegin"   $btNcounter
+    sprintf end_date_label   "bt%dEnd"     $btNcounter
+    sprintf class_count_label "bt%dClass"   $btNcounter
+
+    book getword btBurntool $date $start_date_label -var start_date
+    book getword btBurntool $date $end_date_label -var end_date
+    book getword btBurntool $date $class_count_label -var chip_counter
+
+    # Lookup class_id/host pairs
+    list word -split $hostmatch:$chip_counter
+    $class_id = $word:0
+    $host = $word:1
+    host $host
+#    set.host.for.camera GPC1 $class_id
+    $logfile = "/data/$host.0/burntool_logs/$class_id.$start_date.log"
+
+    $run = ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $start_date --dateobs_end $end_date --dbname gpc1 --logfile $logfile
+	
+    echo "bt.burntool.run: " $date $btN $btNcounter $start_date $end_date $chip_counter $class_id $host $logfile $run
+    echo "bt.burntool.run: " $date $btN $btNcounter $chip_counter $new_state
+
+    book setword btBurntool $date $status_label RUN
+    option $date $new_state $status_label
+
+    command $run
+    periods -exec 0.05
+#     command /bin/true
+   end
+  # success
+  task.exit   0
+#    convert 'stdout' to book format
+    # Set data state based on if we're queueing or waiting
+    # Set the job state for success.
+    book setword btBurntool $options:0 $options:2 FINISHED
+    if ($VERBOSE > 2)
+	book listbook btData
+    end
+  end
+
+  # locked list
+  task.exit    default
+    book setword btBurntool $options:0 $options:2 FAIL
+    showcommand failure
+  end
+  task.exit    crash
+    book setword btBurntool $options:0 $options:2 FAIL
+    showcommand crash
+  end
+  #operation times out?
+  task.exit    timeout
+    book setword btBurntool $options:0 $option:2 FAIL
+    showcommand timeout
+  end
+end
+
+
+#
+# This is all burntool related stuff.
+#
+
+macro burntool
+  if ($0 != 3)
+    echo "USAGE: burntool (dateobs_begin) (dateobs_end)"
+    break
+  end
+
+  for i 0 $hostmatch:n
+    job -host $word:1 ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $1 --dateobs_end $2 --dbname gpc1 --logfile $logfile
+  end
+end
+macro loadhosts
+  for i 0 $allhosts:n
+    controller host add $allhosts:$i
+  end
+end
+
+# sorry this list is messy, it's supposed to be that way to try and keep to burntools from running on the same host at the same time.
+
+list hostmatch
+  XY01    ipp014
+  XY03    ipp038
+  XY05    ipp023
+  XY10    ipp039
+  XY12    ipp024
+  XY15    ipp040
+  XY17    ipp020
+  XY21    ipp041
+  XY23    ipp042
+  XY25    ipp043
+  XY27    ipp028
+  XY31    ipp044
+  XY33    ipp029
+  XY35    ipp045
+  XY37    ipp030
+  XY41    ipp046
+  XY43    ipp031
+  XY45    ipp047
+  XY47    ipp032
+  XY51    ipp048
+  XY53    ipp033
+  XY55    ipp049
+  XY57    ipp034
+  XY61    ipp050
+  XY63    ipp035
+  XY65    ipp051
+  XY67    ipp036
+  XY72    ipp052
+  XY74    ipp015
+  XY76    ipp053
+  XY02    ipp014
+  XY04    ipp038
+  XY06    ipp023
+  XY11    ipp039
+  XY13    ipp024
+  XY14    ipp040
+  XY16    ipp020
+  XY20    ipp041
+  XY22    ipp042
+  XY24    ipp043 
+  XY26    ipp028
+  XY30    ipp044
+  XY32    ipp029
+  XY34    ipp045
+  XY36    ipp030
+  XY40    ipp046
+  XY42    ipp031
+  XY44    ipp047
+  XY46    ipp032
+  XY50    ipp048
+  XY52    ipp033
+  XY54    ipp049
+  XY56    ipp034
+  XY60    ipp050
+  XY62    ipp035
+  XY64    ipp051
+  XY66    ipp036
+  XY71    ipp052
+  XY73    ipp015
+  XY75    ipp053
+end
+
+list allhosts
+  ipp043
+  ipp014
+  ipp015
+  ipp023
+  ipp024
+  ipp020
+  ipp028
+  ipp029
+  ipp030
+  ipp031
+  ipp032
+  ipp033
+  ipp034
+  ipp035
+  ipp036
+  ipp038
+  ipp039
+  ipp040
+  ipp041
+  ipp042
+  ipp044
+  ipp045
+  ipp046
+  ipp047
+  ipp048
+  ipp049
+  ipp050
+  ipp051
+  ipp052
+  ipp053
+end
Index: /branches/eam_branches/ipp-20100621/ippTasks/detrend.norm.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/detrend.norm.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/detrend.norm.pro	(revision 28794)
@@ -64,8 +64,36 @@
 end
 
+macro detnorm.revert.off
+  task detrend.norm.revert
+    active false
+  end
+  task detrend.normexp.revert
+    active false
+  end
+  task detrend.normstat.revert
+    active false
+  end
+end
+
+macro detnorm.revert.on
+  task detrend.norm.revert
+    active true
+  end
+  task detrend.normexp.revert
+    active true
+  end
+  task detrend.normstat.revert
+    active true
+  end
+end
+
+
 # these variables will cycle through the known database names
 $detPendingNormStatImfile_DB = 0
 $detPendingNormImfile_DB = 0
 $detPendingNormExp_DB = 0
+$detPendingNormStatImfile_DB_revert = 0
+$detPendingNormImfile_DB_revert = 0
+$detPendingNormExp_DB_revert = 0
 
 # select images ready for copy 
@@ -440,2 +468,140 @@
   end
 end
+
+task detrend.norm.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+
+    $run = dettool -revertnormalizedimfile -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingNormImfile_DB_revert
+      $run = $run -dbname $DB:$detPendingNormImfile_DB_revert
+      $detPendingNormImfile_DB_revert ++
+      if ($detPendingNormImfile_DB_revert >= $DB:n) set detPendingNormImfile_DB_revert = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task detrend.normexp.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+
+    $run = dettool -revertnormalizedexp -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingNormExp_DB_revert
+      $run = $run -dbname $DB:$detPendingNormExp_DB_revert
+      $detPendingNormExp_DB_revert ++
+      if ($detPendingNormExp_DB_revert >= $DB:n) set detPendingNormExp_DB_revert = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task detrend.normstat.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+  
+    $run = dettool -revertnormalizedstat -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingNormStatImfile_DB_revert
+      $run = $run -dbname $DB:$detPendingNormStatImfile_DB_revert
+      $detPendingNormStatImfile_DB_revert ++
+      if ($detPendingNormStatImfile_DB_revert >= $DB:n) set detPendingNormStatImfile_DB_revert = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
Index: /branches/eam_branches/ipp-20100621/ippTasks/detrend.process.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/detrend.process.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/detrend.process.pro	(revision 28794)
@@ -51,8 +51,27 @@
 end
 
+macro detproc.revert.off
+  task detrend.process.revert
+    active false
+  end
+  task detrend.processexp.revert
+    active false
+  end
+end
+
+macro detproc.revert.on
+  task detrend.process.revert
+    active true
+  end
+  task detrend.processexp.revert
+    active true
+  end
+end
 
 # these variables will cycle through the known database names
 $detPendingProcessedImfile_DB = 0
 $detPendingProcessedExp_DB = 0
+$detPendingProcessedImfile_revert_DB = 0
+$detPendingProcessedExp_revert_DB = 0
 
 # select images ready for copy 
@@ -313,2 +332,94 @@
   end
 end
+
+task detrend.process.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+ 
+    $run = dettool -revertprocessedimfile -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingProcessedImfile_revert_DB
+      $run = $run -dbname $DB:$detPendingProcessedImfile_revert_DB
+      $detPendingProcessedImfile_revert_DB ++
+      if ($detPendingProcessedImfile_revert_DB >= $DB:n) set detPendingProcessedImfile_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task detrend.processexp.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+  
+    $run = dettool -revertprocessedexp -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingProcessedExp_revert_DB
+      $run = $run -dbname $DB:$detPendingProcessedExp_revert_DB
+      $detPendingProcessedExp_revert_DB ++
+      if ($detPendingProcessedExp_revert_DB >= $DB:n) set detPendingProcessedExp_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
Index: /branches/eam_branches/ipp-20100621/ippTasks/detrend.resid.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/detrend.resid.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/detrend.resid.pro	(revision 28794)
@@ -49,7 +49,27 @@
 end
 
+macro detresid.revert.on
+  task detrend.resid.revert
+    active true
+  end
+  task detrend.residexp.revert
+    active true
+  end
+end
+
+macro detresid.revert.off
+  task detrend.resid.revert
+    active false
+  end
+  task detrend.residexp.revert
+    active false
+  end
+end
+
 # these variables will cycle through the known database names
 $detPendingResidImfile_DB = 0
 $detPendingResidExp_DB = 0
+$detPendingResidImfile_revert_DB = 0
+$detPendingResidExp_revert_DB = 0
 
 # select images ready for copy 
@@ -316,2 +336,95 @@
   end
 end
+
+task detrend.resid.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+
+    $run = dettool -revertresidimfile -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingResidImfile_revert_DB
+      $run = $run -dbname $DB:$detPendingResidImfile_revert_DB
+      $detPendingResidImfile_revert_DB ++
+      if ($detPendingResidImfile_revert_DB >= $DB:n) set detPendingResidImfile_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+
+task detrend.residexp.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+  
+
+    $run = dettool -revertresidexp -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingResidExp_revert_DB
+      $run = $run -dbname $DB:$detPendingResidExp_revert_DB
+      $detPendingResidExp_revert_DB ++
+      if ($detPendingResidExp_revert_DB >= $DB:n) set detPendingResidExp_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
Index: /branches/eam_branches/ipp-20100621/ippTasks/detrend.stack.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/detrend.stack.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/detrend.stack.pro	(revision 28794)
@@ -35,6 +35,19 @@
 end
 
+macro detstack.revert.off
+  task detrend.stack.revert
+    active false
+  end
+end
+
+macro detstack.revert.on
+  task detrend.stack.revert
+    active true
+  end
+end
+
 # this variable will cycle through the known database names
 $detPendingStackedImfile_DB = 0
+$detPendingStackedImfile_revert_DB = 0
 
 # select images ready for detrend_stack.pl
@@ -165,2 +178,49 @@
   end
 end
+
+
+task detrend.stack.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGDIR/revert.log
+
+  task.exec
+  
+    $run = dettool -revertstacked -all-run 
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$detPendingStackedImfile_revert_DB
+      $run = $run -dbname $DB:$detPendingStackedImfile_revert_DB
+      $detPendingStackedImfile_revert_DB ++
+      if ($detPendingStackedImfile_revert_DB >= $DB:n) set detPendingStackedImfile_revert_DB = 0
+    end
+    echo $run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
Index: /branches/eam_branches/ipp-20100621/ippTasks/dist.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/dist.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/dist.pro	(revision 28794)
@@ -28,4 +28,6 @@
 list DIST_STAGE -add "stack"
 list DIST_STAGE -add "SSdiff"
+list DIST_STAGE -add "chip_bg"
+list DIST_STAGE -add "warp_bg"
 
 $currentStage = 0
@@ -145,6 +147,11 @@
         # to compute an index into the host table
         strlen $stage_id length
-        $start = $length - 2
-        substr $stage_id $start 2 index
+	if ($length >= 2) 
+	    $start = $length - 2
+	    substr $stage_id $start 2 index
+	else
+	    # stage_id < 10 caused a very annoying couple of hours for bills to debug
+	    substr $stage_id 0 1 index
+	end
         $component_id = $index % $count
         book getword ipphosts distribution $component_id -var myhost
@@ -409,6 +416,7 @@
 
     # using $DIST_ID as the "component" works fine here since we only look
-    # at the last two digits
-    set.dist.workdir.by.component $STAGE_ID "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    # at the last two digits. But make sure that there 2 digits
+    $fake_component = $STAGE_ID + 10
+    set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR 
     if ("$OUTDIR" == "NULL")
         echo ERROR failed to set workdir for $DIST_ID
Index: /branches/eam_branches/ipp-20100621/ippTasks/dqstats.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/dqstats.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/dqstats.pro	(revision 28794)
@@ -37,4 +37,16 @@
   task dqstats.revert
     active false
+  end
+end
+
+macro dqstats.revert.on
+  task dqstats.revert
+    active true
+  end
+end
+
+macro dqstats.revert.off
+  task dqstats.revert
+    active true
   end
 end
@@ -109,5 +121,4 @@
 
   ## we want only a single outstanding dqstats job.  
-  host         local
   npending     1
 
@@ -187,4 +198,5 @@
   periods      -timeout 120.0
   npending     1
+  active       false
 
   stdout NULL
Index: /branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/ipphosts.mhpcc.config	(revision 28794)
@@ -253,36 +253,34 @@
 END
 
-# this list is no longer used
-# XXX : delete if this does not cause trouble
-## ipphosts METADATA
-##   camera STR distribution
-##   count  S32  26
-##   0      STR ipp021
-##   1      STR ipp023
-##   2      STR ipp024
-##   3      STR ipp026
-##   4      STR ipp028
-##   5      STR ipp029
-##   6      STR ipp030
-##   7      STR ipp031
-##   8      STR ipp032
-##   9      STR ipp033
-##  10      STR ipp034
-##  11      STR ipp035
-##  12      STR ipp036
-##  13      STR ipp038
-##  14      STR ipp039
-##  15      STR ipp040
-##  16      STR ipp041
-##  17      STR ipp043
-##  18      STR ipp044
-##  19      STR ipp045
-##  20      STR ipp046
-##  21      STR ipp047
-##  22      STR ipp048
-##  23      STR ipp050
-##  24      STR ipp051
-##  25      STR ipp052
-## END
+ipphosts METADATA
+  camera STR distribution
+  count  S32  26
+  0      STR ipp021
+  1      STR ipp023
+  2      STR ipp024
+  3      STR ipp026
+  4      STR ipp028
+  5      STR ipp029
+  6      STR ipp030
+  7      STR ipp031
+  8      STR ipp032
+  9      STR ipp033
+ 10      STR ipp034
+ 11      STR ipp035
+ 12      STR ipp036
+ 13      STR ipp038
+ 14      STR ipp039
+ 15      STR ipp040
+ 16      STR ipp041
+ 17      STR ipp043
+ 18      STR ipp044
+ 19      STR ipp045
+ 20      STR ipp046
+ 21      STR ipp047
+ 22      STR ipp048
+ 23      STR ipp050
+ 24      STR ipp051
+ 25      STR ipp052
+END
 
 ipphosts METADATA
Index: /branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/nightly_stacks.pro	(revision 28794)
@@ -16,4 +16,5 @@
     ns.initday.off
     ns.detrends.off
+    ns.dqstats.off
     ns.registration.off
     ns.burntool.off
@@ -23,19 +24,9 @@
 
 macro ns.on
-    ns.initday.on
-    ns.detrends.off
-    ns.registration.on
-    ns.burntool.on
-    ns.chips.on
-    ns.stacks.on
+    nightly.stacks.on
 end
 
 macro ns.off
-    ns.initday.off
-    ns.detrends.off
-    ns.registration.off
-    ns.burntool.off
-    ns.chips.off
-    ns.stacks.off
+    nightly.stacks.off
 end
 
Index: /branches/eam_branches/ipp-20100621/ippTasks/pantasks.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/pantasks.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/pantasks.pro	(revision 28794)
@@ -242,4 +242,18 @@
 end
 
+macro detrend.revert.off
+  detproc.revert.off
+  detstack.revert.off
+  detnorm.revert.off
+  detresid.revert.off
+end
+
+macro detrend.revert.on
+  detproc.revert.on
+  detstack.revert.on
+  detnorm.revert.on
+  detresid.revert.on
+end
+
 macro detrend.reset
   detproc.reset
@@ -251,5 +265,4 @@
 
 macro all.on
-  register.on
   detrend.on
   flatcorr.on
@@ -264,5 +277,4 @@
 
 macro all.off
-  register.off
   detrend.off
   flatcorr.off
@@ -721,2 +733,19 @@
 end
 
+macro tasks.revert.off
+    chip.revert.off
+    camera.revert.off
+    fake.revert.off
+    warp.revert.off
+    diff.revert.off
+    stack.revert.off
+end
+macro tasks.revert.on
+    chip.revert.on
+    camera.revert.on
+    fake.revert.on
+    warp.revert.on
+    diff.revert.on
+    stack.revert.on
+end
+
Index: /branches/eam_branches/ipp-20100621/ippTasks/pstamp.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/pstamp.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/pstamp.pro	(revision 28794)
@@ -188,4 +188,7 @@
         add_poll_args run
         add_poll_labels run
+        # limit number of requests in the queue to avoid blocking everything
+        # when jobs take an infinite amount of time to parse
+        $run = $run -limit 10
         command $run
     end
Index: /branches/eam_branches/ipp-20100621/ippTasks/rcserver.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/rcserver.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/rcserver.pro	(revision 28794)
@@ -12,4 +12,5 @@
 ### Database lists
 $rcPendingFS_DB = 0
+$rcRevertFS_DB = 0
 
 ### Check status of tasks
@@ -38,4 +39,13 @@
   task rcserver.makefileset.run
     active false
+  end
+  task rcserver.revert
+    active false
+  end
+end
+
+macro rcserver.revert.on
+  task rcserver.revert
+    active true
   end
 end
@@ -156,2 +166,49 @@
   end
 end
+
+
+task rcserver.revert
+  host         local
+
+  periods      -poll 60.0
+  periods      -exec 1800.0
+  periods      -timeout 120.0
+  npending     1
+
+  stdout NULL
+  stderr $LOGSUBDIR/revert.log
+
+  task.exec
+    if ($LABEL:n == 0) break
+    $run = disttool -revertfileset
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$rcRevertFS_DB
+      $run = $run -dbname $DB:$rcRevertFS_DB
+      $rcRevertFS_DB ++
+      if ($rcRevertFS_DB >= $DB:n) set rcRevertFS_DB = 0
+    end
+    add_poll_labels run
+    command $run
+  end
+
+  # success
+  task.exit    0
+  end
+
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+
+  task.exit    crash
+    showcommand crash
+  end
+
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
Index: /branches/eam_branches/ipp-20100621/ippTasks/simtest.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/simtest.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/simtest.pro	(revision 28794)
@@ -159,19 +159,2 @@
   $SIMTEST_AUTO = simtest.flatcorr.auto
 end
-
-macro tasks.revert.off
-    chip.revert.off
-    camera.revert.off
-    fake.revert.off
-    warp.revert.off
-    diff.revert.off
-    stack.revert.off
-end
-macro tasks.revert.on
-    chip.revert.on
-    camera.revert.on
-    fake.revert.on
-    warp.revert.on
-    diff.revert.on
-    stack.revert.on
-end
Index: /branches/eam_branches/ipp-20100621/ippTasks/stack.pro
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTasks/stack.pro	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTasks/stack.pro	(revision 28794)
@@ -126,5 +126,5 @@
       # save the DB name for the exit tasks
       option $DB:$stackSkycell_DB
-      $run = $run -dbname $DB:$stackSkycell_DB -limit 40
+      $run = $run -dbname $DB:$stackSkycell_DB
       $stackSkycell_DB ++
       if ($stackSkycell_DB >= $DB:n) set stackSkycell_DB = 0
@@ -132,4 +132,6 @@
     add_poll_args run
     add_poll_labels run
+    # reduce the limit (the last one takes precedence)
+    $run = $run -limit 40
     command $run
   end
Index: /branches/eam_branches/ipp-20100621/ippToPsps/config/detection/map.xml
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/config/detection/map.xml	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/config/detection/map.xml	(revision 28794)
@@ -46,4 +46,5 @@
   <map ippName="CERROR" ippType="TFLOAT" pspsName="astroScat" />
   <map ippName="NASTRO" ippType="TLONG" pspsName="numAstroRef" />
+  <map ippName="NASTRO" ippType="TLONG" pspsName="numPhotoRef" />
   <map ippName="CNAXIS1" ippType="TSHORT" pspsName="nx" />
   <map ippName="CNAXIS2" ippType="TSHORT" pspsName="ny" />
@@ -55,4 +56,5 @@
   <map ippName="APMIFIT" ippType="TFLOAT" pspsName="apResid" />
   <map ippName="DAPMIFIT" ippType="TFLOAT" pspsName="dapResid" />
+  <map ippName="DETECTOR" ippType="TSTRING" pspsName="detectorID" />
   <map ippName="DETREND.MASK" ippType="TSTRING" pspsName="detrend1" />
   <map ippName="DETREND.DARK" ippType="TSTRING" pspsName="detrend2" />
@@ -89,16 +91,18 @@
 
  <table name="Detection">
-  <map ippColumn="2" ippName="X_PSF" ippType="TFLOAT" pspsName="xPos" />
-  <map ippColumn="3" ippName="Y_PSF" ippType="TFLOAT" pspsName="yPos" />
-  <map ippColumn="4" ippName="X_PSF_SIG" ippType="TFLOAT" pspsName="xPosErr" />
-  <map ippColumn="5" ippName="Y_PSF_SIG" ippType="TFLOAT" pspsName="yPosErr" />
-  <map ippColumn="22" ippName="PSF_MAJOR" ippType="TFLOAT" pspsName="psfWidMajor" />
-  <map ippColumn="23" ippName="PSF_MINOR" ippType="TFLOAT" pspsName="psfWidMinor" />
-  <map ippColumn="24" ippName="PSF_THETA" ippType="TFLOAT" pspsName="psfTheta" />
-  <map ippColumn="25" ippName="PSF_QF" ippType="TFLOAT" pspsName="psfCf" />
-  <map ippColumn="17" ippName="SKY" ippType="TFLOAT" pspsName="sky" />
-  <map ippColumn="18" ippName="SKY_SIGMA" ippType="TFLOAT" pspsName="skyErr" />
-  <map ippColumn="21" ippName="EXT_NSIGMA" ippType="TFLOAT" pspsName="sgSep" />
-  <map ippColumn="31" ippName="FLAGS" ippType="TLONG" pspsName="infoFlag" />
+  <map ippName="X_PSF" ippType="TFLOAT" pspsName="xPos" />
+  <map ippName="Y_PSF" ippType="TFLOAT" pspsName="yPos" />
+  <map ippName="X_PSF_SIG" ippType="TFLOAT" pspsName="xPosErr" />
+  <map ippName="Y_PSF_SIG" ippType="TFLOAT" pspsName="yPosErr" />
+  <map ippName="PSF_MAJOR" ippType="TFLOAT" pspsName="psfWidMajor" />
+  <map ippName="PSF_MINOR" ippType="TFLOAT" pspsName="psfWidMinor" />
+  <map ippName="PSF_THETA" ippType="TFLOAT" pspsName="psfTheta" />
+  <map ippName="PSF_QF" ippType="TFLOAT" pspsName="psfCf" />
+  <map ippName="MOMENTS_XX" ippType="TFLOAT" pspsName="momentXX" />
+  <map ippName="MOMENTS_XY" ippType="TFLOAT" pspsName="momentXY" />
+  <map ippName="MOMENTS_YY" ippType="TFLOAT" pspsName="momentYY" />
+  <map ippName="SKY" ippType="TFLOAT" pspsName="sky" />
+  <map ippName="SKY_SIGMA" ippType="TFLOAT" pspsName="skyErr" />
+  <map ippName="EXT_NSIGMA" ippType="TFLOAT" pspsName="sgSep" />
  </table>
 
Index: /branches/eam_branches/ipp-20100621/ippToPsps/config/detection/tables.xml
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/config/detection/tables.xml	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/config/detection/tables.xml	(revision 28794)
@@ -79,8 +79,7 @@
     <column name="momentWidMajor" type="TFLOAT" default="-999" comment=" average PSF parameters from moments (unit = arcsec)"></column>
     <column name="momentWidMinor" type="TFLOAT" default="-999" comment=" average PSF parameters from moments (unit = arcsec)"></column>
-    <column name="momentTheta" type="TFLOAT" default="-999" comment=" average PSF parametets from moments (unit = deg)"></column>
     <column name="apResid" type="TFLOAT" default="-999" comment="corrected aperture residual"></column>
     <column name="dapResid" type="TFLOAT" default="-999" comment="scatter of aperture corrections"></column>
-    <column name="detectorID" type="TSHORT" default="-999" comment="identifier for actual CCD chip"></column>
+    <column name="detectorID" type="TSTRING" default=" " comment="identifier for actual CCD chip"></column>
     <column name="qaFlags" type="TLONGLONG" default="-999" comment="Q/A flags for this OTA"></column>
     <column name="detrend1" type="TSTRING" default=" " comment="identifier of detrend image 1"></column>
@@ -144,7 +143,7 @@
     <column name="psfLikelihood" type="TFLOAT" default="-999" comment="PSF likelihood"></column>
     <column name="psfCf" type="TFLOAT" default="-999" comment="PSF coverage factor"></column>
-    <column name="momentWidMajor" type="TFLOAT" default="-999" comment=" PSF width in major axis from moments (unit = arcsec)"></column>
-    <column name="momentWidMinor" type="TFLOAT" default="-999" comment=" PSF width in minot axis from moments (unit = arcsec)"></column>
-    <column name="momentTheta" type="TFLOAT" default="-999" comment=" PSF orientation angle from moments (unit = deg)"></column>
+    <column name="momentXX" type="TFLOAT" default="-999" comment=" moment XX (unit = arcsec)"></column>
+    <column name="momentXY" type="TFLOAT" default="-999" comment=" moment XY (unit = arcsec)"></column>
+    <column name="momentYY" type="TFLOAT" default="-999" comment=" momeny YY (unit = arcsec)"></column>
     <column name="crLikelihood" type="TFLOAT" default="-999" comment="Likelihood the source is a cosmic ray"></column>
     <column name="extendedLikelihood" type="TFLOAT" default="-999" comment="Likelihood the source is extended"></column>
Index: /branches/eam_branches/ipp-20100621/ippToPsps/config/diff/map.xml
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/config/diff/map.xml	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/config/diff/map.xml	(revision 28794)
@@ -22,20 +22,20 @@
 
  <table name="StackHighSigDelta">
-  <map ippColumn="2" ippName="X_PSF" ippType="TFLOAT" pspsName="xPos" />
-  <map ippColumn="3" ippName="Y_PSF" ippType="TFLOAT" pspsName="yPos" />
-  <map ippColumn="4" ippName="X_PSF_SIG" ippType="TFLOAT" pspsName="xPosErr" />
-  <map ippColumn="5" ippName="Y_PSF_SIG" ippType="TFLOAT" pspsName="yPosErr" />
-  <map ippColumn="8" ippName="PSF_INST_MAG" ippType="TFLOAT" pspsName="instMag" />
-  <map ippColumn="9" ippName="PSF_INST_MAG_SIG" ippType="TFLOAT" pspsName="instMagErr" />
-  <map ippColumn="12" ippName="PEAK_FLUX_AS_MAG" ippType="TFLOAT" pspsName="peakFluxMag" />
-  <map ippColumn="22" ippName="PSF_MAJOR" ippType="TFLOAT" pspsName="psfWidMajor" />
-  <map ippColumn="23" ippName="PSF_MINOR" ippType="TFLOAT" pspsName="psfWidMinor" />
-  <map ippColumn="24" ippName="PSF_THETA" ippType="TFLOAT" pspsName="psfTheta" />
-  <map ippColumn="25" ippName="PSF_QF" ippType="TFLOAT" pspsName="psfCf" />
-  <map ippColumn="15" ippName="RA_PSF" ippType="TDOUBLE" pspsName="ra" />
-  <map ippColumn="16" ippName="DEC_PSF" ippType="TDOUBLE" pspsName="dec" />
-  <map ippColumn="17" ippName="SKY" ippType="TFLOAT" pspsName="sky" />
-  <map ippColumn="18" ippName="SKY_SIGMA" ippType="TFLOAT" pspsName="skyErr" />
-  <map ippColumn="21" ippName="EXT_NSIGMA" ippType="TFLOAT" pspsName="sgSep" />
+  <map ippName="X_PSF" ippType="TFLOAT" pspsName="xPos" />
+  <map ippName="Y_PSF" ippType="TFLOAT" pspsName="yPos" />
+  <map ippName="X_PSF_SIG" ippType="TFLOAT" pspsName="xPosErr" />
+  <map ippName="Y_PSF_SIG" ippType="TFLOAT" pspsName="yPosErr" />
+  <map ippName="PSF_INST_MAG" ippType="TFLOAT" pspsName="instMag" />
+  <map ippName="PSF_INST_MAG_SIG" ippType="TFLOAT" pspsName="instMagErr" />
+  <map ippName="PEAK_FLUX_AS_MAG" ippType="TFLOAT" pspsName="peakFluxMag" />
+  <map ippName="PSF_MAJOR" ippType="TFLOAT" pspsName="psfWidMajor" />
+  <map ippName="PSF_MINOR" ippType="TFLOAT" pspsName="psfWidMinor" />
+  <map ippName="PSF_THETA" ippType="TFLOAT" pspsName="psfTheta" />
+  <map ippName="PSF_QF" ippType="TFLOAT" pspsName="psfCf" />
+  <map ippName="RA_PSF" ippType="TDOUBLE" pspsName="ra" />
+  <map ippName="DEC_PSF" ippType="TDOUBLE" pspsName="dec" />
+  <map ippName="SKY" ippType="TFLOAT" pspsName="sky" />
+  <map ippName="SKY_SIGMA" ippType="TFLOAT" pspsName="skyErr" />
+  <map ippName="EXT_NSIGMA" ippType="TFLOAT" pspsName="sgSep" />
  </table>
 
Index: /branches/eam_branches/ipp-20100621/ippToPsps/config/object/tables.xml
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/config/object/tables.xml	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/config/object/tables.xml	(revision 28794)
@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<tableDescriptions type="object">
+  <table name="Object">
+    <column name="objID" type="TLONGLONG" default="0" comment="ODM object identifier index"></column>
+    <column name="ippObjID" type="TLONGLONG" default="0" comment="IPP object number"></column>
+    <column name="projectionCellID" type="TLONG" default="-999" comment="projection cell identifier at discovery time"></column>
+    <column name="htmID" type="TLONGLONG" default="0" comment="HTM index"></column>
+    <column name="zoneID" type="TLONG" default="0" comment="Zone index"></column>
+    <column name="stackVer" type="TSHORT" default="-999 -" comment="static stack version number"></column>
+    <column name="objInfoFlag" type="TLONG" default="-999" comment="flag indicating odd properties"></column>
+    <column name="varFlag" type="TBYTE" default="0" comment="variable in difference image detections"></column>
+    <column name="surveyID" type="TBYTE" default="255" comment="Survey ID"></column>
+    <column name="ra" type="TDOUBLE" default="0" comment="Right Ascension"></column>
+    <column name="dec" type="TDOUBLE" default="0" comment="Declination"></column>
+    <column name="raErr" type="TFLOAT" default="0.0" comment="Right Ascension error"></column>
+    <column name="decErr" type="TFLOAT" default="0.0" comment="Declination error"></column>
+    <column name="cx" type="TDOUBLE" default="0" comment="Cartesian x"></column>
+    <column name="cy" type="TDOUBLE" default="0" comment="Cartesian y"></column>
+    <column name="cz" type="TDOUBLE" default="0" comment="Cartesian z"></column>
+    <column name="lambda" type="TDOUBLE" default="-999" comment="ecliptic longitude"></column>
+    <column name="beta" type="TDOUBLE" default="-999" comment="ecliptic latitude"></column>
+    <column name="l" type="TDOUBLE" default="-999" comment="galactic longitude"></column>
+    <column name="b" type="TDOUBLE" default="-999" comment="galactic latitude"></column>
+    <column name="lsg" type="TDOUBLE" default="-999" comment="supergalactic longitude"></column>
+    <column name="bsg" type="TDOUBLE" default="-999" comment="supergalactic latitude"></column>
+    <column name="nDetections" type="TSHORT" default="-999" comment="total number of detection measurements in all filters"></column>
+    <column name="raMean" type="TDOUBLE" default="-999" comment="mean RA of object"></column>
+    <column name="decMean" type="TDOUBLE" default="-999" comment="mean DEC of object"></column>
+    <column name="raMeanErr" type="TFLOAT" default="-999" comment="estimated error in raMean"></column>
+    <column name="decMeanErr" type="TFLOAT" default="-999" comment="estimated error in decMean"></column>
+    <column name="ng" type="TSHORT" default="-999" comment="number of measures in g filter"></column>
+    <column name="gMeanMag" type="TFLOAT" default="-999" comment="mean g magnitude"></column>
+    <column name="gMeanFlux" type="TFLOAT" default="-999" comment="mean g flux"></column>
+    <column name="gMeanMagErr" type="TFLOAT" default="-999" comment="estimated error of g magnitude mean"></column>
+    <column name="gMeanFluxErr" type="TFLOAT" default="-999" comment="estimated error of g flux mean"></column>
+    <column name="gMin" type="TFLOAT" default="-999" comment="minimum g magnitide"></column>
+    <column name="gMax" type="TFLOAT" default="-999" comment="maximum g magnitude"></column>
+    <column name="nr" type="TSHORT" default="-999" comment="number of measures in r filter"></column>
+    <column name="rMeanMag" type="TFLOAT" default="-999" comment="mean r magnitude"></column>
+    <column name="rMeanFlux" type="TFLOAT" default="-999" comment="mean r flux"></column>
+    <column name="rMeanMagErr" type="TFLOAT" default="-999" comment="estimated error of r magnitude mean"></column>
+    <column name="rMeanFluxErr" type="TFLOAT" default="-999" comment="estimated error of r flux mean"></column>
+    <column name="rMin" type="TFLOAT" default="-999" comment="minimum r magnitude"></column>
+    <column name="rMax" type="TFLOAT" default="-999" comment="maximum r magnitude"></column>
+    <column name="ni" type="TSHORT" default="-999" comment="number of measures in i filter"></column>
+    <column name="iMeanMag" type="TFLOAT" default="-999" comment="mean i magnitude"></column>
+    <column name="iMeanFlux" type="TFLOAT" default="-999" comment="mean i flux"></column>
+    <column name="iMeanMagErr" type="TFLOAT" default="-999" comment="estimated error of i magitude mean"></column>
+    <column name="iMeanFluxErr" type="TFLOAT" default="-999" comment="estimated error of i flux mean"></column>
+    <column name="iMin" type="TFLOAT" default="-999" comment="minimum i magniutde"></column>
+    <column name="iMax" type="TFLOAT" default="-999" comment="maximum i magnitude"></column>
+    <column name="nz" type="TSHORT" default="-999" comment="number of measures in z filter"></column>
+    <column name="zMeanMag" type="TFLOAT" default="-999" comment="mean z magnitude"></column>
+    <column name="zMeanFlux" type="TFLOAT" default="-999" comment="mean z flux"></column>
+    <column name="zMeanMagErr" type="TFLOAT" default="-999" comment="estimated error of z magnitude mean"></column>
+    <column name="zMeanFluxErr" type="TFLOAT" default="-999" comment="estimated error of z flux mean"></column>
+    <column name="zMin" type="TFLOAT" default="-999" comment="minimum z magnitude"></column>
+    <column name="zMax" type="TFLOAT" default="-999" comment="maximum z magnitude"></column>
+    <column name="ny" type="TSHORT" default="-999" comment="number of measures in y filter"></column>
+    <column name="yMeanMag" type="TFLOAT" default="-999" comment="mean y magnitude"></column>
+    <column name="yMeanFlux" type="TFLOAT" default="-999" comment="mean y flux"></column>
+    <column name="yMeanMagErr" type="TFLOAT" default="-999" comment="estimated error of y magnitude mean"></column>
+    <column name="yMeanFluxErr" type="TFLOAT" default="-999" comment="estimated error of y flux mean"></column>
+    <column name="yMin" type="TFLOAT" default="-999" comment="minimum y magnitude"></column>
+    <column name="yMax" type="TFLOAT" default="-999" comment="maximum y magnitude"></column>
+    <column name="nw" type="TSHORT" default="-999" comment="number of measures in the wide (w) solar system filter"></column>
+    <column name="wMeanMag" type="TFLOAT" default="-999" comment="mean w magnitude"></column>
+    <column name="wMeanFlux" type="TFLOAT" default="-999" comment="mean w flux"></column>
+    <column name="wMeanMagErr" type="TFLOAT" default="-999" comment="estimated error of w magnitude mean"></column>
+    <column name="wMeanFluxErr" type="TFLOAT" default="-999" comment="estimated error of w flux mean"></column>
+    <column name="wMin" type="TFLOAT" default="-999" comment="minimum w magnitude"></column>
+    <column name="wMax" type="TFLOAT" default="-999" comment="maximum w magnitude"></column>
+    <column name="grMeanColor" type="TFLOAT" default="-999" comment="g-r color from gMean &amp;amp; rMean"></column>
+    <column name="riMeanColor" type="TFLOAT" default="-999" comment="r-i color from rMean &amp;amp; iMean"></column>
+    <column name="izMeanColor" type="TFLOAT" default="-999" comment="i-z color from iMean &amp;amp; zMean"></column>
+    <column name="zyMeanColor" type="TFLOAT" default="-999" comment="z-y color from zMean &amp;amp; yMean"></column>
+    <column name="grMeanColorErr" type="TFLOAT" default="-999" comment="Estimated error in g-r color from gMean &amp;amp; rMean"></column>
+    <column name="riMeanColorErr" type="TFLOAT" default="-999" comment="Estimated error in r-i color from rMean &amp;amp; iMean"></column>
+    <column name="izMeanColorErr" type="TFLOAT" default="-999" comment="Estimated error in i-z color from iMean &amp;amp; zMean"></column>
+    <column name="zyMeanColorErr" type="TFLOAT" default="-999" comment="Estimated error in z-y color from zMean &amp;amp; yMean"></column>
+    <column name="gMagBest" type="TFLOAT" default="-999" comment="g magnitude from stack frames"></column>
+    <column name="gModID" type="TSHORT" default="-999" comment="index of which g model magnitude was used from stack"></column>
+    <column name="rMagBest" type="TFLOAT" default="-999" comment="r magnitude from stack frames"></column>
+    <column name="rModID" type="TSHORT" default="-999" comment="index of which r model magnitude was used from stack"></column>
+    <column name="iMagBest" type="TFLOAT" default="-999" comment="i magnitude from stack frames"></column>
+    <column name="iModID" type="TSHORT" default="-999" comment="index of which i model magnitude was used from stack"></column>
+    <column name="zMagBest" type="TFLOAT" default="-999" comment="z magnitude from stack frames"></column>
+    <column name="zModID" type="TSHORT" default="-999" comment="index of which z model magnitude was used from stack"></column>
+    <column name="yMagBest" type="TFLOAT" default="-999" comment="y magnitude from stack frames"></column>
+    <column name="yModID" type="TSHORT" default="-999" comment="index of which y model magnitude was used from stack"></column>
+    <column name="grColor" type="TFLOAT" default="-999" comment="g-r color from gmag and rmag"></column>
+    <column name="riColor" type="TFLOAT" default="-999" comment="r-i color from rmag and imag"></column>
+    <column name="izColor" type="TFLOAT" default="-999" comment="i-z color from imag and zmag"></column>
+    <column name="zyColor" type="TFLOAT" default="-999" comment="z-y color from zmag and ymag"></column>
+    <column name="g" type="TFLOAT" default="-999" comment="star/galaxy separator in g filer"></column>
+    <column name="r" type="TFLOAT" default="-999" comment="star/galaxy separator in r filer"></column>
+    <column name="i" type="TFLOAT" default="-999" comment="star/galaxy separator in i filer"></column>
+    <column name="z" type="TFLOAT" default="-999" comment="star/galaxy separator in z filer"></column>
+    <column name="y" type="TFLOAT" default="-999" comment="star/galaxy separator in y filer"></column>
+    <column name="sgSep" type="TFLOAT" default="-999" comment="star/galaxy separator - true if any one color is"></column>
+    <column name="consistencyFlag" type="TSHORT" default="0" comment="Value computed from associated detections, it is originally given by IPP and should agree with the ODM validation"></column>
+    <column name="dataRelease" type="TBYTE" default="0" comment="Data release"></column>
+  </table>
+</tableDescriptions>
Index: /branches/eam_branches/ipp-20100621/ippToPsps/config/test/map.xml
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/config/test/map.xml	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/config/test/map.xml	(revision 28794)
@@ -3,96 +3,7 @@
 <tabledata>
 
- <table name="FrameMeta">
-  <map ippName="ZPT_ERR" ippType="TFLOAT" pspsName="photoScat" />
-  <map ippName="MJD-OBS" ippType="TFLOAT" pspsName="expStart" />
-  <map ippName="EXPREQ" ippType="TFLOAT" pspsName="expTime" />
-  <map ippName="AIRMASS" ippType="TFLOAT" pspsName="airmass" />
-  <map ippName="RA" ippType="TDOUBLE" pspsName="raBore" />
-  <map ippName="DEC" ippType="TDOUBLE" pspsName="decBore" />
-  <map ippName="CTYPE1" ippType="TSTRING" pspsName="ctype1" />
-  <map ippName="CTYPE2" ippType="TSTRING" pspsName="ctype2" />
-  <map ippName="CRVAL1" ippType="TDOUBLE" pspsName="crval1" />
-  <map ippName="CRVAL2" ippType="TDOUBLE" pspsName="crval2" />
-  <map ippName="CRPIX1" ippType="TDOUBLE" pspsName="crpix1" />
-  <map ippName="CRPIX2" ippType="TDOUBLE" pspsName="crpix2" />
-  <map ippName="CDELT1" ippType="TDOUBLE" pspsName="cdelt1" />
-  <map ippName="CDELT2" ippType="TDOUBLE" pspsName="cdelt2" />
-  <map ippName="PC001001" ippType="TDOUBLE" pspsName="pc001001" />
-  <map ippName="PC001002" ippType="TDOUBLE" pspsName="pc001002" />
-  <map ippName="PC002001" ippType="TDOUBLE" pspsName="pc002001" />
-  <map ippName="PC002002" ippType="TDOUBLE" pspsName="pc002002" />
-  <map ippName="NPLYTERM" ippType="TBYTE" pspsName="polyOrder" />
-  <map ippName="PCA1X3Y0" ippType="TDOUBLE" pspsName="pca1x3y0" />
-  <map ippName="PCA1X2Y1" ippType="TDOUBLE" pspsName="pca1x2y1" />
-  <map ippName="PCA1X1Y2" ippType="TDOUBLE" pspsName="pca1x1y2" />
-  <map ippName="PCA1X0Y3" ippType="TDOUBLE" pspsName="pca1x0y3" />
-  <map ippName="PCA1X2Y0" ippType="TDOUBLE" pspsName="pca1x2y0" />
-  <map ippName="PCA1X1Y1" ippType="TDOUBLE" pspsName="pca1x1y1" />
-  <map ippName="PCA1X0Y2" ippType="TDOUBLE" pspsName="pca1x0y2" />
-  <map ippName="PCA2X3Y0" ippType="TDOUBLE" pspsName="pca2x3y0" />
-  <map ippName="PCA2X2Y1" ippType="TDOUBLE" pspsName="pca2x2y1" />
-  <map ippName="PCA2X1Y2" ippType="TDOUBLE" pspsName="pca2x1y2" />
-  <map ippName="PCA2X0Y3" ippType="TDOUBLE" pspsName="pca2x0y3" />
-  <map ippName="PCA2X2Y0" ippType="TDOUBLE" pspsName="pca2x2y0" />
-  <map ippName="PCA2X1Y1" ippType="TDOUBLE" pspsName="pca2x1y1" />
-  <map ippName="PCA2X0Y2" ippType="TDOUBLE" pspsName="pca2x0y2" />
- </table>
-
- <table name="ImageMeta">
-  <map ippName="MSKY_MN" ippType="TFLOAT" pspsName="sky" />
-  <map ippName="MSKY_SIG" ippType="TFLOAT" pspsName="skyScat" />
-  <map ippName="FSATUR" ippType="TFLOAT" pspsName="magSat" />
-  <map ippName="FLIMIT" ippType="TFLOAT" pspsName="completMag" />
-  <map ippName="CERROR" ippType="TFLOAT" pspsName="astroScat" />
-  <map ippName="NASTRO" ippType="TLONG" pspsName="numAstroRef" />
-  <map ippName="CNAXIS1" ippType="TSHORT" pspsName="nx" />
-  <map ippName="CNAXIS2" ippType="TSHORT" pspsName="ny" />
-  <map ippName="FWHM_MAJ" ippType="TFLOAT" pspsName="psfWidMajor" />
-  <map ippName="FWHM_MIN" ippType="TFLOAT" pspsName="psfWidMinor" />
-  <map ippName="ANGLE" ippType="TFLOAT" pspsName="psfTheta" />
-  <map ippName="APMIFIT" ippType="TFLOAT" pspsName="apResid" />
-  <map ippName="DAPMIFIT" ippType="TFLOAT" pspsName="dapResid" />
-  <map ippName="CTYPE1" ippType="TSTRING" pspsName="ctype1" />
-  <map ippName="CTYPE2" ippType="TSTRING" pspsName="ctype2" />
-  <map ippName="CRVAL1" ippType="TDOUBLE" pspsName="crval1" />
-  <map ippName="CRVAL2" ippType="TDOUBLE" pspsName="crval2" />
-  <map ippName="CRPIX1" ippType="TDOUBLE" pspsName="crpix1" />
-  <map ippName="CRPIX2" ippType="TDOUBLE" pspsName="crpix2" />
-  <map ippName="CDELT1" ippType="TDOUBLE" pspsName="cdelt1" />
-  <map ippName="CDELT2" ippType="TDOUBLE" pspsName="cdelt2" />
-  <map ippName="PC001001" ippType="TDOUBLE" pspsName="pc001001" />
-  <map ippName="PC001002" ippType="TDOUBLE" pspsName="pc001002" />
-  <map ippName="PC002001" ippType="TDOUBLE" pspsName="pc002001" />
-  <map ippName="PC002002" ippType="TDOUBLE" pspsName="pc002002" />
-  <map ippName="NPLYTERM" ippType="TBYTE" pspsName="polyOrder" />
-  <map ippName="PCA1X3Y0" ippType="TDOUBLE" pspsName="pca1x3y0" />
-  <map ippName="PCA1X2Y1" ippType="TDOUBLE" pspsName="pca1x2y1" />
-  <map ippName="PCA1X1Y2" ippType="TDOUBLE" pspsName="pca1x1y2" />
-  <map ippName="PCA1X0Y3" ippType="TDOUBLE" pspsName="pca1x0y3" />
-  <map ippName="PCA1X2Y0" ippType="TDOUBLE" pspsName="pca1x2y0" />
-  <map ippName="PCA1X1Y1" ippType="TDOUBLE" pspsName="pca1x1y1" />
-  <map ippName="PCA1X0Y2" ippType="TDOUBLE" pspsName="pca1x0y2" />
-  <map ippName="PCA2X3Y0" ippType="TDOUBLE" pspsName="pca2x3y0" />
-  <map ippName="PCA2X2Y1" ippType="TDOUBLE" pspsName="pca2x2y1" />
-  <map ippName="PCA2X1Y2" ippType="TDOUBLE" pspsName="pca2x1y2" />
-  <map ippName="PCA2X0Y3" ippType="TDOUBLE" pspsName="pca2x0y3" />
-  <map ippName="PCA2X2Y0" ippType="TDOUBLE" pspsName="pca2x2y0" />
-  <map ippName="PCA2X1Y1" ippType="TDOUBLE" pspsName="pca2x1y1" />
-  <map ippName="PCA2X0Y2" ippType="TDOUBLE" pspsName="pca2x0y2" />
- </table>
-
  <table name="Detection">
-  <map ippColumn="2" ippName="X_PSF" ippType="TFLOAT" pspsName="xPos" />
-  <map ippColumn="3" ippName="Y_PSF" ippType="TFLOAT" pspsName="yPos" />
-  <map ippColumn="4" ippName="X_PSF_SIG" ippType="TFLOAT" pspsName="xPosErr" />
-  <map ippColumn="5" ippName="Y_PSF_SIG" ippType="TFLOAT" pspsName="yPosErr" />
-  <map ippColumn="15" ippName="RA_PSF" ippType="TFLOAT" pspsName="ippRa" />
-  <map ippColumn="16" ippName="DEC_PSF" ippType="TFLOAT" pspsName="ippDec" />
-  <map ippColumn="22" ippName="PSF_MAJOR" ippType="TFLOAT" pspsName="psfWidMajor" />
-  <map ippColumn="23" ippName="PSF_MINOR" ippType="TFLOAT" pspsName="psfWidMinor" />
-  <map ippColumn="24" ippName="PSF_THETA" ippType="TFLOAT" pspsName="psfTheta" />
-  <map ippColumn="25" ippName="PSF_QF" ippType="TFLOAT" pspsName="psfCf" />
-  <map ippColumn="17" ippName="SKY" ippType="TFLOAT" pspsName="sky" />
-  <map ippColumn="18" ippName="SKY_SIGMA" ippType="TFLOAT" pspsName="skyErr" />
+  <map ippName="RA_PSF" ippType="TFLOAT" pspsName="ra" />
+  <map ippName="DEC_PSF" ippType="TFLOAT" pspsName="dec" />
  </table>
 
Index: /branches/eam_branches/ipp-20100621/ippToPsps/config/test/tables.xml
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/config/test/tables.xml	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/config/test/tables.xml	(revision 28794)
@@ -2,91 +2,4 @@
 
 <tableDescriptions type="det">
-  <table name="FrameMeta">
-    <column name="frameID" type="TLONG" default="0" comment="unique exposure/frame identifier."></column>
-    <column name="filterID" type="TBYTE" default="0" comment="filter identifier"></column>
-    <column name="nOTA" type="TSHORT" default="-999" comment="number of valid OTA/CCD images in this exposure"></column>
-    <column name="photoScat" type="TFLOAT" default="-999" comment=" global photometric scatter (unit = mag)"></column>
-    <column name="expStart" type="TDOUBLE" default="-999" comment=" exposure start time in MJD (unit = day)"></column>
-    <column name="expTime" type="TFLOAT" default="-999" comment=" exposure time (unit = s)"></column>
-    <column name="airmass" type="TFLOAT" default="-999" comment="airmass at mid-exposure"></column>
-    <column name="raBore" type="TDOUBLE" default="-999" comment=" RA of telescope boresight (unit = deg)"></column>
-    <column name="decBore" type="TDOUBLE" default="-999" comment=" DEC of telescope boresight (unit = deg)"></column>
-    <column name="ctype1" type="TSTRING" default=" " comment="name of astrometric projection in RA"></column>
-    <column name="ctype2" type="TSTRING" default=" " comment="name of astrometric projection in DEC"></column>
-    <column name="crval1" type="TDOUBLE" default="-999" comment=" RA corresponding to reference pixel (unit = deg)"></column>
-    <column name="crval2" type="TDOUBLE" default="-999" comment=" DEC corresponding to reference pixel (unit = deg)"></column>
-    <column name="crpix1" type="TDOUBLE" default="-999" comment="reference pixel value for RA"></column>
-    <column name="crpix2" type="TDOUBLE" default="-999" comment="reference pixel value for DEC"></column>
-    <column name="cdelt1" type="TDOUBLE" default="-999" comment=""></column>
-    <column name="cdelt2" type="TDOUBLE" default="-999" comment=""></column>
-    <column name="pc001001" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="pc001002" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="pc002001" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="pc002002" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="polyOrder" type="TBYTE" default="255" comment="polynomial order of astrometry fit"></column>
-    <column name="pca1x3y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x2y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x1y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x0y3" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x2y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x1y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x0y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x3y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x2y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x1y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x0y3" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x2y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x1y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x0y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-  </table>
-  <table name="ImageMeta">
-    <column name="photoCalID" type="TLONG" default="0" comment="photometry reduction code identifier"></column>
-    <column name="filterID" type="TBYTE" default="0" comment="filter ID"></column>
-    <column name="sky" type="TFLOAT" default="-999" comment=" mean sky level (unit = ADU)"></column>
-    <column name="skyScat" type="TFLOAT" default="-999" comment=" scatter in mean sky level (unit = ADU)"></column>
-    <column name="nDetect" type="TLONG" default="-999" comment="number of detections on CCD"></column>
-    <column name="magSat" type="TFLOAT" default="-999" comment=" saturation magnitude level (unit = mag)"></column>
-    <column name="completMag" type="TFLOAT" default="-999" comment=" 95% completion level in mag (unit = mag)"></column>
-    <column name="astroScat" type="TFLOAT" default="-999" comment=" astrometric scatter for chip (unit = mag)"></column>
-    <column name="photoScat" type="TFLOAT" default="-999" comment="photometric scatter for chip"></column>
-    <column name="numAstroRef" type="TLONG" default="-999" comment="number of astrometric reference sources"></column>
-    <column name="nx" type="TSHORT" default="-999" comment="chip dimension in x"></column>
-    <column name="ny" type="TSHORT" default="-999" comment="chip dimension in y"></column>
-    <column name="psfModelID" type="TLONG" default="-999" comment="PSF model identifier"></column>
-    <column name="psfFwhm" type="TFLOAT" default="-999" comment=" model psf full width at half maximum at chip center (unit = arcsec)"></column>
-    <column name="psfWidMajor" type="TFLOAT" default="-999" comment=" model PSF parameters at chip center (unit = arcsec)"></column>
-    <column name="psfWidMinor" type="TFLOAT" default="-999" comment=" model PSF parameters at chip center (unit = arcsec)"></column>
-    <column name="psfTheta" type="TFLOAT" default="-999" comment=" model PSF parameters at chip center (unit = deg)"></column>
-    <column name="apResid" type="TFLOAT" default="-999" comment="corrected aperture residual"></column>
-    <column name="dapResid" type="TFLOAT" default="-999" comment="scatter of aperture corrections"></column>
-    <column name="photoZero" type="TFLOAT" default="-999" comment=" local derived photometric zero point (unit = mag)"></column>
-    <column name="ctype1" type="TSTRING" default=" " comment="name of astrometric projection in RA"></column>
-    <column name="ctype2" type="TSTRING" default=" " comment="name of astrometric projection in DEC"></column>
-    <column name="crval1" type="TDOUBLE" default="-999" comment=" RA corresponding to reference pixel (unit = deg)"></column>
-    <column name="crval2" type="TDOUBLE" default="-999" comment=" DEC corresponding to reference pixel (unit = deg)"></column>
-    <column name="crpix1" type="TDOUBLE" default="-999" comment=" reference pixel value for RA (unit = pix)"></column>
-    <column name="crpix2" type="TDOUBLE" default="-999" comment=" reference pixel value for DEC (unit = pix)"></column>
-    <column name="cdelt1" type="TDOUBLE" default="-999" comment=""></column>
-    <column name="cdelt2" type="TDOUBLE" default="-999" comment=""></column>
-    <column name="pc001001" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="pc001002" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="pc002001" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="pc002002" type="TDOUBLE" default="-999" comment="elements of rotation/Dcale matrix"></column>
-    <column name="polyOrder" type="TBYTE" default="255" comment="polynomial order of astrometry fit"></column>
-    <column name="pca1x3y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x2y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x1y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x0y3" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x2y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x1y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca1x0y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x3y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x2y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x1y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x0y3" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x2y0" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x1y1" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-    <column name="pca2x0y2" type="TDOUBLE" default="-999" comment="polynomial coefficients for the astrometric fit"></column>
-  </table>
   <table name="Detection">
     <column name="objID" type="TLONGLONG" default="0" comment="ODM object identifier"></column>
@@ -95,19 +8,9 @@
     <column name="ippDetectID" type="TLONGLONG" default="0" comment="detection id generated by IPP"></column>
     <column name="filterID" type="TBYTE" default="0" comment="filter ID: g=1, r=2, i=3, z=4, y=5, w=6, ..."></column>
-    <column name="xPos" type="TFLOAT" default="-999" comment=" measured x on CCD from PSF fit (unit = pix)"></column>
-    <column name="yPos" type="TFLOAT" default="-999" comment=" measured y on CCD from PSF fit (unit = pix)"></column>
-    <column name="xPosErr" type="TFLOAT" default="-999" comment=" estimated error in x (unit = pix)"></column>
-    <column name="yPosErr" type="TFLOAT" default="-999" comment=" estimated error in y (unit = pix)"></column>
-    <column name="ippRa" type="TFLOAT" default="-999" comment="RA"></column>
-    <column name="ippDec" type="TFLOAT" default="-999" comment="dec"></column>
+    <column name="imageID" type="TLONGLONG" default="0" comment="unique ID for each image, hashed from frameID and ccdID (ALEX)"></column>
+    <column name="ra" type="TFLOAT" default="-999" comment="RA"></column>
+    <column name="dec" type="TFLOAT" default="-999" comment="dec"></column>
+    <column name="infoFlag" type="TLONGLONG" default="-999" comment="flag indicating provenance information"></column>
     <column name="instFlux" type="TFLOAT" default="-999" comment=" PSF instrumental flux (unit = adu/s)"></column>
-    <column name="instFluxErr" type="TFLOAT" default="-999" comment=" estimated error in flux (unit = adu/s)"></column>
-    <column name="peakADU" type="TFLOAT" default="-999" comment=" peak count in source pixel (unit = adu)"></column>
-    <column name="psfWidMajor" type="TFLOAT" default="-999" comment=" model PSF width in major axis (unit = arcsec)"></column>
-    <column name="psfWidMinor" type="TFLOAT" default="-999" comment=" model PSF width in minor axis (unit = arcsec)"></column>
-    <column name="psfTheta" type="TFLOAT" default="-999" comment=" model PSF orientation angle (unit = deg)"></column>
-    <column name="psfCf" type="TFLOAT" default="-999" comment="PSF coverage factor"></column>
-    <column name="sky" type="TFLOAT" default="-999" comment=" PSF sky level at source (unit = adu)"></column>
-    <column name="skyErr" type="TFLOAT" default="-999" comment=" estimated error in sky (unit = adu)"></column>
   </table>
 </tableDescriptions>
Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Gpc1Db.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Gpc1Db.pm	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/Gpc1Db.pm	(revision 28794)
@@ -0,0 +1,119 @@
+#!/usr/bin/perl i-w
+
+package ippToPsps::Gpc1Db;
+
+use warnings;
+use strict;
+
+use ippToPsps::MySQLDb;
+our @ISA = qw(ippToPsps::MySQLDb);    # inherits from MySQLDb
+
+###########################################################################
+#
+# Returns diff-stage cmf files for this exposure
+#
+###########################################################################
+sub getDiffStageCmfs {
+    my ($self, $expId) = @_;
+
+    # DESC and LIMIT 1 is to make sure we get most recently processed
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DISTINCT diffSkyfile.path_base
+        FROM warpRun
+        JOIN fakeRun USING(fake_id)
+        JOIN camRun USING(cam_id)
+        JOIN chipRun USING(chip_id)
+        JOIN rawExp USING (exp_id)
+        JOIN diffInputSkyfile ON (warpRun.warp_id = diffInputSkyfile.warp1 OR warpRun.warp_id = diffInputSkyfile.warp2)
+        JOIN diffSkyfile USING (diff_id)
+        WHERE rawExp.exp_id = $expId AND camRun.magicked
+
+SQL
+
+    $query->execute;
+    return $query->fetchall_arrayref();
+}
+
+###########################################################################
+#
+# Returns camera-stage smf files for this exposure
+#
+###########################################################################
+sub getCameraStageSmf {
+    my ($self, $expId) = @_;
+
+    # DESC and LIMIT 1 is to make sure we get most recently processed
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT 
+        camProcessedExp.path_base
+        FROM warpRun
+        JOIN fakeRun USING(fake_id)
+        JOIN camRun USING(cam_id)
+        JOIN camProcessedExp USING(cam_id)
+        JOIN chipRun USING(chip_id)
+        JOIN rawExp USING (exp_id)
+        WHERE rawExp.exp_id = $expId AND camRun.magicked
+        ORDER BY camRun.cam_id DESC LIMIT 1
+SQL
+
+    $query->execute;
+    return $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns a list of exposure IDs for this survey
+#
+###########################################################################
+sub getExposureListFromSurvey {
+    my ($self, $survey) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT rawExp.exp_id, rawExp.exp_name, camRun.dist_group 
+        FROM camRun, chipRun, rawExp 
+        WHERE camRun.state = 'full' 
+        AND camRun.chip_id = chipRun.chip_id 
+        AND chipRun.exp_id = rawExp.exp_id 
+        AND camRun.dist_group = '$survey'   
+        GROUP BY  rawExp.exp_id 
+        ORDER BY rawExp.exp_id ASC;
+SQL
+
+
+    #AND rawExp.exp_id > 133887
+    #AND filter = 'r.00000'
+    #AND rawExp.dateobs <= '2010-03-12'
+    #AND rawExp.exp_id > $lastExpId
+    #AND camRun.label LIKE '%$label%'
+    #AND rawExp.exp_id > 136561
+    #AND camRun.dist_group LIKE 'sas'
+    #AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' Jims Dec range
+
+    $query->execute;
+    return $query->fetchall_arrayref();
+}
+
+###########################################################################
+#
+# Returns info for this exposure ID
+#
+###########################################################################
+sub getExposureListFromExpId {
+    my ($self, $expId) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT DISTINCT rawExp.exp_id,  rawExp.exp_name, dist_group 
+        FROM rawExp, chipRun 
+        WHERE chipRun.exp_id = rawExp.exp_id 
+        AND rawExp.exp_id = $expId
+SQL
+
+    $query->execute;
+    return $query->fetchall_arrayref();
+}
+
+
+
+1;
Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPsps.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPsps.pm	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPsps.pm	(revision 28794)
@@ -0,0 +1,43 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use DBI;
+
+package ippToPsps::IppToPsps;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+
+    my $class = shift;
+    my $self = {
+        _dbName => shift,
+        _dbHost => shift,
+        _dbUser => shift,
+        _dbPass => shift,
+        _verbose => shift,
+    };
+
+    my $dbname = $self->{_dbName};
+    my $dbhost = $self->{_dbHost};
+    my $dbuser = $self->{_dbUser};
+    my $dbpass = $self->{_dbPass};
+
+    use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+
+    $self->{_db} = DBI->connect("DBI:mysql:database=${dbname};host=${dbhost};" .
+            "mysql_socket=" . DB_SOCKET(),
+            ${dbuser},${dbpass},
+            { RaiseError => 1, AutoCommit => 1}
+            );
+
+    if ($self->{_verbose}) {printf("* Connection to " . $dbname . "@" . $dbhost . ": %s\n", $self->{_db} ? "success" : "FAILED ($DBI::errstr)");}
+
+    bless $self, $class;
+    return $self;
+}
+
Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPspsDb.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPspsDb.pm	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/IppToPspsDb.pm	(revision 28794)
@@ -0,0 +1,289 @@
+#!/usr/bin/perl i-w
+
+package ippToPsps::IppToPspsDb;
+
+use warnings;
+use strict;
+
+use ippToPsps::MySQLDb;
+our @ISA = qw(ippToPsps::MySQLDb);    # inherits from MySQLDb
+
+#######################################################################################
+#
+#  Override constructor
+#
+########################################################################################
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class, Person.
+    my $self = $class->SUPER::new($_[1], $_[2], $_[3], $_[4], $_[5], $_[6]);
+
+    bless $self, $class;
+
+    $self->update();
+    return $self;
+}
+
+#######################################################################################
+#
+# Gets the last processed exposure ID 
+#
+########################################################################################
+sub getLastExpId {
+    my ($self) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT exp_id 
+        FROM batches 
+        WHERE processed = 1
+        ORDER BY exp_id 
+        DESC LIMIT 1;
+SQL
+
+    $query->execute;
+
+    my $expId = $query->fetchrow_array();
+
+    if ($self->{_verbose}) {print "* Last successfully processed exposure ID was $expId\n";}
+
+    return $expId;
+}
+
+#######################################################################################
+#
+# Updates an existing database record 
+#
+########################################################################################
+sub updateDb {
+    my ($self, $batchId, $expId, $processed, $published, $totalDetections) = @_;
+
+print "HJHJH '$batchId', '$expId', '$processed', '$published', '$totalDetections'\n";
+
+if (!$totalDetections) {$totalDetections = -1;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    UPDATE batches 
+        SET processed = $processed, on_datastore = $published, total_detections = $totalDetections 
+        WHERE batch_id = $batchId 
+        AND exp_id = $expId;
+SQL
+
+    $query->execute; # TODO check response of this
+}
+
+#######################################################################################
+#
+# Checks whether we have successfully processed this exposure 
+#
+########################################################################################
+sub isExposureAlreadyProcessed {
+    my ($self, $expId) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM batches 
+        WHERE exp_id = $expId 
+        AND processed = 1;
+SQL
+
+    $query->execute;
+
+    my $processed = $query->fetchrow_array();
+
+    if ($processed) {print "* Exposure ID '$expId' has already been processed\n";}
+
+    return $processed;
+}
+
+#######################################################################################
+#
+# Generates a new record in the database with a new batch ID 
+#
+########################################################################################
+sub getNewBatchId {
+    my ($self, $expId, $surveyType, $batchType) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT batch_id 
+        FROM batches 
+        ORDER BY batch_id 
+        DESC LIMIT 1;
+SQL
+
+    $query->execute;
+
+    my $batchId = $query->fetchrow_array();
+
+    $batchId++;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO batches
+        (batch_id, exp_id, survey_id, batch_type)
+        VALUES
+        ($batchId, $expId, '$surveyType', '$batchType');
+
+SQL
+
+    $query->execute;
+
+    print "* New batch '$batchId' for exposure ID = $expId and survey = '$surveyType'\n";
+
+    return $batchId;
+}
+
+###########################################################################
+#
+# Update Db to newest version 
+#
+###########################################################################
+sub update {
+    my ($self) = @_;
+
+    my $currentRevision = -1;
+    my $latestRevision = 4;
+
+    while ($currentRevision != $latestRevision) {
+
+        $currentRevision = $self->getRevision();
+        if ($self->{_verbose}) {print "* Current revision = $currentRevision\n";}
+
+        if ($currentRevision == 0) {$self->createRevision_1();}
+        elsif ($currentRevision == 1) {$self->createRevision_2();}
+        elsif ($currentRevision == 2) {$self->createRevision_3();}
+        elsif ($currentRevision == 3) {$self->createRevision_4();}
+    }
+}
+
+#######################################################################################
+# 
+# Create revision 1 of the database 
+#
+#######################################################################################
+sub createRevision_1 {
+    my ($self) = @_;
+
+    print "* Creating revision 1 of '$dbname'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE revision (
+            revision INT, 
+            created TIMESTAMP DEFAULT NOW(),
+            primary key (revision)
+            );
+SQL
+        $query->execute;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE batches (
+            batch_id BIGINT NOT NULL, 
+            exp_id BIGINT NOT NULL, 
+            survey_id VARCHAR(30) DEFAULT "NONE",
+            processed TINYINT DEFAULT 0,
+            on_datastore TINYINT DEFAULT 0,
+            loaded_to_ODM TINYINT DEFAULT 0,
+            merge_worthy TINYINT DEFAULT 0,
+            deleted TINYINT DEFAULT 0,
+            created TIMESTAMP DEFAULT NOW(),
+            primary key (batch_id, exp_id)
+            );
+
+SQL
+        $query->execute;
+
+    setRevision(1);
+}
+
+#######################################################################################
+# 
+# Create revision 2 of the database 
+#
+#######################################################################################
+sub createRevision_2 {
+    my ($self) = @_;
+
+    print "* Creating revision 2 of '$dbname'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    ALTER TABLE batches 
+        ADD COLUMN merged TINYINT DEFAULT 0
+SQL
+        $query->execute;
+
+    setRevision(2);
+}
+
+#######################################################################################
+# 
+# Create revision 3 of the database 
+#
+#######################################################################################
+sub createRevision_3 {
+    my ($self) = @_;
+
+    print "* Creating revision 3 of '$dbname'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    ALTER TABLE batches 
+        ADD COLUMN total_detections BIGINT DEFAULT 0
+SQL
+        $query->execute;
+
+    setRevision(3);
+}
+
+#######################################################################################
+# 
+# Create revision 4 of the database 
+#
+#######################################################################################
+sub createRevision_4 {
+    my ($self) = @_;
+
+    print "* Creating revision 4 of '$dbname'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    ALTER TABLE batches 
+        ADD COLUMN batch_type VARCHAR(30) DEFAULT "UNKNOWN"
+SQL
+        $query->execute;
+
+    setRevision(4);
+}
+
+#######################################################################################
+# 
+# Sets current revision of ippToPsps database
+#
+#######################################################################################
+sub setRevision {
+    my ($self, $revision) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO revision (revision) VALUES ($revision);
+SQL
+        $query->execute;
+}
+
+#######################################################################################
+# 
+# Gets current revision of ippToPsps database
+#
+#######################################################################################
+sub getRevision {
+    my ($self) = @_;
+
+    if (!$self->SUPER::doesTableExist("revision")) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT revision
+        FROM revision
+        ORDER BY revision DESC LIMIT 1;
+SQL
+
+        $query->execute;
+    my @row = $query->fetchrow_array();
+
+    return $row[0];
+}
+1
Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/MySQLDb.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/MySQLDb.pm	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps/MySQLDb.pm	(revision 28794)
@@ -0,0 +1,135 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use DBI;
+
+package ippToPsps::MySQLDb;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+
+    my $class = shift;
+    my $self = { 
+        _dbName => shift,
+        _dbHost => shift,
+        _dbUser => shift,
+        _dbPass => shift,
+        _verbose => shift,
+        _save_temps => shift,
+    };                              
+
+    my $dbname = $self->{_dbName};
+    my $dbhost = $self->{_dbHost};
+    my $dbuser = $self->{_dbUser};
+    my $dbpass = $self->{_dbPass};
+
+    use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+
+     $self->{_db} = DBI->connect("DBI:mysql:database=${dbname};host=${dbhost};" .
+            "mysql_socket=" . DB_SOCKET(),
+            ${dbuser},${dbpass},
+            { RaiseError => 1, AutoCommit => 1}
+            );
+
+    if ($self->{_verbose}) {printf("* Connection to " . $dbname . "@" . $dbhost . ": %s\n", $self->{_db} ? "success" : "FAILED ($DBI::errstr)");}
+
+    bless $self, $class;                                
+    return $self;                                           
+}                     
+
+###########################################################################
+#
+# Sets the date format to be used
+#
+###########################################################################
+sub setDateFormat {
+    my ($self, $dateFormat) = @_;
+    $self->{_dateFormat} = $dateFormat if defined($dateFormat);
+    return $self->{_dateFormat};
+}
+
+sub setDbHost {
+    my ( $self, $dbHost ) = @_;                              
+    $self->{_dbHost} = $dbHost if defined($dbHost);        
+    return $self->{_dbHost};                            
+}                                                                       
+
+sub getDbHost {
+    my( $self ) = @_;                                                       
+    return $self->{_dbHost};                                                 
+}                                                                               
+
+
+###########################################################################
+#
+# Returns 'now' as a timestamp
+#
+###########################################################################
+sub getIntervalInPast {
+    my ($self, $interval) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT now() - INTERVAL $interval; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+
+###########################################################################
+#
+# Returns 'now' as a timestamp
+#
+###########################################################################
+sub getNowTimestamp {
+    my ($self) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT now(); 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Checks whether a certain table exists 
+#
+#######################################################################################
+sub doesTableExist {
+    my ($self, $table) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM information_schema.tables 
+        WHERE table_schema = '$self->{_dbName}' 
+        AND table_name = '$table';
+SQL
+
+    $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Destructor 
+#
+#######################################################################################
+sub DESTROY {
+    my($self) = @_;                                                       
+
+    if ($self->{_verbose}) {
+        print "* " . ref($self) . " is disconnecting from " . $self->{_dbName} . "@" . $self->{_dbHost} . "\n";
+    }
+    #$self->{_db}->disconnect();
+}
+1;                                        
+
Index: /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/perl/ippToPsps_run.pl	(revision 28794)
@@ -0,0 +1,542 @@
+#!/usr/bin/env perl
+
+use warnings;
+use strict;
+
+use PS::IPP::Config 1.01 qw( :standard );
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use IPC::Cmd 0.36 qw( can_run run );
+use File::Temp qw(tempfile);
+use XML::LibXML;
+
+# local classes
+use ippToPsps::Gpc1Db;
+use ippToPsps::IppToPspsDb;
+
+# globals
+my $camera = 'GPC1';
+my $batchType = undef;
+my $survey = undef;
+my $dvodb = undef;
+my $verbose = undef;
+my $save_temps = undef; 
+my $no_update = undef;
+my $output = undef;
+my $singleExpId = undef;
+my $datastoreProduct = undef;
+my $force = undef;
+my $initBatch = undef;
+my $dontTarNZip = undef;
+
+# get user args
+GetOptions(
+        'output|o=s' => \$output,
+        'batch|b=s' => \$batchType,
+        'dvodb|d=s' => \$dvodb,
+        'survey|s=s' => \$survey,
+        'expid|e=s' => \$singleExpId,
+        'product|p=s' => \$datastoreProduct,
+        'verbose|v' => \$verbose,
+        'save_temps|t' => \$save_temps,
+        'no-update|u' => \$no_update,
+        'force|f' => \$force,
+        'tarnzip|z' => \$dontTarNZip,
+        );
+
+my $quit = 0;
+print "\n*******************************************************************************\n";
+print "* \n";
+if (@ARGV) {
+    print "* UNKNKOWN: option:                         @ARGV\n"; $quit=1;}
+if (!defined $output) {
+    print "* REQUIRED: need to provide an output path: -o <path>\n"; $quit=1;}
+if (!defined $batchType) {
+    print "* REQUIRED: need to define a batch type:    -b <init|det|diff|stack>\n"; $quit=1;}
+if (!defined $dvodb) {
+    print "* REQUIRED: need to provide a DVO Db path:  -d <path>\n"; $quit=1;}
+if (!defined $survey and !defined $singleExpId) {
+    print "* REQUIRED: need to provide a survey type:  -s <ThreePi|STS|SAS|M31|MD01|MD02|etc> ***OR***\n";
+    print "*           an exposure ID                  -e <expID>\n"; $quit=1;}
+if (!defined $datastoreProduct) {
+    print "* OPTIONAL: datastore product:              -p <product>\n";}
+if (!defined $verbose) {
+    print "* OPTIONAL: run in verbose mode:            -v\n"; $verbose = 0;}
+if (!defined $save_temps) {
+    print "* OPTIONAL: keep temp files:                -t\n"; $save_temps = 0;}
+if (!defined $no_update) {
+    print "* OPTIONAL: don't update database:          -u\n"; $no_update = 0;}
+if (!defined $force) {
+    print "* OPTIONAL: force if already processed :    -f\n"; $force = 0;}
+if (!defined $dontTarNZip) {
+    print "* OPTIONAL: don't tar and zip output :      -z\n"; $dontTarNZip = 0;}
+    print "*\n*******************************************************************************\n";
+
+if ($quit) { exit; }
+
+my $gpc1Db = new ippToPsps::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser", $verbose, $save_temps);
+my $ippToPspsDb = new ippToPsps::IppToPspsDb("ippToPsps", "ippdb01", "ipp", "ipp", $verbose, $save_temps);
+
+# check we can run programs and get camera config
+my $ippToPsps = can_run('ippToPsps') or (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR));
+my $dsreg = can_run('dsreg') or (warn "Can't find 'dsreg' program" and exit($PS_EXIT_CONFIG_ERROR));
+my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR)); 
+
+if ($batchType eq "init") {$initBatch = 1;}
+else {$initBatch = 0;}
+process();
+
+#######################################################################################
+# 
+# Generates a PSPS-suitable survey 'type' from the IPP distribution group
+#
+#######################################################################################
+sub getSurveyTypeFromDistGroup {
+    my ($distGroup) = @_;
+
+    if ($distGroup =~ m/^MD([0-1][0-9])$/i) {return "MD$1";}
+    if ($distGroup =~ m/^M31$/i) {return "M31";}
+    if ($distGroup =~ m/^sts$/i) {return "STS1";}
+    if ($distGroup =~ m/^SweetSpot$/i) {return "SSS";}
+    if ($distGroup =~ m/^(3PI)|(ThreePi)|(SAS)$/i) {return "3PI";}
+
+    print "* Unknown distribution group: '$distGroup'\n";
+    return undef;
+}
+
+#######################################################################################
+# 
+# Finds all exposures for the provided TODO and generates PSPS FITS data for them
+#
+#######################################################################################
+sub process {
+
+    my ($resultsFile, $resultsFilePath) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps);
+    my $lastExpId = $ippToPspsDb->getLastExpId();
+    my $rows = undef;
+
+    my $query;
+    if ($initBatch) {
+   
+        my @row = (0, 'NULL', 'ThreePi');
+        @{$rows} = (\@row);
+    }
+    elsif ($singleExpId) { $rows = $gpc1Db->getExposureListFromExpId($singleExpId); }
+    else { $rows = $gpc1Db->getExposureListFromSurvey($survey); }
+
+# TODO check if there are no exposures and give a warning
+
+    my $batchId = 0;
+    my $batchesPerJob = 1;
+    my $published = 0;
+
+    #my $batchId = -1;
+    my $row;
+    foreach $row ( @{$rows} ) {
+        my ($expId, $expName, $distGroup) = @{$row};
+    #        print "JHGHGHGHGH2 $expId, $expName, $distGroup\n";
+
+    # loop round exposures
+    #while (my @row = $query->fetchrow_array()) {
+     #   my ($expId, $expName, $distGroup) = @row;
+
+        if (!$initBatch && $ippToPspsDb->isExposureAlreadyProcessed($expId)) {
+                
+                if ($force) {print "* Forcing....\n";} 
+                else {next};
+        }
+
+        my $surveyType = getSurveyTypeFromDistGroup($distGroup);
+        if (!$surveyType) {next;}
+
+        $batchId = $ippToPspsDb->getNewBatchId($expId, $surveyType, $batchType);
+
+        # TODO quit here if no sensible batch ID
+
+        # generate batch path from batch IDs
+        my $batch = sprintf("B%08d", $batchId);
+        my $batchDir = sprintf("$output/$batch");
+        mkdir($batchDir, 0777);
+        $published = 0;
+
+        print "* Preparing exposure $expId as batch '$batch'...\n";
+
+        # run IppToPsps program
+        if (runIppToPsps($expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath)) {
+
+            my ($filename, $minObjId, $maxObjId, $totalDetections);
+            parseResults($resultsFilePath, \$filename, \$minObjId, \$maxObjId, \$totalDetections);
+
+            # write batch manifest and tar and zip up
+            if (writeBatchManifest($batchDir, $batch, $batchType, $surveyType, $filename, $minObjId, $maxObjId)) {
+
+                if (!$dontTarNZip) {
+                    my $tarball = tarAndZipBatch($output, $batch);
+                    if ($tarball && $datastoreProduct && publishToDatastore($batch, $output, $tarball)) {$published = 1;}
+                }
+            }
+
+            $ippToPspsDb->updateDb($batchId, $expId, 1, $published, $totalDetections);
+        }
+
+        if ($initBatch) {print "* Wrote initialisation batch\n";}
+
+        print "*\n*******************************************************************************\n\n";
+    }
+
+    close $resultsFile;
+
+    return 1;
+}
+
+#######################################################################################
+#
+# run IppToPsps 
+#
+#######################################################################################
+sub runIppToPsps {
+    my ($expId, $expName, $surveyType, $batchDir, $batchType, $resultsFilePath) = @_;
+
+    my $success = 0;
+
+    if ($batchType =~ /init/) {
+        $success = produceInit($batchDir, $resultsFilePath);
+    }
+    elsif ($batchType =~ /det/) {
+        $success = produceDetections($expId, $expName, $surveyType, $batchDir, $resultsFilePath);
+    }
+    elsif ($batchType =~ /diff/) {
+        $success = produceDiffs($expId, $expName, $surveyType, $batchDir, $resultsFilePath);
+    }
+    elsif ($batchType =~ /test/) {
+        $success = produceDetections($expId, $expName, $surveyType, $batchDir, $resultsFilePath);
+    }
+
+    return $success;
+}
+
+#######################################################################################
+#
+# tar 'n zip a batch TODO handle errors
+#
+#######################################################################################
+sub tarAndZipBatch {
+    my ($path,$batch) = @_;
+
+    my $batchDir = "$path/$batch";
+    my $tar = "$batch.tar";
+    my $tarball = "$tar.gz";
+    my $batchTar = "$path/$tar";
+    my $batchZip = "$path/$tarball";
+
+    # tar
+    my $command = "tar -cvf $batchTar -C $path $batch";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    if (!$success) { print "* Unable to tar up contents of dir: $batchDir\n" and return undef; }
+
+    # zip
+    $command = "gzip -c $batchTar  > $batchZip";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    if (!$success) { print "* Unable to gzip: $batchTar\n" and return undef; }
+
+    # remove tar file
+    $command = "rm $batchTar";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    if (!$success) { print "* Unable to remove: $batchTar\n" };
+
+    # remove batch dir
+    $command = "rm -r $batchDir";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    if (!$success) { print "* Unable to remove: $batch\n"};
+
+    return $tarball;
+}
+
+#######################################################################################
+#
+# zip a file then delete it 
+#
+#######################################################################################
+sub zipFile {
+    my ($path,$origFile) = @_;
+
+    my $file;
+
+    # zip
+    my $zippedName =  "$origFile.gz";
+    my $command = "gzip -c $path/$origFile  > $path/$zippedName";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
+        run(command => $command, verbose => $verbose);
+
+    if (!$success) { 
+
+        $file = $origFile;
+        print "* Unable to gzip: $path/$origFile\n"; 
+    }
+    else {
+
+        $file = $zippedName;
+
+        # now delete original
+        $command = "rm -r $path/$origFile";
+        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+
+        if (!$success) { print "* Unable to remove: $path/$origFile\n"};
+    }
+
+    return $file;
+}
+
+
+#######################################################################################
+#
+# parse results XML
+#
+#######################################################################################
+sub parseResults {
+    my ($resultsFilePath, $filename, $minObjId, $maxObjId, $totalDetections) = @_;
+
+    my $parser = XML::LibXML->new;
+    my $doc = $parser->parse_file($resultsFilePath);
+    ${$filename} = $doc->findvalue('//filename');
+    ${$minObjId} = $doc->findvalue('//minObjID');
+    ${$maxObjId} = $doc->findvalue('//maxObjID');
+    ${$totalDetections} = $doc->findvalue('//totalDetections');
+}
+
+
+#######################################################################################
+#
+# writes a batch manifest file for the one FITS file that should be in the path passed in
+#
+#######################################################################################
+sub writeBatchManifest {
+    my ($path,$batch,$batchType,$surveyType,$filename,$minObjId,$maxObjId) = @_;
+
+    use XML::Writer;
+    use Digest::MD5::File qw( file_md5_hex );
+
+    # sort out paths
+    my $outputPath = "$path/BatchManifest.xml";
+    my $output = new IO::File(">$outputPath");
+
+    # create a timestamp
+    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
+    my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec;
+
+    # determine batch 'type'
+    my $type;
+    if ($batchType eq 'init') {$type = "IN";}
+    elsif ($batchType eq 'det') {$type = "P2";}
+    elsif ($batchType eq 'stack') {$type = "ST";}
+    elsif ($batchType eq 'diff') {$type = "OB";}
+    else {$type = "UNKNOWN";}
+
+    # create XML file
+    my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
+    $writer->xmlDecl('UTF-8');
+    $writer->doctype('manifest', "", "psps-manifest.dtd");
+
+    # open directory to hunt for FITS files
+    opendir(DIR, $path) or print "* Cannot open '$path' to write batch manifest\n" and return 0;
+    my @thefiles= readdir(DIR);
+    closedir(DIR);
+
+    my $foundFits = 0;
+    foreach my $f (@thefiles) {
+
+        if ($f =~ /\.FITS/) {
+
+            if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;}
+
+            # my $file = zipFile($path, $f); 
+
+            # get md5sum of file
+            my $md5sum  = file_md5_hex("$path/$f");
+
+            # get size of file in bytes
+            my $filesize = -s "$path/$f";
+
+            # write manifest element TODO untidy
+            if($batchType eq 'det' && $filename eq $f) {
+
+                my $pspsSurvey = undef;
+                if ($surveyType =~ m/^MD[0-1][0-9]$/i) {$pspsSurvey = "MDF";}
+                else {$pspsSurvey = $surveyType;}
+
+                $writer->startTag('manifest',
+                        "name" => "$batch",
+                        "type" => $type,
+                        "survey" => $pspsSurvey,
+                        "timestamp" => "$timeStamp",
+                        "minObjId" => $minObjId,
+                        "maxObjId" => $maxObjId);
+            }
+            else {
+
+                $writer->startTag('manifest',
+                        "name" => "$batch",
+                        "type" => $type,
+                        "timestamp" => "$timeStamp");
+            }
+            # write the tag for this batch
+            $writer->startTag("file", 
+                    "name" => $f,
+                    "bytes" => $filesize,
+                    "md5" => $md5sum);
+
+            $writer->endTag();
+            $writer->endTag();
+            $writer->end();
+            $foundFits = 1;
+        }
+
+    }
+
+    if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;}
+    return 1;
+}
+
+#######################################################################################
+#
+# register new job with the datastore
+#
+########################################################################################
+sub publishToDatastore {
+    my ($batch, $path, $tarball) = @_;
+
+    my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps);
+
+    print $tempFile $tarball . '|||tgz' . "\n";
+
+    # build dsreg command command
+    my $command  = "$dsreg";
+    $command .= " --add $batch";
+    $command .= " --copy";
+    $command .= " --datapath $path";
+    $command .= " --type IPP_PSPS";
+    $command .= " --product $datastoreProduct";
+    $command .= " --list $tempName";
+
+    # run command
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+
+    if (!$success) { print "* Unable to publish $tarball to datastore\n" and return 0 };
+
+    print "* Successfully published $tarball to datastore\n";
+
+    close($tempFile);
+
+    return 1;
+}
+
+#######################################################################################
+# 
+# runs ippToPsps to produce init batch 
+#
+#######################################################################################
+sub produceInit {
+    my ($outputPath, $resultsFilePath) = @_;
+
+    my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType, $resultsFilePath);
+    return $success;
+}
+
+#######################################################################################
+# 
+# runs ippToPsps for detections for this exposure 
+#
+#######################################################################################
+sub produceDetections {
+    my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_;
+
+    my $nebPath = $gpc1Db->getCameraStageSmf($expId);
+    if (!$nebPath) { return 0; }
+
+    my $success = 0;
+
+    # get real filename from neb 'key'
+    my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",$nebPath) or return 0;
+    my $realFile = $ipprc->file_resolve($fpaObjects) or return 0;
+
+    # now write the path to this file out to temp file
+    my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_inputFileList.XXXX", UNLINK => !$save_temps);
+    print $tempFile $realFile . "\n";
+    close $tempFile;
+
+    $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath);
+    return $success;
+}
+
+#######################################################################################
+# 
+# runs ippToPsps for diffs for this exposure 
+#
+#######################################################################################
+sub produceDiffs {
+    my ($expId, $expName, $surveyType, $outputPath, $resultsFilePath) = @_;
+
+    my $rows = $gpc1Db->getDiffStageCmfs($expId);
+
+    my $size = @{$rows};
+    if ($size < 1) {print "* No cmf files found for this exposure\n"; return 0;}
+
+    my ($tempFile, $tempName) = tempfile( "inputFileList.XXXX", UNLINK => !$save_temps);
+
+    # loop round db results should be one file per skycell
+    my $success = 0;
+
+    my $row;
+    foreach $row ( @{$rows} ) {
+        my ($nebPath) = @{$row};
+
+        print "NEB PATH $nebPath\n";
+
+        # get real filename from neb 'key'
+        my $fpaObjects = $ipprc->filename("PPSUB.OUTPUT.SOURCES",$nebPath) or return 0;
+        my $realFile = $ipprc->file_resolve($fpaObjects) or return 0;
+        #print "REALFILE $realFile\n";
+
+        # now write the path to this file out to temp file
+        print $tempFile $realFile . "\n";
+    }
+
+    my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType, $resultsFilePath);
+
+    close $tempFile;
+
+    return $ret;
+}
+
+#######################################################################################
+#
+# runs the ippToPsps program
+#
+#######################################################################################
+sub runProgram {
+    my ($input, $output, $expid, $expName, $surveyType, $batchType, $resultsFilePath) = @_;
+
+    # build command
+    my $command = "$ippToPsps"; 
+    $command .= " -input $input";
+    $command .= " -output $output";
+    $command .= " -D CATDIR $dvodb";
+    $command .= " -config ../config"; # TODO
+    $command .= " -expid $expid";
+    $command .= " -expname $expName";
+    $command .= " -survey $surveyType";
+    $command .= " -batch $batchType";
+    $command .= " -results $resultsFilePath";
+
+    # run command
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf )
+        = run(command => $command, verbose => $verbose);
+
+    return $success;
+}
Index: /branches/eam_branches/ipp-20100621/ippToPsps/scripts/createDb.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/scripts/createDb.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/scripts/createDb.pl	(revision 28794)
@@ -25,12 +25,23 @@
 print "* Connected to '$dbname' on 'dbserver'\n";
 
-if (!doesTableExist("revision")) {
-    createRevision_1();
+my $currentRevision = -1;
+
+my $latestRevision = 3;
+    print "* Latest revision = $latestRevision\n";
+
+while ($currentRevision != $latestRevision) {
+
+    $currentRevision = getRevision();
+    print "* Current revision = $currentRevision\n";
+
+    if ($currentRevision == 0) {createRevision_1();}
+    elsif ($currentRevision == 1) {createRevision_2();}
+    elsif ($currentRevision == 2) {createRevision_3();}
+
 }
-
-
 $db->disconnect();
 print "* Disconnected from '$dbname' on 'dbserver'\n";
 print "*\n*******************************************************************************\n\n";
+
 
 #######################################################################################
@@ -45,5 +56,4 @@
     my $query = $db->prepare(<<SQL);
     CREATE TABLE revision (
-
             revision INT, 
             created TIMESTAMP DEFAULT NOW(),
@@ -54,12 +64,6 @@
 
     $query = $db->prepare(<<SQL);
-    INSERT INTO revision (revision) VALUES (1);
-SQL
-        $query->execute;
-
-    $query = $db->prepare(<<SQL);
 
     CREATE TABLE batches (
-
             batch_id BIGINT NOT NULL, 
             exp_id BIGINT NOT NULL, 
@@ -77,4 +81,59 @@
     $query->execute;
 
+setRevision(1);
+
+}
+
+#######################################################################################
+# 
+# Create revision 2 of the database 
+#
+#######################################################################################
+sub createRevision_2 {
+
+    print "* Creating revision 2 of '$dbname'\n";
+
+        my $query = $db->prepare(<<SQL);
+
+        ALTER TABLE batches 
+        ADD COLUMN merged TINYINT DEFAULT 0
+SQL
+    $query->execute;
+
+setRevision(2);
+}
+
+#######################################################################################
+# 
+# Create revision 3 of the database 
+#
+#######################################################################################
+sub createRevision_3 {
+
+    print "* Creating revision 3 of '$dbname'\n";
+
+        my $query = $db->prepare(<<SQL);
+
+        ALTER TABLE batches 
+        ADD COLUMN total_detections BIGINT DEFAULT 0
+SQL
+    $query->execute;
+
+setRevision(3);
+}
+
+
+#######################################################################################
+# 
+# Sets current revision of ippToPsps database
+#
+#######################################################################################
+sub setRevision {
+    my ($revision) = @_;
+
+    my $query = $db->prepare(<<SQL);
+    INSERT INTO revision (revision) VALUES ($revision);
+SQL
+        $query->execute;
 }
 
@@ -82,4 +141,25 @@
 # 
 # Gets current revision of ippToPsps database
+#
+#######################################################################################
+sub getRevision {
+
+    if (!doesTableExist("revision")) {return 0;}
+
+    my $query = $db->prepare(<<SQL);
+
+    SELECT revision
+        FROM revision
+        ORDER BY revision DESC LIMIT 1;
+SQL
+   $query->execute;
+   my @row = $query->fetchrow_array();
+
+   return $row[0];
+}
+
+#######################################################################################
+# 
+# Checks whether a certain table exists 
 #
 #######################################################################################
@@ -99,6 +179,4 @@
     my $count = $query->fetchrow_array();
 
-    printf( "* Table '$table' %s\n", $count ? "exists" : "does not exist");
-
     return $count;
 }
Index: anches/eam_branches/ipp-20100621/ippToPsps/scripts/ippToPsps_run.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/scripts/ippToPsps_run.pl	(revision 28793)
+++ 	(revision )
@@ -1,806 +1,0 @@
-#!/usr/bin/env perl
-
-use warnings;
-use strict;
-use PS::IPP::Config 1.01 qw( :standard );
-use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
-use Pod::Usage qw( pod2usage );
-use IPC::Cmd 0.36 qw( can_run run );
-use DBI;
-use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
-use File::Temp qw(tempfile);
-
-# globals
-my ($verbose, $save_temps, $no_op, $no_update, $camera, $batchType, $output, $dvodb, $datastoreProduct, $survey, $singleExpId, $no_publish, $force, $initBatch);
-
-# default values for certain globals
-$verbose = 0;
-$save_temps = 0; 
-$no_op = 0;
-$no_update = 0;
-$no_publish = 0;
-$camera = 'GPC1';
-$output = undef;
-$singleExpId = undef;
-$datastoreProduct = "PSPS_test";
-$force = 0;
-$initBatch = 0;
-
-# get user args
-GetOptions(
-        'output|o=s' => \$output,
-        'batch|b=s' => \$batchType,
-        'dvodb|d=s' => \$dvodb,
-        'survey|s=s' => \$survey,
-        'expid|e=s' => \$singleExpId,
-        'product|p=s' => \$datastoreProduct,
-        'no_publish' => \$no_publish,
-        'verbose|v' => \$verbose,
-        'save_temps' => \$save_temps,
-        'no-op' => \$no_op,
-        'no-update' => \$no_update,
-        'force|f' => \$force,
-        ) or pod2usage( 2 );
-
-pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-
-# tell off user for not providing proper args
-pod2usage(
-        -msg => "\n   Required options:\n\n".
-        "--batch <init|det|diff|stack>\n".
-        "--output <path>\n".
-        "--survey <ThreePi|MD01,2 etc|STS|SAS|SweetSpot> or --expid <expid>\n" .
-        "--dvodb <path>\n".
-        "\n   Optional:\n\n".
-        "--product <datastoreProduct>\n".
-        "--no_publish - won't publish to datastore\n".
-        "--save_temps - will save temporary files\n".
-        "--no-op - ?\n".
-        "--no-update - will not update database\n",
-        -exitval => 3
-        ) unless 
-defined $batchType and
-defined $output and
-defined $dvodb and
-(( defined $survey or defined $singleExpId ) or ( $batchType eq "init"));
-
-if ($batchType eq "init") {$initBatch = 1;}
-
-# check we can run programs and get camera config
-my $ippToPsps = can_run('ippToPsps') or (warn "Can't find 'ippToPsps' program" and exit($PS_EXIT_CONFIG_ERROR));
-my $dsreg = can_run('dsreg') or (warn "Can't find 'dsreg' program" and exit($PS_EXIT_CONFIG_ERROR));
-my $ipprc = PS::IPP::Config->new($camera) or (warn "Can't get camera configuration" and exit($PS_EXIT_CONFIG_ERROR)); 
-
-# make a temporary file for saving results
-my ($resultsFile, $resultsFileName) = tempfile( "/tmp/ippToPsps_results.XXXX", UNLINK => !$save_temps);
-
-process();
-
-close $resultsFile;
-
-#######################################################################################
-# 
-# Generates a PSPS-suitable survey 'type' from the IPP distribution group
-#
-#######################################################################################
-sub getSurveyTypeFromDistGroup {
-    my ($distGroup) = @_;
-
-    if ($distGroup =~ m/^MD([0-1][0-9])$/i) {return "MD$1";}
-    if ($distGroup =~ m/^M31$/i) {return "M31";}
-    if ($distGroup =~ m/^sts$/i) {return "STS1";}
-    if ($distGroup =~ m/^SweetSpot$/i) {return "SSS";}
-    if ($distGroup =~ m/^(3PI)|(ThreePi)|(SAS)$/i) {return "3PI";}
-
-    print "* Unknown distribution group: '$distGroup'\n";
-    return undef;
-}
-
-
-#######################################################################################
-# 
-# Connects to a db
-# 
-########################################################################################
-sub connectToDb {
-    my ($dbname,$dbserver,$dbuser,$dbpass) = @_;
-
-
-    my $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
-            "mysql_socket=" . DB_SOCKET(),
-            ${dbuser},${dbpass},
-            { RaiseError => 1, AutoCommit => 1}
-            );
-    
-    printf("* Connection to '$dbname' on '$dbserver': %s\n", $db ? "success" : "FAILED ($DBI::errstr)");
-
-    return $db;
-}
-
-
-#######################################################################################
-# 
-# Finds all exposures for the provided TODO and generates PSPS FITS data for them
-#
-#######################################################################################
-sub process {
-
-    print "*******************************************************************************\n*\n";
-
-    my $gpc1Db = connectToDb("gpc1", "ippdb01", "ippuser", "ippuser") or return 0;
-    my $ippToPspsDb = connectToDb("ippToPsps", "ippdb01", "ipp", "ipp") or return 0;
-    my $lastExpId = getLastExpId($ippToPspsDb);
-
-    my $query;
-    if ($initBatch) {
-    
-        $query = $gpc1Db->prepare(<<SQL);
-
-        SELECT 0, 'NULL', 'ThreePi';
-SQL
-    }
-    elsif ($singleExpId) {
-
-        $query = $gpc1Db->prepare(<<SQL);
-
-        SELECT DISTINCT rawExp.exp_id,  rawExp.exp_name, dist_group 
-            FROM rawExp, chipRun 
-            WHERE chipRun.exp_id = rawExp.exp_id 
-            AND rawExp.exp_id = $singleExpId
-SQL
-    }
-    else {
-
-        $query = $gpc1Db->prepare(<<SQL);
-
-        SELECT rawExp.exp_id, rawExp.exp_name, camRun.dist_group 
-            FROM camRun, chipRun, rawExp 
-            WHERE camRun.state = 'full' 
-            AND camRun.chip_id = chipRun.chip_id 
-            AND chipRun.exp_id = rawExp.exp_id 
-            AND camRun.dist_group = '$survey'   
-            GROUP BY  rawExp.exp_id 
-            ORDER BY rawExp.exp_id ASC;
-SQL
-
-            #AND rawExp.dateobs <= '2010-03-12'
-            #AND rawExp.exp_id > $lastExpId
-            #AND camRun.label LIKE '%$label%'
-            #AND rawExp.exp_id > 136561
-            #AND camRun.dist_group LIKE 'sas'
-            #AND rawExp.decl >= '-0.157079633' AND rawExp.decl <= '0.157079633' Jims Dec range
-    }
-
-    my $batchId = 0;
-    my $batchesPerJob = 1;
-    my $anyBatches = 0;
-    my $published = 0;
-
-    $query->execute;
-
-    my $jobId = -1;
-
-    # loop round exposures
-    while (my @row = $query->fetchrow_array()) {
-        my ($expId, $expName, $distGroup) = @row;
-
-        if (!$initBatch && isExposureAlreadyProcessed($ippToPspsDb, $expId)) {
-                
-                if ($force) {print "* Forcing....\n";} 
-                else {next};
-        }
-
-        my $surveyType = getSurveyTypeFromDistGroup($distGroup);
-        if (!$surveyType) {next;}
-
-        $jobId = getNewBatchId($ippToPspsDb, $expId, $surveyType);
-
-        # TODO quit here if no sensible job ID
-
-        # generate batch and job paths from job and batch IDs
-        my $job = sprintf("J%06d", $jobId);
-        my $jobOutputPath = sprintf("$output/$job");
-        my $batchDir = sprintf("B%03d", $batchId);
-        my $batchOutputPath = sprintf("$jobOutputPath/%s", $batchDir);
-        mkdir($jobOutputPath, 0777);
-        mkdir($batchOutputPath, 0777);
-
-        print "* Preparing exposure $expId as job '$job'...\n";
-
-        # run IppToPsps program
-        if (runIppToPsps($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath, $batchType)) {
-
-            # write batch manifest and tar and zip up
-            if (writeBatchManifest($batchOutputPath, $batchDir, $batchType)) {
-
-                if (tarAndZipDirectory($jobOutputPath, $batchDir)) {
-
-                    $anyBatches = 1
-                }
-            }
-
-            if (($batchId+1) < $batchesPerJob) {
-                $batchId++;
-            }
-            # job is complete, now publish
-            elsif ($anyBatches) {
-
-                $published = 0;
-                if (writeJobManifest($output, $job, $surveyType)) {
-
-                    if (!$no_publish && publishToDatastore($output, $job)) { 
-                        $published = 1;
-                    }
-                }
-
-                updateDb($ippToPspsDb, $jobId, $expId, 1, $published);
-
-                $batchId = 0;
-                $anyBatches = 0;
-            }
-            else {
-
-                print "* No batches to publish\n";
-            }
-        }
-
-        if ($initBatch) {print "* Wrote initialisation batch\n";}
-
-        print "*\n*******************************************************************************\n\n";
-    }
-
-    $gpc1Db->disconnect();
-    $ippToPspsDb->disconnect();
-
-    return 1;
-}
-
-#######################################################################################
-#
-# Gets the last processed exposure ID (ini file for now - will be Db eventually TODO) 
-#
-########################################################################################
-sub getLastExpId {
-    my ($db) = @_;
-
-    my $query = $db->prepare(<<SQL);
-
-    SELECT exp_id 
-        FROM batches 
-        WHERE processed = 1
-        ORDER BY exp_id 
-        DESC LIMIT 1;
-SQL
-
-    $query->execute;
-
-    my $expId = $query->fetchrow_array();
-
-    print "* Last successfully processed exposure ID was $expId\n";
-
-    return $expId; 
-}
-
-#######################################################################################
-#
-# Updates an existing database record 
-#
-########################################################################################
-sub updateDb {
-    my ($db, $batchId, $expId, $processed, $published) = @_;
-
-    my $query = $db->prepare(<<SQL);
-
-    UPDATE batches 
-        SET processed = $processed, on_datastore = $published 
-        WHERE batch_id = $batchId 
-        AND exp_id = $expId;
-SQL
-
-    $query->execute; # TODO check response of these
-}
-
-#######################################################################################
-#
-# Checks whether we have successfully processed this exposure 
-#
-########################################################################################
-sub isExposureAlreadyProcessed {
-    my ($db, $expId) = @_;
-
-    my $query = $db->prepare(<<SQL);
-
-SELECT COUNT(*) 
-    FROM batches 
-    WHERE exp_id = $expId 
-    AND processed = 1;
-
-SQL
-
-    $query->execute;
-
-    my $processed = $query->fetchrow_array();
-
-    if ($processed) {print "* Exposure ID '$expId' has already been processed\n";}
-
-    return $processed;
-}
-
-#######################################################################################
-#
-# Generates a new record in the database with a new jobId 
-#
-########################################################################################
-sub getNewBatchId {
-    my ($db, $expId,$surveyType) = @_;
-
-    my $query = $db->prepare(<<SQL);
-
-    SELECT batch_id 
-        FROM batches 
-        ORDER BY batch_id 
-        DESC LIMIT 1;
-SQL
-
-    $query->execute;
-
-    my $batchId = $query->fetchrow_array();
-
-    $batchId++;
-
-    $query = $db->prepare(<<SQL);
-    INSERT INTO batches
-        (batch_id, exp_id, survey_id)
-        VALUES
-        ($batchId, $expId, '$surveyType');
-
-SQL
-
-    $query->execute;
-
-    print "* New batch '$batchId' for exposure ID = $expId and survey = '$surveyType'\n";
-
-    return $batchId;
-}
-
-#######################################################################################
-#
-# run IppToPsps 
-#
-#######################################################################################
-sub runIppToPsps {
-    my ($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath, $batchType) = @_;
-
-    my $success = 0;
-
-    if ($batchType =~ /init/) {
-        $success = produceInit($batchOutputPath);
-    }
-    elsif ($batchType =~ /det/) {
-        $success = produceDetections($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath);
-    }
-    elsif ($batchType =~ /diff/) {
-        $success = produceDiffs($gpc1Db, $expId, $expName, $surveyType, $batchOutputPath);
-    }
-
-    return $success;
-}
-
-#######################################################################################
-#
-# tar 'n zip up a directory TODO handle errors
-#
-#######################################################################################
-sub tarAndZipDirectory {
-    my ($jobOutputPath,$batch) = @_;
-
-    my $batchDir = "$jobOutputPath/$batch";
-    my $batchTar = "$batchDir.tar";
-    my $batchZip = "$batchTar.gz";
-
-    # tar
-    my $command = "tar -cf $batchTar -C $jobOutputPath $batch";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if (!$success) { print "* Unable to tar up dir: $batch\n" and return 0 };
-
-    # zip
-    $command = "gzip -c $batchTar  > $batchZip";
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if (!$success) { print "* Unable to gzip: $batchTar\n" and return 0 };
-
-    $command = "rm $batchTar";
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if (!$success) { print "* Unable to remove: $batchTar\n" };
-
-    $command = "rm -r $batchDir";
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if (!$success) { print "* Unable to remove: $batch\n"};
-
-    return 1;
-}
-
-#######################################################################################
-#
-# writes a batch manifest file for the one FITS file that should be in the path passed in
-#
-#######################################################################################
-sub writeBatchManifest {
-
-    my ($path,$batch,$batchType) = @_;
-
-    use XML::Writer;
-    use Digest::MD5::File qw( file_md5_hex );
-
-    # sort out paths
-    my $outputPath = "$path/BatchManifest.xml";
-    my $output = new IO::File(">$outputPath");
-
-    # create a timestamp
-    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
-    my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec;
-
-    # determine batch 'type'
-    my $type;
-    if($batchType eq 'init') {$type = "IN";}
-    elsif($batchType eq 'det') {$type = "P2";}
-    elsif ($batchType eq 'stack') {$type = "ST";}
-    elsif ($batchType eq 'diff') {$type = "OB";}
-    else {$type = "UNKNOWN";}
-
-    # read results
-    open (MYFILE, $resultsFileName);
-    my $i = 0;
-
-    # detection results
-    my $minObjId;
-    my $maxObjId;
-    my $filename;
-    if($batchType eq 'det') {
-
-        while (<MYFILE>) {
-            chomp;
-            if ($i == 0) { $filename = $_; }
-            if ($i == 1) { $minObjId = $_; }
-            if ($i == 2) { $maxObjId = $_; }
-            $i++;
-        }
-    }
-    close (MYFILE);
-
-    # create XML file
-    my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
-    $writer->xmlDecl('UTF-8');
-    $writer->doctype('manifest', "", "psps-manifest.dtd");
-
-    # open directory to hunt for FITS files
-    opendir(DIR, $path) or print "* Cannot open '$path' to write batch manifest\n" and return 0;
-    my @thefiles= readdir(DIR);
-    closedir(DIR);
-
-    my $foundFits = 0;
-
-    foreach my $f (@thefiles) {
-
-        if ($f =~ /\.FITS/) {
-
-            if ($foundFits) { print "* More than one FITS file found for this batch\n"; return 0;}
-
-            # get md5sum of file
-            my $md5sum  = file_md5_hex("$path/$f");
-
-            # get size of file in bytes
-            my $filesize = -s "$path/$f";
-
-            # write manifest element TODO untidy
-            if($batchType eq 'det' && $filename eq $f) {
-
-                $writer->startTag('manifest',
-                        "name" => "$batch",
-                        "type" => $type,
-                        "timestamp" => "$timeStamp",
-                        "minObjId" => $minObjId,
-                        "maxObjId" => $maxObjId);
-            }
-            else {
-
-                $writer->startTag('manifest',
-                        "name" => "$batch",
-                        "type" => $type,
-                        "timestamp" => "$timeStamp");
-            }
-            # write the tag for this batch
-            $writer->startTag("file", 
-                    "name" => $f,
-                    "bytes" => $filesize,
-                    "md5" => $md5sum);
-
-            #$writer->startTag('file' );
-            #$writer->startTag('name'); $writer->characters("$f"); $writer->endTag();
-            #$writer->startTag('bytes'); $writer->characters("$filesize"); $writer->endTag();
-            #$writer->startTag('md5'); $writer->characters("$md5sum"); $writer->endTag();
-
-            #if($batchType eq 'det' && $filename eq $f) {
-
-                #    $writer->startTag('minObjId'); $writer->characters("$minObjId"); $writer->endTag();
-                #    $writer->startTag('maxObjId'); $writer->characters("$maxObjId"); $writer->endTag();
-                #}
-            $writer->endTag();
-            # finish up
-            $writer->endTag();
-            $writer->end();
-            $foundFits = 1;
-        }
-
-    }
-
-
-    if (!$foundFits) { print "* Could not find any FITS files for this batch\n"; return 0;}
-    return 1;
-}
-
-#######################################################################################
-#
-# write job manifest
-#
-#######################################################################################
-sub writeJobManifest {
-    my ($path, $job, $surveyType) = @_;
-
-    use XML::Writer;
-    use Digest::MD5::File qw( file_md5_hex );
-
-    # sort out paths
-    my $outputPath = "$path/$job/JobManifest.xml";
-    my $output = new IO::File(">$outputPath");
-
-    # create a timestamp
-    my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)=localtime(time);
-    my $timeStamp = sprintf "%4d-%02d-%02d %02d:%02d:%02d", $year+1900,$mon+1,$mday,$hour,$min,$sec;
-
-    # create XML file
-    my $writer = new XML::Writer(OUTPUT => $output, DATA_MODE => 1, DATA_INDENT=>2);
-    $writer->xmlDecl('UTF-8');
-    $writer->doctype('manifest', "", "psps-manifest.dtd");
-    $writer->startTag('manifest',
-            "name" => "$job",
-            "type" => $surveyType,
-            "timestamp" => "$timeStamp");
-
-    my $jobPath = "$path/$job";
-
-    # open directory to hunt for FITS files
-    opendir(DIR, $jobPath) or print "* Cannot open '$path' to write job manifest\n" and return 0;
-    my @thefiles= readdir(DIR);
-    closedir(DIR);
-
-    my $foundBatch = 0;
-
-    foreach my $f (@thefiles) {
-
-        if (($f =~ /\.tar\.gz/)) {
-
-            my $filePath = "$jobPath/$f";
-
-            # get md5sum of file
-            my $md5sum  = file_md5_hex($filePath);
-
-            # get size of file in bytes
-            my $filesize = -s $filePath;
-
-            # write the tag for this batch
-            $writer->startTag('file',
-                    "name" => $f,
-                    "bytes" => "$filesize",
-                    "md5" => "$md5sum",
-                    );
-            $writer->endTag();
-            $foundBatch = 1;
-        }
-    }
-
-    # finish up
-    $writer->endTag();
-    $writer->end();
-
-    if (!$foundBatch) { print "* Could not find any batches for this job\n"; return 0;}
-    return 1;
-}
-
-#######################################################################################
-#
-# register new job with the datastore
-#
-########################################################################################
-sub publishToDatastore {
-    my ($path, $job) = @_;
-
-    my $fullJobPath = "$path/$job";
-
-    my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_dsregList.XXXX", UNLINK => !$save_temps);
-
-    # loop through all batch files in this job directory
-    opendir(DIR, $fullJobPath) or print "* Cannot open '$fullJobPath' in order to publish to datastore\n" and return 0;
-    my @thefiles= readdir(DIR);
-    closedir(DIR);
-
-    my $foundBatch = 0;
-
-    foreach my $f (@thefiles) {
-
-        # get all batch tgz files
-        if ($f =~ /\.tar.gz/) {print $tempFile $f . '|||tgz' . "\n"; $foundBatch = 1;}
-
-        # get JobManifest XML file
-        elsif ($f =~ /\.xml/) {print $tempFile $f . '|||xml' . "\n";}
-    }
-
-    if (!$foundBatch) { print "Could not find any batches to publish\n"; return 0;}
-
-    # build dsreg command command
-    my $command  = "$dsreg";
-    $command .= " --add $job";
-    $command .= " --copy";
-    $command .= " --datapath $fullJobPath";
-    $command .= " --type IPP_PSPS";
-    $command .= " --product $datastoreProduct";
-    $command .= " --list $tempName";
-
-    # run command
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if (!$success) { print "* Unable to publish $job to datastore\n" and return 0 };
-
-    print "* Successfully published $job to datastore\n";
-
-    close($tempFile);
-
-    return 1;
-}
-
-#######################################################################################
-# 
-# runs ippToPsps to produce init batch 
-#
-#######################################################################################
-sub produceInit {
-    my ($outputPath) = @_;
-
-    my $success = runProgram("NULL", $outputPath, 0, "NULL", "NULL", $batchType);
-    return $success;
-}
-#######################################################################################
-# 
-# runs ippToPsps for detections for this exposure 
-#
-#######################################################################################
-sub produceDetections {
-    my ($gpc1Db, $expId, $expName, $surveyType, $outputPath) = @_;
-
-    # query to retrieve nebulous key of camera smf file for this exposure
-    my $query = 
-        "SELECT camProcessedExp.path_base ".
-        "FROM warpRun ".
-        "JOIN fakeRun USING(fake_id) ".
-        "JOIN camRun USING(cam_id) ".
-        "JOIN camProcessedExp USING(cam_id) ".
-        "JOIN chipRun USING(chip_id) ".
-        "JOIN rawExp USING (exp_id) ".
-        "WHERE rawExp.exp_id = $expId AND camRun.magicked ".
-        "ORDER BY camRun.cam_id DESC LIMIT 1";  # make sure we get most recently processed
-
-    if ($verbose) { print"* $query\n";}
-
-
-    # execute query and check results
-    my $results = $gpc1Db->selectall_arrayref( $query );
-    my $size = @$results;
-    if ($size < 1) {print "* No smf files found for this exposure\n"; return 0;}
-
-    # loop round db results TODO should only be one result
-    my $success = 0;
-    for my $row (@$results) {
-
-        # get real filename from neb 'key'
-        my ($nebPath) = @$row;
-        my $fpaObjects = $ipprc->filename("PSASTRO.OUTPUT",$nebPath) or return 0;
-        my $realFile = $ipprc->file_resolve($fpaObjects) or return 0;
-
-        # now write the path to this file out to temp file
-        my ($tempFile, $tempName) = tempfile( "/tmp/ippToPsps_inputFileList.XXXX", UNLINK => !$save_temps);
-        print $tempFile $realFile . "\n";
-        close $tempFile;
-
-        $success = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType);
-        return $success;
-    }
-}
-
-#######################################################################################
-# 
-# runs ippToPsps for diffs for this exposure 
-#
-#######################################################################################
-sub produceDiffs {
-    my ($gpc1Db, $expId, $expName, $surveyType, $outputPath) = @_;
-
-    my $query = 
-        "SELECT DISTINCT diffSkyfile.path_base ".
-        "FROM warpRun ".
-        "JOIN fakeRun USING(fake_id) ".
-        "JOIN camRun USING(cam_id) ".
-        "JOIN chipRun USING(chip_id) ".
-        "JOIN rawExp USING (exp_id) ".
-        "JOIN diffInputSkyfile ON (warpRun.warp_id = diffInputSkyfile.warp1 OR warpRun.warp_id = diffInputSkyfile.warp2) ".
-        "JOIN diffSkyfile USING (diff_id) ".
-        "WHERE rawExp.exp_id = $expId AND camRun.magicked";
-
-    if ($verbose) { print"* $query\n";}
-
-    # execute query and check results
-    my $results = $gpc1Db->selectall_arrayref( $query );
-    my $size = @$results;
-    if ($size < 1) {print "* No cmf files found for this exposure\n"; return 0;}
-
-    my ($tempFile, $tempName) = tempfile( "inputFileList.XXXX", UNLINK => !$save_temps);
-
-    # loop round db results should be one file per skycell
-    my $success = 0;
-    for my $row (@$results) {
-
-        my ($nebPath) = @$row;
-        #print "NEB PATH $nebPath\n";
-
-        # get real filename from neb 'key'
-        my $fpaObjects = $ipprc->filename("PPSUB.OUTPUT.SOURCES",$nebPath) or return 0;
-        my $realFile = $ipprc->file_resolve($fpaObjects) or return 0;
-        #print "REALFILE $realFile\n";
-
-        # now write the path to this file out to temp file
-        print $tempFile $realFile . "\n";
-    }
-
-    my $ret = runProgram($tempName, $outputPath, $expId, $expName, $surveyType, $batchType);
-
-    close $tempFile;
-
-    return $ret;
-}
-
-#######################################################################################
-#
-# runs the ippToPsps program
-#
-#######################################################################################
-sub runProgram {
-    my ($input, $output, $expid, $expName, $surveyType, $batchType) = @_;
-
-    # build command
-    my $command = "$ippToPsps"; 
-    $command .= " -input $input";
-    $command .= " -output $output";
-    $command .= " -D CATDIR $dvodb";
-    $command .= " -config config"; # TODO
-    $command .= " -expid $expid";
-    $command .= " -expname $expName";
-    $command .= " -survey $surveyType";
-    $command .= " -batch $batchType";
-    $command .= " -results $resultsFileName";
-
-    # run command
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf )
-        = run(command => $command, verbose => $verbose);
-
-    return $success;
-}
-
-
-
Index: /branches/eam_branches/ipp-20100621/ippToPsps/scripts/pspsSchema2xml.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/scripts/pspsSchema2xml.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/scripts/pspsSchema2xml.pl	(revision 28794)
@@ -31,5 +31,5 @@
         -msg => "\n   Required options:\n\n".
         "--schema <path to PSPS schema>\n".
-        "--type <init|det|diff|stack>\n".
+        "--type <init|det|diff|stack|object>\n".
         -exitval => 3
         ) unless
@@ -37,5 +37,5 @@
 defined $type;
 
-if ($type ne "init" && $type ne "det" && $type ne "diff" && $type ne "stack") {
+if ($type ne "init" && $type ne "det" && $type ne "diff" && $type ne "stack" && $type ne "object" ) {
 
     print "Don't understand type '$type'\n"; 
@@ -60,4 +60,5 @@
 elsif ($type eq "diff") {createDiffs();}
 elsif ($type eq "stack") {createStacks();}
+elsif ($type eq "object") {createObjects();}
 
 # finish up XML
@@ -170,4 +171,14 @@
 #######################################################################################
 #
+# Creates object batch tables
+#
+#######################################################################################
+sub createObjects {
+
+    parseTable("Object");
+}
+
+#######################################################################################
+#
 # Parses a particular table from the SQL file and converts it to an XML description 
 #
Index: /branches/eam_branches/ipp-20100621/ippToPsps/scripts/removeFromDatastore.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/scripts/removeFromDatastore.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/scripts/removeFromDatastore.pl	(revision 28794)
@@ -19,5 +19,5 @@
 pod2usage(
         -msg => "\n   Required options:\n\n".
-        "--start <jobStartNumber> [--end <jobEndNumber>]\n".
+        "--start <batchStartNumber> [--end <batchEndNumber>]\n".
         "--product <datastoreProduct>\n",
         -exitval => 3
@@ -33,6 +33,6 @@
 for ($i=$start; $i<$end+1; $i++) {
 
-    my $job = sprintf("J%06d", $i);
-    my $command  = "dsreg --del $job --product $product";
+    my $tarball = sprintf("B%08d", $i);
+    my $command  = "dsreg --del $tarball --product $product";
 
     run(command => $command, verbose => 1);
Index: /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPsps.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPsps.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPsps.c	(revision 28794)
@@ -53,4 +53,14 @@
     }
 
+    // save XML document for results
+    if (this->resultsXmlDoc) {
+    
+        xmlSaveFormatFileEnc(this->resultsPath, this->resultsXmlDoc, "UTF-8", 1);
+
+        xmlFreeDoc(this->resultsXmlDoc);
+        xmlCleanupParser();
+        xmlMemoryDump();
+    }
+
     psFree(this->fitsInPath);
     psFree(this->resultsPath);
@@ -59,5 +69,4 @@
     psFree(this->pmconfig);
 
-    if (this->resultsFile) fclose(this->resultsFile);
 
     for(uint32_t i=0; i<this->numOfInputFiles; i++)
@@ -246,5 +255,5 @@
     this->fitsInPath = NULL;
     this->resultsPath = NULL;
-    this->resultsFile = NULL;
+    this->resultsXmlDoc = NULL;
     this->numOfInputFiles = 0;
     this->inputFiles = NULL;
@@ -304,4 +313,5 @@
     }
 
+    // create a config object
     this->config = ippToPspsConfig_Constructor(this->configsDir);
     if (this->config == NULL) {
@@ -311,18 +321,11 @@
     }
 
-    // ready results file, if necessary
+    // create XML document for results 
     if (this->resultsPath) {
 
-        this->resultsFile = fopen (this->resultsPath, "w+");
-
-        if (this->resultsFile == NULL) {
-
-            psError(PS_ERR_UNKNOWN, false, "Unable to open results file at %s", this->resultsPath);
-            // not essential to write results, so don't bail out
-        }
-        else {
-            if (fprintf(this->resultsFile, "%s\n", outputName) < 1 )     
-                psError(PS_ERR_IO, false, "Unable to write FITS output filename to'%s'\n", this->resultsPath);
-        }
+        this->resultsXmlDoc = xmlNewDoc(BAD_CAST "1.0");
+        xmlNodePtr rootNode = xmlNewNode(NULL, BAD_CAST "ippToPsps_Results");
+        xmlDocSetRootElement(this->resultsXmlDoc, rootNode);
+        xmlNewChild(rootNode, NULL, BAD_CAST "filename", BAD_CAST outputName);
     }
 
Index: /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPsps.h
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPsps.h	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPsps.h	(revision 28794)
@@ -15,4 +15,6 @@
 #include <dvo_util.h>
 #include "ippToPspsConfig.h"
+#include <libxml/parser.h>
+#include <libxml/tree.h>
 
 #define MAXDETECT 30000 //TODO limit ok?
@@ -20,20 +22,20 @@
 typedef struct {
 
-    uint32_t expId;            // the exposure ID to be used
-    psString expName;          // the exposure name
-    psString surveyType;       // the survey type, eg 3PI, MD01, STS, SSS
-    uint8_t batchType;         // PSPS batch type
-    psString fitsInPath;       // path to FITS input
-    psString resultsPath;      // path to results file
-    FILE* resultsFile;         // file for results  
-    uint16_t numOfInputFiles;  // number of input files
-    char** inputFiles;         // array of input file names
-    psString fitsOutPath;      // path to FITS output
-    fitsfile *fitsOut;         // output FITS file
-    psString configsDir;       // path to IPP/PSPS mapping file
-    pmConfig* pmconfig;        // pmConfig
-    dvoConfig* dvoConfig;      // dvo database
-    IppToPspsConfig* config;   // config structure
-    int exitCode;              // ps exit code
+    uint32_t expId;             // the exposure ID to be used
+    psString expName;           // the exposure name
+    psString surveyType;        // the survey type, eg 3PI, MD01, STS, SSS
+    uint8_t batchType;          // PSPS batch type
+    psString fitsInPath;        // path to FITS input
+    psString resultsPath;       // path to results file
+    xmlDocPtr resultsXmlDoc;    // pointer to XML document for results
+    uint16_t numOfInputFiles;   // number of input files
+    char** inputFiles;          // array of input file names
+    psString fitsOutPath;       // path to FITS output
+    fitsfile *fitsOut;          // output FITS file
+    psString configsDir;        // path to IPP/PSPS mapping file
+    pmConfig* pmconfig;         // pmConfig
+    dvoConfig* dvoConfig;       // dvo database
+    IppToPspsConfig* config;    // config structure
+    int exitCode;               // ps exit code
 
     int (*run)();
Index: /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsBatchDetection.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsBatchDetection.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsBatchDetection.c	(revision 28794)
@@ -14,8 +14,12 @@
 
 // Gets uncalibrated instrumental flux from magnitude
-static __inline bool ippToPsps_getFlux(const float zeroPoint, const float exposureTime, const float magnitude, float* flux) {
+static __inline bool ippToPsps_getFlux(const float exposureTime, const float magnitude, float* flux, const float magnitudeErr, float* fluxErr) {
 
     *flux = powf(10.0, -0.4*magnitude) / exposureTime;
-    return (!isfinite(*flux) || *flux < 0.000001) ? false : true;
+    if (!isfinite(*flux) || *flux < 0.000001) return false;
+    if (fluxErr) *fluxErr = fabsf((magnitudeErr * *flux)/1.085736);
+    //  if (fluxErr)    printf("Mag = %03.03f, Flux = %03.03f, Mag err = %03.03f, Flux Err = %03.03f\n", magnitude, *flux, magnitudeErr, *fluxErr);
+         
+    return true;
 }
 
@@ -46,5 +50,5 @@
     fits_get_hdrspace(fitsIn, &nKeys, NULL, &status);
 
-    float zptObs, zeroPoint, exposureTime;
+    float zptObs, exposureTime;
     char filterType[20];
     double obsTime;
@@ -52,10 +56,9 @@
     double expTime;
     status=0; fits_read_key(fitsIn, TFLOAT, "ZPT_OBS", &zptObs, NULL, &status);
-    status=0; fits_read_key(fitsIn, TFLOAT, "ZPT_REF", &zeroPoint, NULL, &status);
     status=0; fits_read_key(fitsIn, TFLOAT, "EXPREQ", &exposureTime, NULL, &status);
     status=0; fits_read_key(fitsIn, TSTRING, "FILTERID", filterType, NULL, &status);
     status=0; fits_read_key(fitsIn, TDOUBLE, "MJD-OBS", &expStart, NULL, &status);
     status=0; fits_read_key(fitsIn, TDOUBLE, "EXPTIME", &expTime, NULL, &status);
-    obsTime = expStart + (expTime/172800); // exp start plus half exp time (converted from secs to days)
+    obsTime = expStart + (expTime/172800.0); // exp start plus half exp time (converted from secs to days)
    
     ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, 1, "FrameMeta", true);
@@ -83,12 +86,6 @@
 
     // stuff to keep from psf.hdr header
-    int sourceId = -1;
-    int imageId = -1;
-    float fwhmMaj;
-    float fwhmMin;
-    float momentMaj;
-    float momentMin;
-    float psfFwhm;
-    float momentFwhm;
+    int sourceId = -1, imageId = -1;
+    float fwhmMaj, fwhmMin, momentMaj, momentMin, psfFwhm, momentFwhm;
     long pspsImageId = -1;
 
@@ -96,6 +93,5 @@
     SkyList *skylist = NULL;
     dvoDetection *dvoDetections = NULL;
-    int maxDvoDetId = -1;
-    int numDvoDetections = -1;
+    int maxDvoDetId = -1, numDvoDetections = -1;
     Image *image = NULL;
 
@@ -108,9 +104,5 @@
     strftime(timeStr, sizeof(timeStr), "%Y-%m-%d", ts);
 
-    uint32_t totalDetections = 0;
-    uint32_t s,d;
-    uint32_t invalidDvoRows;
-    uint32_t smfJumps;
-    uint32_t unmatched;
+    uint32_t s,d, invalidDvoRows, smfJumps, unmatched, totalDetections = 0;
 
     long longnull = -999;
@@ -118,23 +110,14 @@
     int anynull = 0;
 
-    char ccdNumber[3];
-    char extensionName[15];
-
+    char ccdNumber[3], extensionName[15];
+
+    // for storing FITS column data
     long ippIDet[MAXDETECT];
-    float instMag[MAXDETECT];
-    float instMagErr[MAXDETECT];
-    float peakMag[MAXDETECT];
+    float instMag[MAXDETECT], instMagErr[MAXDETECT], peakMag[MAXDETECT];
     uint64_t flags[MAXDETECT];
-    long objID[MAXDETECT];
-    long detectID[MAXDETECT];
-    long ippObjID[MAXDETECT];
-    long ippDetectID[MAXDETECT];
-    long imageID[MAXDETECT];
+    long objID[MAXDETECT], detectID[MAXDETECT], ippObjID[MAXDETECT], ippDetectID[MAXDETECT], imageID[MAXDETECT];
     double obsTimes[MAXDETECT];
-    float instFlux[MAXDETECT];
-    float instFluxErr[MAXDETECT];
-    float peakFlux[MAXDETECT];
-    int8_t filterIDs[MAXDETECT];
-    int8_t surveyIDs[MAXDETECT];
+    float instFlux[MAXDETECT], instFluxErr[MAXDETECT], peakFlux[MAXDETECT];
+    int8_t filterIDs[MAXDETECT], surveyIDs[MAXDETECT];
 
     char** assocDate = (char**)calloc(MAXDETECT, sizeof(char**));
@@ -149,16 +132,15 @@
     }
 
-    long maxObjID = LONG_MIN; 
-    long minObjID = LONG_MAX;
+    long maxObjID = LONG_MIN, minObjID = LONG_MAX;
     uint64_t imageFlags;
     short nOta = 0;
     long i;
     bool isDuplicate;
-    uint32_t numOfDuplicates;
-    uint32_t numInvalidFlux;
-    long numDetectionsOut;
-    long totalDetectionsOut = 0;
+    uint32_t numOfDuplicates, numInvalidFlux;
+    long numDetectionsOut, totalDetectionsOut = 0, numPhotoRef, totalNumPhotoRef = 0;
     long removeList[MAXDETECT];
     long thisObjId;
+    bool firstTimeIn = true, peakFluxOk, instFluxOk;
+    int ippIDetNum, instMagNum, instMagErrNum, peakMagNum;
 
     // loop round all 60 chips
@@ -197,4 +179,6 @@
             status=0; fits_read_key(fitsIn, TLONG, "IMAGEID", &imageId, NULL, &status);
             status=0; fits_read_key(fitsIn, TLONG, "SOURCEID", &sourceId, NULL, &status);
+            status=0; fits_read_key(fitsIn, TLONG, "NASTRO", &numPhotoRef, NULL, &status);
+            totalNumPhotoRef += numPhotoRef; // total up for storing in FrameMeta
 
             // access DVO database
@@ -202,5 +186,5 @@
             if (skylist == NULL) {
                 psError(PS_ERR_IO, false, 
-                        "DVO: can't find SkyList for sourceId='%d' imageId='%d' (CCD = XY%s): skipping\n", 
+                        "DVO: can't find SkyList for sourceId='%d' imageId='%d' (CCD = XY%s): skipping this chip\n",
                         sourceId, imageId, ccdNumber);
                 continue;
@@ -251,9 +235,24 @@
             s = d = totalDetections = invalidDvoRows = smfJumps = unmatched = 0;
 
+            // determine column numbers for certain IPP detection columns
+            if (firstTimeIn) {
+
+                status=0;fits_get_colnum(fitsIn, CASESEN, "IPP_IDET", &ippIDetNum, &status);
+                if (status) psError(PS_ERR_IO, false, "Unable to read col num for IPP_IDET");
+                status=0;fits_get_colnum(fitsIn, CASESEN, "PSF_INST_MAG", &instMagNum, &status);
+                if (status) psError(PS_ERR_IO, false, "Unable to read col num for PSF_INST_MAG");
+                status=0;fits_get_colnum(fitsIn, CASESEN, "PSF_INST_MAG_SIG", &instMagErrNum, &status);
+                if (status) psError(PS_ERR_IO, false, "Unable to read col num for PSF_INST_MAG_SIG");
+                status=0;fits_get_colnum(fitsIn, CASESEN, "PEAK_FLUX_AS_MAG", &peakMagNum, &status);
+                if (status) psError(PS_ERR_IO, false, "Unable to read col num for PEAK_FLUX_AS_MAG");
+
+                firstTimeIn=false;
+            }
+
             anynull = 0;
-            fits_read_col(fitsIn, TLONG, 1, 1, 1, nDet, &longnull, ippIDet, &anynull, &status);
-            fits_read_col(fitsIn, TFLOAT, 8, 1, 1, nDet, &floatnull, instMag, &anynull, &status);
-            fits_read_col(fitsIn, TFLOAT, 9, 1, 1, nDet, &floatnull, instMagErr, &anynull, &status);
-            fits_read_col(fitsIn, TFLOAT, 12, 1, 1, nDet, &floatnull, peakMag, &anynull, &status);
+            fits_read_col(fitsIn, TLONG, ippIDetNum, 1, 1, nDet, &longnull, ippIDet, &anynull, &status);
+            fits_read_col(fitsIn, TFLOAT, instMagNum, 1, 1, nDet, &floatnull, instMag, &anynull, &status);
+            fits_read_col(fitsIn, TFLOAT, instMagErrNum, 1, 1, nDet, &floatnull, instMagErr, &anynull, &status);
+            fits_read_col(fitsIn, TFLOAT, peakMagNum, 1, 1, nDet, &floatnull, peakMag, &anynull, &status);
 
             // DVO detections are ordered by IPP_IDET, which increments from 0 in SMF table
@@ -297,9 +296,10 @@
                     obsTimes[s] = obsTime;
 
-                    ippToPsps_getFlux(zeroPoint, exposureTime, instMagErr[s], &instFluxErr[s]);
+                    // calculate flux and flux errors from magnitudes
+                    peakFluxOk = ippToPsps_getFlux(exposureTime, peakMag[s], &peakFlux[s], 0.0, NULL);
+                    instFluxOk = ippToPsps_getFlux(exposureTime, instMag[s], &instFlux[s], instMagErr[s], &instFluxErr[s]);
 
                     // check for invalid flux values (if not already labelled as a duplicate)
-                    if ((!ippToPsps_getFlux(zeroPoint, exposureTime, peakMag[s], &peakFlux[s]) ||
-                                !ippToPsps_getFlux(zeroPoint, exposureTime, instMag[s], &instFlux[s])) && !isDuplicate) {
+                    if ( (!peakFluxOk || !instFluxOk) && !isDuplicate) {
                         removeList[numOfDuplicates+numInvalidFlux] = s+1;
                         numInvalidFlux++;
@@ -379,19 +379,22 @@
     if (fits_movnam_hdu(this->fitsOut, BINARY_TBL, "FrameMeta", 0, &status))
         psError(PS_ERR_IO, false, "Can't move back to FrameMeta extension to write number of OTAs\n");
-    else 
+    else { 
         fits_write_col(this->fitsOut, TSHORT, FRAMEMETA_NOTA, 1, 1, 1, &nOta, &status);
-
+        fits_write_col(this->fitsOut, TSHORT, FRAMEMETA_NUMPHOTOREF, 1, 1, 1, &totalNumPhotoRef, &status);
+    }
     status=0;
     if (fits_close_file(fitsIn, &status)) fits_report_error(stderr, status);
 
     // write results
-    if (this->resultsFile) {
-
-        if (fprintf(this->resultsFile, "%ld\n", minObjID) < 1 ) 
-            psError(PS_ERR_IO, false, "Unable to write min Object ID to'%s'\n", this->resultsPath);
-        if (fprintf(this->resultsFile, "%ld\n", maxObjID) < 1 ) 
-            psError(PS_ERR_IO, false, "Unable to write max Object ID to'%s'\n", this->resultsPath);
-        if (fprintf(this->resultsFile, "%ld\n", totalDetectionsOut) < 1 ) 
-            psError(PS_ERR_IO, false, "Unable to write totalDetectionsOut to'%s'\n", this->resultsPath);
+    if (this->resultsXmlDoc) {
+
+        xmlNodePtr rootNode = xmlDocGetRootElement(this->resultsXmlDoc);
+        char tmp[100];
+        sprintf(tmp, "%ld", minObjID);
+        xmlNewChild(rootNode, NULL, BAD_CAST "minObjID", BAD_CAST tmp);
+        sprintf(tmp, "%ld", maxObjID);
+        xmlNewChild(rootNode, NULL, BAD_CAST "maxObjID", BAD_CAST tmp);
+        sprintf(tmp, "%ld", totalDetectionsOut);
+        xmlNewChild(rootNode, NULL, BAD_CAST "totalDetections", BAD_CAST tmp);
     }
 
@@ -403,3 +406,2 @@
 }
 
-
Index: /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsBatchTest.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsBatchTest.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsBatchTest.c	(revision 28794)
@@ -13,15 +13,13 @@
 #include "fitsio.h"
 
-// Gets flux from magnitude
-static __inline bool ippToPsps_getFlux(const float zeroPoint, const float exposureTime, const float magnitude, float* flux) {
-
-    // use uncalibrated instrumental flux
+// Gets uncalibrated instrumental flux from magnitude
+static __inline bool ippToPsps_getFlux(const float exposureTime, const float magnitude, float* flux, const float magnitudeErr, float* fluxErr) {
+
     *flux = powf(10.0, -0.4*magnitude) / exposureTime;
-
-    // use calibrated flux in Janskys, where 3631 Jy is the zero point flux for the filter: constant over all filters because we're using AB magnitudes
-    //float flux = 3631 * powf(10.0, -0.4*(magnitude + zeroPoint)) / exposureTime;
-
-    //    printf("mag=%f\texpTime=%f\tzeroPoint=%f\tflux=%f\n", magnitude, exposureTime, zeroPoint, flux);
-    return (!isfinite(*flux) || *flux < 0.000001) ? false : true;
+    if (!isfinite(*flux) || *flux < 0.000001) return false;
+    if (fluxErr) *fluxErr = fabsf((magnitudeErr * *flux)/1.085736);
+    //  if (fluxErr)    printf("Mag = %03.03f, Flux = %03.03f, Mag err = %03.03f, Flux Err = %03.03f\n", magnitude, *flux, magnitudeErr, *fluxErr);
+
+    return true;
 }
 
@@ -29,5 +27,5 @@
   \brief test data for checking PSPS values correspond with IPP 
 
-  */
+*/
 int ippToPsps_batchTest(IppToPsps *this) {
 
@@ -53,13 +51,13 @@
     status=0; fits_read_key(fitsIn, TFLOAT, "EXPREQ", &exposureTime, NULL, &status);
     status=0; fits_read_key(fitsIn, TSTRING, "FILTERID", filterType, NULL, &status);
-    
-    ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, 1, "FrameMeta", true);
+
+//    ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, 1, "FrameMeta", true);
 
     // FrameMeta values
-    fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->expId, &status);
+//    fits_write_col(this->fitsOut, TLONG, FRAMEMETA_FRAMEID, 1, 1, 1, &this->expId, &status);
 
     int8_t filterID = -1;
     ippToPspsConfig_getFilterId(this->config, filterType, &filterID);
-    fits_write_col(this->fitsOut, TBYTE, FRAMEMETA_FILTERID, 1, 1, 1, &filterID, &status);
+//    fits_write_col(this->fitsOut, TBYTE, FRAMEMETA_FILTERID, 1, 1, 1, &filterID, &status);
 
     // stuff to keep from psf.hdr header
@@ -107,4 +105,5 @@
     float instMagErr[MAXDETECT];
     float peakMag[MAXDETECT];
+    uint64_t flags[MAXDETECT];
     long objID[MAXDETECT];
     long detectID[MAXDETECT];
@@ -130,4 +129,5 @@
     long maxObjID = LONG_MIN; 
     long minObjID = LONG_MAX;
+    uint64_t imageFlags;
     short nOta = 0;
     long i;
@@ -138,8 +138,10 @@
     long removeList[MAXDETECT];
     long thisObjId;
+    bool firstTimeIn = true, peakFluxOk, instFluxOk;
+    int ippIDetNum, instMagNum, instMagErrNum, peakMagNum;
 
     // loop round all 60 chips
-    for (int x=3; x<4; x++) {
-        for (int y=3; y<4; y++) {
+    for (int x=0; x<8; x++) {
+        for (int y=0; y<8; y++) {
 
             // dodge the corners
@@ -194,12 +196,14 @@
 
             // create ImageMeta
-            ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, 1, "ImageMeta", true);
+    //        ippToPspsConfig_writeTable(this->config, fitsIn, this->fitsOut, 1, "ImageMeta", true);
             psfFwhm = (fwhmMaj+fwhmMin)/2;
             momentFwhm = (momentMaj+momentMin)/2;
-            fits_write_col(this->fitsOut, TLONG, IMAGEMETA_PHOTOCALID, 1, 1, 1, &pImage->photcode, &status); 
-            fits_write_col(this->fitsOut, TBYTE, IMAGEMETA_FILTERID, 1, 1, 1, &filterID, &status);
-            fits_write_col(this->fitsOut, TFLOAT, IMAGEMETA_PHOTOSCAT, 1, 1, 1, &zptObs, &status);
-            fits_write_col(this->fitsOut, TFLOAT, IMAGEMETA_PSFFWHM, 1, 1, 1, &psfFwhm, &status);
-            fits_write_col(this->fitsOut, TFLOAT, IMAGEMETA_PHOTOZERO, 1, 1, 1, &zptObs, &status);
+            imageFlags = (uint64_t)pImage->flags;
+      //      fits_write_col(this->fitsOut, TLONG, IMAGEMETA_PHOTOCALID, 1, 1, 1, &pImage->photcode, &status); 
+        //    fits_write_col(this->fitsOut, TBYTE, IMAGEMETA_FILTERID, 1, 1, 1, &filterID, &status);
+        //    fits_write_col(this->fitsOut, TFLOAT, IMAGEMETA_PHOTOSCAT, 1, 1, 1, &zptObs, &status);
+        //    fits_write_col(this->fitsOut, TFLOAT, IMAGEMETA_PSFFWHM, 1, 1, 1, &psfFwhm, &status);
+        //    fits_write_col(this->fitsOut, TFLOAT, IMAGEMETA_PHOTOZERO, 1, 1, 1, &zptObs, &status);
+        //    fits_write_col(this->fitsOut, TLONGLONG, IMAGEMETA_QAFLAGS, 1, 1, 1, &imageFlags, &status);
 
             // now move BACK to detections table in smf
@@ -222,9 +226,24 @@
             s = d = totalDetections = invalidDvoRows = smfJumps = unmatched = 0;
 
+            // determine column numbers for certain IPP detection columns
+            if (firstTimeIn) {
+
+                status=0;fits_get_colnum(fitsIn, CASESEN, "IPP_IDET", &ippIDetNum, &status);
+                if (status) psError(PS_ERR_IO, false, "Unable to read col num for IPP_IDET");
+                status=0;fits_get_colnum(fitsIn, CASESEN, "PSF_INST_MAG", &instMagNum, &status);
+                if (status) psError(PS_ERR_IO, false, "Unable to read col num for PSF_INST_MAG");
+                status=0;fits_get_colnum(fitsIn, CASESEN, "PSF_INST_MAG_SIG", &instMagErrNum, &status);
+                if (status) psError(PS_ERR_IO, false, "Unable to read col num for PSF_INST_MAG_SIG");
+                status=0;fits_get_colnum(fitsIn, CASESEN, "PEAK_FLUX_AS_MAG", &peakMagNum, &status);
+                if (status) psError(PS_ERR_IO, false, "Unable to read col num for PEAK_FLUX_AS_MAG");
+
+                firstTimeIn=false;
+            }
+
             anynull = 0;
-            fits_read_col(fitsIn, TLONG, 1, 1, 1, nDet, &longnull, ippIDet, &anynull, &status);
-            fits_read_col(fitsIn, TFLOAT, 8, 1, 1, nDet, &floatnull, instMag, &anynull, &status);
-            fits_read_col(fitsIn, TFLOAT, 9, 1, 1, nDet, &floatnull, instMagErr, &anynull, &status);
-            fits_read_col(fitsIn, TFLOAT, 12, 1, 1, nDet, &floatnull, peakMag, &anynull, &status);
+            fits_read_col(fitsIn, TLONG, ippIDetNum, 1, 1, nDet, &longnull, ippIDet, &anynull, &status);
+            fits_read_col(fitsIn, TFLOAT, instMagNum, 1, 1, nDet, &floatnull, instMag, &anynull, &status);
+            fits_read_col(fitsIn, TFLOAT, instMagErrNum, 1, 1, nDet, &floatnull, instMagErr, &anynull, &status);
+            fits_read_col(fitsIn, TFLOAT, peakMagNum, 1, 1, nDet, &floatnull, peakMag, &anynull, &status);
 
             // DVO detections are ordered by IPP_IDET, which increments from 0 in SMF table
@@ -262,12 +281,13 @@
                     ippObjID[s] = (uint64_t)dvoDetections[d].ave.catID*1000000000 + (uint64_t)dvoDetections[d].ave.objID;
                     ippDetectID[s] = dvoDetections[d].meas.detID;
+                    flags[s] = ((uint64_t)dvoDetections[d].meas.dbFlags << 32) | (uint64_t)dvoDetections[d].meas.photFlags;
                     imageID[s] = pspsImageId;
                     obsTimes[s] = obsTime;
 
-                    ippToPsps_getFlux(zeroPoint, exposureTime, instMagErr[s], &instFluxErr[s]);
+                    peakFluxOk = ippToPsps_getFlux(exposureTime, peakMag[s], &peakFlux[s], 0.0, NULL);
+                    instFluxOk = ippToPsps_getFlux(exposureTime, instMag[s], &instFlux[s], instMagErr[s], &instFluxErr[s]);
 
                     // check for invalid flux values (if not already labelled as a duplicate)
-                    if ((!ippToPsps_getFlux(zeroPoint, exposureTime, peakMag[s], &peakFlux[s]) ||
-                                !ippToPsps_getFlux(zeroPoint, exposureTime, instMag[s], &instFlux[s])) && !isDuplicate) {
+                    if ( (!peakFluxOk || !instFluxOk) && !isDuplicate) {
                         removeList[numOfDuplicates+numInvalidFlux] = s+1;
                         numInvalidFlux++;
@@ -292,5 +312,5 @@
             // write number of rows (detections) to ImageMeta
             numDetectionsOut = totalDetections - numOfDuplicates - numInvalidFlux;
-            fits_write_col(this->fitsOut, TLONG, IMAGEMETA_NDETECT, 1, 1, 1, &numDetectionsOut, &status);
+          //  fits_write_col(this->fitsOut, TLONG, IMAGEMETA_NDETECT, 1, 1, 1, &numDetectionsOut, &status);
 
             // detections
@@ -301,7 +321,9 @@
             fits_write_col(this->fitsOut, TLONGLONG, DETECTION_IPPDETECTID, 1, 1, nDet, ippDetectID, &status);
             fits_write_col(this->fitsOut, TBYTE, DETECTION_FILTERID, 1, 1, nDet, filterIDs, &status);
+            fits_write_col(this->fitsOut, TLONGLONG, DETECTION_IMAGEID, 1, 1, nDet, imageID, &status);
             fits_write_col(this->fitsOut, TFLOAT, DETECTION_INSTFLUX, 1, 1, nDet, instFlux, &status);
-            fits_write_col(this->fitsOut, TFLOAT, DETECTION_INSTFLUXERR, 1, 1, nDet, instFluxErr, &status);
-            fits_write_col(this->fitsOut, TFLOAT, DETECTION_PEAKADU, 1, 1, nDet, peakFlux, &status);
+            //fits_write_col(this->fitsOut, TFLOAT, DETECTION_INSTFLUXERR, 1, 1, nDet, instFluxErr, &status);
+            //fits_write_col(this->fitsOut, TFLOAT, DETECTION_PEAKADU, 1, 1, nDet, peakFlux, &status);
+            fits_write_col(this->fitsOut, TLONGLONG, DETECTION_INFOFLAG, 1, 1, nDet, flags, &status);
             if (numOfDuplicates||numInvalidFlux) fits_delete_rowlist(this->fitsOut, removeList, numOfDuplicates+numInvalidFlux, &status);
 
@@ -323,21 +345,12 @@
     // write number of images we have found into FrameMeta table
     status=0; 
-    if (fits_movnam_hdu(this->fitsOut, BINARY_TBL, "FrameMeta", 0, &status))
-        psError(PS_ERR_IO, false, "Can't move back to FrameMeta extension to write number of OTAs\n");
-    else 
-        fits_write_col(this->fitsOut, TSHORT, FRAMEMETA_NOTA, 1, 1, 1, &nOta, &status);
+    //if (fits_movnam_hdu(this->fitsOut, BINARY_TBL, "FrameMeta", 0, &status))
+      //  psError(PS_ERR_IO, false, "Can't move back to FrameMeta extension to write number of OTAs\n");
+    //else 
+    //    fits_write_col(this->fitsOut, TSHORT, FRAMEMETA_NOTA, 1, 1, 1, &nOta, &status);
 
     status=0;
     if (fits_close_file(fitsIn, &status)) fits_report_error(stderr, status);
 
-    // write results
-    if (this->resultsFile) {
-
-        if (fprintf(this->resultsFile, "%ld\n", minObjID) < 1 ) 
-            psError(PS_ERR_IO, false, "Unable to write min Object ID to'%s'\n", this->resultsPath);
-        if (fprintf(this->resultsFile, "%ld\n", maxObjID) < 1 ) 
-            psError(PS_ERR_IO, false, "Unable to write max Object ID to'%s'\n", this->resultsPath);
-    }
-
     psLogMsg("ippToPsps", PS_LOG_INFO, "Data written for a total of %d chips/OTAs", nOta);
 
Index: /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsConfig.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsConfig.c	(revision 28794)
@@ -616,11 +616,7 @@
                 column->ippType = ippToPsps_GetDataType(buffer); 
 
-                // IPP column
-                ippToPspsConfig_getAttribute(node, "ippColumn", buffer);
-                column->ippColNum = atoi(buffer);
-
                 column->usingDefault = false;
 
-                //psLogMsg("ippToPsps", PS_LOG_INFO, "...mapping PSPS:'%s' to '%s'", column->pspsName, column->ippName );
+                //psLogMsg("ippToPsps", PS_LOG_INFO, "...mapping PSPS:'%s' to '%s' '%d'", column->pspsName, column->ippName, column->ippColNum );
             }
         }
@@ -734,5 +730,16 @@
     int readStatus = 0;
     int writeStatus = 0;
-
+   
+    // first loop round all columns and get IPP col numbers for provided column names TODO only do once, first time in
+    if(!fromHeader) {
+
+        for (uint32_t i=0; i<table->numOfColumns; i++) {
+
+            if (strlen(table->columns[i].ippName) < 1) continue;
+            readStatus = 0;
+            fits_get_colnum(fitsIn, CASESEN, table->columns[i].ippName, &table->columns[i].ippColNum, &readStatus);
+            if (readStatus) psError(PS_ERR_IO, false, "%d Unable to read col num for '%s' '%s' %d", i, table->columns[i].pspsName, table->columns[i].ippName, table->columns[i].ippColNum);
+        }
+    }
 
     int col;
Index: /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsDetEnums.h
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsDetEnums.h	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsDetEnums.h	(revision 28794)
@@ -81,47 +81,46 @@
   IMAGEMETA_MOMENTWIDMAJOR = 25,
   IMAGEMETA_MOMENTWIDMINOR = 26,
-  IMAGEMETA_MOMENTTHETA = 27,
-  IMAGEMETA_APRESID = 28,
-  IMAGEMETA_DAPRESID = 29,
-  IMAGEMETA_DETECTORID = 30,
-  IMAGEMETA_QAFLAGS = 31,
-  IMAGEMETA_DETREND1 = 32,
-  IMAGEMETA_DETREND2 = 33,
-  IMAGEMETA_DETREND3 = 34,
-  IMAGEMETA_DETREND4 = 35,
-  IMAGEMETA_DETREND5 = 36,
-  IMAGEMETA_DETREND6 = 37,
-  IMAGEMETA_DETREND7 = 38,
-  IMAGEMETA_DETREND8 = 39,
-  IMAGEMETA_PHOTOZERO = 40,
-  IMAGEMETA_CTYPE1 = 41,
-  IMAGEMETA_CTYPE2 = 42,
-  IMAGEMETA_CRVAL1 = 43,
-  IMAGEMETA_CRVAL2 = 44,
-  IMAGEMETA_CRPIX1 = 45,
-  IMAGEMETA_CRPIX2 = 46,
-  IMAGEMETA_CDELT1 = 47,
-  IMAGEMETA_CDELT2 = 48,
-  IMAGEMETA_PC001001 = 49,
-  IMAGEMETA_PC001002 = 50,
-  IMAGEMETA_PC002001 = 51,
-  IMAGEMETA_PC002002 = 52,
-  IMAGEMETA_POLYORDER = 53,
-  IMAGEMETA_PCA1X3Y0 = 54,
-  IMAGEMETA_PCA1X2Y1 = 55,
-  IMAGEMETA_PCA1X1Y2 = 56,
-  IMAGEMETA_PCA1X0Y3 = 57,
-  IMAGEMETA_PCA1X2Y0 = 58,
-  IMAGEMETA_PCA1X1Y1 = 59,
-  IMAGEMETA_PCA1X0Y2 = 60,
-  IMAGEMETA_PCA2X3Y0 = 61,
-  IMAGEMETA_PCA2X2Y1 = 62,
-  IMAGEMETA_PCA2X1Y2 = 63,
-  IMAGEMETA_PCA2X0Y3 = 64,
-  IMAGEMETA_PCA2X2Y0 = 65,
-  IMAGEMETA_PCA2X1Y1 = 66,
-  IMAGEMETA_PCA2X0Y2 = 67,
-  IMAGEMETA_CALIBMODNUM = 68,
-  IMAGEMETA_DATARELEASE = 69,
+  IMAGEMETA_APRESID = 27,
+  IMAGEMETA_DAPRESID = 28,
+  IMAGEMETA_DETECTORID = 29,
+  IMAGEMETA_QAFLAGS = 30,
+  IMAGEMETA_DETREND1 = 31,
+  IMAGEMETA_DETREND2 = 32,
+  IMAGEMETA_DETREND3 = 33,
+  IMAGEMETA_DETREND4 = 34,
+  IMAGEMETA_DETREND5 = 35,
+  IMAGEMETA_DETREND6 = 36,
+  IMAGEMETA_DETREND7 = 37,
+  IMAGEMETA_DETREND8 = 38,
+  IMAGEMETA_PHOTOZERO = 39,
+  IMAGEMETA_CTYPE1 = 40,
+  IMAGEMETA_CTYPE2 = 41,
+  IMAGEMETA_CRVAL1 = 42,
+  IMAGEMETA_CRVAL2 = 43,
+  IMAGEMETA_CRPIX1 = 44,
+  IMAGEMETA_CRPIX2 = 45,
+  IMAGEMETA_CDELT1 = 46,
+  IMAGEMETA_CDELT2 = 47,
+  IMAGEMETA_PC001001 = 48,
+  IMAGEMETA_PC001002 = 49,
+  IMAGEMETA_PC002001 = 50,
+  IMAGEMETA_PC002002 = 51,
+  IMAGEMETA_POLYORDER = 52,
+  IMAGEMETA_PCA1X3Y0 = 53,
+  IMAGEMETA_PCA1X2Y1 = 54,
+  IMAGEMETA_PCA1X1Y2 = 55,
+  IMAGEMETA_PCA1X0Y3 = 56,
+  IMAGEMETA_PCA1X2Y0 = 57,
+  IMAGEMETA_PCA1X1Y1 = 58,
+  IMAGEMETA_PCA1X0Y2 = 59,
+  IMAGEMETA_PCA2X3Y0 = 60,
+  IMAGEMETA_PCA2X2Y1 = 61,
+  IMAGEMETA_PCA2X1Y2 = 62,
+  IMAGEMETA_PCA2X0Y3 = 63,
+  IMAGEMETA_PCA2X2Y0 = 64,
+  IMAGEMETA_PCA2X1Y1 = 65,
+  IMAGEMETA_PCA2X0Y2 = 66,
+  IMAGEMETA_CALIBMODNUM = 67,
+  IMAGEMETA_DATARELEASE = 68,
 } ImageMeta;
 
@@ -147,7 +146,7 @@
   DETECTION_PSFLIKELIHOOD = 19,
   DETECTION_PSFCF = 20,
-  DETECTION_MOMENTWIDMAJOR = 21,
-  DETECTION_MOMENTWIDMINOR = 22,
-  DETECTION_MOMENTTHETA = 23,
+  DETECTION_MOMENTXX = 21,
+  DETECTION_MOMENTXY = 22,
+  DETECTION_MOMENTYY = 23,
   DETECTION_CRLIKELIHOOD = 24,
   DETECTION_EXTENDEDLIKELIHOOD = 25,
Index: /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsTestEnums.h
===================================================================
--- /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsTestEnums.h	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippToPsps/src/ippToPspsTestEnums.h	(revision 28794)
@@ -3,88 +3,4 @@
 
 
-typedef enum {
-  FRAMEMETA_FRAMEID = 1,
-  FRAMEMETA_FILTERID,
-  FRAMEMETA_NOTA,
-  FRAMEMETA_PHOTOSCAT,
-  FRAMEMETA_EXPSTART,
-  FRAMEMETA_EXPTIME,
-  FRAMEMETA_AIRMASS,
-  FRAMEMETA_RABORE,
-  FRAMEMETA_DECBORE,
-  FRAMEMETA_CTYPE1,
-  FRAMEMETA_CTYPE2,
-  FRAMEMETA_CRVAL1,
-  FRAMEMETA_CRVAL2,
-  FRAMEMETA_CRPIX1,
-  FRAMEMETA_CRPIX2,
-  FRAMEMETA_PC001001,
-  FRAMEMETA_PC001002,
-  FRAMEMETA_PC002001,
-  FRAMEMETA_PC002002,
-  FRAMEMETA_POLYORDER,
-  FRAMEMETA_PCA1X3Y0,
-  FRAMEMETA_PCA1X2Y1,
-  FRAMEMETA_PCA1X1Y2,
-  FRAMEMETA_PCA1X0Y3,
-  FRAMEMETA_PCA1X2Y0,
-  FRAMEMETA_PCA1X1Y1,
-  FRAMEMETA_PCA1X0Y2,
-  FRAMEMETA_PCA2X3Y0,
-  FRAMEMETA_PCA2X2Y1,
-  FRAMEMETA_PCA2X1Y2,
-  FRAMEMETA_PCA2X0Y3,
-  FRAMEMETA_PCA2X2Y0,
-  FRAMEMETA_PCA2X1Y1,
-  FRAMEMETA_PCA2X0Y2,
-} FrameMeta;
-
-typedef enum {
-  IMAGEMETA_PHOTOCALID = 1,
-  IMAGEMETA_FILTERID,
-  IMAGEMETA_SKY,
-  IMAGEMETA_SKYSCAT,
-  IMAGEMETA_NDETECT,
-  IMAGEMETA_MAGSAT,
-  IMAGEMETA_COMPLETMAG,
-  IMAGEMETA_ASTROSCAT,
-  IMAGEMETA_PHOTOSCAT,
-  IMAGEMETA_NUMASTROREF,
-  IMAGEMETA_NX,
-  IMAGEMETA_NY,
-  IMAGEMETA_PSFMODELID,
-  IMAGEMETA_PSFFWHM,
-  IMAGEMETA_PSFWIDMAJOR,
-  IMAGEMETA_PSFWIDMINOR,
-  IMAGEMETA_PSFTHETA,
-  IMAGEMETA_APRESID,
-  IMAGEMETA_DAPRESID,
-  IMAGEMETA_PHOTOZERO,
-  IMAGEMETA_CTYPE1,
-  IMAGEMETA_CTYPE2,
-  IMAGEMETA_CRVAL1,
-  IMAGEMETA_CRVAL2,
-  IMAGEMETA_CRPIX1,
-  IMAGEMETA_CRPIX2,
-  IMAGEMETA_PC001001,
-  IMAGEMETA_PC001002,
-  IMAGEMETA_PC002001,
-  IMAGEMETA_PC002002,
-  IMAGEMETA_POLYORDER,
-  IMAGEMETA_PCA1X3Y0,
-  IMAGEMETA_PCA1X2Y1,
-  IMAGEMETA_PCA1X1Y2,
-  IMAGEMETA_PCA1X0Y3,
-  IMAGEMETA_PCA1X2Y0,
-  IMAGEMETA_PCA1X1Y1,
-  IMAGEMETA_PCA1X0Y2,
-  IMAGEMETA_PCA2X3Y0,
-  IMAGEMETA_PCA2X2Y1,
-  IMAGEMETA_PCA2X1Y2,
-  IMAGEMETA_PCA2X0Y3,
-  IMAGEMETA_PCA2X2Y0,
-  IMAGEMETA_PCA2X1Y1,
-  IMAGEMETA_PCA2X0Y2,
-} ImageMeta;
 
 typedef enum {
@@ -94,19 +10,9 @@
   DETECTION_IPPDETECTID,
   DETECTION_FILTERID,
-  DETECTION_XPOS,
-  DETECTION_YPOS,
-  DETECTION_XPOSERR,
-  DETECTION_YPOSERR,
-  DETECTION_IPPRA ,
-  DETECTION_IPPDEC ,
+  DETECTION_IMAGEID,
+  DETECTION_RA ,
+  DETECTION_DEC ,
+  DETECTION_INFOFLAG,
   DETECTION_INSTFLUX,
-  DETECTION_INSTFLUXERR,
-  DETECTION_PEAKADU,
-  DETECTION_PSFWIDMAJOR,
-  DETECTION_PSFWIDMINOR,
-  DETECTION_PSFTHETA,
-  DETECTION_PSFCF,
-  DETECTION_SKY,
-  DETECTION_SKYERR,
 } Detection;
 
Index: /branches/eam_branches/ipp-20100621/ippTools/configure.ac
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/configure.ac	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/configure.ac	(revision 28794)
@@ -19,6 +19,7 @@
 PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
 PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.63]) 
+PKG_CHECK_MODULES([PPSTAMP], [ippdb >= 1.1.63]) 
 
-PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}"
+PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${PPSTAMP_CFLAGS=} ${IPPDB_CFLAGS=}"
 PXTOOLS_LIBS="${PSLIB_LIBS=} ${PSMODULES_LIBS=} ${IPPDB_LIBS=}"
 AC_SUBST(PXTOOLS_CFLAGS,[$PXTOOLS_CFLAGS])
Index: /branches/eam_branches/ipp-20100621/ippTools/share/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/Makefile.am	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/Makefile.am	(revision 28794)
@@ -4,334 +4,356 @@
 
 dist_pkgdata_DATA = \
-     addtool_checkminidvodbaddrun.sql \
-     addtool_donecleanup.sql \
-     addtool_find_cam_id.sql \
-     addtool_find_pendingexp.sql \
-     addtool_find_pendingmergeprocess.sql \
-     addtool_find_processedexp.sql \
-     addtool_find_minidvodbprocessed.sql \
-     addtool_find_minidvodbrun.sql \
-     addtool_pendingcleanupexp.sql \
-     addtool_pendingcleanuprun.sql \
-     addtool_queue_cam_id.sql \
-     addtool_queue_minidvodbrun.sql \
-     addtool_revertminidvodbprocessed.sql \
-     addtool_revertprocessedexp.sql \
-     camtool_addprocessedexp.sql \
-     camtool_donecleanup.sql \
-     camtool_find_chip_id.sql \
-     camtool_pendingexp.sql \
-     camtool_find_pendingimfile.sql \
-     camtool_find_processedexp.sql \
-     camtool_pendingcleanupexp.sql \
-     camtool_pendingcleanuprun.sql \
-     camtool_queue_chip_id.sql \
-     camtool_revertprocessedexp.sql \
-     camtool_export_run.sql \
-     camtool_export_processed_exp.sql \
-     chiptool_change_exp_state.sql \
-     chiptool_change_imfile_data_state.sql \
-     chiptool_completely_processed_exp.sql \
-     chiptool_coalesce_run.sql \
-     chiptool_definecopy.sql \
-     chiptool_donecleanup.sql \
-     chiptool_find_rawexp.sql \
-     chiptool_listrun.sql \
-     chiptool_pendingcleanupimfile.sql \
-     chiptool_pendingcleanuprun.sql \
-     chiptool_pendingimfile.sql \
-     chiptool_processedimfile.sql \
-     chiptool_revertprocessedimfile.sql \
-     chiptool_revertupdatedimfile.sql \
-     chiptool_run.sql \
-     chiptool_runstate.sql \
-     chiptool_setimfiletoupdate.sql \
-     chiptool_export_run.sql \
-     chiptool_export_imfile.sql \
-     chiptool_export_processed_imfile.sql \
-     chiptool_unmasked.sql \
-     detselect_search.sql \
-     detselect_select.sql \
-     dettool_addprocessedexp.sql \
-     dettool_childlessrun.sql \
-     dettool_definebydetrun.sql \
-     dettool_detrunsummary.sql \
-     dettool_find_completed_runs.sql \
-     dettool_input.sql \
-     dettool_normalizedexp.sql \
-     dettool_normalizedimfile.sql \
-     dettool_normalizedstat.sql \
-     dettool_pending.sql \
-     dettool_pendingcleanup_detrunsummary.sql \
-     dettool_pendingcleanup_normalizedexp.sql \
-     dettool_pendingcleanup_normalizedimfile.sql \
-     dettool_pendingcleanup_normalizedstat.sql \
-     dettool_pendingcleanup_processedexp.sql \
-     dettool_pendingcleanup_processedimfile.sql \
-     dettool_pendingcleanup_residexp.sql \
-     dettool_pendingcleanup_residimfile.sql \
-     dettool_pendingcleanup_stacked.sql \
-     dettool_processedimfile.sql \
-     dettool_raw.sql \
-     dettool_residexp.sql \
-     dettool_residimfile.sql \
-     dettool_revertdetrunsummary.sql \
-     dettool_revertnormalizedexp.sql \
-     dettool_revertnormalizedimfile.sql \
-     dettool_revertnormalizedstat.sql \
-     dettool_revertprocessedexp.sql \
-     dettool_revertprocessedimfile.sql \
-     dettool_revertresidexp.sql \
-     dettool_revertresidimfile.sql \
-     dettool_revertstacked.sql \
-     dettool_runs.sql \
-     dettool_stacked.sql \
-     dettool_start_new_iteration.sql \
-     dettool_stop_completed_correct_runs.sql \
-     dettool_tocorrectexp.sql \
-     dettool_tocorrectimfile.sql \
-     dettool_todetrunsummary.sql \
-     dettool_tonormalize.sql \
-     dettool_tonormalizedexp.sql \
-     dettool_tonormalizedstat.sql \
-     dettool_toprocessedexp.sql \
-     dettool_toprocessedimfile.sql \
-     dettool_toresidexp.sql \
-     dettool_toresidimfile.sql \
-     dettool_tostacked.sql \
-     difftool_change_skyfile_data_state.sql \
-     difftool_change_run_state.sql \
-     difftool_completed_runs.sql \
-     difftool_coalesce_run.sql \
-     difftool_definewarpstack_part1.sql \
-     difftool_definewarpstack_part2.sql \
-     difftool_definewarpstack_temp_create.sql \
-     difftool_definewarpwarp_temp_create.sql \
-     difftool_definewarpwarp_temp_insert.sql \
-     difftool_definewarpwarp_select.sql \
-     difftool_definewarpwarp_insert.sql \
-     difftool_definestackstack_part0.sql \
-     difftool_definestackstack_part1.sql \
-     difftool_donecleanup.sql \
-     difftool_export_input_skyfile.sql \
-     difftool_export_run.sql \
-     difftool_export_skyfile.sql \
-     difftool_inputskyfile.sql \
-     difftool_listrun.sql \
-     difftool_pendingcleanuprun.sql \
-     difftool_pendingcleanupskyfile.sql \
-     difftool_revertdiffskyfile_delete.sql \
-     difftool_revertdiffskyfile_updated.sql \
-     difftool_setskyfiletoupdate.sql \
-     difftool_skyfile.sql \
-     difftool_todiffskyfile.sql \
-     difftool_tosummary.sql \
-     difftool_addsummary.sql \
-     disttool_definebyquery_camera.sql \
-     disttool_definebyquery_chip.sql \
-     disttool_definebyquery_diff.sql \
-     disttool_definebyquery_fake.sql \
-     disttool_definebyquery_raw.sql \
-     disttool_definebyquery_stack.sql \
-     disttool_definebyquery_warp.sql \
-     disttool_definebyquery_SSdiff.sql \
-     disttool_defineinterest.sql \
-     disttool_listfilesets.sql \
-     disttool_listinterests.sql \
-     disttool_pending_camera.sql \
-     disttool_pending_chip.sql \
-     disttool_pending_diff.sql \
-     disttool_pending_fake.sql \
-     disttool_pending_raw.sql \
-     disttool_pending_stack.sql \
-     disttool_pending_warp.sql \
-     disttool_pending_SSdiff.sql \
-     disttool_pendingfileset.sql \
-     disttool_pendingdest.sql \
-     disttool_pendingcleanup.sql \
-     disttool_processedcomponent.sql \
-     disttool_queuercrun.sql \
-     disttool_revertrcrun.sql \
-     disttool_revertrun.sql \
-     disttool_revertcomponent.sql \
-     disttool_revertfileset.sql \
-     disttool_toadvance.sql \
-     disttool_updateinterest.sql \
-     disttool_updatercrun.sql \
-     dqstatstool_createbundle.sql \
-     dqstatstool_definebyquery.sql \
-     dqstatstool_get_contents.sql \
-     dqstatstool_get_run.sql \
-     faketool_change_exp_state.sql \
-     faketool_change_imfile_data_state.sql \
-     faketool_completely_processed_exp.sql \
-     faketool_donecleanup.sql \
-     faketool_export_processed_imfile.sql \
-     faketool_export_run.sql \
-     faketool_find_camrun.sql \
-     faketool_find_pendingexp.sql \
-     faketool_pendingcleanupimfile.sql \
-     faketool_pendingcleanuprun.sql \
-     faketool_pendingimfile.sql \
-     faketool_processedimfile.sql \
-     faketool_queue_cam_id.sql \
-     faketool_revertprocessedimfile.sql \
-     faketool_unmasked.sql \
-     flatcorr_chiprundone.sql \
-     flatcorr_camerarundone.sql \
-     flatcorr_pendingprocess.sql \
-     flatcorr_inputimfile.sql \
-     flatcorr_dropchip.sql \
-     flatcorr_dropcamera.sql \
-     magictool_addmask.sql \
-     magictool_restore_camera.sql \
-     magictool_restore_chip.sql \
-     magictool_restore_diff.sql \
-     magictool_restore_raw.sql \
-     magictool_restore_warp.sql \
-     magictool_create_tmp_warpcomplete.sql \
-     magictool_definebyquery_insert.sql \
-     magictool_definebyquery_select.sql \
-     magictool_inputs.sql \
-     magictool_inputskyfile.sql \
-     magictool_mask.sql \
-     magictool_tomask.sql \
-     magictool_toprocess_inputs.sql \
-     magictool_toprocess_runs.sql \
-     magictool_toprocess_tree.sql \
-     magictool_toskyfilemask.sql \
-     magictool_totree.sql \
-     magictool_diffskyfile.sql \
-     magictool_warpskyfile.sql \
-     magictool_chipprocessedimfile.sql \
-     magictool_rawimfile.sql \
-     magictool_revertnode.sql \
-     magictool_exposure.sql \
-     magicdstool_clearstatefaults.sql \
-     magicdstool_completed_runs.sql \
-     magicdstool_completedrevert.sql \
-     magicdstool_definebyquery_raw.sql \
-     magicdstool_definebyquery_chip.sql \
-     magicdstool_definebyquery_camera.sql \
-     magicdstool_definebyquery_warp.sql \
-     magicdstool_definebyquery_diff.sql \
-     magicdstool_definecopy_chip.sql \
-     magicdstool_definecopy_warp.sql \
-     magicdstool_getrunids.sql \
-     magicdstool_getskycells.sql \
-     magicdstool_revertdestreakedfile.sql \
-     magicdstool_tocleanup.sql \
-     magicdstool_todestreak_camera.sql \
-     magicdstool_todestreak_chip.sql \
-     magicdstool_todestreak_diff.sql \
-     magicdstool_todestreak_raw.sql \
-     magicdstool_todestreak_warp.sql \
-     magicdstool_toremove.sql \
-     magicdstool_torevert_raw.sql \
-     magicdstool_torevert_chip.sql \
-     magicdstool_torevert_camera.sql \
-     magicdstool_torevert_warp.sql \
-     magicdstool_torevert_diff.sql \
-     pstamptool_completedreq.sql \
-     pstamptool_datastore.sql \
-     pstamptool_getdependent.sql \
-     pstamptool_listjob.sql \
-     pstamptool_pendingcleanup.sql \
-     pstamptool_pendingdependent.sql \
-     pstamptool_pendingjob.sql \
-     pstamptool_pendingreq.sql \
-     pstamptool_project.sql \
-     pstamptool_revertdependent.sql \
-     pstamptool_revertjob.sql \
-     pstamptool_revertreq.sql \
-     pstamptool_revertreq_deletejobs.sql \
-     pstamptool_updatejob.sql \
-     pxadmin_create_tables.sql \
-     pxadmin_create_mirror_tables.sql \
-     pxadmin_drop_tables.sql \
-     pxadmin_update_version.sql \
-     pubtool_definerun.sql \
-     pubtool_pending.sql \
-     pubtool_revert.sql \
-     pztool_find_completed_exp.sql \
-     pztool_pendingimfile.sql \
-     pztool_revert_downloadimfile_faults.sql \
-     pztool_revert_fileset_faults.sql \
-     pztool_revertcopied.sql \
-     receivetool_list.sql \
-     receivetool_addfileset.sql \
-     receivetool_pendingfileset.sql \
-     receivetool_pendingfile.sql \
-     receivetool_revert.sql \
-     receivetool_toadvance.sql \
-     regtool_create_dup_table.sql \
-     regtool_export_exp.sql \
-     regtool_export_imfile.sql \
-     regtool_pendingexp.sql \
-     regtool_pendingimfile.sql \
-     regtool_populate_dup_table.sql \
-     regtool_processedexp.sql \
-     regtool_processedimfile.sql \
-     regtool_revertprocessedexp.sql \
-     regtool_revertprocessedimfile.sql \
-     regtool_updateprocessedimfile.sql \
-     regtool_pendingcompressimfile.sql \
-     regtool_finishcompressexp.sql \
-     regtool_updatebyquery.sql \
-     regtool_updatebyqueryimfile.sql \
-     stacktool_associationdefine_select.sql \
-     stacktool_definebyquery_insert.sql \
-     stacktool_definebyquery_insert_random_part1.sql \
-     stacktool_definebyquery_insert_random_part2.sql \
-     stacktool_definebyquery_select.sql \
-     stacktool_definebyquery_test.sql \
-     stacktool_donecleanup.sql \
-     stacktool_export_input_skyfile.sql \
-     stacktool_export_run.sql \
-     stacktool_export_sum_skyfile.sql \
-     stacktool_find_complete_warps.sql \
-     stacktool_inputskyfile.sql \
-     stacktool_pendingcleanuprun.sql \
-     stacktool_pendingcleanupskyfile.sql \
-     stacktool_revertsumskyfile_delete.sql \
-     stacktool_sumskyfile.sql \
-     stacktool_sassskyfile.sql \
-     stacktool_tosum.sql \
-     stacktool_tosummary.sql \
-     stacktool_addsummary.sql \
-     staticskytool_definebyquery_select.sql \
-     staticskytool_definebyquery_inputs.sql \
-     staticskytool_inputs.sql \
-     staticskytool_todo.sql \
-     staticskytool_result.sql \
-     staticskytool_revert.sql \
-     warptool_change_skyfile_data_state.sql \
-     warptool_change_run_state.sql \
-     warptool_definebyquery.sql \
-     warptool_donecleanup.sql \
-     warptool_exp.sql \
-     warptool_export_imfile.sql \
-     warptool_export_run.sql \
-     warptool_export_skyfile.sql \
-     warptool_export_skycell_map.sql \
-     warptool_finished_run_select.sql \
-     warptool_finish_run.sql \
-     warptool_coalesce_run.sql \
-     warptool_imfile.sql \
-     warptool_listrun.sql \
-     warptool_pendingcleanuprun.sql \
-     warptool_pendingcleanupskyfile.sql \
-     warptool_revertoverlap.sql \
-     warptool_revertwarped_delete.sql \
-     warptool_revertwarped_updated.sql \
-     warptool_runstate.sql \
-     warptool_scmap.sql \
-     warptool_setskyfiletoupdate.sql \
-     warptool_tooverlap.sql \
-     warptool_towarped.sql \
-     warptool_updateskyfile.sql \
-     warptool_warped.sql \
-     warptool_tosummary.sql \
-     warptool_addsummary.sql \
+	addtool_checkminidvodbaddrun.sql \
+	addtool_donecleanup.sql \
+	addtool_find_cam_id.sql \
+	addtool_find_pendingexp.sql \
+	addtool_find_pendingmergeprocess.sql \
+	addtool_find_processedexp.sql \
+	addtool_find_minidvodbprocessed.sql \
+	addtool_find_minidvodbrun.sql \
+	addtool_pendingcleanupexp.sql \
+	addtool_pendingcleanuprun.sql \
+	addtool_queue_cam_id.sql \
+	addtool_queue_minidvodbrun.sql \
+	addtool_revertminidvodbprocessed.sql \
+	addtool_revertprocessedexp.sql \
+	bgtool_advancechip.sql \
+	bgtool_advancewarp.sql \
+	bgtool_chipinputs.sql \
+	bgtool_chip.sql \
+	bgtool_cleanedchip.sql \
+	bgtool_cleanedwarp.sql \
+	bgtool_definechip.sql \
+	bgtool_definewarp.sql \
+	bgtool_revertchip.sql \
+	bgtool_revertwarp.sql \
+	bgtool_tochip.sql \
+	bgtool_tocleanchip.sql \
+	bgtool_tocleanwarp.sql \
+	bgtool_towarp.sql \
+	bgtool_warpinputs.sql \
+	bgtool_warp.sql \
+	camtool_addprocessedexp.sql \
+	camtool_donecleanup.sql \
+	camtool_find_chip_id.sql \
+	camtool_pendingexp.sql \
+	camtool_find_pendingimfile.sql \
+	camtool_find_processedexp.sql \
+	camtool_pendingcleanupexp.sql \
+	camtool_pendingcleanuprun.sql \
+	camtool_queue_chip_id.sql \
+	camtool_revertprocessedexp.sql \
+	camtool_export_run.sql \
+	camtool_export_processed_exp.sql \
+	chiptool_change_exp_state.sql \
+	chiptool_change_imfile_data_state.sql \
+	chiptool_completely_processed_exp.sql \
+	chiptool_coalesce_run.sql \
+	chiptool_definecopy.sql \
+	chiptool_donecleanup.sql \
+	chiptool_find_rawexp.sql \
+	chiptool_listrun.sql \
+	chiptool_pendingcleanupimfile.sql \
+	chiptool_pendingcleanuprun.sql \
+	chiptool_pendingimfile.sql \
+	chiptool_processedimfile.sql \
+	chiptool_revertprocessedimfile.sql \
+	chiptool_revertupdatedimfile.sql \
+	chiptool_run.sql \
+	chiptool_runstate.sql \
+	chiptool_setimfiletoupdate.sql \
+	chiptool_export_run.sql \
+	chiptool_export_imfile.sql \
+	chiptool_export_processed_imfile.sql \
+	chiptool_unmasked.sql \
+	detselect_search.sql \
+	detselect_select.sql \
+	dettool_addprocessedexp.sql \
+	dettool_childlessrun.sql \
+	dettool_definebydetrun.sql \
+	dettool_detrunsummary.sql \
+	dettool_find_completed_runs.sql \
+	dettool_input.sql \
+	dettool_normalizedexp.sql \
+	dettool_normalizedimfile.sql \
+	dettool_normalizedstat.sql \
+	dettool_pending.sql \
+	dettool_pendingcleanup_detrunsummary.sql \
+	dettool_pendingcleanup_normalizedexp.sql \
+	dettool_pendingcleanup_normalizedimfile.sql \
+	dettool_pendingcleanup_normalizedstat.sql \
+	dettool_pendingcleanup_processedexp.sql \
+	dettool_pendingcleanup_processedimfile.sql \
+	dettool_pendingcleanup_residexp.sql \
+	dettool_pendingcleanup_residimfile.sql \
+	dettool_pendingcleanup_stacked.sql \
+	dettool_processedimfile.sql \
+	dettool_raw.sql \
+	dettool_residexp.sql \
+	dettool_residimfile.sql \
+	dettool_revertdetrunsummary.sql \
+	dettool_revertnormalizedexp.sql \
+	dettool_revertnormalizedimfile.sql \
+	dettool_revertnormalizedstat.sql \
+	dettool_revertprocessedexp.sql \
+	dettool_revertprocessedimfile.sql \
+	dettool_revertresidexp.sql \
+	dettool_revertresidimfile.sql \
+	dettool_revertstacked.sql \
+	dettool_runs.sql \
+	dettool_stacked.sql \
+	dettool_start_new_iteration.sql \
+	dettool_stop_completed_correct_runs.sql \
+	dettool_tocorrectexp.sql \
+	dettool_tocorrectimfile.sql \
+	dettool_todetrunsummary.sql \
+	dettool_tonormalize.sql \
+	dettool_tonormalizedexp.sql \
+	dettool_tonormalizedstat.sql \
+	dettool_toprocessedexp.sql \
+	dettool_toprocessedimfile.sql \
+	dettool_toresidexp.sql \
+	dettool_toresidimfile.sql \
+	dettool_tostacked.sql \
+	difftool_change_skyfile_data_state.sql \
+	difftool_change_run_state.sql \
+	difftool_completed_runs.sql \
+	difftool_coalesce_run.sql \
+	difftool_definewarpstack_part1.sql \
+	difftool_definewarpstack_part2.sql \
+	difftool_definewarpstack_temp_create.sql \
+	difftool_definewarpwarp_temp_create.sql \
+	difftool_definewarpwarp_temp_insert.sql \
+	difftool_definewarpwarp_select.sql \
+	difftool_definewarpwarp_insert.sql \
+	difftool_definestackstack_part0.sql \
+	difftool_definestackstack_part1.sql \
+	difftool_donecleanup.sql \
+	difftool_export_input_skyfile.sql \
+	difftool_export_run.sql \
+	difftool_export_skyfile.sql \
+	difftool_inputskyfile.sql \
+	difftool_listrun.sql \
+	difftool_pendingcleanuprun.sql \
+	difftool_pendingcleanupskyfile.sql \
+	difftool_revertdiffskyfile_delete.sql \
+	difftool_revertdiffskyfile_updated.sql \
+	difftool_setskyfiletoupdate.sql \
+	difftool_skyfile.sql \
+	difftool_todiffskyfile.sql \
+	difftool_tosummary.sql \
+	difftool_addsummary.sql \
+	disttool_definebyquery_camera.sql \
+	disttool_definebyquery_chip.sql \
+	disttool_definebyquery_chip_bg.sql \
+	disttool_definebyquery_diff.sql \
+	disttool_definebyquery_fake.sql \
+	disttool_definebyquery_raw.sql \
+	disttool_definebyquery_stack.sql \
+	disttool_definebyquery_warp.sql \
+	disttool_definebyquery_warp_bg.sql \
+	disttool_definebyquery_SSdiff.sql \
+	disttool_defineinterest.sql \
+	disttool_listfilesets.sql \
+	disttool_listinterests.sql \
+	disttool_pending_camera.sql \
+	disttool_pending_chip.sql \
+	disttool_pending_chip_bg.sql \
+	disttool_pending_diff.sql \
+	disttool_pending_fake.sql \
+	disttool_pending_raw.sql \
+	disttool_pending_stack.sql \
+	disttool_pending_warp.sql \
+	disttool_pending_warp_bg.sql \
+	disttool_pending_SSdiff.sql \
+	disttool_pendingfileset.sql \
+	disttool_pendingdest.sql \
+	disttool_pendingcleanup.sql \
+	disttool_processedcomponent.sql \
+	disttool_queuercrun.sql \
+	disttool_revertrcrun.sql \
+	disttool_revertrun.sql \
+	disttool_revertcomponent.sql \
+	disttool_revertfileset.sql \
+	disttool_toadvance.sql \
+	disttool_updateinterest.sql \
+	disttool_updatercrun.sql \
+	dqstatstool_createbundle.sql \
+	dqstatstool_definebyquery.sql \
+	dqstatstool_get_contents.sql \
+	dqstatstool_get_run.sql \
+	faketool_change_exp_state.sql \
+	faketool_change_imfile_data_state.sql \
+	faketool_completely_processed_exp.sql \
+	faketool_donecleanup.sql \
+	faketool_export_processed_imfile.sql \
+	faketool_export_run.sql \
+	faketool_find_camrun.sql \
+	faketool_find_pendingexp.sql \
+	faketool_pendingcleanupimfile.sql \
+	faketool_pendingcleanuprun.sql \
+	faketool_pendingimfile.sql \
+	faketool_processedimfile.sql \
+	faketool_queue_cam_id.sql \
+	faketool_revertprocessedimfile.sql \
+	faketool_unmasked.sql \
+	flatcorr_chiprundone.sql \
+	flatcorr_camerarundone.sql \
+	flatcorr_pendingprocess.sql \
+	flatcorr_inputimfile.sql \
+	flatcorr_dropchip.sql \
+	flatcorr_dropcamera.sql \
+	magictool_addmask.sql \
+	magictool_restore_camera.sql \
+	magictool_restore_chip.sql \
+	magictool_restore_diff.sql \
+	magictool_restore_raw.sql \
+	magictool_restore_warp.sql \
+	magictool_create_tmp_warpcomplete.sql \
+	magictool_definebyquery_insert.sql \
+	magictool_definebyquery_select.sql \
+	magictool_inputs.sql \
+	magictool_inputskyfile.sql \
+	magictool_mask.sql \
+	magictool_tomask.sql \
+	magictool_toprocess_inputs.sql \
+	magictool_toprocess_runs.sql \
+	magictool_toprocess_tree.sql \
+	magictool_toskyfilemask.sql \
+	magictool_totree.sql \
+	magictool_diffskyfile.sql \
+	magictool_warpskyfile.sql \
+	magictool_chipprocessedimfile.sql \
+	magictool_rawimfile.sql \
+	magictool_revertnode.sql \
+	magictool_exposure.sql \
+	magicdstool_clearstatefaults.sql \
+	magicdstool_completed_runs.sql \
+	magicdstool_completedrevert.sql \
+	magicdstool_definebyquery_raw.sql \
+	magicdstool_definebyquery_chip.sql \
+	magicdstool_definebyquery_chip_bg.sql \
+	magicdstool_definebyquery_camera.sql \
+	magicdstool_definebyquery_warp.sql \
+	magicdstool_definebyquery_warp_bg.sql \
+	magicdstool_definebyquery_diff.sql \
+	magicdstool_definecopy_chip.sql \
+	magicdstool_definecopy_warp.sql \
+	magicdstool_getrunids.sql \
+	magicdstool_getskycells.sql \
+	magicdstool_revertdestreakedfile.sql \
+	magicdstool_tocleanup.sql \
+	magicdstool_todestreak_camera.sql \
+	magicdstool_todestreak_chip.sql \
+	magicdstool_todestreak_diff.sql \
+	magicdstool_todestreak_raw.sql \
+	magicdstool_todestreak_warp.sql \
+	magicdstool_toremove.sql \
+	magicdstool_torevert_raw.sql \
+	magicdstool_torevert_chip.sql \
+	magicdstool_torevert_camera.sql \
+	magicdstool_torevert_warp.sql \
+	magicdstool_torevert_diff.sql \
+	pstamptool_completedreq.sql \
+	pstamptool_datastore.sql \
+	pstamptool_getdependent.sql \
+	pstamptool_listjob.sql \
+	pstamptool_pendingcleanup.sql \
+	pstamptool_pendingdependent.sql \
+	pstamptool_pendingjob.sql \
+	pstamptool_pendingreq.sql \
+	pstamptool_project.sql \
+	pstamptool_revertdependent.sql \
+	pstamptool_revertjob.sql \
+	pstamptool_revertreq.sql \
+	pstamptool_revertreq_deletejobs.sql \
+	pstamptool_updatejob.sql \
+	pxadmin_create_tables.sql \
+	pxadmin_create_mirror_tables.sql \
+	pxadmin_drop_tables.sql \
+	pxadmin_update_version.sql \
+	pubtool_definerun.sql \
+	pubtool_pending.sql \
+	pubtool_revert.sql \
+	pztool_find_completed_exp.sql \
+	pztool_pendingimfile.sql \
+	pztool_revert_downloadimfile_faults.sql \
+	pztool_revert_fileset_faults.sql \
+	pztool_revertcopied.sql \
+	receivetool_list.sql \
+	receivetool_addfileset.sql \
+	receivetool_pendingfileset.sql \
+	receivetool_pendingfile.sql \
+	receivetool_revert.sql \
+	receivetool_toadvance.sql \
+	regtool_create_dup_table.sql \
+	regtool_export_exp.sql \
+	regtool_export_imfile.sql \
+	regtool_pendingexp.sql \
+	regtool_pendingimfile.sql \
+	regtool_populate_dup_table.sql \
+	regtool_processedexp.sql \
+	regtool_processedimfile.sql \
+	regtool_revertprocessedexp.sql \
+	regtool_revertprocessedimfile.sql \
+	regtool_updateprocessedimfile.sql \
+	regtool_pendingcompressimfile.sql \
+	regtool_finishcompressexp.sql \
+	regtool_updatebyquery.sql \
+	regtool_updatebyqueryimfile.sql \
+	stacktool_associationdefine_select.sql \
+	stacktool_definebyquery_insert.sql \
+	stacktool_definebyquery_insert_random_part1.sql \
+	stacktool_definebyquery_insert_random_part2.sql \
+	stacktool_definebyquery_select.sql \
+	stacktool_definebyquery_test.sql \
+	stacktool_donecleanup.sql \
+	stacktool_export_input_skyfile.sql \
+	stacktool_export_run.sql \
+	stacktool_export_sum_skyfile.sql \
+	stacktool_find_complete_warps.sql \
+	stacktool_inputskyfile.sql \
+	stacktool_pendingcleanuprun.sql \
+	stacktool_pendingcleanupskyfile.sql \
+	stacktool_revertsumskyfile_delete.sql \
+	stacktool_sumskyfile.sql \
+	stacktool_sassskyfile.sql \
+	stacktool_tosum.sql \
+	stacktool_tosummary.sql \
+	stacktool_addsummary.sql \
+	staticskytool_definebyquery_select.sql \
+	staticskytool_definebyquery_inputs.sql \
+	staticskytool_inputs.sql \
+	staticskytool_todo.sql \
+	staticskytool_result.sql \
+	staticskytool_revert.sql \
+	warptool_change_skyfile_data_state.sql \
+	warptool_change_run_state.sql \
+	warptool_definebyquery.sql \
+	warptool_donecleanup.sql \
+	warptool_exp.sql \
+	warptool_export_imfile.sql \
+	warptool_export_run.sql \
+	warptool_export_skyfile.sql \
+	warptool_export_skycell_map.sql \
+	warptool_finished_run_select.sql \
+	warptool_finish_run.sql \
+	warptool_coalesce_run.sql \
+	warptool_imfile.sql \
+	warptool_listrun.sql \
+	warptool_pendingcleanuprun.sql \
+	warptool_pendingcleanupskyfile.sql \
+	warptool_revertoverlap.sql \
+	warptool_revertwarped_delete.sql \
+	warptool_revertwarped_updated.sql \
+	warptool_runstate.sql \
+	warptool_scmap.sql \
+	warptool_setskyfiletoupdate.sql \
+	warptool_tooverlap.sql \
+	warptool_towarped.sql \
+	warptool_updateskyfile.sql \
+	warptool_warped.sql \
+	warptool_tosummary.sql \
+	warptool_addsummary.sql \
 	diffphottool_definerun.sql \
 	diffphottool_input.sql \
Index: /branches/eam_branches/ipp-20100621/ippTools/share/addtool_revertminidvodbprocessed.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/addtool_revertminidvodbprocessed.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/addtool_revertminidvodbprocessed.sql	(revision 28794)
@@ -5,3 +5,3 @@
     AND addRun.minidvodb_name = minidvodbRun.minidvodb_name
     AND minidvodbProcessed.fault != 0
-    AND minidvodbRun.state = 'new'
+    AND minidvodbRun.state = 'merged'
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_advancechip.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_advancechip.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_advancechip.sql	(revision 28794)
@@ -0,0 +1,15 @@
+SELECT DISTINCT
+    chipBackgroundRun.chip_bg_id,
+    chipBackgroundImfile.magicked
+FROM chipBackgroundRun
+JOIN chipProcessedImfile USING(chip_id)
+LEFT JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
+WHERE chipBackgroundRun.state = 'new'
+    AND chipProcessedImfile.quality = 0
+    AND chipProcessedImfile.fault = 0
+-- WHERE hook %s
+GROUP BY chip_bg_id
+HAVING
+    COUNT(chipBackgroundImfile.class_id) = COUNT(chipProcessedImfile.class_id)
+    AND SUM(IF(chipBackgroundImfile.fault > 0, 1, 0)) = 0
+    AND SUM(IF(chipBackgroundImfile.quality > 0, 1, 0)) != COUNT(*)
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_advancewarp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_advancewarp.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_advancewarp.sql	(revision 28794)
@@ -0,0 +1,15 @@
+SELECT DISTINCT
+    warpBackgroundRun.warp_bg_id,
+    warpBackgroundSkyfile.magicked
+FROM warpBackgroundRun
+JOIN warpSkyfile USING(warp_id)
+LEFT JOIN warpBackgroundSkyfile USING(warp_bg_id, skycell_id)
+WHERE warpBackgroundRun.state = 'new'
+    AND warpSkyfile.quality = 0
+    AND warpSkyfile.fault = 0
+-- WHERE hook %s
+GROUP BY warp_bg_id
+HAVING
+    COUNT(warpBackgroundSkyfile.skycell_id) = COUNT(warpSkyfile.skycell_id)
+    AND SUM(IF(warpBackgroundSkyfile.fault > 0, 1, 0)) = 0
+    AND SUM(IF(warpBackgroundSkyfile.quality > 0, 1, 0)) != COUNT(*)
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_chip.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_chip.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_chip.sql	(revision 28794)
@@ -0,0 +1,17 @@
+SELECT
+    chipBackgroundImfile.*,
+    chipBackgroundRun.state,
+    chipBackgroundRun.workdir,
+    chipBackgroundRun.label,
+    rawExp.exp_id,
+    rawExp.exp_name,
+    rawExp.camera,
+    rawExp.filter,
+    rawExp.dateobs,
+    rawExp.ra,
+    rawExp.decl,
+    rawExp.exp_time
+FROM chipBackgroundRun
+JOIN chipBackgroundImfile USING(chip_bg_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_chipinputs.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_chipinputs.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_chipinputs.sql	(revision 28794)
@@ -0,0 +1,8 @@
+SELECT
+    chipProcessedImfile.*
+FROM chipBackgroundRun
+JOIN chipRun USING(chip_id)
+JOIN chipProcessedImfile USING(chip_id)
+WHERE chipRun.state = 'full'
+    AND chipProcessedImfile.fault = 0
+    AND chipProcessedImfile.quality = 0
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_cleanedchip.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_cleanedchip.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_cleanedchip.sql	(revision 28794)
@@ -0,0 +1,3 @@
+UPDATE chipBackgroundRun
+SET state = '%s'
+WHERE state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged')
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_cleanedwarp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_cleanedwarp.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_cleanedwarp.sql	(revision 28794)
@@ -0,0 +1,3 @@
+UPDATE warpBackgroundRun
+SET state = '%s'
+WHERE state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged')
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_definechip.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_definechip.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_definechip.sql	(revision 28794)
@@ -0,0 +1,7 @@
+SELECT
+    chipRun.*
+FROM chipRun
+JOIN rawExp USING(exp_id)
+LEFT JOIN chipBackgroundRun USING(chip_id)
+WHERE chipRun.state = 'full'
+    AND chipRun.magicked > 0 -- require destreaked to stop race condition in destreaking
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_definewarp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_definewarp.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_definewarp.sql	(revision 28794)
@@ -0,0 +1,12 @@
+SELECT
+    warpRun.*,
+    chipBackgroundRun.chip_bg_id
+FROM warpRun
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN chipBackgroundRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+LEFT JOIN warpBackgroundRun USING(chip_bg_id)
+WHERE chipBackgroundRun.state = 'full'
+    AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_revertchip.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_revertchip.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_revertchip.sql	(revision 28794)
@@ -0,0 +1,4 @@
+DELETE chipBackgroundImfile
+FROM chipBackgroundRun
+JOIN chipBackgroundImfile USING(chip_bg_id)
+WHERE chipBackgroundImfile.fault != 0
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_revertwarp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_revertwarp.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_revertwarp.sql	(revision 28794)
@@ -0,0 +1,4 @@
+DELETE warpBackgroundSkyfile
+FROM warpBackgroundRun
+JOIN warpBackgroundSkyfile USING(warp_bg_id)
+WHERE warpBackgroundSkyfile.fault != 0
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_tochip.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_tochip.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_tochip.sql	(revision 28794)
@@ -0,0 +1,18 @@
+SELECT
+    chipBackgroundRun.*,
+    class_id,
+    rawExp.exp_tag,
+    rawExp.camera
+FROM chipBackgroundRun
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+JOIN chipProcessedImfile USING(chip_id)
+LEFT JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
+LEFT JOIN Label ON chipBackgroundRun.label = Label.label
+WHERE chipBackgroundImfile.chip_bg_id IS NULL
+    AND chipRun.state = 'full'
+    AND chipProcessedImfile.fault = 0
+    AND chipProcessedImfile.quality = 0
+    AND (Label.active OR Label.active IS NULL)
+-- WHERE hook %s
+ORDER BY priority DESC, chip_bg_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_tocleanchip.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_tocleanchip.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_tocleanchip.sql	(revision 28794)
@@ -0,0 +1,9 @@
+SELECT
+    chipBackgroundRun.chip_bg_id,
+    chipBackgroundRun.state,
+    rawExp.camera
+FROM chipBackgroundRun
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+WHERE chipBackgroundRun.state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged')
+
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_tocleanwarp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_tocleanwarp.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_tocleanwarp.sql	(revision 28794)
@@ -0,0 +1,12 @@
+SELECT
+    warpBackgroundRun.warp_bg_id,
+    warpBackgroundRun.state,
+    rawExp.camera
+FROM warpBackgroundRun
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+WHERE warpBackgroundRun.state IN ('goto_cleaned', 'goto_scrubbed', 'goto_purged')
+
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_towarp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_towarp.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_towarp.sql	(revision 28794)
@@ -0,0 +1,26 @@
+SELECT
+    warpBackgroundRun.*,
+    warpRun.tess_id,
+    skycell_id,
+    rawExp.exp_tag,
+    rawExp.camera
+FROM warpBackgroundRun
+JOIN warpRun USING(warp_id)
+JOIN warpSkyCellMap USING(warp_id)
+JOIN warpSkyfile USING(warp_id, skycell_id)
+JOIN chipBackgroundRun USING(chip_bg_id)
+JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+LEFT JOIN warpBackgroundSkyfile USING(warp_bg_id, skycell_id)
+LEFT JOIN Label ON Label.label = warpBackgroundRun.label
+WHERE warpBackgroundSkyfile.warp_bg_id IS NULL
+    AND chipBackgroundRun.state = 'full'
+    AND chipBackgroundImfile.fault = 0
+    AND chipBackgroundImfile.quality = 0
+    AND warpRun.state IN ('full', 'cleaned', 'goto_cleaned')
+    AND warpSkyfile.fault = 0
+    AND warpSkyfile.quality = 0
+    AND (Label.active OR Label.active IS NULL)
+-- WHERE hook %s
+ORDER BY priority DESC, chip_bg_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_warp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_warp.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_warp.sql	(revision 28794)
@@ -0,0 +1,20 @@
+SELECT
+    warpBackgroundSkyfile.*,
+    warpBackgroundRun.state,
+    warpBackgroundRun.workdir,
+    warpBackgroundRun.label,
+    rawExp.exp_id,
+    rawExp.exp_name,
+    rawExp.camera,
+    rawExp.filter,
+    rawExp.dateobs,
+    rawExp.ra,
+    rawExp.decl,
+    rawExp.exp_time
+FROM warpBackgroundRun
+JOIN warpBackgroundSkyfile USING(warp_bg_id)
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
Index: /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_warpinputs.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_warpinputs.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/bgtool_warpinputs.sql	(revision 28794)
@@ -0,0 +1,19 @@
+SELECT
+    chipBackgroundImfile.path_base AS chip_path_base,
+    chipBackgroundImfile.class_id,
+    chipBackgroundImfile.magicked,
+    camProcessedExp.path_base AS cam_path_base
+FROM warpBackgroundRun
+JOIN warpRun USING(warp_id)
+JOIN warpSkyCellMap USING(warp_id)
+JOIN warpSkyfile USING(warp_id, skycell_id)
+JOIN chipBackgroundRun USING(chip_bg_id)
+JOIN chipBackgroundImfile USING(chip_bg_id, class_id)
+JOIN fakeRun USING(fake_id)
+JOIN camProcessedExp USING(cam_id)
+WHERE warpRun.state IN ('full', 'cleaned', 'goto_cleaned') -- only need it to have been completed
+    AND warpSkyfile.fault = 0
+    AND warpSkyfile.quality = 0
+    AND chipBackgroundRun.state = 'full'
+    AND chipBackgroundImfile.fault = 0
+    AND chipBackgroundImfile.quality = 0
Index: /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_listrun.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_listrun.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_listrun.sql	(revision 28794)
@@ -21,5 +21,6 @@
     rawExp.exp_time,
     rawExp.magicked AS raw_magicked,
-    magicDSRun.state AS dsRun_state
+    magicDSRun.state AS dsRun_state,
+    IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id
 FROM chipRun
 JOIN rawExp
Index: /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_processedimfile.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_processedimfile.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_processedimfile.sql	(revision 28794)
@@ -30,5 +30,5 @@
     rawImfile.burntool_state,
     magicDSRun.state AS dsRun_state,
-    magicDSRun.magic_ds_id
+    IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id
 FROM chipRun
 JOIN chipImfile
Index: /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_revertprocessedimfile.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_revertprocessedimfile.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/chiptool_revertprocessedimfile.sql	(revision 28794)
@@ -2,5 +2,6 @@
 USING chipProcessedImfile, chipRun, rawExp
 WHERE
-    rawExp.exp_id = chipProcessedImfile.exp_id
+    chipRun.chip_id = chipProcessedImfile.chip_id
+    AND rawExp.exp_id = chipProcessedImfile.exp_id
     AND chipRun.state = 'new'
     AND chipProcessedImfile.fault != 0
Index: /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertdetrunsummary.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertdetrunsummary.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertdetrunsummary.sql	(revision 28794)
@@ -1,3 +1,8 @@
 DELETE FROM detRunSummary
+USING detRunSummary, detRun
 WHERE
     fault != 0
+AND
+    state = 'run'
+AND
+    detRunSummary.det_id = detRun.det_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertnormalizedexp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertnormalizedexp.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertnormalizedexp.sql	(revision 28794)
@@ -1,3 +1,8 @@
 DELETE FROM detNormalizedExp
+USING detNormalizedExp, detRun
 WHERE
     fault != 0
+AND
+    state = 'run'
+AND
+    detNormalizedExp.det_id = detRun.det_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertnormalizedimfile.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertnormalizedimfile.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertnormalizedimfile.sql	(revision 28794)
@@ -1,3 +1,8 @@
 DELETE FROM detNormalizedImfile
+USING detNormalizedImfile, detRun
 WHERE
     fault != 0
+AND
+    state = 'run'
+AND
+    detNormalizedImfile.det_id = detRun.det_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertnormalizedstat.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertnormalizedstat.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertnormalizedstat.sql	(revision 28794)
@@ -1,3 +1,8 @@
 DELETE FROM detNormalizedStatImfile
+USING detNormalizedStatImfile, detRun
 WHERE
     fault != 0
+AND
+    state = 'run'
+AND
+    detNormalizedStatImfile.det_id = detRun.det_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertprocessedexp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertprocessedexp.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertprocessedexp.sql	(revision 28794)
@@ -1,3 +1,8 @@
 DELETE FROM detProcessedExp
+USING detProcessedExp, detRun
 WHERE
     fault != 0
+AND
+    state = 'run'
+AND
+    detProcessedExp.det_id = detRun.det_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertprocessedimfile.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertprocessedimfile.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertprocessedimfile.sql	(revision 28794)
@@ -1,3 +1,8 @@
 DELETE FROM detProcessedImfile
+USING detProcessedImfile, detRun
 WHERE
     fault != 0
+AND
+    state = 'run'
+AND
+    detProcessedImfile.det_id = detRun.det_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertresidexp.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertresidexp.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertresidexp.sql	(revision 28794)
@@ -1,3 +1,8 @@
 DELETE FROM detResidExp
+USING detResidExp, detRun
 WHERE
     fault != 0
+AND
+    state = 'run'
+AND
+    detResidExp.det_id = detRun.det_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertresidimfile.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertresidimfile.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertresidimfile.sql	(revision 28794)
@@ -1,3 +1,8 @@
 DELETE FROM detResidImfile
+USING detResidImfile, detRun
 WHERE
     fault != 0
+AND
+    state = 'run'
+AND
+    detResidImfile.det_id = detRun.det_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertstacked.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertstacked.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/dettool_revertstacked.sql	(revision 28794)
@@ -1,3 +1,8 @@
 DELETE FROM detStackedImfile
+USING detStackedImfile, detRun
 WHERE
     fault != 0
+AND
+    state = 'run'
+AND
+    detStackedImfile.det_id = detRun.det_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_advance.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_advance.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_advance.sql	(revision 28794)
@@ -1,4 +1,5 @@
 SELECT
-    diff_phot_id
+    diff_phot_id,
+    diffPhotSkyfile.magicked
 FROM diffPhotRun
 JOIN diffSkyfile USING(diff_id)
Index: /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_data.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_data.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_data.sql	(revision 28794)
@@ -5,4 +5,5 @@
     diffPhotRun.workdir,
     diffPhotRun.label,
+    diffPhotRun.diff_id,
     diffRun.bothways,
     diffRun.diff_mode,
@@ -22,4 +23,5 @@
     camInput.cam_id AS cam_id_1,
     fakeInput.fake_id AS fake_id_1,
+    warp1,
     camProcessedInput.sigma_ra AS sigma_ra_1,
     camProcessedInput.sigma_dec AS sigma_dec_1,
@@ -29,4 +31,5 @@
     camTemplate.cam_id AS cam_id_2,
     fakeTemplate.fake_id AS fake_id_2,
+    warp2,
     camProcessedTemplate.sigma_ra AS sigma_ra_2,
     camProcessedTemplate.sigma_dec AS sigma_dec_2
Index: /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_input.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_input.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/diffphottool_input.sql	(revision 28794)
@@ -1,4 +1,5 @@
 SELECT
-    diffPhotRun.*,
+    diff_phot_id,
+    skycell_id,
     diffSkyfile.path_base,
     diffSkyfile.magicked,
Index: /branches/eam_branches/ipp-20100621/ippTools/share/difftool_setskyfiletoupdate.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/difftool_setskyfiletoupdate.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/difftool_setskyfiletoupdate.sql	(revision 28794)
@@ -12,4 +12,4 @@
     -- don't queue update if the associated magicDSFile still exists
     AND (diffRun.magicked = 0 
-      OR (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'new')
+      OR (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'new' OR magicDSRun.state IS NULL)
             AND magicDSFile.component IS NULL)
Index: /branches/eam_branches/ipp-20100621/ippTools/share/disttool_definebyquery_chip_bg.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/disttool_definebyquery_chip_bg.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/disttool_definebyquery_chip_bg.sql	(revision 28794)
@@ -0,0 +1,26 @@
+SELECT DISTINCT
+    'chip_bg' as stage,
+    chipBackgroundRun.chip_bg_id as stage_id,
+    chipBackgroundRun.magicked,
+    rawExp.exp_name as run_tag,
+    chipBackgroundRun.label,
+    chipBackgroundRun.data_group,
+    chipBackgroundRun.dist_group,
+    distTarget.target_id,
+    distTarget.clean
+FROM chipBackgroundRun
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+JOIN distTarget
+    ON distTarget.stage = 'chip_bg'
+    AND chipBackgroundRun.dist_group = distTarget.dist_group
+    AND rawExp.filter = distTarget.filter
+JOIN rcInterest USING(target_id)
+LEFT JOIN distRun
+    ON distRun.stage_id = chipBackgroundRun.chip_bg_id
+    AND distRun.target_id = distTarget.target_id
+    -- JOIN hook %s
+WHERE distTarget.state = 'enabled'
+    AND rcInterest.state = 'enabled'
+    AND distRun.dist_id IS NULL      -- no existing distRun
+    AND ((chipBackgroundRun.state = 'full') OR (distTarget.clean AND chipBackgroundRun.state = 'cleaned'))
Index: /branches/eam_branches/ipp-20100621/ippTools/share/disttool_definebyquery_warp_bg.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/disttool_definebyquery_warp_bg.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/disttool_definebyquery_warp_bg.sql	(revision 28794)
@@ -0,0 +1,30 @@
+SELECT DISTINCT
+    'warp_bg' as stage,
+    warpBackgroundRun.warp_bg_id AS stage_id,
+    warpBackgroundRun.magicked,
+    rawExp.exp_name as run_tag,
+    warpBackgroundRun.label,
+    warpBackgroundRun.data_group,
+    distTarget.dist_group,
+    distTarget.target_id,
+    distTarget.clean
+FROM warpBackgroundRun
+JOIN warpRun USING(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun ON fakeRun.cam_id = camRun.cam_id
+JOIN chipRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+JOIN distTarget
+    ON distTarget.stage = 'warp_bg'
+    AND warpBackgroundRun.dist_group = distTarget.dist_group
+    AND rawExp.filter = distTarget.filter
+JOIN rcInterest USING(target_id)
+LEFT JOIN distRun ON (distRun.stage_id = warp_bg_id)
+    AND distRun.target_id = distTarget.target_id
+WHERE  distTarget.state = 'enabled'
+    AND rcInterest.state = 'enabled'
+    AND distRun.dist_id IS NULL
+    AND ((warpBackgroundRun.state = 'full') OR (distTarget.clean AND warpBackgroundRun.state = 'cleaned'))
+
+
+
Index: /branches/eam_branches/ipp-20100621/ippTools/share/disttool_pending_chip_bg.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/disttool_pending_chip_bg.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/disttool_pending_chip_bg.sql	(revision 28794)
@@ -0,0 +1,32 @@
+SELECT
+    distRun.dist_id,
+    distRun.label,
+    distTarget.dist_group,
+    stage,
+    stage_id,
+    chipBackgroundImfile.class_id AS component,
+    distRun.clean,
+    rawExp.camera,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
+    chipBackgroundImfile.path_base,
+    chipBackgroundImfile.path_base as chip_path_base,
+    chipBackgroundRun.state,
+    chipBackgroundRun.state AS data_state,
+    0 AS quality,
+    distRun.no_magic,
+    chipBackgroundImfile.magicked
+FROM distRun
+JOIN distTarget USING(target_id, stage, clean)
+JOIN chipBackgroundRun ON chipBackgroundRun.chip_bg_id = distRun.stage_id
+JOIN chipRun using(chip_id)
+JOIN rawExp using(exp_id)
+JOIN chipBackgroundImfile ON chipBackgroundImfile.chip_bg_id = stage_id
+LEFT JOIN distComponent 
+    ON distRun.dist_id = distComponent.dist_id 
+    AND chipBackgroundImfile.class_id = distComponent.component
+WHERE
+    distRun.state = 'new'
+    AND distRun.stage = 'chip_bg'
+    AND distComponent.dist_id IS NULL
+    AND ((chipBackgroundRun.magicked > 0) OR distRun.no_magic)
+    AND (chipBackgroundRun.state = 'full')
Index: /branches/eam_branches/ipp-20100621/ippTools/share/disttool_pending_warp_bg.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/disttool_pending_warp_bg.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/disttool_pending_warp_bg.sql	(revision 28794)
@@ -0,0 +1,35 @@
+SELECT
+    distRun.dist_id,
+    distRun.label,
+    distTarget.dist_group,
+    stage,
+    stage_id,
+    warpBackgroundSkyfile.skycell_id AS component,
+    clean,
+    rawExp.camera,
+    CONCAT_WS('.', outroot, CONVERT(distRun.dist_id, CHAR)) as outdir,
+    warpBackgroundSkyfile.path_base,
+    CAST(NULL AS CHAR(255)) as chip_path_base,
+    warpBackgroundRun.state,
+    warpBackgroundRun.state as data_state,
+    0 as quality,
+    distRun.no_magic,
+    warpBackgroundSkyfile.magicked
+FROM distRun
+JOIN distTarget USING(target_id, stage, clean)
+JOIN warpBackgroundRun ON stage_id = warp_bg_id
+JOIN warpBackgroundSkyfile using(warp_bg_id)
+JOIN warpRun using(warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun ON camRun.chip_id = chipRun.chip_id
+JOIN rawExp using(exp_id)
+LEFT JOIN distComponent 
+    ON distRun.dist_id = distComponent.dist_id 
+    AND warpBackgroundSkyfile.skycell_id = distComponent.component
+WHERE
+    distRun.state = 'new'
+    AND distRun.stage = 'warp_bg'
+    AND distComponent.dist_id IS NULL
+    AND ((warpBackgroundRun.magicked > 0) OR distRun.no_magic)
+    AND (warpBackgroundRun.state = 'full')
Index: /branches/eam_branches/ipp-20100621/ippTools/share/disttool_toadvance.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/disttool_toadvance.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/disttool_toadvance.sql	(revision 28794)
@@ -74,4 +74,28 @@
     HAVING
         COUNT(chipProcessedImfile.class_id) = COUNT(distComponent.component)
+        AND SUM(distComponent.fault) = 0
+UNION
+-- chip_bg stage
+SELECT
+    distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    label,
+    clean
+    FROM distRun
+    JOIN chipBackgroundImfile ON stage_id = chipBackgroundImfile.chip_bg_id
+    LEFT JOIN distComponent
+        ON distComponent.dist_id = distRun.dist_id
+        AND distComponent.component = chipBackgroundImfile.class_id
+    WHERE
+        distRun.state = 'new'
+        AND distRun.fault = 0
+        AND distRun.stage = 'chip_bg'
+    GROUP BY
+        dist_id,
+        chipBackgroundImfile.chip_bg_id
+    HAVING
+        COUNT(chipBackgroundImfile.class_id) = COUNT(distComponent.component)
         AND SUM(distComponent.fault) = 0
 UNION
@@ -145,4 +169,30 @@
         AND SUM(distComponent.fault) = 0
 UNION
+-- warp_bg stage
+SELECT
+    distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    label,
+    clean
+    FROM distRun
+    JOIN warpBackgroundSkyfile on stage_id = warp_bg_id
+    LEFT JOIN distComponent
+        ON distRun.dist_id = distComponent.dist_id
+        AND distComponent.component = warpBackgroundSkyfile.skycell_id
+    WHERE
+        distRun.state = 'new'
+        AND distRun.fault = 0
+        AND distRun.stage = 'warp_bg'
+--        AND warpSkyfile.fault = 0
+--        AND warpSkyfile.quality = 0
+    GROUP BY
+        distRun.dist_id,
+        warp_bg_id
+    HAVING
+        COUNT(warpBackgroundSkyfile.skycell_id) = COUNT(distComponent.component)
+        AND SUM(distComponent.fault) = 0
+UNION
 -- diff stage
 SELECT
Index: /branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_definebyquery_chip_bg.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_definebyquery_chip_bg.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_definebyquery_chip_bg.sql	(revision 28794)
@@ -0,0 +1,33 @@
+SELECT DISTINCT       -- DISTINCT because we are going though diffInputSkyfile
+    magicRun.magic_id,
+    exp_id,
+    'chip_bg' AS stage,
+    chip_bg_id AS stage_id,
+    chipBackgroundRun.data_group,
+    camRun.cam_id,
+    magicRun.label,
+    magicRun.workdir,
+    CAST(NULL AS SIGNED) AS inv_magic_id,
+    CAST(NULL AS SIGNED) AS inv_exp_id
+FROM magicRun
+JOIN magicMask USING(magic_id)
+JOIN diffRun USING(diff_id)
+JOIN diffInputSkyfile USING(diff_id)
+JOIN warpRun
+    ON (!magicRun.inverse AND warp1 = warp_id)
+    OR ( magicRun.inverse AND warp2 = warp_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id, exp_id)
+JOIN chipBackgroundRun USING(chip_id)
+JOIN rawExp USING(exp_id)
+LEFT JOIN magicDSRun
+    ON magicRun.magic_id = magicDSRun.magic_id
+    AND magicDSRun.stage = 'chip_bg'
+WHERE magicRun.state = 'full'
+    AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s
+    OR magicDSRun.magic_ds_id IS NULL)
+    AND chipBackgroundRun.magicked = 0
+    AND chipBackgroundRun.state = 'full'
+    AND camRun.state = 'full'
+
Index: /branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_definebyquery_warp_bg.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_definebyquery_warp_bg.sql	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/magicdstool_definebyquery_warp_bg.sql	(revision 28794)
@@ -0,0 +1,32 @@
+SELECT DISTINCT       -- DISTINCT because we are going though diffInputSkyfile
+    magicRun.magic_id,
+    exp_id,
+    'warp_bg' AS stage,
+    warp_bg_id AS stage_id,
+    warpBackgroundRun.data_group,
+    0 AS cam_id,
+    magicRun.label,
+    magicRun.workdir,
+    CAST(NULL AS SIGNED) AS inv_magic_id,
+    CAST(NULL AS SIGNED) AS inv_exp_id
+FROM magicRun
+JOIN magicMask USING(magic_id)
+JOIN diffRun USING(diff_id)
+JOIN diffInputSkyfile USING(diff_id)
+JOIN warpRun
+    ON (!magicRun.inverse AND warp1 = warp_id)
+    OR ( magicRun.inverse AND warp2 = warp_id)
+JOIN warpBackgroundRun USING(warp_bg_id)
+JOIN fakeRun USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id, exp_id)
+JOIN rawExp USING(exp_id)
+LEFT JOIN magicDSRun
+    ON magicRun.magic_id = magicDSRun.magic_id
+    AND magicDSRun.stage = 'warp_bg'
+WHERE magicRun.state = 'full'
+    AND ( -- rerun HOOK magicdstool sends "\n1 " if rerun else "\n0 " %s
+    OR magicDSRun.magic_ds_id IS NULL)
+    AND warpBackgroundRun.magicked  = 0
+    AND warpBackgroundRun.state = 'full'
+
Index: /branches/eam_branches/ipp-20100621/ippTools/share/magictool_definebyquery_select.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/magictool_definebyquery_select.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/magictool_definebyquery_select.sql	(revision 28794)
@@ -4,5 +4,6 @@
     MAX(diffWarps.diff_id) AS diff_id,
     -- The following trick pulls out the 'inverse' value for the maximum diff_id
-    CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id), ',', 1), UNSIGNED) AS inverse
+    CONVERT(SUBSTRING_INDEX(GROUP_CONCAT(diffWarps.inverse ORDER BY diffWarps.diff_id), ',', 1), UNSIGNED) AS inverse,
+    diff_data_group
 FROM (
     -- Forward diffs
@@ -10,5 +11,6 @@
         diffRun.diff_id,
         warp1 AS warp_id,
-        0 AS inverse
+        0 AS inverse,
+        diffRun.data_group AS diff_data_group
     FROM diffRun
     JOIN diffInputSkyfile USING(diff_id)
@@ -22,5 +24,6 @@
         diffRun.diff_id,
         warp2 AS warp_id,
-        1 AS inverse
+        1 AS inverse,
+        diffRun.data_group AS diff_data_group
     FROM diffRun
     JOIN diffInputSkyfile USING(diff_id)
Index: /branches/eam_branches/ipp-20100621/ippTools/share/magictool_toprocess_inputs.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/magictool_toprocess_inputs.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/magictool_toprocess_inputs.sql	(revision 28794)
@@ -7,5 +7,6 @@
     -- note that the type stays a 64 bit int
     magicNodeResult.magic_id IS TRUE AS done,
-    magicNodeResult.fault IS TRUE AS bad
+    magicNodeResult.fault IS TRUE AS bad,
+    IFNULL(Label.priority, 10000) AS priority
 FROM magicRun
 JOIN magicTree USING(magic_id)
@@ -18,4 +19,5 @@
     ON diffRun.diff_id =  diffSkyfile.diff_id
 LEFT JOIN magicNodeResult USING(magic_id, node)
+LEFT JOIN Label ON magicRun.label = Label.label
 WHERE
     magicRun.state = 'new'
@@ -23,3 +25,4 @@
     AND magicNodeResult.node IS NULL
     AND diffRun.state = 'full'
+    AND (Label.active OR Label.active IS NULL)
 -- WHERE hook %s
Index: /branches/eam_branches/ipp-20100621/ippTools/share/magictool_toprocess_runs.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/magictool_toprocess_runs.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/magictool_toprocess_runs.sql	(revision 28794)
@@ -1,7 +1,9 @@
 SELECT DISTINCT
-    magic_id
+    magic_id,
+    IFNULL(priority, 10000) AS priority
 FROM magicTree
 JOIN magicRun USING(magic_id)
 LEFT JOIN magicNodeResult USING(magic_id, node)
+LEFT JOIN Label ON magicRun.label = Label.label
 WHERE
     magicRun.state = 'new'
@@ -9,3 +11,3 @@
 -- WHERE hook %s
 ORDER BY
-    magicRun.magic_id
+    priority DESC, magicRun.magic_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/magictool_toprocess_tree.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/magictool_toprocess_tree.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/magictool_toprocess_tree.sql	(revision 28794)
@@ -7,12 +7,14 @@
     -- note that the type stays a 64 bit int
     magicNodeResult.magic_id IS TRUE AS done,
-    magicNodeResult.fault IS TRUE AS bad
+    magicNodeResult.fault IS TRUE AS bad,
+    IFNULL(Label.priority, 10000) AS priority
 FROM magicTree
 JOIN magicRun USING(magic_id)
 JOIN rawExp USING(exp_id)
 LEFT JOIN magicNodeResult USING(magic_id, node)
+LEFT JOIN Label ON magicRun.label = Label.label
 WHERE
     magicRun.state = 'new'
 -- WHERE hook %s
 ORDER BY
-    magicRun.magic_id
+    priority, magicRun.magic_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/magictool_totree.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/magictool_totree.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/magictool_totree.sql	(revision 28794)
@@ -6,5 +6,6 @@
     ra,
     decl,
-    diffRun.tess_id
+    diffRun.tess_id,
+    IFNULL(priority, 10000) AS priority
 FROM magicRun
 JOIN diffRun USING(diff_id)
@@ -12,4 +13,5 @@
 LEFT JOIN magicTree
     USING(magic_id)
+LEFT JOIN Label ON magicRun.label = Label.label
 WHERE
     magicRun.state = 'new'
Index: /branches/eam_branches/ipp-20100621/ippTools/share/pubtool_pending.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/pubtool_pending.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/pubtool_pending.sql	(revision 28794)
@@ -50,4 +50,5 @@
         AND (camRun.magicked != 0 OR publishClient.magicked = 0)
         -- WHERE hook %s
+    UNION
     SELECT DISTINCT
         publishRun.pub_id,
Index: /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_create_tables.sql	(revision 28794)
@@ -375,4 +375,5 @@
     KEY(fault),
     KEY(quality),
+    KEY(data_state),
     FOREIGN KEY(chip_id, exp_id) REFERENCES chipRun(chip_id, exp_id),
     FOREIGN KEY(exp_id, class_id) REFERENCES rawImfile(exp_id, class_id)
@@ -973,4 +974,5 @@
     KEY(fault),
     KEY(quality),
+    KEY(data_state),
     FOREIGN KEY(warp_id, skycell_id, tess_id) REFERENCES warpSkyCellMap(warp_id, skycell_id, tess_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -1182,4 +1184,5 @@
         KEY(fault),
         KEY(quality),
+        KEY(data_state),
         FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -1715,4 +1718,6 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+
+-- Processing labels with their priorities
 CREATE TABLE Label (
     label       VARCHAR(64),
@@ -1723,4 +1728,101 @@
     KEY(priority),
     KEY(active)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Tables to support background restoration
+
+-- Background replacement on a chipRun
+CREATE TABLE chipBackgroundRun (
+    chip_bg_id BIGINT AUTO_INCREMENT, -- unique identifier
+    chip_id BIGINT NOT NULL,          -- link to chipRun
+    state VARCHAR(64) NOT NULL,       -- state of run (new, full, etc.)
+    workdir VARCHAR(255) NOT NULL,    -- working directory
+    label VARCHAR(64),                -- processing label
+    data_group VARCHAR(64),           -- group for data
+    dist_group VARCHAR(64),           -- group for distribution
+    reduction VARCHAR(64),    -- reduction class (for altering recipe)
+    note VARCHAR(255),        -- note
+    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
+    magicked BIGINT DEFAULT 0 NOT NULL, -- magic status
+    PRIMARY KEY(chip_bg_id),
+    KEY(chip_id),
+    KEY(state),
+    KEY(label),
+    KEY(data_group),
+    KEY(dist_group),
+    FOREIGN KEY(chip_id) REFERENCES chipRun(chip_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Results of background replacement from chipBackgroundRun
+CREATE TABLE chipBackgroundImfile (
+    chip_bg_id BIGINT NOT NULL,        -- unique identifier
+    class_id VARCHAR(64) NOT NULL,     -- class (component) identifier
+    path_base VARCHAR(255) NOT NULL,   -- root name for outputs
+    magicked BIGINT,                   -- magic_id if magicked
+    dtime_script FLOAT,                -- elapsed time for script
+    hostname VARCHAR(64) NOT NULL,     -- host that executed script
+    quality SMALLINT NOT NULL,         -- bad quality flag
+    fault SMALLINT NOT NULL,           -- fault code
+    software_ver VARCHAR(16),          -- software version
+    bg FLOAT,                          -- background level
+    bg_stdev FLOAT,                    -- stdev of background
+    maskfrac_npix FLOAT,               -- Number of pixels masked
+    maskfrac_static FLOAT,             -- Fraction masked static
+    maskfrac_dynamic FLOAT,            -- Fraction masked dynamic
+    maskfrac_magic FLOAT,              -- Fraction masked magic
+    maskfrac_advisory FLOAT,           -- Fraction masked advisory
+    PRIMARY KEY(chip_bg_id,class_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Background replacement on a warpRun (utilising chipBackgroundRun)
+CREATE TABLE warpBackgroundRun (
+    warp_bg_id BIGINT AUTO_INCREMENT, -- unique identifier
+    warp_id BIGINT NOT NULL,          -- link to warpRun
+    chip_bg_id BIGINT NOT NULL,       -- link to chipBackgroundRun
+    state VARCHAR(64) NOT NULL,       -- state of run (new, full, etc.)
+    workdir VARCHAR(255) NOT NULL,    -- working directory
+    label VARCHAR(64),                -- processing label
+    data_group VARCHAR(64),           -- group for data
+    dist_group VARCHAR(64),           -- group for distribution
+    reduction VARCHAR(64),    -- reduction class (for altering recipe)
+    note VARCHAR(255),        -- note
+    registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
+    magicked BIGINT DEFAULT 0 NOT NULL, -- magic status
+    PRIMARY KEY(warp_bg_id),
+    KEY(warp_id),
+    KEY(chip_bg_id),
+    KEY(state),
+    KEY(label),
+    KEY(data_group),
+    KEY(dist_group),
+    FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id),
+    FOREIGN KEY(chip_bg_id) REFERENCES chipBackgroundRun(chip_bg_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+-- Results of background replacement from warpBackgroundRun
+CREATE TABLE warpBackgroundSkyfile (
+    warp_bg_id BIGINT NOT NULL,        -- unique identifier
+    skycell_id VARCHAR(64) NOT NULL,   -- skycell identifier
+    path_base VARCHAR(255) NOT NULL,   -- root name for outputs
+    magicked BIGINT,                   -- magic_id if magicked
+    dtime_script FLOAT,                -- elapsed time for script
+    hostname VARCHAR(64) NOT NULL,     -- host that executed script
+    quality SMALLINT NOT NULL,         -- bad quality flag
+    fault SMALLINT NOT NULL,           -- fault code
+    software_ver VARCHAR(16),          -- software version
+    bg FLOAT,                          -- background level
+    bg_stdev FLOAT,                    -- stdev of background
+    maskfrac_npix FLOAT,               -- Number of pixels masked
+    maskfrac_static FLOAT,             -- Fraction masked static
+    maskfrac_dynamic FLOAT,            -- Fraction masked dynamic
+    maskfrac_magic FLOAT,              -- Fraction masked magic
+    maskfrac_advisory FLOAT,           -- Fraction masked advisory
+    PRIMARY KEY(warp_bg_id,skycell_id),
+    KEY(fault),
+    KEY(quality),
+    FOREIGN KEY(warp_bg_id) REFERENCES warpBackgroundRun(warp_bg_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
Index: /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_drop_tables.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_drop_tables.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/pxadmin_drop_tables.sql	(revision 28794)
@@ -91,4 +91,8 @@
 DROP TABLE IF EXISTS Label;
 DROP TABLE IF EXISTS pstampWebRequest;
+DROP TABLE IF EXISTS chipBackgroundRun;
+DROP TABLE IF EXISTS chipBackgroundImfile;
+DROP TABLE IF EXISTS warpBackgroundRun;
+DROP TABLE IF EXISTS warpBackgroundSkyfile;
 DROP TABLE IF EXISTS diffPhotRun;
 DROP TABLE IF EXISTS diffPhotSkyfile;
Index: /branches/eam_branches/ipp-20100621/ippTools/share/warptool_listrun.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/warptool_listrun.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/warptool_listrun.sql	(revision 28794)
@@ -10,5 +10,7 @@
     rawExp.ra,
     rawExp.decl,
-    rawExp.exp_time
+    rawExp.exp_time,
+    magicDSRun.state AS dsRun_state,
+    IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id
 FROM warpRun
 JOIN fakeRun
@@ -20,2 +22,4 @@
 JOIN rawExp
     ON chipRun.exp_id  = rawExp.exp_id
+LEFT JOIN magicDSRun
+    ON stage = 'warp' AND stage_id = warp_id
Index: /branches/eam_branches/ipp-20100621/ippTools/share/warptool_setskyfiletoupdate.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/warptool_setskyfiletoupdate.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/warptool_setskyfiletoupdate.sql	(revision 28794)
@@ -11,4 +11,4 @@
     AND (warpSkyfile.data_state = 'cleaned')
     AND (warpRun.magicked = 0
-      OR (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'new')
+      OR (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'new' OR magicDSRun.state IS NULL)
       AND magicDSFile.component IS NULL)
Index: /branches/eam_branches/ipp-20100621/ippTools/share/warptool_warped.sql
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/share/warptool_warped.sql	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/share/warptool_warped.sql	(revision 28794)
@@ -12,5 +12,7 @@
     rawExp.ra,
     rawExp.decl,
-    rawExp.exp_time
+    rawExp.exp_time,
+    magicDSRun.state AS dsRun_state,
+    IFNULL(magicDSRun.magic_ds_id, 0) AS magic_ds_id
 FROM warpRun
 JOIN warpSkyfile
@@ -26,2 +28,4 @@
 JOIN rawExp
     ON chipRun.exp_id  = rawExp.exp_id
+LEFT JOIN magicDSRun
+    ON stage = 'warp' AND stage_id = warp_id
Index: /branches/eam_branches/ipp-20100621/ippTools/src/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/Makefile.am	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/Makefile.am	(revision 28794)
@@ -1,4 +1,5 @@
 bin_PROGRAMS = \
 	addtool \
+	bgtool \
 	caltool \
 	camtool \
@@ -48,4 +49,5 @@
 noinst_HEADERS = \
 	addtool.h \
+	bgtool.h \
 	caltool.h \
 	camtool.h \
@@ -117,4 +119,10 @@
     addtoolConfig.c
 
+bgtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+bgtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+bgtool_SOURCES = \
+    bgtool.c \
+    bgtoolConfig.c
+
 caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
 caltool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
Index: /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.c	(revision 28794)
@@ -0,0 +1,1809 @@
+/*
+ * bgtool.c
+ *
+ * Copyright (C) 2006-2010  Joshua Hoblitt, Paul Price
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * program; see the file COPYING. If not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifdef HAVB_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <math.h>
+#include <ippdb.h>
+
+#include "pxtools.h"
+#include "bgtool.h"
+
+static bool definechipMode(pxConfig *config);
+static bool updatechipMode(pxConfig *config);
+static bool tochipMode(pxConfig *config);
+static bool chipinputsMode(pxConfig *config);
+static bool addchipMode(pxConfig *config);
+static bool chipMode(pxConfig *config);
+static bool advancechipMode(pxConfig *config);
+static bool revertchipMode(pxConfig *config);
+static bool definewarpMode(pxConfig *config);
+static bool updatewarpMode(pxConfig *config);
+static bool towarpMode(pxConfig *config);
+static bool warpinputsMode(pxConfig *config);
+static bool addwarpMode(pxConfig *config);
+static bool warpMode(pxConfig *config);
+static bool advancewarpMode(pxConfig *config);
+static bool revertwarpMode(pxConfig *config);
+static bool tocleanchipMode(pxConfig *config);
+static bool cleanedchipMode(pxConfig *config);
+static bool tocleanwarpMode(pxConfig *config);
+static bool cleanedwarpMode(pxConfig *config);
+static bool exportchipMode(pxConfig *config);
+static bool importchipMode(pxConfig *config);
+static bool exportwarpMode(pxConfig *config);
+static bool importwarpMode(pxConfig *config);
+
+// Tables to import/export
+typedef struct {
+    const char *name;                   // Table name
+    void* (*parse)();                   // Parsing function
+    bool (*insert)();                   // Insertion function
+} tableData;
+static const tableData chipTables[] = {
+    { "chipBackgroundRun", (void*)&chipBackgroundRunObjectFromMetadata, &chipBackgroundRunInsertObject },
+    { "chipBackgroundImfile", (void*)&chipBackgroundImfileObjectFromMetadata, &chipBackgroundImfileInsertObject },
+    { NULL, NULL, NULL }
+};
+static const tableData warpTables[] = {
+    { "warpBackgroundRun", (void*)&warpBackgroundRunObjectFromMetadata, &warpBackgroundRunInsertObject },
+    { "warpBackgroundSkyfile", (void*)&warpBackgroundSkyfileObjectFromMetadata, &warpBackgroundSkyfileInsertObject },
+    { NULL, NULL, NULL }
+};
+
+
+# define MODECASE(caseName, func) \
+    case caseName: \
+    if (!func(config)) { \
+        goto FAIL; \
+    } \
+    break;
+
+int main(int argc, char **argv)
+{
+    psLibInit(NULL);
+
+    pxConfig *config = bgtoolConfig(NULL, argc, argv);
+    if (!config) {
+        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
+        goto FAIL;
+    }
+
+    switch (config->mode) {
+        MODECASE(BGTOOL_MODE_DEFINECHIP,  definechipMode);
+        MODECASE(BGTOOL_MODE_UPDATECHIP,  updatechipMode);
+        MODECASE(BGTOOL_MODE_TOCHIP,      tochipMode);
+        MODECASE(BGTOOL_MODE_CHIPINPUTS,  chipinputsMode);
+        MODECASE(BGTOOL_MODE_ADDCHIP,     addchipMode);
+        MODECASE(BGTOOL_MODE_CHIP,        chipMode);
+        MODECASE(BGTOOL_MODE_ADVANCECHIP, advancechipMode);
+        MODECASE(BGTOOL_MODE_REVERTCHIP,  revertchipMode);
+        MODECASE(BGTOOL_MODE_DEFINEWARP,  definewarpMode);
+        MODECASE(BGTOOL_MODE_UPDATEWARP,  updatewarpMode);
+        MODECASE(BGTOOL_MODE_TOWARP,      towarpMode);
+        MODECASE(BGTOOL_MODE_WARPINPUTS,  warpinputsMode);
+        MODECASE(BGTOOL_MODE_ADDWARP,     addwarpMode);
+        MODECASE(BGTOOL_MODE_WARP,        warpMode);
+        MODECASE(BGTOOL_MODE_ADVANCEWARP, advancewarpMode);
+        MODECASE(BGTOOL_MODE_REVERTWARP,  revertwarpMode);
+        MODECASE(BGTOOL_MODE_TOCLEANCHIP, tocleanchipMode);
+        MODECASE(BGTOOL_MODE_CLEANEDCHIP, cleanedchipMode);
+        MODECASE(BGTOOL_MODE_TOCLEANWARP, tocleanwarpMode);
+        MODECASE(BGTOOL_MODE_CLEANEDWARP, cleanedwarpMode);
+        MODECASE(BGTOOL_MODE_EXPORTCHIP,  exportchipMode);
+        MODECASE(BGTOOL_MODE_IMPORTCHIP,  importchipMode);
+        MODECASE(BGTOOL_MODE_EXPORTWARP,  exportwarpMode);
+        MODECASE(BGTOOL_MODE_IMPORTWARP,  importwarpMode);
+
+        default:
+            psAbort("invalid option (this should not happen)");
+    }
+
+    psFree(config);
+    pmConfigDone();
+    psLibFinalize();
+
+    exit(EXIT_SUCCESS);
+
+FAIL:
+    psErrorStackPrint(stderr, "\n");
+    int exit_status = pxerrorGetExitStatus();
+
+    psFree(config);
+    pmConfigDone();
+    psLibFinalize();
+
+    exit(exit_status);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// General functions
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static bool exportTables(pxConfig *config, // Configuration (with DB handle)
+                         const char *filename, // Filename to which to write
+                         const tableData tables[], // Tables to export (NULL terminated)
+                         const psMetadata *where, // WHERE restrictions
+                         bool clean               // Write as cleaned?
+                         )
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    FILE *file = fopen(filename, "w");
+    if (!file) {
+        psError(PXTOOLS_ERR_SYS, true, "failed to open output file %s", filename);
+        return false;
+    }
+
+    if (!pxExportVersion(config, file)) {
+        psError(psErrorCodeLast(), false, "failed to write dbversion to output file %s", filename);
+        return false;
+    }
+
+    for (int i = 0; tables[i].name; i++) {
+        const char *name = tables[i].name; // Name of table
+        psString query = NULL;
+        psStringAppend(&query, "SELECT * FROM %s", name);
+
+        if (psListLength(where->list)) {
+            psString whereClause = psDBGenerateWhereSQL(where, NULL);
+            psStringAppend(&query, " %s", whereClause);
+            psFree(whereClause);
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(psErrorCodeLast(), false, "database error");
+            psFree(query);
+            return false;
+        }
+        psFree(query);
+
+        psArray *output = p_psDBFetchResult(config->dbh);
+        if (!output) {
+            psError(psErrorCodeLast(), false, "database error");
+            return false;
+        }
+        if (!psArrayLength(output)) {
+            psError(PXTOOLS_ERR_CONFIG, true, "no rows found");
+            psFree(output);
+            return false;
+        }
+
+        if (clean &&
+            (strcmp(name, "chipBackgroundRun") == 0 ||
+             strcmp(name, "warpBackgroundRun") == 0) &&
+            !pxSetStateCleaned(name, "state", output)) {
+            psFree(output);
+            psError(psErrorCodeLast(), false, "pxSetStateClean failed for table %s", name);
+            return false;
+        }
+
+        if (!ippdbPrintMetadatas(file, output, name, true)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+    }
+    fclose(file);
+
+    return true;
+}
+
+static bool importTables(pxConfig *config, // Configuration
+                        const char *filename, //
+                        const tableData tables[] // Tables to read in
+                        )
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    unsigned int badLines = 0;          // Number of bad lines
+    psMetadata *input = psMetadataConfigRead(NULL, &badLines, filename, false); // Input file contents
+    if (!input) {
+        psError(psErrorCodeLast(), false, "Unable to parse input file %s", filename);
+        return false;
+    }
+    if (badLines > 0) {
+        psWarning("%d bad lines encountered when parsing %s", badLines, filename);
+    }
+
+    if (!pxCheckImportVersion(config, input)) {
+        psError(psErrorCodeLast(), false, "pxCheckImportVersion failed");
+        return false;
+    }
+
+    // Import primary table
+    for (int i = 0; tables[i].name; i++) {
+        const char *name = tables[i].name; // Name of table
+        psMetadataItem *item = psMetadataLookup(input, name); // Item from input
+        psAssert(item, "%s not in input", name);
+        psAssert(item->type == PS_DATA_METADATA_MULTI, "%s not MULTI type", name);
+        psAssert(psListLength(item->data.list) == 1, "%s has multiple entries", name);
+        psMetadataItem *entry = psListGet(item->data.list, PS_LIST_HEAD); // Entry of interest
+        void *data = tables[i].parse(entry);                             // Parsed entry
+        if (!data) {
+            psError(PXTOOLS_ERR_CONFIG, false, "Unable to parse entry %s", name);
+            psFree(input);
+            return false;
+        }
+        if (!tables[0].insert(config->dbh, data)) {
+            psError(psErrorCodeLast(), false, "Unable to insert entry %s", name);
+            psFree(input);
+            return false;
+        }
+    }
+
+    psFree(input);
+
+    return true;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Functions for chip stage
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static bool definechipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args,   where, "-chip_id",            "chipRun.chip_id",       "==");
+    PXOPT_COPY_S64(config->args,   where, "-exp_id",             "rawExp.exp_id",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-exp_name",           "rawExp.exp_name",       "==");
+    PXOPT_COPY_STR(config->args,   where, "-inst",               "rawExp.camera",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-telescope",          "rawExp.telescope",      "==");
+    PXOPT_COPY_TIME(config->args,  where, "-dateobs_begin",      "rawExp.dateobs",        ">=");
+    PXOPT_COPY_TIME(config->args,  where, "-dateobs_end",        "rawExp.dateobs",        "<=");
+    PXOPT_COPY_STR(config->args,   where, "-exp_tag",            "rawExp.exp_tag",        "==");
+    PXOPT_COPY_STR(config->args,   where, "-exp_type",           "rawExp.exp_type",       "==");
+    PXOPT_COPY_STR(config->args,   where, "-filelevel",          "rawExp.filelevel",      "==");
+    PXOPT_COPY_STR(config->args,   where, "-filter",             "rawExp.filter",         "==");
+    PXOPT_COPY_F64(config->args,   where, "-airmass_min",        "rawExp.airmass",        ">=");
+    PXOPT_COPY_F64(config->args,   where, "-airmass_max",        "rawExp.airmass",        "<");
+    PXOPT_COPY_RADEC(config->args, where, "-ra_min",             "rawExp.ra",             ">=");
+    PXOPT_COPY_RADEC(config->args, where, "-ra_max",             "rawExp.ra",             "<");
+    PXOPT_COPY_RADEC(config->args, where, "-decl_min",           "rawExp.decl",           ">=");
+    PXOPT_COPY_RADEC(config->args, where, "-decl_max",           "rawExp.decl",           "<");
+    PXOPT_COPY_F32(config->args,   where, "-exp_time_min",       "rawExp.exp_time",       ">=");
+    PXOPT_COPY_F32(config->args,   where, "-exp_time_max",       "rawExp.exp_time",       "<");
+    PXOPT_COPY_F32(config->args,   where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args,   where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args,   where, "-bg_min",             "rawExp.bg",             ">=");
+    PXOPT_COPY_F64(config->args,   where, "-bg_max",             "rawExp.bg",             "<");
+    PXOPT_COPY_F64(config->args,   where, "-bg_stdev_min",       "rawExp.bg_stdev",       ">=");
+    PXOPT_COPY_F64(config->args,   where, "-bg_stdev_max",       "rawExp.bg_stdev",       "<");
+    PXOPT_COPY_F64(config->args,   where, "-bg_mean_stdev_min",  "rawExp.bg_mean_stdev",  ">=");
+    PXOPT_COPY_F64(config->args,   where, "-bg_mean_stdev_max",  "rawExp.bg_mean_stdev",  "<");
+    PXOPT_COPY_F64(config->args,   where, "-alt_min",            "rawExp.alt",            ">=");
+    PXOPT_COPY_F64(config->args,   where, "-alt_max",            "rawExp.alt",            "<");
+    PXOPT_COPY_F64(config->args,   where, "-az_min",             "rawExp.az",             ">=");
+    PXOPT_COPY_F64(config->args,   where, "-az_max",             "rawExp.az",             "<");
+    PXOPT_COPY_F32(config->args,   where, "-ccd_temp_min",       "rawExp.ccd_temp",       ">=");
+    PXOPT_COPY_F32(config->args,   where, "-ccd_temp_max",       "rawExp.ccd_temp",       "<");
+    PXOPT_COPY_F64(config->args,   where, "-posang_min",         "rawExp.posang",         ">=");
+    PXOPT_COPY_F64(config->args,   where, "-posang_max",         "rawExp.posang",         "<");
+    PXOPT_COPY_STR(config->args,   where, "-object",             "rawExp.object",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-comment",            "rawExp.comment",        "LIKE");
+    PXOPT_COPY_STR(config->args,   where, "-obs_mode",           "rawExp.obs_mode",       "LIKE");
+    PXOPT_COPY_F32(config->args,   where, "-sun_angle_min",      "rawExp.sun_angle",      ">=");
+    PXOPT_COPY_F32(config->args,   where, "-sun_angle_max",      "rawExp.sun_angle",      "<");
+    pxAddLabelSearchArgs(config,   where, "-label",              "chipRun.label",         "==");
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required");
+        return false;
+    }
+
+    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
+    PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    // Get chip runs to promote to chipBackgroundRun
+
+    psString query = pxDataGet("bgtool_definechip.sql"); // Query to execute
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\nAND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (destreaked) {
+        psStringAppend(&query, " AND chipRun.magicked > 0");
+    }
+
+    if (!rerun) {
+        psStringAppend(&query, "\nAND chipBackgroundRun.chip_bg_id IS NULL");
+        if (label) {
+            psStringAppend(&query,
+                           "\nAND (chipBackgroundRun.label = '%s'"
+                           " OR chipBackgroundRun.label IS NULL)",
+                           label);
+        }
+        if (data_group) {
+            psStringAppend(&query,
+                           "\nAND (chipBackgroundRun.data_group = '%s'"
+                           " OR chipBackgroundRun.data_group IS NULL)",
+                           data_group);
+        }
+        if (dist_group) {
+            psStringAppend(&query,
+                           "\nAND (chipBackgroundRun.dist_group = '%s'"
+                           " OR chipBackgroundRun.dist_group IS NULL)",
+                           dist_group);
+        }
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh); // Matching rows
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return true;
+    }
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        chipRunRow *row = chipRunObjectFromMetadata(md);
+        if (!row) {
+            psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        if (!chipBackgroundRunInsert(config->dbh, 0, row->chip_id, "new",
+                                     workdir     ? workdir    : row->workdir,
+                                     label       ? label      : row->label,
+                                     data_group  ? data_group : row->data_group,
+                                     dist_group  ? dist_group : row->dist_group,
+                                     reduction   ? reduction  : row->reduction,
+                                     note        ? note       : row->note,
+                                     NULL, 0)) {
+            psError(psErrorCodeLast(), false, "database error");
+            psFree(row);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psFree(row);
+    }
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+        return false;
+    }
+
+    return true;
+}
+
+static bool updatechipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id",   "==");
+    PXOPT_COPY_STR(config->args, where, "-state",      "chipBackgroundRun.state",     "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "chipBackgroundRun.data_group","LIKE");
+    PXOPT_COPY_STR(config->args, where, "-dist_group", "chipBackgroundRun.dist_group","LIKE");
+    pxAddLabelSearchArgs(config,  where, "-label",     "chipBackgroundRun.label",     "LIKE");
+    PXOPT_COPY_TIME(config->args, where, "-registered_begin", "chipBackgroundRun.registered",  ">=");
+    PXOPT_COPY_TIME(config->args, where, "-registered_end",   "chipBackgroundRun.registered",  "<");
+
+    PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false);
+    if (destreaked) {
+        psMetadataAddS64(where, PS_LIST_TAIL, "chipBackgroundRun.magicked", PS_META_DUPLICATE_OK, ">", 0);
+    }
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = psStringCopy("UPDATE chipBackgroundRun");
+    bool result = pxUpdateRun(config, where, &query, "chipBackgroundRun", "chip_bg_id",
+                              "chipBackgroundImfile", true);
+
+    psFree(query);
+    psFree(where);
+
+    return result;
+}
+
+static bool tochipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("bgtool_tochip.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString whereStr = psStringCopy("");
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&whereStr, "\n AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, whereStr)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(whereStr);
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query);
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool chipinputsMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // find all rawImfiles matching the default query
+    psString query = pxDataGet("bgtool_chipinputs.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool addchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false);
+    PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
+
+    // optional
+
+    PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
+    PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
+    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false);
+    PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false);
+    PXOPT_LOOKUP_STR(ver_ppbackground, config->args, "-ver_ppbackground", false, false);
+    PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false);
+    PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
+    PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
+    PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false);
+
+    psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules);
+    ver_code = pxMergeCodeVersions(ver_code, ver_ppbackground);
+    ver_code = pxMergeCodeVersions(ver_code, ver_ppstats);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script,
+                                    hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
+                                    maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool chipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id",    "chipBackgroundRun.chip_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label",   "chipBackgroundRun.label", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-data_group",   "chipBackgroundRun.data_group", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-dist_group",   "chipBackgroundRun.data_group", "LIKE");
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // find all rawImfiles matching the default query
+    psString query = pxDataGet("bgtool_chip.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString magicWhere = NULL;
+    if (!pxmagicAddWhere(config, &magicWhere, "chipBackgroundImfile")) {
+        psError(psErrorCodeLast(), false, "pxMagicAddWhere failed");
+        return false;
+    }
+    if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) {
+        psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else if (!all && !magicWhere) {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+        return false;
+    }
+    if (magicWhere) {
+        psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere);
+    }
+    psFree(magicWhere);
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    if (psArrayLength(output)) {
+        if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(output);
+
+    return true;
+}
+
+
+static bool advancechipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "label", "==");
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+
+    psString select = pxDataGet("bgtool_advancechip.sql");
+    if (!select) {
+        psError(psErrorCodeLast(), false, "failed to retrieve SQL statement");
+        return false;
+    }
+
+    psString selectWhere = psStringCopy("");
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&selectWhere, "\n AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&select, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(select);
+        psFree(selectWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    psFree(select);
+    psFree(selectWhere);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+        bool status = true;             // Status of MD lookup
+        psS64 chip_bg_id = psMetadataLookupS64(&status, row, "chip_bg_id");
+        if (!status) {
+            psError(PXTOOLS_ERR_PROG, true, "failed to look up value for chip_bg_id");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psS64 magicked = psMetadataLookupS64(&status, row, "magicked");
+        if (!status) {
+            psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        if (!p_psDBRunQueryF(config->dbh,
+                             "UPDATE chipBackgroundRun "
+                             "SET state = 'full', magicked = %" PRId64 " "
+                             " WHERE chip_bg_id = %" PRId64,
+                             magicked, chip_bg_id)) {
+            psError(psErrorCodeLast(), false, "database error");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        psS64 numUpdated = psDBAffectedRows(config->dbh);
+        if (numUpdated != 1) {
+            psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row");
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool revertchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "chipBackgroundImfile.fault", "==");
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required");
+        return false;
+    }
+
+    psString query = pxDataGet("bgtool_revertchip.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    int numDeleted = psDBAffectedRows(config->dbh);
+    psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d chipBackgroundImfiles", numDeleted);
+
+    return true;
+}
+
+static bool tocleanchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psMetadata *where = psMetadataAlloc();
+    pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "==");
+
+    psString query = pxDataGet("bgtool_tocleanchip.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (where && psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool cleanedchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
+    if (!pxIsValidCleanedState(state)) {
+        psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state);
+        return false;
+    }
+
+    if (!psListLength(where->list)) {
+        psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set.");
+        return false;
+    }
+
+    psString query = pxDataGet("bgtool_cleanedchip.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, state)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
+static bool exportchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true,  false);
+    PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+    PXOPT_LOOKUP_BOOL(clean,  config->args, "-clean", false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "==");
+
+    bool status = exportTables(config, outfile, chipTables, where, clean);
+
+    psFree(where);
+    return status;
+}
+
+static bool importchipMode(pxConfig *config)
+{
+    PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+
+    return importTables(config, infile, chipTables);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Functions for warp stage
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static bool definewarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args,   where, "-warp_id",            "warpRun.warp_id",       "==");
+    PXOPT_COPY_S64(config->args,   where, "-exp_id",             "rawExp.exp_id",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-exp_name",           "rawExp.exp_name",       "==");
+    PXOPT_COPY_STR(config->args,   where, "-inst",               "rawExp.camera",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-telescope",          "rawExp.telescope",      "==");
+    PXOPT_COPY_TIME(config->args,  where, "-dateobs_begin",      "rawExp.dateobs",        ">=");
+    PXOPT_COPY_TIME(config->args,  where, "-dateobs_end",        "rawExp.dateobs",        "<=");
+    PXOPT_COPY_STR(config->args,   where, "-exp_tag",            "rawExp.exp_tag",        "==");
+    PXOPT_COPY_STR(config->args,   where, "-exp_type",           "rawExp.exp_type",       "==");
+    PXOPT_COPY_STR(config->args,   where, "-filelevel",          "rawExp.filelevel",      "==");
+    PXOPT_COPY_STR(config->args,   where, "-filter",             "rawExp.filter",         "==");
+    PXOPT_COPY_F64(config->args,   where, "-airmass_min",        "rawExp.airmass",        ">=");
+    PXOPT_COPY_F64(config->args,   where, "-airmass_max",        "rawExp.airmass",        "<");
+    PXOPT_COPY_RADEC(config->args, where, "-ra_min",             "rawExp.ra",             ">=");
+    PXOPT_COPY_RADEC(config->args, where, "-ra_max",             "rawExp.ra",             "<");
+    PXOPT_COPY_RADEC(config->args, where, "-decl_min",           "rawExp.decl",           ">=");
+    PXOPT_COPY_RADEC(config->args, where, "-decl_max",           "rawExp.decl",           "<");
+    PXOPT_COPY_F32(config->args,   where, "-exp_time_min",       "rawExp.exp_time",       ">=");
+    PXOPT_COPY_F32(config->args,   where, "-exp_time_max",       "rawExp.exp_time",       "<");
+    PXOPT_COPY_F32(config->args,   where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args,   where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args,   where, "-bg_min",             "rawExp.bg",             ">=");
+    PXOPT_COPY_F64(config->args,   where, "-bg_max",             "rawExp.bg",             "<");
+    PXOPT_COPY_F64(config->args,   where, "-bg_stdev_min",       "rawExp.bg_stdev",       ">=");
+    PXOPT_COPY_F64(config->args,   where, "-bg_stdev_max",       "rawExp.bg_stdev",       "<");
+    PXOPT_COPY_F64(config->args,   where, "-bg_mean_stdev_min",  "rawExp.bg_mean_stdev",  ">=");
+    PXOPT_COPY_F64(config->args,   where, "-bg_mean_stdev_max",  "rawExp.bg_mean_stdev",  "<");
+    PXOPT_COPY_F64(config->args,   where, "-alt_min",            "rawExp.alt",            ">=");
+    PXOPT_COPY_F64(config->args,   where, "-alt_max",            "rawExp.alt",            "<");
+    PXOPT_COPY_F64(config->args,   where, "-az_min",             "rawExp.az",             ">=");
+    PXOPT_COPY_F64(config->args,   where, "-az_max",             "rawExp.az",             "<");
+    PXOPT_COPY_F32(config->args,   where, "-ccd_temp_min",       "rawExp.ccd_temp",       ">=");
+    PXOPT_COPY_F32(config->args,   where, "-ccd_temp_max",       "rawExp.ccd_temp",       "<");
+    PXOPT_COPY_F64(config->args,   where, "-posang_min",         "rawExp.posang",         ">=");
+    PXOPT_COPY_F64(config->args,   where, "-posang_max",         "rawExp.posang",         "<");
+    PXOPT_COPY_STR(config->args,   where, "-object",             "rawExp.object",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-comment",            "rawExp.comment",        "LIKE");
+    PXOPT_COPY_STR(config->args,   where, "-obs_mode",           "rawExp.obs_mode",       "LIKE");
+    PXOPT_COPY_F32(config->args,   where, "-sun_angle_min",      "rawExp.sun_angle",      ">=");
+    PXOPT_COPY_F32(config->args,   where, "-sun_angle_max",      "rawExp.sun_angle",      "<");
+    pxAddLabelSearchArgs(config,   where, "-warp_label",         "warpRun.label",         "==");
+    pxAddLabelSearchArgs(config,   where, "-chip_label",         "chipBackgroundRun.label", "==");
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required");
+        return false;
+    }
+
+    PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    // Get warp runs to promote to warpBackgroundRun
+
+    psString query = pxDataGet("bgtool_definewarp.sql"); // Query to execute
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (destreaked) {
+        psStringAppend(&query, " AND warpRun.magicked > 0");
+    }
+
+    if (!rerun) {
+        psStringAppend(&query, "\nAND warpBackgroundRun.warp_bg_id IS NULL");
+        if (label) {
+            psStringAppend(&query,
+                           "\nAND (warpBackgroundRun.label = '%s'"
+                           " OR warpBackgroundRun.label IS NULL)",
+                           label);
+        }
+        if (data_group) {
+            psStringAppend(&query,
+                           "\nAND (warpBackgroundRun.data_group = '%s'"
+                           " OR warpBackgroundRun.data_group IS NULL)",
+                           data_group);
+        }
+        if (dist_group) {
+            psStringAppend(&query,
+                           "\nAND (warpBackgroundRun.dist_group = '%s'"
+                           " OR warpBackgroundRun.dist_group IS NULL)",
+                           dist_group);
+        }
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh); // Matching rows
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return true;
+    }
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        psS64 chip_bg_id = psMetadataLookupS64(NULL, md, "chip_bg_id");
+
+        warpRunRow *row = warpRunObjectFromMetadata(md);
+        if (!row) {
+            psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        if (!warpBackgroundRunInsert(config->dbh, 0, row->warp_id, chip_bg_id, "new",
+                                     workdir     ? workdir    : row->workdir,
+                                     label       ? label      : row->label,
+                                     data_group  ? data_group : row->data_group,
+                                     dist_group  ? dist_group : row->dist_group,
+                                     reduction   ? reduction  : row->reduction,
+                                     note        ? note       : row->note,
+                                     NULL, 0)) {
+            psError(psErrorCodeLast(), false, "database error");
+            psFree(row);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psFree(row);
+    }
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+        return false;
+    }
+
+    return true;
+}
+
+static bool updatewarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id",   "==");
+    PXOPT_COPY_STR(config->args, where, "-state",      "warpBackgroundRun.state",     "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "warpBackgroundRun.data_group","LIKE");
+    PXOPT_COPY_STR(config->args, where, "-dist_group", "warpBackgroundRun.dist_group","LIKE");
+    pxAddLabelSearchArgs(config,  where, "-label",     "warpBackgroundRun.label",     "LIKE");
+    PXOPT_COPY_TIME(config->args, where, "-registered_begin", "warpBackgroundRun.registered",  ">=");
+    PXOPT_COPY_TIME(config->args, where, "-registered_end",   "warpBackgroundRun.registered",  "<");
+
+    PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false);
+    if (destreaked) {
+        psMetadataAddS64(where, PS_LIST_TAIL, "warpBackgroundRun.magicked", PS_META_DUPLICATE_OK, ">", 0);
+    }
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = psStringCopy("UPDATE warpBackgroundRun");
+    bool result = pxUpdateRun(config, where, &query, "warpBackgroundRun", "warp_bg_id",
+                              "warpBackgroundSkyfile", true);
+
+    psFree(query);
+    psFree(where);
+
+    return result;
+}
+
+static bool towarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "warpBackgroundRun.label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("bgtool_towarp.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString whereStr = psStringCopy("");
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&whereStr, "\n AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, whereStr)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(whereStr);
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query);
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool warpinputsMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // find all rawImfiles matching the default query
+    psString query = pxDataGet("bgtool_warpinputs.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundSkyfile", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool addwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false);
+    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
+
+    // optional
+
+    PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
+    PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
+    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false);
+    PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false);
+    PXOPT_LOOKUP_STR(ver_pswarp, config->args, "-ver_pswarp", false, false);
+    PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false);
+    PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
+    PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
+    PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false);
+
+    psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules);
+    ver_code = pxMergeCodeVersions(ver_code, ver_pswarp);
+    ver_code = pxMergeCodeVersions(ver_code, ver_ppstats);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script,
+                                    hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
+                                    maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool warpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id",    "warpBackgroundRun.warp_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label",   "warpBackgroundRun.label", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-data_group",   "warpBackgroundRun.data_group", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-dist_group",   "warpBackgroundRun.data_group", "LIKE");
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // find all rawImfiles matching the default query
+    psString query = pxDataGet("bgtool_warp.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString magicWhere = NULL;
+    if (!pxmagicAddWhere(config, &magicWhere, "warpBackgroundSkyfile")) {
+        psError(psErrorCodeLast(), false, "pxMagicAddWhere failed");
+        return false;
+    }
+    if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) {
+        psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else if (!all && !magicWhere) {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+        return false;
+    }
+    if (magicWhere) {
+        psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere);
+    }
+    psFree(magicWhere);
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    if (psArrayLength(output)) {
+        if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(output);
+
+    return true;
+}
+
+
+static bool advancewarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "label", "==");
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+
+    psString select = pxDataGet("bgtool_advancewarp.sql");
+    if (!select) {
+        psError(psErrorCodeLast(), false, "failed to retrieve SQL statement");
+        return false;
+    }
+
+    psString selectWhere = psStringCopy("");
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&selectWhere, "\n AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&select, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(select);
+        psFree(selectWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    psFree(select);
+    psFree(selectWhere);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+        bool status = true;             // Status of MD lookup
+        psS64 warp_bg_id = psMetadataLookupS64(&status, row, "warp_bg_id");
+        if (!status) {
+            psError(PXTOOLS_ERR_PROG, true, "failed to look up value for warp_bg_id");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psS64 magicked = psMetadataLookupS64(&status, row, "magicked");
+        if (!status) {
+            psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        if (!p_psDBRunQueryF(config->dbh,
+                             "UPDATE warpBackgroundRun "
+                             "SET state = 'full', magicked = %" PRId64 " "
+                             " WHERE warp_bg_id = %" PRId64,
+                             magicked, warp_bg_id)) {
+            psError(psErrorCodeLast(), false, "database error");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        psS64 numUpdated = psDBAffectedRows(config->dbh);
+        if (numUpdated != 1) {
+            psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row");
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool revertwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "warpBackgroundSkyfile.fault", "==");
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required");
+        return false;
+    }
+
+    psString query = pxDataGet("bgtool_revertwarp.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    int numDeleted = psDBAffectedRows(config->dbh);
+    psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d warpBackgroundSkyfiles", numDeleted);
+
+    return true;
+}
+
+static bool tocleanwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psMetadata *where = psMetadataAlloc();
+    pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "==");
+
+    psString query = pxDataGet("bgtool_tocleanwarp.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (where && psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool cleanedwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
+    if (!pxIsValidCleanedState(state)) {
+        psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state);
+        return false;
+    }
+
+    if (!psListLength(where->list)) {
+        psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set.");
+        return false;
+    }
+
+    psString query = pxDataGet("bgtool_cleanedwarp.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, state)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
+static bool exportwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true,  false);
+    PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+    PXOPT_LOOKUP_BOOL(clean,  config->args, "-clean", false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "==");
+
+    bool status = exportTables(config, outfile, warpTables, where, clean);
+
+    psFree(where);
+    return status;
+}
+
+static bool importwarpMode(pxConfig *config)
+{
+    PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+
+    return importTables(config, infile, warpTables);
+}
Index: /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/bgtool.h	(revision 28794)
@@ -0,0 +1,59 @@
+/*
+ * bgtool.h
+ *
+ * Copyright (C) 2006-2010  Joshua Hoblitt, Paul Price
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * program; see the file COPYING. If not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifndef BGTOOL_H
+#define BGTOOL_H 1
+
+#include "pxtools.h"
+
+typedef enum {
+    BGTOOL_MODE_NONE           = 0x0,
+    // Chip stage
+    BGTOOL_MODE_DEFINECHIP,
+    BGTOOL_MODE_UPDATECHIP,
+    BGTOOL_MODE_TOCHIP,
+    BGTOOL_MODE_CHIPINPUTS,
+    BGTOOL_MODE_ADDCHIP,
+    BGTOOL_MODE_CHIP,
+    BGTOOL_MODE_ADVANCECHIP,
+    BGTOOL_MODE_REVERTCHIP,
+    // Warp stage
+    BGTOOL_MODE_DEFINEWARP,
+    BGTOOL_MODE_UPDATEWARP,
+    BGTOOL_MODE_TOWARP,
+    BGTOOL_MODE_WARPINPUTS,
+    BGTOOL_MODE_ADDWARP,
+    BGTOOL_MODE_WARP,
+    BGTOOL_MODE_ADVANCEWARP,
+    BGTOOL_MODE_REVERTWARP,
+    // Cleanups
+    BGTOOL_MODE_TOCLEANCHIP,
+    BGTOOL_MODE_CLEANEDCHIP,
+    BGTOOL_MODE_TOCLEANWARP,
+    BGTOOL_MODE_CLEANEDWARP,
+    // Exporting
+    BGTOOL_MODE_EXPORTCHIP,
+    BGTOOL_MODE_IMPORTCHIP,
+    BGTOOL_MODE_EXPORTWARP,
+    BGTOOL_MODE_IMPORTWARP,
+} bgtoolMode;
+
+pxConfig *bgtoolConfig(pxConfig *config, int argc, char **argv);
+
+#endif // BGTOOL_H
Index: /branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/bgtoolConfig.c	(revision 28794)
@@ -0,0 +1,424 @@
+/*
+ * bgtoolConfig.c
+ *
+ * Copyright (C) 2006-2010  Joshua Hoblitt, Paul Price
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * program; see the file COPYING. If not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdint.h>
+
+#include <psmodules.h>
+
+#include "pxtools.h"
+#include "bgtool.h"
+
+pxConfig *bgtoolConfig(pxConfig *config, int argc, char **argv)
+{
+    if (!config) {
+        config = pxConfigAlloc();
+    }
+
+    pmConfigReadParamsSet(false);
+
+    // setup site config
+    config->modules = pmConfigRead(&argc, argv, NULL);
+    if (!config->modules) {
+        psError(psErrorCodeLast(), false, "Can't find site configuration");
+        psFree(config);
+        return NULL;
+    }
+
+    psTime *now = psTimeGetNow(PS_TIME_TAI);
+
+    // -definechip
+    psMetadata *definechipArgs = psMetadataAlloc();
+    psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip_id", 0);
+    psMetadataAddS64(definechipArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL);
+    psMetadataAddTime(definechipArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(definechipArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN);
+    psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN);
+    psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN);
+    psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN);
+    psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN);
+    psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN);
+    psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN);
+    psMetadataAddF64(definechipArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL);
+    psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN);
+    psMetadataAddF32(definechipArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search on chipRun label", NULL);
+    psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-rerun", 0, "re-run data?", false);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL);
+    psMetadataAddStr(definechipArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL);
+
+    psMetadataAddTime(definechipArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(definechipArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false);
+
+    // -updatechip
+    psMetadata *updatechipArgs = psMetadataAlloc();
+    psMetadataAddS64(updatechipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0);
+    psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
+    psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label (LIKE comparison)", NULL);
+    psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group (LIKE comparison)", NULL);
+    psMetadataAddTime(updatechipArgs, PS_LIST_TAIL, "-registered_begin", 0, "search by registration time (>=)", NULL);
+    psMetadataAddTime(updatechipArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL);
+    psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddBool(updatechipArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false);
+    psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);
+    psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0);
+    psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL);
+    psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL);
+    psMetadataAddStr(updatechipArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL);
+
+    // -tochip
+    psMetadata *tochipArgs = psMetadataAlloc();
+    psMetadataAddS64(tochipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0);
+    psMetadataAddStr(tochipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
+    psMetadataAddU64(tochipArgs, PS_LIST_TAIL, "-limit",  0, "limit result set to N items", 0);
+    psMetadataAddBool(tochipArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+
+    // -chipinputs
+    psMetadata *chipinputsArgs = psMetadataAlloc();
+    psMetadataAddS64(chipinputsArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0);
+    psMetadataAddStr(chipinputsArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", 0);
+    psMetadataAddU64(chipinputsArgs, PS_LIST_TAIL, "-limit",  0, "limit result set to N items", 0);
+    psMetadataAddBool(chipinputsArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+
+    // -addchip
+    psMetadata *addchipArgs = psMetadataAlloc();
+    psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "define chip_bg_id (required)", 0);
+    psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-class_id", 0, "define class_id (required)", NULL);
+
+    psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location (required)", NULL);
+    psMetadataAddS64(addchipArgs, PS_LIST_TAIL, "-set_magicked",  0, "define if this skycell has been magicked", 0);
+    psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN);
+    psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL);
+    psMetadataAddS16(addchipArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0);
+    psMetadataAddS16(addchipArgs, PS_LIST_TAIL, "-fault",  0, "set fault code", 0);
+    psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL);
+    psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL);
+    psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_ppbackground", 0, "define ppBackground version", NULL);
+    psMetadataAddStr(addchipArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL);
+    psMetadataAddF64(addchipArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN);
+    psMetadataAddF64(addchipArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN);
+    psMetadataAddS32(addchipArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0);
+    psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN);
+    psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN);
+    psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN);
+    psMetadataAddF32(addchipArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN);
+
+    // -chip
+    psMetadata *chipArgs = psMetadataAlloc();
+    psMetadataAddS64(chipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0);
+    psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-class_id", 0, "search by class_id", NULL);
+    psMetadataAddS16(chipArgs, PS_LIST_TAIL, "-fault",  0, "search by fault code", 0);
+    psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
+    psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", NULL);
+    psMetadataAddStr(chipArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by dist_group", NULL);
+    pxmagicAddArguments(chipArgs);
+    pxspaceAddArguments(chipArgs);
+    psMetadataAddBool(chipArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false);
+    psMetadataAddU64(chipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(chipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -advancechip
+    psMetadata *advancechipArgs = psMetadataAlloc();
+    psMetadataAddS64(advancechipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0);
+    psMetadataAddStr(advancechipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label ", NULL);
+    psMetadataAddU64(advancechipArgs, PS_LIST_TAIL, "-limit", 0, "search limit", 0);
+
+    // -revertchip
+    psMetadata *revertchipArgs = psMetadataAlloc();
+    psMetadataAddS64(revertchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0);
+    psMetadataAddStr(revertchipArgs, PS_LIST_TAIL, "-class_id",  0, "search by class_id", NULL);
+    psMetadataAddStr(revertchipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
+    psMetadataAddS16(revertchipArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
+    psMetadataAddBool(revertchipArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false);
+
+    // -tocleanchip
+    psMetadata *tocleanchipArgs = psMetadataAlloc();
+    psMetadataAddStr(tocleanchipArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
+    psMetadataAddBool(tocleanchipArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddU64(tocleanchipArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+
+    // -cleanedchip
+    psMetadata *cleanedchipArgs = psMetadataAlloc();
+    psMetadataAddS64(cleanedchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "search by chip_bg_id", 0);
+    psMetadataAddStr(cleanedchipArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL);
+
+    // -exportchip
+    psMetadata *exportchipArgs = psMetadataAlloc();
+    psMetadataAddS64(exportchipArgs, PS_LIST_TAIL, "-chip_bg_id", 0, "export this chip_bg_id (required)", 0);
+    psMetadataAddStr(exportchipArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL);
+    psMetadataAddBool(exportchipArgs, PS_LIST_TAIL, "-clean", 0, "export run in cleaned state", false);
+
+    // -importchip
+    psMetadata *importchipArgs = psMetadataAlloc();
+    psMetadataAddStr(importchipArgs, PS_LIST_TAIL, "-infile",  0, "import from this file (required)", NULL);
+
+
+
+    // -definewarp
+    psMetadata *definewarpArgs = psMetadataAlloc();
+    psMetadataAddS64(definewarpArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", 0);
+    psMetadataAddS64(definewarpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL);
+    psMetadataAddTime(definewarpArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(definewarpArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-airmass_min", 0, "search by min airmass", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-airmass_max", 0, "search by max airmass", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-ra_min", 0, "search by min RA (degrees) ", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-ra_max", 0, "search by max RA (degrees) ", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-decl_min", 0, "search by min DEC (degrees)", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-decl_max", 0, "search by max DEC (degrees)", NAN);
+    psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-exp_time_min", 0, "search by min exposure time", NAN);
+    psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-exp_time_max", 0, "search by max exposure time", NAN);
+    psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by min fraction of saturated pixels", NAN);
+    psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by max fraction of saturated pixels", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_min", 0, "search by min background", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_max", 0, "search by max background", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_stdev_min", 0, "search by min background standard deviation", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_stdev_max", 0, "search by max background standard deviation", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "search by min background mean standard deviation (across imfiles)", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "search by max background mean standard deviation (across imfiles)", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-alt_min", 0, "search by min altitude", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-alt_max", 0, "search by max altitude", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-az_min", 0, "search by min azimuth ", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-az_max", 0, "search by max azimuth ", NAN);
+    psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-ccd_temp_min", 0, "search by min ccd tempature", NAN);
+    psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-ccd_temp_max", 0, "search by max ccd tempature", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-posang_min", 0, "search by min rotator position angle", NAN);
+    psMetadataAddF64(definewarpArgs, PS_LIST_TAIL, "-posang_max", 0, "search by max rotator position angle", NAN);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-obs_mode", 0, "search by observation mode", NULL);
+    psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "search by min solar angle", NAN);
+    psMetadataAddF32(definewarpArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "search by max solar angle", NAN);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-warp_label", PS_META_DUPLICATE_OK, "search on warpRun label", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-chip_label", PS_META_DUPLICATE_OK, "search on chipBackgroundRun label", NULL);
+    psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-rerun", 0, "rerun data?", false);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL);
+    psMetadataAddStr(definewarpArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL);
+
+    psMetadataAddTime(definewarpArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(definewarpArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false);
+
+    // -updatewarp
+    psMetadata *updatewarpArgs = psMetadataAlloc();
+    psMetadataAddS64(updatewarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0);
+    psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL);
+    psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label (LIKE comparison)", NULL);
+    psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group (LIKE comparison)", NULL);
+    psMetadataAddTime(updatewarpArgs, PS_LIST_TAIL, "-registered_begin", 0, "search by registration time (>=)", NULL);
+    psMetadataAddTime(updatewarpArgs, PS_LIST_TAIL, "-registered_end", 0, "search by registration time (<)", NULL);
+    psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-destreaked", 0, "search for runs that have been destreaked", false);
+    psMetadataAddBool(updatewarpArgs, PS_LIST_TAIL, "-pretend", 0, "only pretend to run the query", false);
+    psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL);
+    psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0);
+    psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL);
+    psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL);
+    psMetadataAddStr(updatewarpArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL);
+
+    // -towarp
+    psMetadata *towarpArgs = psMetadataAlloc();
+    psMetadataAddS64(towarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0);
+    psMetadataAddStr(towarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
+    psMetadataAddU64(towarpArgs, PS_LIST_TAIL, "-limit",  0, "limit result set to N items", 0);
+    psMetadataAddBool(towarpArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+
+    // -warpinputs
+    psMetadata *warpinputsArgs = psMetadataAlloc();
+    psMetadataAddS64(warpinputsArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0);
+    psMetadataAddStr(warpinputsArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", 0);
+    psMetadataAddU64(warpinputsArgs, PS_LIST_TAIL, "-limit",  0, "limit result set to N items", 0);
+    psMetadataAddBool(warpinputsArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+
+    // -addwarp
+    psMetadata *addwarpArgs = psMetadataAlloc();
+    psMetadataAddS64(addwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "define warp_bg_id (required)", 0);
+    psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell_id (required)", NULL);
+
+    psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0);
+    psMetadataAddS64(addwarpArgs, PS_LIST_TAIL, "-set_magicked",  0, "define if this skycell has been magicked", 0);
+    psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN);
+    psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0);
+    psMetadataAddS16(addwarpArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0);
+    psMetadataAddS16(addwarpArgs, PS_LIST_TAIL, "-fault",  0, "set fault code", 0);
+    psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_pslib", 0, "define psLib version", NULL);
+    psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_psmodules", 0, "define psModules version", NULL);
+    psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_pswarp", 0, "define pswarp version", NULL);
+    psMetadataAddStr(addwarpArgs, PS_LIST_TAIL, "-ver_ppstats", 0, "define ppStats version", NULL);
+    psMetadataAddF64(addwarpArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN);
+    psMetadataAddF64(addwarpArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN);
+    psMetadataAddS32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_npix", 0, "define number of pixels used for maskstats", 0);
+    psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_static", 0, "define static mask fraction", NAN);
+    psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_dynamic", 0, "define dynamic mask fraction", NAN);
+    psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_magic", 0, "define magic mask fraction", NAN);
+    psMetadataAddF32(addwarpArgs, PS_LIST_TAIL, "-maskfrac_advisory", 0, "define advisory mask fraction", NAN);
+
+    // -warp
+    psMetadata *warpArgs = psMetadataAlloc();
+    psMetadataAddS64(warpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0);
+    psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell_id", NULL);
+    psMetadataAddS16(warpArgs, PS_LIST_TAIL, "-fault",  0, "search by fault code", 0);
+    psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
+    psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by data_group", NULL);
+    psMetadataAddStr(warpArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "search by dist_group", NULL);
+    pxmagicAddArguments(warpArgs);
+    pxspaceAddArguments(warpArgs);
+    psMetadataAddBool(warpArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false);
+    psMetadataAddU64(warpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(warpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -advancewarp
+    psMetadata *advancewarpArgs = psMetadataAlloc();
+    psMetadataAddS64(advancewarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0);
+    psMetadataAddStr(advancewarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label ", NULL);
+    psMetadataAddU64(advancewarpArgs, PS_LIST_TAIL, "-limit", 0, "search limit", 0);
+
+    // -revertwarp
+    psMetadata *revertwarpArgs = psMetadataAlloc();
+    psMetadataAddS64(revertwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0);
+    psMetadataAddStr(revertwarpArgs, PS_LIST_TAIL, "-skycell_id",  0, "search by skycell_id", NULL);
+    psMetadataAddStr(revertwarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL);
+    psMetadataAddS16(revertwarpArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
+    psMetadataAddBool(revertwarpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false);
+
+    // -tocleanwarp
+    psMetadata *tocleanwarpArgs = psMetadataAlloc();
+    psMetadataAddStr(tocleanwarpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL);
+    psMetadataAddBool(tocleanwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddU64(tocleanwarpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+
+    // -cleanedwarp
+    psMetadata *cleanedwarpArgs = psMetadataAlloc();
+    psMetadataAddS64(cleanedwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "search by warp_bg_id", 0);
+    psMetadataAddStr(cleanedwarpArgs, PS_LIST_TAIL, "-state", 0, "cleaned state to set", NULL);
+
+    // -exportwarp
+    psMetadata *exportwarpArgs = psMetadataAlloc();
+    psMetadataAddS64(exportwarpArgs, PS_LIST_TAIL, "-warp_bg_id", 0, "export this warp_bg_id (required)", 0);
+    psMetadataAddStr(exportwarpArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL);
+    psMetadataAddBool(exportwarpArgs, PS_LIST_TAIL, "-clean", 0, "export run in cleaned state", false);
+
+    // -importwarp
+    psMetadata *importwarpArgs = psMetadataAlloc();
+    psMetadataAddStr(importwarpArgs, PS_LIST_TAIL, "-infile",  0, "import from this file (required)", NULL);
+
+
+
+    psFree(now);
+    psMetadata *argSets = psMetadataAlloc();
+    psMetadata *modes   = psMetadataAlloc();
+
+    PXOPT_ADD_MODE("-definechip",  "", BGTOOL_MODE_DEFINECHIP,  definechipArgs);
+    PXOPT_ADD_MODE("-updatechip",  "", BGTOOL_MODE_UPDATECHIP,  updatechipArgs);
+    PXOPT_ADD_MODE("-tochip",      "", BGTOOL_MODE_TOCHIP,      tochipArgs);
+    PXOPT_ADD_MODE("-chipinputs",  "", BGTOOL_MODE_CHIPINPUTS,  chipinputsArgs);
+    PXOPT_ADD_MODE("-addchip",     "", BGTOOL_MODE_ADDCHIP,     addchipArgs);
+    PXOPT_ADD_MODE("-chip",        "", BGTOOL_MODE_CHIP,        chipArgs);
+    PXOPT_ADD_MODE("-advancechip", "", BGTOOL_MODE_ADVANCECHIP, advancechipArgs);
+    PXOPT_ADD_MODE("-revertchip",  "", BGTOOL_MODE_REVERTCHIP,  revertchipArgs);
+    PXOPT_ADD_MODE("-tocleanchip", "", BGTOOL_MODE_TOCLEANCHIP, tocleanchipArgs);
+    PXOPT_ADD_MODE("-cleanedchip", "", BGTOOL_MODE_CLEANEDCHIP, cleanedchipArgs);
+    PXOPT_ADD_MODE("-exportchip",  "", BGTOOL_MODE_EXPORTCHIP,  exportchipArgs);
+    PXOPT_ADD_MODE("-importchip",  "", BGTOOL_MODE_IMPORTCHIP,  importchipArgs);
+
+    PXOPT_ADD_MODE("-definewarp",  "", BGTOOL_MODE_DEFINEWARP,  definewarpArgs);
+    PXOPT_ADD_MODE("-updatewarp",  "", BGTOOL_MODE_UPDATEWARP,  updatewarpArgs);
+    PXOPT_ADD_MODE("-towarp",      "", BGTOOL_MODE_TOWARP,      towarpArgs);
+    PXOPT_ADD_MODE("-warpinputs",  "", BGTOOL_MODE_WARPINPUTS,  warpinputsArgs);
+    PXOPT_ADD_MODE("-addwarp",     "", BGTOOL_MODE_ADDWARP,     addwarpArgs);
+    PXOPT_ADD_MODE("-warp",        "", BGTOOL_MODE_WARP,        warpArgs);
+    PXOPT_ADD_MODE("-advancewarp", "", BGTOOL_MODE_ADVANCEWARP, advancewarpArgs);
+    PXOPT_ADD_MODE("-revertwarp",  "", BGTOOL_MODE_REVERTWARP,  revertwarpArgs);
+    PXOPT_ADD_MODE("-tocleanwarp", "", BGTOOL_MODE_TOCLEANWARP, tocleanwarpArgs);
+    PXOPT_ADD_MODE("-cleanedwarp", "", BGTOOL_MODE_CLEANEDWARP, cleanedwarpArgs);
+    PXOPT_ADD_MODE("-exportwarp",  "", BGTOOL_MODE_EXPORTWARP,  exportwarpArgs);
+    PXOPT_ADD_MODE("-importwarp",  "", BGTOOL_MODE_IMPORTWARP,  importwarpArgs);
+
+    if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {
+        psError(PS_ERR_UNKNOWN, true, "option parsing failed");
+        psFree(argSets);
+        psFree(modes);
+        psFree(config);
+        return NULL;
+    }
+
+    psFree(argSets);
+    psFree(modes);
+
+    // define Database handle, if used
+    // do this last so we don't setup a connection before CLI options are
+    // validated
+    config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules));
+    if (!config->dbh) {
+        psError(PS_ERR_UNKNOWN, false, "Can't configure database");
+        psFree(config);
+        return NULL;
+    }
+
+    return config;
+}
Index: /branches/eam_branches/ipp-20100621/ippTools/src/chiptool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/chiptool.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/chiptool.c	(revision 28794)
@@ -335,4 +335,5 @@
     pxchipGetSearchArgs (config, where); // rawExp only
     pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group", "chipOld.data_group", "LIKE");
 
     // psListLength(where->list) is at least 1 because exp_type defaults to "object"
Index: /branches/eam_branches/ipp-20100621/ippTools/src/chiptoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/chiptoolConfig.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/chiptoolConfig.c	(revision 28794)
@@ -67,5 +67,6 @@
     psMetadata *definecopyArgs = psMetadataAlloc();
     pxchipSetSearchArgs (definecopyArgs);
-    psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by rawExp label (LIKE comparison)", NULL);
+    psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL);
+    psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-data_group", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL);
 
     psMetadataAddStr(definecopyArgs, PS_LIST_TAIL, "-exp_type", PS_META_REPLACE, "search by exp_type", "object");
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool.c	(revision 28794)
@@ -255,6 +255,6 @@
 
     // get -ref_det_id and -ref_iter : required for 'verify' mode / disallowed otherwise
-    PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", true, false);
-    PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", true, false);
+    PXOPT_LOOKUP_S64(ref_det_id, config->args, "-ref_det_id", false, false);
+    PXOPT_LOOKUP_S32(ref_iter, config->args, "-ref_iter", false, false);
     if (!strcmp(mode, "verify") && ((ref_det_id == 0) || (ref_iter == -1))) {
         psError(PS_ERR_UNKNOWN, false, "verify mode requires both -ref_det_id and -ref_iter");
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettoolConfig.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettoolConfig.c	(revision 28794)
@@ -299,9 +299,9 @@
     // -revertprocessedimfile
     psMetadata *revertprocessedimfileArgs = psMetadataAlloc();
-    psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-det_id",  0,            "search for detrend ID (required)", 0);
+    psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-det_id",  0,            "search for detrend ID", 0);
     psMetadataAddS64(revertprocessedimfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
     psMetadataAddStr(revertprocessedimfileArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
     psMetadataAddS16(revertprocessedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
-
+    psMetadataAddBool(revertprocessedimfileArgs, PS_LIST_TAIL, "-all-run",  0,            "revert all in state 'run'", false);
     // -updateprocessedimfile
     psMetadata *updateprocessedimfileArgs = psMetadataAlloc();
@@ -361,8 +361,8 @@
     // -revertprocessedexp
     psMetadata *revertprocessedexpArgs = psMetadataAlloc();
-    psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-det_id",  0,            "search by detrend ID (required)", 0);
+    psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-det_id",  0,            "search by detrend ID", 0);
     psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
     psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
-
+    psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all-run",  0,            "revert all in state 'run'", false);
     // -updateprocessedexp
     psMetadata *updateprocessedexpArgs = psMetadataAlloc();
@@ -425,8 +425,9 @@
     // -revertstacked
     psMetadata *revertstackedArgs= psMetadataAlloc();
-    psMetadataAddS64(revertstackedArgs, PS_LIST_TAIL, "-det_id",  0,            "search for detrend ID (required)", 0);
+    psMetadataAddS64(revertstackedArgs, PS_LIST_TAIL, "-det_id",  0,            "search for detrend ID", 0);
     psMetadataAddS32(revertstackedArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
     psMetadataAddStr(revertstackedArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
     psMetadataAddS16(revertstackedArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
+    psMetadataAddBool(revertstackedArgs, PS_LIST_TAIL, "-all-run",  0,            "revert all in state 'run'", false);
 
     // -updatestacked
@@ -477,9 +478,9 @@
     // -revertnormalizedstat
     psMetadata *revertnormalizedstatArgs= psMetadataAlloc();
-    psMetadataAddS64(revertnormalizedstatArgs, PS_LIST_TAIL, "-det_id",  0,            "search by detrend ID (required)", 0);
+    psMetadataAddS64(revertnormalizedstatArgs, PS_LIST_TAIL, "-det_id",  0,            "search by detrend ID", 0);
     psMetadataAddS32(revertnormalizedstatArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
     psMetadataAddStr(revertnormalizedstatArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
     psMetadataAddS16(revertnormalizedstatArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
-
+    psMetadataAddBool(revertnormalizedstatArgs, PS_LIST_TAIL, "-all-run",  0,            "revert all in state 'run'", false);
     // -updatenormalizedstat
     psMetadata *updatenormalizedstatArgs = psMetadataAlloc();
@@ -539,9 +540,9 @@
     // -revertnormalizedimfile
     psMetadata *revertnormalizedimfileArgs = psMetadataAlloc();
-    psMetadataAddS64(revertnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID (required)", 0);
+    psMetadataAddS64(revertnormalizedimfileArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID", 0);
     psMetadataAddS32(revertnormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
     psMetadataAddStr(revertnormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0,            "search by class ID", NULL);
     psMetadataAddS16(revertnormalizedimfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
-
+    psMetadataAddBool(revertnormalizedimfileArgs, PS_LIST_TAIL, "-all-run",  0,            "revert all in state 'run'", false);
     // -updatenormalizedimfile
     psMetadata *updatenormalizedimfileArgs = psMetadataAlloc();
@@ -599,7 +600,8 @@
     // -revertnormalizedexp
     psMetadata *revertnormalizedexpArgs = psMetadataAlloc();
-    psMetadataAddS64(revertnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID (required)", 0);
+    psMetadataAddS64(revertnormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID", 0);
     psMetadataAddS32(revertnormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
     psMetadataAddS16(revertnormalizedexpArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
+    psMetadataAddBool(revertnormalizedexpArgs, PS_LIST_TAIL, "-all-run",  0,            "revert all in state 'run'", false);
 
     // -updatenormalizedexp
@@ -675,9 +677,10 @@
     // -revertresidimfile
     psMetadata *revertresidimfileArgs =  psMetadataAlloc();
-    psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-det_id", 0,          "search by detrend ID (required)", 0);
+    psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-det_id", 0,          "search by detrend ID", 0);
     psMetadataAddS32(revertresidimfileArgs, PS_LIST_TAIL, "-iteration", 0,       "search by iteration number", 0);
     psMetadataAddS64(revertresidimfileArgs, PS_LIST_TAIL, "-exp_id",  0,         "search by detrend ID", 0);
     psMetadataAddStr(revertresidimfileArgs, PS_LIST_TAIL, "-class_id",  0,       "search for class ID", NULL);
     psMetadataAddS16(revertresidimfileArgs, PS_LIST_TAIL, "-fault",  0,           "search by fault code", 0);
+    psMetadataAddBool(revertresidimfileArgs, PS_LIST_TAIL, "-all-run",  0,            "revert all in state 'run'", false);
 
     // -updateresidimfile
@@ -752,9 +755,9 @@
     // -revertresidexp
     psMetadata *revertresidexpArgs = psMetadataAlloc();
-    psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID (required)", 0);
+    psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID", 0);
     psMetadataAddS32(revertresidexpArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
     psMetadataAddS64(revertresidexpArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by detrend ID", 0);
     psMetadataAddS16(revertresidexpArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
-
+    psMetadataAddBool(revertresidexpArgs, PS_LIST_TAIL, "-all-run",  0,            "revert all in state 'run'", false);
     // -updateresidexp
     psMetadata *updateresidexpArgs = psMetadataAlloc();
@@ -819,7 +822,8 @@
     // -revertdetrunsummary
     psMetadata *revertdetrunsummaryArgs = psMetadataAlloc();
-    psMetadataAddS64(revertdetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID (required)", 0);
+    psMetadataAddS64(revertdetrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0,            "search by detrend ID", 0);
     psMetadataAddS32(revertdetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
     psMetadataAddS16(revertdetrunsummaryArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
+    psMetadataAddBool(revertdetrunsummaryArgs, PS_LIST_TAIL, "-all-run",  0,            "revert all in state 'run'", false);
 
     // -updatedetrunsummary
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool_detrunsummary.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool_detrunsummary.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool_detrunsummary.c	(revision 28794)
@@ -306,5 +306,11 @@
     PXOPT_COPY_S64(config->args, where, "-det_id",    "det_id", "==");
     PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "==");
-    PXOPT_COPY_STR(config->args, where, "-fault",      "fault", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
+    
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
 
     psString query = pxDataGet("dettool_revertdetrunsummary.sql");
@@ -327,9 +333,4 @@
     }
     psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
 
     return true;
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedexp.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedexp.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedexp.c	(revision 28794)
@@ -222,4 +222,10 @@
     PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "==");
     PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
+    
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
 
     psString query = pxDataGet("dettool_revertnormalizedexp.sql");
@@ -242,9 +248,4 @@
     }
     psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
 
     return true;
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedimfile.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedimfile.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedimfile.c	(revision 28794)
@@ -202,4 +202,10 @@
     PXOPT_COPY_STR(config->args, where, "-class_id",  "class_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
+    
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
 
     psString query = pxDataGet("dettool_revertnormalizedimfile.sql");
@@ -222,9 +228,4 @@
     }
     psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
 
     return true;
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedstat.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedstat.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool_normalizedstat.c	(revision 28794)
@@ -182,4 +182,10 @@
     PXOPT_COPY_STR(config->args, where, "-class_id",  "class_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
+    
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
 
     psString query = pxDataGet("dettool_revertnormalizedstat.sql");
@@ -202,9 +208,4 @@
     }
     psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
 
     return true;
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedexp.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedexp.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedexp.c	(revision 28794)
@@ -232,4 +232,10 @@
     PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault",   "fault", "==");
+    
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
 
     psString query = pxDataGet("dettool_revertprocessedexp.sql");
@@ -252,9 +258,4 @@
     }
     psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
 
     return true;
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedimfile.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedimfile.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool_processedimfile.c	(revision 28794)
@@ -264,4 +264,10 @@
     PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
+    
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
 
     psString query = pxDataGet("dettool_revertprocessedimfile.sql");
@@ -284,9 +290,4 @@
     }
     psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
 
     return true;
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool_residexp.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool_residexp.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool_residexp.c	(revision 28794)
@@ -275,4 +275,10 @@
     PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
 
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
     psString query = pxDataGet("dettool_revertresidexp.sql");
     if (!query) {
@@ -294,9 +300,4 @@
     }
     psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
 
     return true;
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool_residimfile.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool_residimfile.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool_residimfile.c	(revision 28794)
@@ -250,4 +250,10 @@
     PXOPT_COPY_STR(config->args, where, "-class_id",  "class_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
+   
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
 
     psString query = pxDataGet("dettool_revertresidimfile.sql");
@@ -270,9 +276,4 @@
     }
     psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
 
     return true;
Index: /branches/eam_branches/ipp-20100621/ippTools/src/dettool_stack.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/dettool_stack.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/dettool_stack.c	(revision 28794)
@@ -287,4 +287,10 @@
     PXOPT_COPY_STR(config->args, where, "-class_id",  "class_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault",      "fault", "==");
+    
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all-run")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
 
     psString query = pxDataGet("dettool_revertstacked.sql");
@@ -307,9 +313,4 @@
     }
     psFree(query);
-
-    if (psDBAffectedRows(config->dbh) < 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
-        return false;
-    }
 
     return true;
Index: /branches/eam_branches/ipp-20100621/ippTools/src/diffphottool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/diffphottool.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/diffphottool.c	(revision 28794)
@@ -177,5 +177,5 @@
                                                   set_reduction ? set_reduction : reduction,
                                                   registered,
-                                                  set_note ? set_note : note);
+                                                  set_note ? set_note : note, 0);
         if (!diffPhotRunInsertObject(config->dbh, run)) {
             psError(psErrorCodeLast(), false, "database error");
@@ -430,7 +430,9 @@
         psMetadata *row = output->data[i]; // Row of interest
         psS64 diff_phot_id = psMetadataLookupS64(NULL, row, "diff_phot_id");
-
-        const char *query = "UPDATE diffPhotRun SET state = 'full' WHERE diff_phot_id = %" PRId64;
-        if (!p_psDBRunQueryF(config->dbh, query, diff_phot_id)) {
+        psS64 magicked = psMetadataLookupS64(NULL, row, "magicked");
+
+        const char *query = "UPDATE diffPhotRun SET state = 'full', magicked = %" PRId64
+            " WHERE diff_phot_id = %" PRId64;
+        if (!p_psDBRunQueryF(config->dbh, query, magicked, diff_phot_id)) {
             psError(psErrorCodeLast(), false,
                     "failed to change state for diff_phot_id %" PRId64, diff_phot_id);
Index: /branches/eam_branches/ipp-20100621/ippTools/src/disttool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/disttool.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/disttool.c	(revision 28794)
@@ -210,4 +210,20 @@
             psStringAppend(&query, " AND (chipRun.dist_group = '%s')", dist_group);
         }
+    } else if (!strcmp(stage, "chip_bg")) {
+        magicRunType = "chipBackgroundRun";
+        runJoinStr = "chipBackgroundRun.chip_bg_id";
+        query = pxDataGet("disttool_definebyquery_chip_bg.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        if (label) {
+            psStringAppend(&query, " AND (chipBackgroundRun.label = '%s')", label);
+        }
+        if (dist_group) {
+            psStringAppend(&query, " AND (chipBackgroundRun.dist_group = '%s')", dist_group);
+        }
     } else if (!strcmp(stage, "camera")) {
         magicRunType = "camRun";    // This is used below to set the magicked business
@@ -258,4 +274,21 @@
         if (dist_group) {
             psStringAppend(&query, " AND (warpRun.dist_group = '%s')", dist_group);
+        }
+
+    } else if (!strcmp(stage, "warp_bg")) {
+        magicRunType = "warpBackgroundRun";
+        runJoinStr = "warpBackgroundRun.warp_bg_id";
+        query = pxDataGet("disttool_definebyquery_warp_bg.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+            psFree(where);
+            return false;
+        }
+
+        if (label) {
+            psStringAppend(&query, " AND (warpBackgroundRun.label = '%s')", label);
+        }
+        if (dist_group) {
+            psStringAppend(&query, " AND (warpBackgroundRun.dist_group = '%s')", dist_group);
         }
 
Index: /branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/magicdstool.c	(revision 28794)
@@ -122,6 +122,8 @@
     PXOPT_COPY_S64(config->args, where, "-exp_id",  "exp_id", "==");
     PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "==");
     PXOPT_COPY_S64(config->args, where, "-cam_id",  "cam_id", "==");
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "==");
     PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
     PXOPT_COPY_S64(config->args, where, "-magic_id","magicRun.magic_id", "==");
@@ -140,4 +142,7 @@
         query = pxDataGet("magicdstool_definebyquery_chip.sql");
         break;
+    case IPP_STAGE_CHIP_BG:
+        query = pxDataGet("magicdstool_definebyquery_chip_bg.sql");
+        break;
     case IPP_STAGE_CAMERA:
         query = pxDataGet("magicdstool_definebyquery_camera.sql");
@@ -145,4 +150,7 @@
     case IPP_STAGE_WARP:
         query = pxDataGet("magicdstool_definebyquery_warp.sql");
+        break;
+    case IPP_STAGE_WARP_BG:
+        query = pxDataGet("magicdstool_definebyquery_warp_bg.sql");
         break;
     case IPP_STAGE_DIFF:
@@ -720,4 +728,7 @@
         query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'";
         break;
+    case IPP_STAGE_CHIP_BG:
+        query = "UPDATE chipBackgroundImfile SET magicked = %" PRId64 " where chip_bg_id = %" PRId64 " AND class_id = '%s'";
+        break;
     case IPP_STAGE_CAMERA:
         // no there is no magicked column in camProcessedExp so we have nothing to do
@@ -727,4 +738,7 @@
         query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'";
         break;
+    case IPP_STAGE_WARP_BG:
+        query = "UPDATE warpBackgroundSkyfile SET magicked = %" PRId64 " where warp_bg_id = %" PRId64 " AND skycell_id = '%s'";
+        break;
     case IPP_STAGE_DIFF:
         query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'";
@@ -794,4 +808,7 @@
         query = "UPDATE chipRun SET magicked = %" PRId64 " where chip_id = %" PRId64;
         break;
+    case IPP_STAGE_CHIP_BG:
+        query = "UPDATE chipBackgroundRun SET magicked = %" PRId64 " where chip_bg_id = %" PRId64;
+        break;
     case IPP_STAGE_CAMERA:
         query = "UPDATE camRun SET magicked = %" PRId64 " where cam_id = %" PRId64;
@@ -799,4 +816,7 @@
     case IPP_STAGE_WARP:
         query = "UPDATE warpRun SET magicked = %" PRId64 " where warp_id = %" PRId64;
+        break;
+    case IPP_STAGE_WARP_BG:
+        query = "UPDATE warpBackgroundRun SET magicked = %" PRId64 " where warp_bg_id = %" PRId64;
         break;
     case IPP_STAGE_DIFF:
Index: /branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/magicdstoolConfig.c	(revision 28794)
@@ -60,6 +60,8 @@
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id",   0, "search by exp_id", 0);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id",  0, "search by chip_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_bg_id",  0, "search by chip_bg_id", 0);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id",  0, "search by cam_id", 0);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id",  0, "search by warp_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_bg_id",  0, "search by warp_bg_id", 0);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id",  0, "search by diff_id", 0);
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic_id", 0);
Index: /branches/eam_branches/ipp-20100621/ippTools/src/magictool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/magictool.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/magictool.c	(revision 28794)
@@ -246,4 +246,5 @@
         psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); // difference identifier
         bool inverse = psMetadataLookupU64(NULL, row, "inverse"); // Inverse subtraction? Note types!
+        psString diff_data_group = psMetadataLookupStr(NULL, row, "diff_data_group");
 
         // create a new magicRun for this group
@@ -256,5 +257,5 @@
                                             "dirty",    // workdir_state
                                             label,
-                                            data_group ? data_group : label,
+                                            data_group ? data_group : (diff_data_group ? diff_data_group : label),
                                             dvodb,
                                             registered,
@@ -521,4 +522,6 @@
     psFree(where);
 
+    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
+
     // treat limit == 0 as "no limit"
     if (limit) {
@@ -830,4 +833,6 @@
     psFree(where);
 
+    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
+
     // treat limit == 0 as "no limit"
     if (limit) {
@@ -1365,8 +1370,8 @@
     // check that state is a valid string value
     if (!(
-            (strncmp(state, "new", 4) == 0)
-            || (strncmp(state, "full", 5) == 0)
-            || (strncmp(state, "drop", 5) == 0)
-            || (strncmp(state, "reg", 4) == 0)
+            (strncmp(state, "new", 3) == 0)
+            || (strncmp(state, "full", 4) == 0)
+            || (strncmp(state, "drop", 4) == 0)
+            || (strncmp(state, "reg", 3) == 0)
         )
     ) {
Index: /branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/pstamptool.c	(revision 28794)
@@ -206,4 +206,5 @@
     PXOPT_LOOKUP_S64(ds_id,       config->args, "-ds_id",         true, false);
     PXOPT_LOOKUP_STR(lastFileset, config->args, "-set_last_fileset",  false, false);
+    PXOPT_LOOKUP_STR(uri, config->args,         "-set_uri",  false, false);
     PXOPT_LOOKUP_STR(state,       config->args, "-set_state",         false, false);
     PXOPT_LOOKUP_STR(label,       config->args, "-set_label",         false, false);
@@ -211,5 +212,5 @@
     PXOPT_LOOKUP_BOOL(update_timestamp, config->args, "-update_timestamp", false);
 
-    if (!state && !lastFileset && !pollInterval && !update_timestamp && !label) {
+    if (!state && !lastFileset && !pollInterval && !update_timestamp && !label &&!uri) {
         psError(PS_ERR_UNKNOWN, true, "at least one of -last_fileset or -set_state is required");
         return false;
@@ -222,4 +223,7 @@
     }
 
+    if (uri) {
+        psStringAppend(&query, ", uri = '%s'", uri);
+    }
     if (state) {
         psStringAppend(&query, ", state = '%s'", state);
@@ -738,4 +742,5 @@
     PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
     PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
     PXOPT_COPY_S64(config->args, where, "-fault",  "fault", "==");
 
@@ -891,4 +896,5 @@
     PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
     PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
+    PXOPT_COPY_S32(config->args, where, "-fault",  "pstampJob.fault", "==");
     PXOPT_COPY_STR(config->args, where, "-state",  "pstampJob.state", "==");
 
Index: /branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/pstamptoolConfig.c	(revision 28794)
@@ -62,4 +62,5 @@
     psMetadataAddS64(moddatastoreArgs, PS_LIST_TAIL, "-ds_id", 0,            "define ds_id", 0);
     psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_last_fileset", 0,     "set last_fileset seen", NULL);
+    psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_uri", 0,     "set uri for data store", NULL);
     psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_state", 0,            "set state", NULL);
     psMetadataAddStr(moddatastoreArgs, PS_LIST_TAIL, "-set_label", 0,            "set label", NULL);
@@ -146,4 +147,5 @@
     psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0,          "select by request ID", 0);
     psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0,          "select by job ID", 0);
+    psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-dep_id", 0,          "select by dependent ID", 0);
     psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0,           "select by fault", 0);
     psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit",  0,          "limit result set to N items", 0);
@@ -163,5 +165,6 @@
     psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0,            "job_id of jobs to update", 0);
     psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0,            "dep_id of jobs to update", 0);
-    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0,            "current state of jobs to update", 0);
+    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0,             "current state of jobs to update", 0);
+    psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-fault", 0,             "current value for job fault", 0);
     psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-set_state", 0,            "new state", NULL);
     psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-set_fault", 0,            "new result", 0);
Index: /branches/eam_branches/ipp-20100621/ippTools/src/pubtool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/pubtool.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/pubtool.c	(revision 28794)
@@ -175,6 +175,6 @@
 
     PXOPT_COPY_S64(config->args, diffphotWhere, "-client_id", "client_id", "==");
-    pxAddLabelSearchArgs(config, diffphotWhere, "-label", "diffphotRun.label", "==");
-    pxAddLabelSearchArgs(config, diffphotWhere, "-data_group", "diffphotRun.data_group", "LIKE");
+    pxAddLabelSearchArgs(config, diffphotWhere, "-label", "diffPhotRun.label", "==");
+    pxAddLabelSearchArgs(config, diffphotWhere, "-data_group", "diffPhotRun.data_group", "LIKE");
     PXOPT_COPY_TIME(config->args, diffphotWhere, "-dateobs_begin", "rawExp.dateobs", ">=");
     PXOPT_COPY_TIME(config->args, diffphotWhere, "-dateobs_end", "rawExp.dateobs", "<=");
Index: /branches/eam_branches/ipp-20100621/ippTools/src/pxtools.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/pxtools.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/pxtools.c	(revision 28794)
@@ -48,4 +48,14 @@
 }
 
+bool pxIsValidCleanedState(const char *state)
+{
+    PS_ASSERT_PTR_NON_NULL(state, false);
+
+    if (!strcmp(state, "cleaned")) return true;
+    if (!strcmp(state, "purged")) return true;
+    if (!strcmp(state, "scrubbed")) return true;
+    return false;
+}
+
 psString pxMergeCodeVersions(psString version1, psString version2)
 {
@@ -211,9 +221,9 @@
 
 // change the value for tableName.columName from 'full' to 'cleaned' if necessary
-bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows)
+bool pxSetStateCleaned(const char *tableName, const char *columnName, psArray *rows)
 {
     for (long i = 0; i < psArrayLength(rows); i++) {
         psMetadata *row = rows->data[i];
-        psString state = psMetadataLookupStr(NULL, row, columnName);
+        const char *state = psMetadataLookupStr(NULL, row, columnName);
         if (!state) {
             psError(PS_ERR_PROGRAMMING, false, "%s not found in row %ld of table %s",
Index: /branches/eam_branches/ipp-20100621/ippTools/src/pxtools.h
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/pxtools.h	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/pxtools.h	(revision 28794)
@@ -53,20 +53,21 @@
 
 bool pxIsValidState(const char *state);
+bool pxIsValidCleanedState(const char *state);
 psString pxMergeCodeVersions(psString version1, psString version2);
 bool pxCoalesceRunStatus(pxConfig *config, const psString dbQFile, psS64 stage_id, psString *software_ver,
-			 psS64 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic,
-			 psF32 *maskfrac_magic, psF32 *maskfrac_advisory);
+                         psS64 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic,
+                         psF32 *maskfrac_magic, psF32 *maskfrac_advisory);
 bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id,
-		      psString software_ver);
+                      psString software_ver);
 bool pxSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id,
-		      psS64 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic,
-		      psF32 maskfrac_magic, psF32 maskfrac_advisory);
+                      psS64 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic,
+                      psF32 maskfrac_magic, psF32 maskfrac_advisory);
 bool pxCamSetRunMaskfrac(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id,
-			 psS64 maskfrac_ref_npix, psF32 maskfrac_ref_static, psF32 maskfrac_ref_dynamic,
-			 psF32 maskfrac_ref_magic, psF32 maskfrac_ref_advisory,
-			 psS64 maskfrac_max_npix, psF32 maskfrac_max_static, psF32 maskfrac_max_dynamic,
-			 psF32 maskfrac_max_magic, psF32 maskfrac_max_advisory);
-
-bool pxSetStateCleaned(const psString tableName, const psString columnName, psArray *rows);
+                         psS64 maskfrac_ref_npix, psF32 maskfrac_ref_static, psF32 maskfrac_ref_dynamic,
+                         psF32 maskfrac_ref_magic, psF32 maskfrac_ref_advisory,
+                         psS64 maskfrac_max_npix, psF32 maskfrac_max_static, psF32 maskfrac_max_dynamic,
+                         psF32 maskfrac_max_magic, psF32 maskfrac_max_advisory);
+
+bool pxSetStateCleaned(const char *tableName, const char *columnName, psArray *rows);
 bool pxAddLabelSearchArgs (pxConfig *config, psMetadata *where, char *field, char *name, char *op);
 
Index: /branches/eam_branches/ipp-20100621/ippTools/src/stacktool.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/stacktool.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/stacktool.c	(revision 28794)
@@ -78,8 +78,8 @@
         MODECASE(STACKTOOL_MODE_ADDSUMSKYFILE,         addsumskyfileMode);
         MODECASE(STACKTOOL_MODE_SUMSKYFILE,            sumskyfileMode);
-	MODECASE(STACKTOOL_MODE_SASSSKYFILE,            sassskyfileMode);
+        MODECASE(STACKTOOL_MODE_SASSSKYFILE,            sassskyfileMode);
         MODECASE(STACKTOOL_MODE_REVERTSUMSKYFILE,      revertsumskyfileMode);
-	MODECASE(STACKTOOL_MODE_TOSUMMARY,             tosummaryMode);
-	MODECASE(STACKTOOL_MODE_ADDSUMMARY,            addsummaryMode);
+        MODECASE(STACKTOOL_MODE_TOSUMMARY,             tosummaryMode);
+        MODECASE(STACKTOOL_MODE_ADDSUMMARY,            addsummaryMode);
         MODECASE(STACKTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
         MODECASE(STACKTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
@@ -155,8 +155,8 @@
 
     printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,row,"sass_id"),
-	   psMetadataLookupStr(NULL,row,"data_group"),
-	   psMetadataLookupStr(NULL,row,"tess_id"),
-	   psMetadataLookupStr(NULL,row,"filter"),
-	   psMetadataLookupStr(NULL,row,"projection_cell"));
+           psMetadataLookupStr(NULL,row,"data_group"),
+           psMetadataLookupStr(NULL,row,"tess_id"),
+           psMetadataLookupStr(NULL,row,"filter"),
+           psMetadataLookupStr(NULL,row,"projection_cell"));
 
     if (psMetadataLookupS64(NULL,row,"sass_id") == PS_MAX_S64) {
@@ -170,12 +170,12 @@
     psMetadataAddStr(outrow,PS_LIST_TAIL,"filter",PS_META_REPLACE,"",psMetadataLookupStr(NULL,row,"filter"));
     psMetadataAddStr(outrow,PS_LIST_TAIL,"projection_cell",PS_META_REPLACE,"",
-		     psMetadataLookupStr(NULL,row,"projection_cell"));
+                     psMetadataLookupStr(NULL,row,"projection_cell"));
   }
   printf("%" PRId64 " %s %s %s %s\n",psMetadataLookupS64(NULL,outrow,"sass_id"),
-	 psMetadataLookupStr(NULL,outrow,"data_group"),
-	 psMetadataLookupStr(NULL,outrow,"tess_id"),
-	 psMetadataLookupStr(NULL,outrow,"filter"),
-	 psMetadataLookupStr(NULL,outrow,"projection_cell"));
-  
+         psMetadataLookupStr(NULL,outrow,"data_group"),
+         psMetadataLookupStr(NULL,outrow,"tess_id"),
+         psMetadataLookupStr(NULL,outrow,"filter"),
+         psMetadataLookupStr(NULL,outrow,"projection_cell"));
+
 
   psFree(output);
@@ -184,8 +184,8 @@
   return(sassRow);
 }
-      
-      
-  
-					      
+
+
+
+
 
 
@@ -239,4 +239,5 @@
     PXOPT_COPY_F32(config->args,  where, "-select_zpt_obs_min",        "camProcessedExp.zpt_obs", ">=");
     PXOPT_COPY_F32(config->args,  where, "-select_zpt_obs_max",        "camProcessedExp.zpt_obs", "<=");
+    PXOPT_COPY_F32(config->args,  where, "-select_astrom",             "sqrt(POWER(camProcessedExp.sigma_ra, 2) + POWER(camProcessedExp.sigma_dec, 2))", "<=");
 
     PXOPT_COPY_STR(config->args,  where, "-select_exp_type",           "rawExp.exp_type", "==");
@@ -483,48 +484,48 @@
         psFree(run);
 
-	//CZW Add an association entry here.
-	// Define the requested association, and insert it if it doesn't already exist
-	stackAssociationRow *association = pxStackAssociationDefine(config,stack_id);
-	psS64 sass_id;
-	if (!association->sass_id) {
-	  psTrace("stacktool.association",2,"No required Association found. Adding.");
-
-	  if (!stackAssociationInsertObject(config->dbh,association)) {
-	    if (!psDBRollback(config->dbh)) {
-	      psError(PS_ERR_UNKNOWN, false, "database error");
-	    }
-	    psError(PS_ERR_UNKNOWN, false, "database error");
-	    psFree(output);
-	    psFree(run);
-	    psFree(insert);
-	    psFree(list);
-	    psFree(association);
-	    if (!psDBRollback(config->dbh)) {
-	      psError(PS_ERR_UNKNOWN, false, "database error");
-	    }
-	    return(false);
-	  }
-	  sass_id = psDBLastInsertID(config->dbh);
-	  association->sass_id = sass_id;
-	}
-	// Insert the map entry for this row.
-	stackAssociationMapRow *maprow = stackAssociationMapRowAlloc(sass_id,stack_id);
-	if (!stackAssociationMapInsertObject(config->dbh,maprow)) {
-	  if (!psDBRollback(config->dbh)) {
-	    psError(PS_ERR_UNKNOWN, false, "database error");
-	  }
-	  psError(PS_ERR_UNKNOWN, false, "database error");
-	  psFree(output);
-	  psFree(run);
-	  psFree(insert);
-	  psFree(list);
-	  psFree(association);
-	  if (!psDBRollback(config->dbh)) {
-	    psError(PS_ERR_UNKNOWN, false, "database error");
-	  }
-	  return(false);
-	}
-	
-	
+        //CZW Add an association entry here.
+        // Define the requested association, and insert it if it doesn't already exist
+        stackAssociationRow *association = pxStackAssociationDefine(config,stack_id);
+        psS64 sass_id;
+        if (!association->sass_id) {
+          psTrace("stacktool.association",2,"No required Association found. Adding.");
+
+          if (!stackAssociationInsertObject(config->dbh,association)) {
+            if (!psDBRollback(config->dbh)) {
+              psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            psFree(run);
+            psFree(insert);
+            psFree(list);
+            psFree(association);
+            if (!psDBRollback(config->dbh)) {
+              psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return(false);
+          }
+          sass_id = psDBLastInsertID(config->dbh);
+          association->sass_id = sass_id;
+        }
+        // Insert the map entry for this row.
+        stackAssociationMapRow *maprow = stackAssociationMapRowAlloc(sass_id,stack_id);
+        if (!stackAssociationMapInsertObject(config->dbh,maprow)) {
+          if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+          }
+          psError(PS_ERR_UNKNOWN, false, "database error");
+          psFree(output);
+          psFree(run);
+          psFree(insert);
+          psFree(list);
+          psFree(association);
+          if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+          }
+          return(false);
+        }
+
+
         // Create a suitable insertion query for this run
         psString thisInsert = psStringCopy(insert);
@@ -685,5 +686,5 @@
 
     //CZW Add an association entry here.
-    
+
     // insert the stackInputSkyfile rows
     psListIterator *iter = psListIteratorAlloc(warp_ids->data.list, 0, false);
@@ -740,5 +741,5 @@
     PXOPT_COPY_STR(config->args, where, "-label",     "stackRun.label",     "==");
     PXOPT_COPY_STR(config->args, where, "-state",     "stackRun.state",     "==");
-    PXOPT_COPY_STR(config->args, where, "-sass_id",   "stackAssociationMap.sass_id",  "==");
+    PXOPT_COPY_S64(config->args, where, "-sass_id",   "stackAssociationMap.sass_id",  "==");
     if (!psListLength(where->list)) {
         psFree(where);
@@ -1348,5 +1349,5 @@
 static bool tosummaryMode(pxConfig *config) {
   PS_ASSERT_PTR_NON_NULL(config, NULL);
-  
+
   psMetadata *where = psMetadataAlloc();
   PXOPT_COPY_S64(config->args, where, "-stack_id",    "stackSumSkyfile.warp_id", "==");
@@ -1357,10 +1358,10 @@
   pxAddLabelSearchArgs (config, where, "-label",   "stackRun.label", "LIKE");
   pxAddLabelSearchArgs (config, where, "-data_group",   "stackRun.data_group", "LIKE");
-  
+
   PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
 
   PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
   PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-  
+
   // find all rawImfiles matching the default query
   psString query = pxDataGet("stacktool_tosummary.sql");
@@ -1380,5 +1381,5 @@
     return false;
   }
-  
+
   psFree(where);
 
@@ -1408,5 +1409,5 @@
       psError(PXTOOLS_ERR_PROG, false, "unknown error");
     }
-    
+
     return false;
   }
@@ -1416,5 +1417,5 @@
     return true;
   }
-  
+
   if (psArrayLength(output)) {
     // negative simple so the default is true
@@ -1425,5 +1426,5 @@
     }
   }
-  
+
   psFree(output);
   return(true);
@@ -1447,5 +1448,5 @@
   }
   psS64 numUpdated = psDBAffectedRows(config->dbh);
-  
+
   if (numUpdated != 1) {
     psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
@@ -1453,9 +1454,9 @@
     return false;
   }
-  
+
   psFree(query);
 
   // Print anything here?
-  
+
   return(true);
 }
@@ -1471,5 +1472,5 @@
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "==");
     PXOPT_COPY_S64(config->args, where, "-sass_id", "stackAssociationMap.sass_id", "==");
-    
+
     pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
 
Index: /branches/eam_branches/ipp-20100621/ippTools/src/stacktoolConfig.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ippTools/src/stacktoolConfig.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippTools/src/stacktoolConfig.c	(revision 28794)
@@ -90,4 +90,5 @@
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_zpt_obs_min", 0, "define min zero point", NAN);
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_zpt_obs_max", 0, "define max zero point", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_astrom", 0, "define max astrometry rms", NAN);
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0);
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0);
@@ -115,4 +116,5 @@
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0,            "search by state", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0,            "search by label", 0);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-sass_id", 0,           "search by stack association ID", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0,        "define new value for label", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0,        "define new state", NULL);
@@ -237,6 +239,6 @@
     psMetadataAddS64(addsummaryArgs, PS_LIST_TAIL, "-sass_id", 0,      "set stack Association ID", 0);
     psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-projection_cell", 0, "set projection cell", NULL);
-    psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0,     "set summary path base", NULL);		     
-    
+    psMetadataAddStr(addsummaryArgs, PS_LIST_TAIL, "-path_base", 0,     "set summary path base", NULL);
+
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
Index: /branches/eam_branches/ipp-20100621/ippconfig/gpc1/format_20100122.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/gpc1/format_20100122.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/gpc1/format_20100122.config	(revision 28794)
@@ -220,4 +220,5 @@
         CELL.YPARITY    STR     ATM2_2
 	CHIP.TEMP	STR	CDETTEM
+	CHIP.TEMPERATURE STR	DETTEM
         FPA.EXPOSURE    STR     EXPREQ          # Requested exposure time, presumably camera exposure time
         CELL.EXPOSURE   STR     EXPTIME         # Exposure time measured by shutter
Index: /branches/eam_branches/ipp-20100621/ippconfig/gpc1/ppMerge.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/gpc1/ppMerge.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/gpc1/ppMerge.config	(revision 28794)
@@ -75,9 +75,24 @@
 # the 0th order term is implied, right?
 # Note: GPC1 has bad values for DARKTIME in unpredictable cells; use EXPOSURE for now
+#DARK.ORDINATES	METADATA
+#	CELL.EXPOSURE	S32	1
+# 	TEMP	METADATA
+# 		ORDER	S32	1
+#		RULE	STR	CHIP.TEMP * CELL.EXPOSURE
+# 	END
+#END
+ #
+#Dark for 2010-07-02 with new temp concept.
 DARK.ORDINATES	METADATA
 	CELL.EXPOSURE	S32	1
- 	TEMP	METADATA
- 		ORDER	S32	1
-		RULE	STR	CHIP.TEMP * CELL.EXPOSURE
- 	END
+  	TEMP	METADATA
+  		ORDER	S32	1
+ 		RULE	STR	CHIP.TEMPERATURE * CELL.EXPOSURE
+  	END
+  	TEMP2	METADATA
+  		ORDER	S32	1
+ 		RULE	STR	CHIP.TEMPERATURE * CHIP.TEMPERATURE * CELL.EXPOSURE
+  	END
 END
+
+
Index: /branches/eam_branches/ipp-20100621/ippconfig/gpc1/psastro.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/gpc1/psastro.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/gpc1/psastro.config	(revision 28794)
@@ -286,2 +286,7 @@
   ## END
 END
+
+### MOPS wants dynamic masking turned off
+MOPS.TEST	METADATA
+	REFSTAR_MASK	BOOL	FALSE
+END
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/Makefile.am	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/Makefile.am	(revision 28794)
@@ -30,5 +30,6 @@
 	ppSkycell.config \
 	ppVizPSF.mdc \
-	nightly_science.config
+	nightly_science.config \
+	ppBackground.mdc
 
 install_DATA = $(install_files)
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/filerules-mef.mdc
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/filerules-mef.mdc	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/filerules-mef.mdc	(revision 28794)
@@ -338,2 +338,4 @@
 PPBACKGROUND.OUTPUT	     OUTPUT {OUTPUT}.{CHIP.NAME}.fits    IMAGE     COMP_IMG   CHIP       TRUE      NONE
 PPBACKGROUND.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits MASK      COMP_MASK  CHIP       TRUE      NONE
+PPBACKGROUND.CONFIG          OUTPUT {OUTPUT}.{CHIP.NAME}.ppBackground.mdc TEXT        NONE       CHIP       TRUE      NONE
+PPBACKGROUND.STATS           OUTPUT {OUTPUT}.{CHIP.NAME}.stats        STATS           NONE       CHIP       TRUE      NONE
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/filerules-simple.mdc
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/filerules-simple.mdc	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/filerules-simple.mdc	(revision 28794)
@@ -303,2 +303,4 @@
 PPBACKGROUND.OUTPUT	     OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE           NONE       FPA        TRUE      NONE
 PPBACKGROUND.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits      MASK            NONE       FPA        TRUE      NONE
+PPBACKGROUND.CONFIG          OUTPUT {OUTPUT}.{CHIP.NAME}.ppBackground.mdc TEXT        NONE       CHIP       TRUE      NONE
+PPBACKGROUND.STATS           OUTPUT {OUTPUT}.{CHIP.NAME}.stats        STATS           NONE       CHIP       TRUE      NONE
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/filerules-split.mdc
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/filerules-split.mdc	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/filerules-split.mdc	(revision 28794)
@@ -295,8 +295,8 @@
 PPARITH.OUTPUT.MASK          OUTPUT {OUTPUT}.fits                     MASK            COMP_MASK  CHIP       TRUE      NONE
                                                                                                         
-PPSKYCELL.JPEG1              OUTPUT {OUTPUT}.{FILE.INDEX}.b1.jpeg     JPEG            NONE       CHIP       TRUE      NONE
-PPSKYCELL.JPEG2              OUTPUT {OUTPUT}.{FILE.INDEX}.b2.jpeg     JPEG            NONE       CHIP       TRUE      NONE
-PPSKYCELL.BIN1    	OUTPUT {OUTPUT}.{FILE.INDEX}.b1.fits     IMAGE     COMP_IMG       FPA        TRUE      NONE
-PPSKYCELL.BIN2    	OUTPUT {OUTPUT}.{FILE.INDEX}.b2.fits     IMAGE     COMP_IMG       FPA       TRUE      NONE
+PPSKYCELL.JPEG1              OUTPUT {OUTPUT}.0.b1.jpeg     JPEG            NONE       CHIP       TRUE      NONE
+PPSKYCELL.JPEG2              OUTPUT {OUTPUT}.0.b2.jpeg     JPEG            NONE       CHIP       TRUE      NONE
+PPSKYCELL.BIN1    	OUTPUT {OUTPUT}.0.b1.fits     IMAGE     COMP_IMG       FPA        TRUE      NONE
+PPSKYCELL.BIN2    	OUTPUT {OUTPUT}.0.b2.fits     IMAGE     COMP_IMG       FPA       TRUE      NONE
 
 LOG.IMFILE                   OUTPUT {OUTPUT}.{CHIP.NAME}.log          TEXT            NONE       CHIP       TRUE      NONE
@@ -328,4 +328,7 @@
 PPBACKGROUND.OUTPUT	     OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE           COMP_IMG   CHIP       TRUE      NONE
 PPBACKGROUND.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits      MASK            COMP_MASK  CHIP       TRUE      NONE
+PPBACKGROUND.CONFIG          OUTPUT {OUTPUT}.{CHIP.NAME}.ppBackground.mdc TEXT        NONE       CHIP       TRUE      NONE
+PPBACKGROUND.STATS           OUTPUT {OUTPUT}.{CHIP.NAME}.stats        STATS           NONE       CHIP       TRUE      NONE
+
 
 # FILERULE naming operators:
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/nightly_science.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/nightly_science.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/nightly_science.config	(revision 28794)
@@ -6,5 +6,4 @@
   COMMAND STR chiptool
   RETENTION_TIME U16 30
-  ALTERNATE_CMD BOOL F
 END
 CLEAN_MODES METADATA
@@ -12,18 +11,22 @@
   COMMAND STR warptool
   RETENTION_TIME U16 7
-  ALTERNATE_CMD BOOL F
 END
 CLEAN_MODES METADATA
   MODE STR DIFF
   COMMAND STR difftool
-# RETENTION_TIME U16 30
-  RETENTION_TIME S16 -1
-  ALTERNATE_CMD BOOL F
+  RETENTION_TIME S16 30
+#  RETENTION_TIME S16 -1
 END
 CLEAN_MODES METADATA
   MODE STR DIST
-  COMMAND STR disttool
+  COMMAND STR disttool 
   RETENTION_TIME S16 7
-  ALTERNATE_CMD BOOL T
+  ALTERNATE_CMD STR A
+END
+CLEAN_MODES METADATA
+  MODE STR MAGICDS
+  COMMAND STR magicdstool
+  RETENTION_TIME S16 1
+  ALTERNATE_CMD STR B
 END
 
@@ -102,13 +105,16 @@
 TARGETS METADATA
   NAME STR MD08
-  TESS STR MD08
+  TESS STR MD08.V2
   OBSMODE STR MD
   OBJECT STR MD08%
 #  COMMENT STR %MD08%
   STACKABLE BOOL TRUE
+  CHIP S16 9000
+  WARP S16 9000
+  DIFF S16 9000
 END
 TARGETS METADATA
   NAME STR MD09
-  TESS STR MD09
+  TESS STR MD09.V2
   OBSMODE STR MD
   OBJECT STR MD09%
@@ -118,5 +124,5 @@
 TARGETS METADATA
   NAME STR MD10
-  TESS STR MD10
+  TESS STR MD10.V2
   OBSMODE STR MD
   OBJECT STR MD10%
@@ -316,5 +322,5 @@
   DETTYPE  STR FLAT
   EXPTYPE  STR skyflat
-  REF_ID   S64 300
+  REF_ID   S64 278
   REF_ITER S32 0
   FILTER   STR g.00000
@@ -326,5 +332,5 @@
   DETTYPE  STR FLAT
   EXPTYPE  STR skyflat
-  REF_ID   S64 301
+  REF_ID   S64 279
   REF_ITER S32 0
   FILTER   STR r.00000
@@ -336,5 +342,5 @@
   DETTYPE  STR FLAT
   EXPTYPE  STR skyflat
-  REF_ID   S64 302
+  REF_ID   S64 280
   REF_ITER S32 0
   FILTER   STR i.00000
@@ -346,5 +352,5 @@
   DETTYPE  STR FLAT
   EXPTYPE  STR skyflat
-  REF_ID   S64 303
+  REF_ID   S64 281
   REF_ITER S32 0
   FILTER   STR z.00000
@@ -356,5 +362,5 @@
   DETTYPE  STR FLAT
   EXPTYPE  STR skyflat
-  REF_ID   S64 304
+  REF_ID   S64 282
   REF_ITER S32 0
   FILTER   STR y.00000
@@ -366,5 +372,5 @@
   DETTYPE  STR FLAT
   EXPTYPE  STR skyflat
-  REF_ID   S64 305
+  REF_ID   S64 283
   REF_ITER S32 0
   FILTER   STR w.00000
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/ppBackground.mdc
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/ppBackground.mdc	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/ppBackground.mdc	(revision 28794)
@@ -0,0 +1,5 @@
+# Recipe options for ppBackground
+
+
+BACKGROUND	 METADATA
+END
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/ppStack.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/ppStack.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/ppStack.config	(revision 28794)
@@ -9,8 +9,5 @@
 COMBINE.DISCARD	F32	0.2		# Discard fraction for Olympic weighted mean
 
-# XXX which of these is right?
 MASK.VAL	STR	MASK.VALUE,CONV.BAD,GHOST	# Mask value of input bad pixels
-MASK.IN		STR	MASK.VALUE,CONV.BAD,GHOST	# Mask value of input bad pixels
-
 MASK.SUSPECT	STR	SUSPECT,CONV.POOR	# Mask value of suspect pixels
 MASK.BAD	STR	BLANK		# Mask value to give bad pixels
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/ppStatsFromMetadata.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/ppStatsFromMetadata.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/ppStatsFromMetadata.config	(revision 28794)
@@ -326,4 +326,27 @@
 END
 
+
+BACKGROUND_WARP	METADATA
+  ENTRY MULTI
+  TYPE   VAL  KEYWORD             TYPE STATISTIC         FLAG 
+  ENTRY  VAL  ROBUST_MEDIAN       F32  ROBUST_MEDIAN     -bg            
+  ENTRY  VAL  ROBUST_STDEV        F32  RMS               -bg_stdev      
+  ENTRY  VAL  QUALITY             S32  CONSTANT          -quality             # Bad quality flag
+
+  # Revision values
+  ENTRY	 VAL  PSLIB_V		  STR  CONSTANT		-ver_pslib
+  ENTRY  VAL  MODULE_V		  STR  CONSTANT		-ver_psmodules
+  ENTRY	 VAL  STATS_V		  STR  CONSTANT		-ver_ppstats
+  ENTRY	 VAL  WARP_V		  STR  CONSTANT		-ver_pswarp
+
+  # Mask stats values
+  ENTRY VAL MASKFRAC_NPIX	  S32  CONSTANT		-maskfrac_npix
+  ENTRY VAL MASKFRAC_STATIC	  F32  CONSTANT		-maskfrac_static
+  ENTRY VAL MASKFRAC_DYNAMIC	  F32  CONSTANT		-maskfrac_dynamic
+  ENTRY VAL MASKFRAC_MAGIC	  F32  CONSTANT		-maskfrac_magic
+  ENTRY VAL MASKFRAC_ADVISORY	  F32  CONSTANT		-maskfrac_advisory
+END
+
+
 STACK_SKYCELL METADATA
   ENTRY MULTI
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/psastro.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/psastro.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/psastro.config	(revision 28794)
@@ -235,2 +235,5 @@
 PS1_REFERENCE METADATA
 END
+
+MOPS.TEST	METADATA
+END
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/psphot.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/psphot.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/psphot.config	(revision 28794)
@@ -242,5 +242,5 @@
 PCM_BOX_SIZE                        S32   8
 
-NOISE.FACTOR                        F32   5.0
+NOISE.FACTOR                        F32  10.0
 NOISE.SIZE                          F32   2.0
 
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/pswarp.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/pswarp.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/pswarp.config	(revision 28794)
@@ -11,4 +11,5 @@
 INTERPOLATION.NUM	S32	1000		# Number of interpolation kernels to pre-calculate
 PSF			BOOL	TRUE		# Measure PSF for warped image?
+SOURCES			BOOL	TRUE		# Write source list for warped image?
 
 MASK.STATS         BOOL    TRUE            # calculate mask statistics.
@@ -27,2 +28,8 @@
 	PSF	BOOL	FALSE
 END
+
+# Background restoration
+BACKGROUND	METADATA
+	SOURCES		BOOL	FALSE		# Write source list for warped image?
+	PSF		BOOL	FALSE		# Measure PSF for warped image?
+END
Index: /branches/eam_branches/ipp-20100621/ippconfig/recipes/reductionClasses.mdc
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/recipes/reductionClasses.mdc	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/recipes/reductionClasses.mdc	(revision 28794)
@@ -158,4 +158,7 @@
 	PSASTRO		STR	DEFAULT_RECIPE
 	STACKPHOT       STR     STACKPHOT
+	BACKGROUND_PPBACKGROUND	STR	BACKGROUND
+	BACKGROUND_PSWARP	STR	BACKGROUND
+
 END
 
@@ -448,2 +451,25 @@
 	PSASTRO		STR	DEFAULT_RECIPE
 END
+
+
+# basic science analysis
+MOPS.TEST	METADATA
+	CHIP_PPIMAGE	STR	CHIP
+	CHIP_PSPHOT	STR	CHIP
+	WARP_PSWARP	STR	WARP
+	STACK_PPSTACK	STR	STACK
+	STACK_PPSUB	STR	STACK
+	STACK_PSPHOT	STR	STACK
+	DIFF_PPSUB	STR	DIFF
+	DIFF_PSPHOT	STR	DIFF
+	JPEG_BIN1	STR	PPIMAGE_J1
+	JPEG_BIN2	STR	PPIMAGE_J2
+	FAKEPHOT	STR	FAKEPHOT
+	ADDSTAR		STR	ADDSTAR
+	PSASTRO		STR	MOPS.TEST
+	STACKPHOT       STR     STACKPHOT
+	BACKGROUND_PPBACKGROUND	STR	BACKGROUND
+	BACKGROUND_PSWARP	STR	BACKGROUND
+
+END
+
Index: /branches/eam_branches/ipp-20100621/ippconfig/system.config
===================================================================
--- /branches/eam_branches/ipp-20100621/ippconfig/system.config	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ippconfig/system.config	(revision 28794)
@@ -59,3 +59,4 @@
 	PPVIZPSF	STR		recipes/ppVizPSF.mdc # PSF visualisation
         NIGHTLY_SCIENCE STR             recipes/nightly_science.config # Nightly Science
+	PPBACKGROUND	STR		recipes/ppBackground.mdc       # Background restoration
 END
Index: /branches/eam_branches/ipp-20100621/magic/remove/src/streaksremove.c
===================================================================
--- /branches/eam_branches/ipp-20100621/magic/remove/src/streaksremove.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/magic/remove/src/streaksremove.c	(revision 28794)
@@ -577,5 +577,5 @@
 }
 
-static void 
+static void
 setStreakBits(psImage *maskImage, psU32 maskStreak)
 {
@@ -644,5 +644,5 @@
             if (exciseAll) {
                 strkGetMaskValues(sf);
-                
+
                 // add the STREAK bit to the mask image pixels
                 setStreakBits(sf->inMask->image, sf->maskStreak);
@@ -941,9 +941,5 @@
         psArray *inTable = psFitsReadTable(in->fits);
         if (!inTable) {
-            psErrorStackPrint(stderr, "failed to read tablle in %s", in->resolved_name);
-            streaksExit("", PS_EXIT_DATA_ERROR);
-        }
-        if (!inTable->n) {
-            psErrorStackPrint(stderr, "table in %s is empty", in->resolved_name);
+            psErrorStackPrint(stderr, "failed to read table in %s", in->resolved_name);
             streaksExit("", PS_EXIT_DATA_ERROR);
         }
@@ -983,4 +979,12 @@
                 streaksExit("", PS_EXIT_DATA_ERROR);
             }
+        } else if (psArrayLength(inTable) == 0) {
+            printf("No input sources\n");
+            // We'd like to write a blank table, but this is as good as it gets without more work to parse the
+            // header and create dummy columns.
+            if (!psFitsWriteBlank(out->fits, header, extname)) {
+                psErrorStackPrint(stderr, "failed to write empty header to %s", out->resolved_name);
+                streaksExit("", PS_EXIT_DATA_ERROR);
+            }
         } else {
             printf("Censored ALL %d sources\n", numCensored);
Index: /branches/eam_branches/ipp-20100621/ppBackground/src/ppBackgroundArguments.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ppBackground/src/ppBackgroundArguments.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppBackground/src/ppBackgroundArguments.c	(revision 28794)
@@ -77,9 +77,16 @@
     const char *statsName = psMetadataLookupStr(NULL, arguments, "-stats");
     if (statsName) {
-        data->statsFile = fopen(statsName, "w");
-        if (!data->statsFile) {
-            psError(PPBACKGROUND_ERR_IO, true, "Unable to open statistics file %s", statsName);
+        psString resolved = pmConfigConvertFilename(statsName, data->config, true, true); // Resolved filename
+        if (!resolved) {
+            psError(psErrorCodeLast(), false, "Unable to resolve statistics file %s", statsName);
             return false;
         }
+        data->statsFile = fopen(resolved, "w");
+        if (!data->statsFile) {
+            psError(PPBACKGROUND_ERR_IO, true, "Unable to open statistics file %s = %s", statsName, resolved);
+            psFree(resolved);
+            return false;
+        }
+        psFree(resolved);
         data->stats = psMetadataAlloc();
         pmConfigRunFilenameAddWrite(data->config, "STATS", statsName);
Index: /branches/eam_branches/ipp-20100621/ppBackground/src/ppBackgroundErrorCodes.dat
===================================================================
--- /branches/eam_branches/ipp-20100621/ppBackground/src/ppBackgroundErrorCodes.dat	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppBackground/src/ppBackgroundErrorCodes.dat	(revision 28794)
@@ -7,5 +7,5 @@
 ARGUMENTS		Incorrect arguments
 CONFIG			Problem in configure files
-IO			Problem in FITS I/O
+IO			Problem in I/O
 DATA                    Problem in data values
 PROG			Programming error
Index: /branches/eam_branches/ipp-20100621/ppStack/src/ppStackArguments.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ppStack/src/ppStackArguments.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppStack/src/ppStackArguments.c	(revision 28794)
@@ -261,5 +261,5 @@
     VALUE_ARG_RECIPE_FLOAT("-poor-frac",       "POOR.FRACTION",   F32);
 
-    valueArgRecipeStr(arguments, recipe, "-mask-val",  "MASK.IN",   recipe);
+    valueArgRecipeStr(arguments, recipe, "-mask-val",  "MASK.VAL",   recipe);
     valueArgRecipeStr(arguments, recipe, "-mask-bad",  "MASK.BAD",  recipe);
     valueArgRecipeStr(arguments, recipe, "-mask-poor", "MASK.POOR", recipe);
Index: /branches/eam_branches/ipp-20100621/ppSub/src/ppSubConvolve.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ppSub/src/ppSubConvolve.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppSub/src/ppSubConvolve.c	(revision 28794)
@@ -29,4 +29,7 @@
     )
 {
+    if (!name) {
+        return;
+    }
     psArray *files = psArrayAlloc(1); // Array with file names
     files->data[0] = psStringCopy(name);
@@ -53,7 +56,9 @@
         psMetadataAddStr(arguments, PS_LIST_TAIL, "-image", 0, "Input image", NULL);
         psMetadataAddStr(arguments, PS_LIST_TAIL, "-mask", 0, "Input mask", NULL);
+        psMetadataAddStr(arguments, PS_LIST_TAIL, "-variance", 0, "Input variance", NULL);
         psMetadataAddStr(arguments, PS_LIST_TAIL, "-kernel", 0, "Convolution kernel", NULL);
         psMetadataAddBool(arguments, PS_LIST_TAIL, "-reference", 0, "Input is actually reference?", false);
         psMetadataAddS32(arguments, PS_LIST_TAIL, "-threads", 0, "Threads to use", 0);
+        psMetadataAddBool(arguments, PS_LIST_TAIL, "-save-all", 0, "Save all outputs?", false);
 
         if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 2) {
@@ -63,4 +68,5 @@
         const char *inImage = psMetadataLookupStr(NULL, arguments, "-image"); // Input image
         const char *inMask = psMetadataLookupStr(NULL, arguments, "-mask"); // Input mask
+        const char *inVariance = psMetadataLookupStr(NULL, arguments, "-variance"); // Input variance
         const char *inKernel = psMetadataLookupStr(NULL, arguments, "-kernel"); // Input kernel
         if (!inImage || !inKernel) {
@@ -72,4 +78,5 @@
         fileList("PPSUB.INPUT", inImage, "Input image", config);
         fileList("PPSUB.INPUT.MASK", inMask, "Input mask", config);
+        fileList("PPSUB.INPUT.VARIANCE", inVariance, "Input variance", config);
         fileList("PPSUB.INPUT.KERNEL", inKernel, "Input kernel", config);
         psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[1]);
@@ -81,9 +88,19 @@
             goto die;
         }
-        pmFPAfile *mask = pmFPAfileBindFromArgs(&status, image, config, "PPSUB.INPUT.MASK",
-                                                "PPSUB.INPUT.MASK");
-        if (!mask || !status || mask->type != PM_FPA_FILE_MASK) {
-            psError(PPSUB_ERR_CONFIG, false, "Unable to define input mask file.");
-            goto die;
+        if (inMask) {
+            pmFPAfile *mask = pmFPAfileBindFromArgs(&status, image, config, "PPSUB.INPUT.MASK",
+                                                    "PPSUB.INPUT.MASK");
+            if (!mask || !status || mask->type != PM_FPA_FILE_MASK) {
+                psError(PPSUB_ERR_CONFIG, false, "Unable to define input mask file.");
+                goto die;
+            }
+        }
+        if (inVariance) {
+            pmFPAfile *variance = pmFPAfileBindFromArgs(&status, image, config, "PPSUB.INPUT.VARIANCE",
+                                                        "PPSUB.INPUT.VARIANCE");
+            if (!variance || !status || variance->type != PM_FPA_FILE_VARIANCE) {
+                psError(PPSUB_ERR_CONFIG, false, "Unable to define input variance file.");
+                goto die;
+            }
         }
 
@@ -101,4 +118,20 @@
         }
         output->save = true;
+
+        if (psMetadataLookupBool(NULL, arguments, "-save-all")) {
+            pmFPAfile *outMask = pmFPAfileDefineOutput(config, output->fpa, "PPSUB.INPUT.CONV.MASK");
+            if (!outMask || outMask->type != PM_FPA_FILE_MASK) {
+                psError(PPSUB_ERR_CONFIG, false, "Unable to define output mask file.");
+                goto die;
+            }
+            outMask->save = true;
+
+            pmFPAfile *outVar = pmFPAfileDefineOutput(config, output->fpa, "PPSUB.INPUT.CONV.VARIANCE");
+            if (!outVar || outVar->type != PM_FPA_FILE_VARIANCE) {
+                psError(PPSUB_ERR_CONFIG, false, "Unable to define output variance file.");
+                goto die;
+            }
+            outVar->save = true;
+        }
     }
 
@@ -206,4 +239,5 @@
         if (threads > 0) {
             pmSubtractionThreadsInit();
+            psThreadPoolInit(threads);
         }
         if (!pmSubtractionMatchPrecalc(inConv, refConv, input, ref, inRO->analysis,
Index: /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMops.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMops.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMops.c	(revision 28794)
@@ -20,15 +20,17 @@
     }
 
-    if (!ppMopsPurgeDuplicates(detections)) {
+    ppMopsDetections *merged = ppMopsMerge(detections); // Merged detections
+    psFree(detections);
+    if (!merged) {
         psErrorStackPrint(stderr, "Unable to merge detections");
         exit(PS_EXIT_SYS_ERROR);
     }
 
-    if (!ppMopsWrite(detections, args)) {
+    if (!ppMopsWrite(merged, args)) {
         psErrorStackPrint(stderr, "Unable to write detections");
         exit(PS_EXIT_SYS_ERROR);
     }
 
-    psFree(detections);
+    psFree(merged);
     psFree(args);
 
Index: /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMops.h
===================================================================
--- /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMops.h	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMops.h	(revision 28794)
@@ -7,4 +7,7 @@
 #define IN_EXTNAME "SkyChip.psf"        // Extension name for data in input
 #define OBSERVATORY_CODE "F51"          // IAU Observatory Code
+#define OUT_EXTNAME "MOPS_TRANSIENT_DETECTIONS" // Extension name for data in output
+#define SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_BADPSF | PM_SOURCE_MODE_SATURATED | \
+                     PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_SKY_FAILURE) // Flags to exclude
 
 // Configuration data
@@ -24,10 +27,35 @@
 } ppMopsArguments;
 
+#if 0
+TTYPE19 = 'PSF_CHISQ'          / label for field  19
+TFORM19 = '1E      '           / data format of field: 4-byte REAL
+TTYPE20 = 'CR_NSIGMA'          / label for field  20
+TFORM20 = '1E      '           / data format of field: 4-byte REAL
+TTYPE21 = 'EXT_NSIGMA'         / label for field  21
+TFORM21 = '1E      '           / data format of field: 4-byte REAL
+TTYPE22 = 'PSF_MAJOR'          / label for field  22
+TFORM22 = '1E      '           / data format of field: 4-byte REAL
+TTYPE23 = 'PSF_MINOR'          / label for field  23
+TFORM23 = '1E      '           / data format of field: 4-byte REAL
+TTYPE24 = 'PSF_THETA'          / label for field  24
+TFORM24 = '1E      '           / data format of field: 4-byte REAL
+TTYPE25 = 'PSF_QF  '           / label for field  25
+TFORM25 = '1E      '           / data format of field: 4-byte REAL
+TTYPE26 = 'PSF_NDOF'           / label for field  26
+TFORM26 = '1J      '           / data format of field: 4-byte INTEGER
+TTYPE27 = 'PSF_NPIX'           / label for field  27
+TFORM27 = '1J      '           / data format of field: 4-byte INTEGER
+TTYPE28 = 'MOMENTS_XX'         / label for field  28
+TFORM28 = '1E      '           / data format of field: 4-byte REAL
+TTYPE29 = 'MOMENTS_XY'         / label for field  29
+TFORM29 = '1E      '           / data format of field: 4-byte REAL
+TTYPE30 = 'MOMENTS_YY'         / label for field  30
+TFORM30 = '1E      '           / data format of field: 4-byte REAL
+#endif
+
 /// Parse arguments
 ppMopsArguments *ppMopsArgumentsParse(int argc, char *argv[]);
 
 typedef struct {
-    psString component;                 // Component name
-    psMetadata *header;                 // FITS header (extension names *.hdr)
     psString raBoresight, decBoresight; // RA,Dec of telescope boresight
     psString filter;                    // Filter for exposure
@@ -36,27 +64,45 @@
     double posangle;                    // Position angle
     double alt, az;                     // Telescope altitude and azimuth
-    double mjd;                         // Modified Julian Date of exposure mid-point
+    double mjd;                         // Modified Julian Date
     float seeing;                       // Seeing of exposure
-    int naxis1, naxis2;                 // Size of image
     long num;                           // Number of detections
-    psMetadata *psfHeader;              // FITS header (extension names *.psf)
-    psMetadata *table;                  // Columns of data
     psVector *x, *y;                    // Image coordinates
     psVector *ra, *dec;                 // Sky coordinates
-    psMetadata *deteffHeader;           // Detection efficiency header (extension names *.deteff)
-    psMetadata *deteffTable;            // Detection efficiency table
+    psVector *raErr, *decErr;           // Error in sky coordinates
+    psVector *mag, *magErr;             // Magnitude and associated error
+    psVector *chi2, *dof;               // Chi^2 from fitting, with associated degrees of freedom
+    psVector *cr, *extended;            // Measures of CR-ness and extendedness
+    psVector *psfMajor, *psfMinor, *psfTheta; // PSF major and minor axes, and position angle
+    psVector *quality, *numPix;               // PSF quality factor and number of pixels
+    psVector *xxMoment, *xyMoment, *yyMoment; // Moments
+    psVector *flags;                    // psphot flags
+    psVector *diffSkyfileId;            // Identifier for source image
+    psVector *naxis1, *naxis2;          // Size of image
+    psVector *mask;                     // Mask for detections
+    psVector *nPos;                     // Number of positive pixels
+    psVector *fPos;                     // Fraction of positive flux
+    psVector *nRatioBad;                // Fraction of positive pixels to negative
+    psVector *nRatioMask;               // Fraction of positive pixels to masked
+    psVector *nRatioAll;                // Fraction of positive pixels to all
 } ppMopsDetections;
 
-// Allocator
-ppMopsDetections *ppMopsDetectionsAlloc(void);
+
+ppMopsDetections *ppMopsDetectionsAlloc(long num);
+
+/// Copy a detection
+bool ppMopsDetectionsCopySingle(ppMopsDetections *target, const ppMopsDetections *source, long index);
+
+/// Purge the detections list of masked detections
+bool ppMopsDetectionsPurge(ppMopsDetections *detections);
+
 
 /// Read detections
 psArray *ppMopsRead(const ppMopsArguments *args);
 
-/// Purge duplicate detections
-bool ppMopsPurgeDuplicates(const psArray *detections);
+/// Merge detections
+ppMopsDetections *ppMopsMerge(const psArray *detections);
 
 /// Write detections
-bool ppMopsWrite(const psArray *detections, const ppMopsArguments *args);
+bool ppMopsWrite(const ppMopsDetections *detections, const ppMopsArguments *args);
 
 #endif
Index: /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsDetections.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsDetections.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsDetections.c	(revision 28794)
@@ -10,26 +10,46 @@
 static void mopsDetectionsFree(ppMopsDetections *det)
 {
-    psFree(det->component);
     psFree(det->raBoresight);
     psFree(det->decBoresight);
     psFree(det->filter);
-    psFree(det->header);
-    psFree(det->table);
     psFree(det->x);
     psFree(det->y);
     psFree(det->ra);
     psFree(det->dec);
-    psFree(det->deteffHeader);
-    psFree(det->deteffTable);
+    psFree(det->raErr);
+    psFree(det->decErr);
+    psFree(det->mag);
+    psFree(det->magErr);
+    psFree(det->chi2);
+    psFree(det->dof);
+    psFree(det->cr);
+    psFree(det->extended);
+    psFree(det->psfMajor);
+    psFree(det->psfMinor);
+    psFree(det->psfTheta);
+    psFree(det->quality);
+    psFree(det->numPix);
+    psFree(det->xxMoment);
+    psFree(det->xyMoment);
+    psFree(det->yyMoment);
+    psFree(det->flags);
+    psFree(det->diffSkyfileId);
+    psFree(det->naxis1);
+    psFree(det->naxis2);
+    psFree(det->mask);
+    psFree(det->nPos);
+    psFree(det->fPos);
+    psFree(det->nRatioBad);
+    psFree(det->nRatioMask);
+    psFree(det->nRatioAll);
 
     return;
 }
 
-ppMopsDetections *ppMopsDetectionsAlloc(void)
+ppMopsDetections *ppMopsDetectionsAlloc(long num)
 {
     ppMopsDetections *det = psAlloc(sizeof(ppMopsDetections)); // Detections, to return
     psMemSetDeallocator(det, (psFreeFunc)mopsDetectionsFree);
 
-    det->component = NULL;
     det->raBoresight = NULL;
     det->decBoresight = NULL;
@@ -43,13 +63,234 @@
     det->seeing = NAN;
     det->num = 0;
-    det->header = NULL;
-    det->table = NULL;
-    det->x = NULL;
-    det->y = NULL;
-    det->ra = NULL;
-    det->dec = NULL;
-    det->deteffHeader = NULL;
-    det->deteffTable = NULL;
+    det->x = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->y = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->ra = psVectorAllocEmpty(num, PS_TYPE_F64);
+    det->dec = psVectorAllocEmpty(num, PS_TYPE_F64);
+    det->raErr = psVectorAllocEmpty(num, PS_TYPE_F64);
+    det->decErr = psVectorAllocEmpty(num, PS_TYPE_F64);
+    det->mag = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->magErr = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->chi2 = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->dof = psVectorAllocEmpty(num, PS_TYPE_S32);
+    det->cr = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->extended = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->psfMajor = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->psfMinor = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->psfTheta = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->quality = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->numPix = psVectorAllocEmpty(num, PS_TYPE_S32);
+    det->xxMoment = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->xyMoment = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->yyMoment = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->flags = psVectorAllocEmpty(num, PS_TYPE_U32);
+    det->diffSkyfileId = psVectorAllocEmpty(num, PS_TYPE_S64);
+    det->naxis1 = psVectorAllocEmpty(num, PS_TYPE_S32);
+    det->naxis2 = psVectorAllocEmpty(num, PS_TYPE_S32);
+    det->mask = psVectorAllocEmpty(num, PS_TYPE_U8);
+    det->nPos = psVectorAllocEmpty(num, PS_TYPE_S32);
+    det->fPos = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->nRatioBad = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->nRatioMask = psVectorAllocEmpty(num, PS_TYPE_F32);
+    det->nRatioAll = psVectorAllocEmpty(num, PS_TYPE_F32);
 
     return det;
 }
+
+
+ppMopsDetections *ppMopsDetectionsRealloc(ppMopsDetections *det, long num)
+{
+    det->x = psVectorRealloc(det->x, num);
+    det->y = psVectorRealloc(det->y, num);
+    det->ra = psVectorRealloc(det->ra, num);
+    det->dec = psVectorRealloc(det->dec, num);
+    det->raErr = psVectorRealloc(det->raErr, num);
+    det->decErr = psVectorRealloc(det->decErr, num);
+    det->mag = psVectorRealloc(det->mag, num);
+    det->magErr = psVectorRealloc(det->magErr, num);
+    det->chi2 = psVectorRealloc(det->chi2, num);
+    det->dof = psVectorRealloc(det->dof, num);
+    det->cr = psVectorRealloc(det->cr, num);
+    det->extended = psVectorRealloc(det->extended, num);
+    det->psfMajor = psVectorRealloc(det->psfMajor, num);
+    det->psfMinor = psVectorRealloc(det->psfMinor, num);
+    det->psfTheta = psVectorRealloc(det->psfTheta, num);
+    det->quality = psVectorRealloc(det->quality, num);
+    det->numPix = psVectorRealloc(det->numPix, num);
+    det->xxMoment = psVectorRealloc(det->xxMoment, num);
+    det->xyMoment = psVectorRealloc(det->xyMoment, num);
+    det->yyMoment = psVectorRealloc(det->yyMoment, num);
+    det->flags = psVectorRealloc(det->flags, num);
+    det->diffSkyfileId = psVectorRealloc(det->diffSkyfileId, num);
+    det->naxis1 = psVectorRealloc(det->naxis1, num);
+    det->naxis2 = psVectorRealloc(det->naxis2, num);
+    det->mask = psVectorRealloc(det->mask, num);
+    det->nPos = psVectorRealloc(det->nPos, num);
+    det->fPos = psVectorRealloc(det->fPos, num);
+    det->nRatioBad = psVectorRealloc(det->nRatioBad, num);
+    det->nRatioMask = psVectorRealloc(det->nRatioMask, num);
+    det->nRatioAll = psVectorRealloc(det->nRatioAll, num);
+
+    return det;
+}
+
+
+bool ppMopsDetectionsAdd(ppMopsDetections *det, float x, float y, double ra, double dec,
+                         double raErr, double decErr, float mag, float magErr,
+                         float chi2, int dof, float cr, float extended, float psfMajor,
+                         float psfMinor, float psfTheta, float quality, int numPix,
+                         float xxMoment, float xyMoment, float yyMoment,
+                         psU32 flags, psS64 diffSkyfileId, int naxis1, int naxis2,
+                         int nPos, float fPos, float nRatioBad, float nRatioMask, float nRatioAll)
+{
+    psVectorAppend(det->x, x);
+    psVectorAppend(det->y, y);
+    psVectorAppend(det->ra, ra);
+    psVectorAppend(det->dec, dec);
+    psVectorAppend(det->raErr, raErr);
+    psVectorAppend(det->decErr, decErr);
+    psVectorAppend(det->mag, mag);
+    psVectorAppend(det->magErr, magErr);
+    psVectorAppend(det->chi2, chi2);
+    psVectorAppend(det->dof, dof);
+    psVectorAppend(det->cr, cr);
+    psVectorAppend(det->extended, extended);
+    psVectorAppend(det->psfMajor, psfMajor);
+    psVectorAppend(det->psfMinor, psfMinor);
+    psVectorAppend(det->psfTheta, psfTheta);
+    psVectorAppend(det->quality, quality);
+    psVectorAppend(det->numPix, numPix);
+    psVectorAppend(det->xxMoment, xxMoment);
+    psVectorAppend(det->xyMoment, xyMoment);
+    psVectorAppend(det->yyMoment, yyMoment);
+    psVectorAppend(det->flags, flags);
+    psVectorAppend(det->diffSkyfileId, diffSkyfileId);
+    psVectorAppend(det->naxis1, naxis1);
+    psVectorAppend(det->naxis2, naxis2);
+    psVectorAppend(det->mask, 0);
+    psVectorAppend(det->nPos, nPos);
+    psVectorAppend(det->fPos, fPos);
+    psVectorAppend(det->nRatioBad, nRatioBad);
+    psVectorAppend(det->nRatioMask, nRatioMask);
+    psVectorAppend(det->nRatioAll, nRatioAll);
+
+    return true;
+}
+
+
+bool ppMopsDetectionsCopySingle(ppMopsDetections *target, const ppMopsDetections *source, long index)
+{
+    psVectorAppend(target->x, source->x->data.F32[index]);
+    psVectorAppend(target->y, source->y->data.F32[index]);
+    psVectorAppend(target->ra, source->ra->data.F64[index]);
+    psVectorAppend(target->dec, source->dec->data.F64[index]);
+    psVectorAppend(target->raErr, source->raErr->data.F64[index]);
+    psVectorAppend(target->decErr, source->decErr->data.F64[index]);
+    psVectorAppend(target->mag, source->mag->data.F32[index]);
+    psVectorAppend(target->magErr, source->magErr->data.F32[index]);
+    psVectorAppend(target->chi2, source->chi2->data.F32[index]);
+    psVectorAppend(target->dof, source->dof->data.S32[index]);
+    psVectorAppend(target->cr, source->cr->data.F32[index]);
+    psVectorAppend(target->extended, source->extended->data.F32[index]);
+    psVectorAppend(target->psfMajor, source->psfMajor->data.F32[index]);
+    psVectorAppend(target->psfMinor, source->psfMinor->data.F32[index]);
+    psVectorAppend(target->psfTheta, source->psfTheta->data.F32[index]);
+    psVectorAppend(target->quality, source->quality->data.F32[index]);
+    psVectorAppend(target->numPix, source->numPix->data.S32[index]);
+    psVectorAppend(target->xxMoment, source->xxMoment->data.F32[index]);
+    psVectorAppend(target->xyMoment, source->xyMoment->data.F32[index]);
+    psVectorAppend(target->yyMoment, source->yyMoment->data.F32[index]);
+    psVectorAppend(target->flags, source->flags->data.U32[index]);
+    psVectorAppend(target->diffSkyfileId, source->diffSkyfileId->data.S64[index]);
+    psVectorAppend(target->naxis1, source->naxis1->data.S32[index]);
+    psVectorAppend(target->naxis2, source->naxis2->data.S32[index]);
+    psVectorAppend(target->mask, 0);
+    psVectorAppend(target->nPos, source->nPos->data.S32[index]);
+    psVectorAppend(target->fPos, source->fPos->data.F32[index]);
+    psVectorAppend(target->nRatioBad, source->nRatioBad->data.F32[index]);
+    psVectorAppend(target->nRatioMask, source->nRatioMask->data.F32[index]);
+    psVectorAppend(target->nRatioAll, source->nRatioAll->data.F32[index]);
+
+    target->num++;
+
+    return true;
+}
+
+
+bool ppMopsDetectionsPurge(ppMopsDetections *det)
+{
+    long num = 0;
+    for (long i = 0; i < det->num; i++) {
+        if (!det->mask->data.U8[i]) {
+            if (i == num) {
+                // No need to copy
+                num++;
+                continue;
+            }
+            det->x->data.F32[num] = det->x->data.F32[i];
+            det->y->data.F32[num] = det->y->data.F32[i];
+            det->ra->data.F64[num] = det->ra->data.F64[i];
+            det->dec->data.F64[num] = det->dec->data.F64[i];
+            det->raErr->data.F64[num] = det->raErr->data.F64[i];
+            det->decErr->data.F64[num] = det->decErr->data.F64[i];
+            det->mag->data.F32[num] = det->mag->data.F32[i];
+            det->magErr->data.F32[num] = det->magErr->data.F32[i];
+            det->chi2->data.F32[num] = det->chi2->data.F32[i];
+            det->dof->data.S32[num] = det->dof->data.S32[i];
+            det->cr->data.F32[num] = det->cr->data.F32[i];
+            det->extended->data.F32[num] = det->extended->data.F32[i];
+            det->psfMajor->data.F32[num] = det->psfMajor->data.F32[i];
+            det->psfMinor->data.F32[num] = det->psfMinor->data.F32[i];
+            det->psfTheta->data.F32[num] = det->psfTheta->data.F32[i];
+            det->quality->data.F32[num] = det->quality->data.F32[i];
+            det->numPix->data.S32[num] = det->numPix->data.S32[i];
+            det->xxMoment->data.F32[num] = det->xxMoment->data.F32[i];
+            det->xyMoment->data.F32[num] = det->xyMoment->data.F32[i];
+            det->yyMoment->data.F32[num] = det->yyMoment->data.F32[i];
+            det->flags->data.U32[num] = det->flags->data.U32[i];
+            det->diffSkyfileId->data.S64[num] = det->diffSkyfileId->data.S64[i];
+            det->naxis1->data.S32[num] = det->naxis1->data.S32[i];
+            det->naxis2->data.S32[num] = det->naxis2->data.S32[i];
+            det->mask->data.U8[num] = 0;
+            det->nPos->data.S32[num] = det->nPos->data.S32[i];
+            det->fPos->data.F32[num] = det->fPos->data.F32[i];
+            det->nRatioBad->data.F32[num] = det->nRatioBad->data.F32[i];
+            det->nRatioMask->data.F32[num] = det->nRatioMask->data.F32[i];
+            det->nRatioAll->data.F32[num] = det->nRatioAll->data.F32[i];
+            num++;
+        }
+    }
+    det->x->n = num;
+    det->y->n = num;
+    det->ra->n = num;
+    det->dec->n = num;
+    det->raErr->n = num;
+    det->decErr->n = num;
+    det->mag->n = num;
+    det->magErr->n = num;
+    det->chi2->n = num;
+    det->dof->n = num;
+    det->cr->n = num;
+    det->extended->n = num;
+    det->psfMajor->n = num;
+    det->psfMinor->n = num;
+    det->psfTheta->n = num;
+    det->quality->n = num;
+    det->numPix->n = num;
+    det->xxMoment->n = num;
+    det->xyMoment->n = num;
+    det->yyMoment->n = num;
+    det->flags->n = num;
+    det->diffSkyfileId->n = num;
+    det->naxis1->n = num;
+    det->naxis2->n = num;
+    det->mask->n = num;
+    det->num = num;
+    det->nPos->n = num;
+    det->fPos->n = num;
+    det->nRatioBad->n = num;
+    det->nRatioMask->n = num;
+    det->nRatioAll->n = num;
+
+    return true;
+}
+
Index: /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsMerge.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsMerge.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsMerge.c	(revision 28794)
@@ -9,5 +9,5 @@
 #include "ppMops.h"
 
-#define LEAF_SIZE 2                     // Size of leaf
+#define LEAF_SIZE 4                     // Size of leaf
 #define MATCH_RADIUS SEC_TO_RAD(1.0)    // Matching radius
 #define MJD_TOL 1.0/3600.0/24.0         // Tolerance for MJD matching
@@ -17,12 +17,4 @@
 #define AIRMASS_TOL 1.0e-3              // Tolerance for airmass matching
 
-#if 0
-#undef psTrace
-#define psTrace(facil, level, ...)              \
-    if (level <= 5) {                           \
-        fprintf(stderr, __VA_ARGS__);           \
-    }
-#endif
-
 // Get distance from detection to centre of image
 static float mergeDistance(const ppMopsDetections *detections, // Detections of interest
@@ -30,45 +22,19 @@
     )
 {
-    float dx = detections->x->data.F32[index] - detections->naxis1 / 2.0;
-    float dy = detections->y->data.F32[index] - detections->naxis2 / 2.0;
+    float dx = detections->x->data.F32[index] - detections->naxis1->data.S32[index] / 2.0;
+    float dy = detections->y->data.F32[index] - detections->naxis2->data.S32[index] / 2.0;
     return PS_SQR(dx) + PS_SQR(dy);
 }
 
 
-bool ppMopsPurgeDuplicates(const psArray *detections)
+ppMopsDetections *ppMopsMerge(const psArray *detections)
 {
     PS_ASSERT_ARRAY_NON_NULL(detections, NULL);
 
-    int numInputs = detections->n;                // Number of inputs
-    psTrace("ppMops.merge", 1, "Checking detections from %d inputs\n", numInputs);
+    psTrace("ppMops.merge", 1, "Merging detections from %ld inputs\n", detections->n);
 
-    long total = 0;                                // Total number of sources
-    psArray *duplicates = psArrayAlloc(numInputs); // Vector of duplicate bits for each input
-    psVector *dupNum = psVectorAlloc(numInputs, PS_TYPE_U32); // Number of duplicates for each input
-    psVectorInit(dupNum, 0);
-    for (int i = 0; i < numInputs; i++) {
-        ppMopsDetections *det = detections->data[i]; // Detections from
-        if (!det || det->num == 0) {
-            continue;
-        }
-        psVector *dupes = duplicates->data[i] = psVectorAlloc(det->num, PS_TYPE_U8);
-        psVectorInit(dupes, 0);
-        total += det->num;
-    }
-    psTrace("ppMops.merge", 2, "Total detections: %ld\n", total);
-
-    psVector *raMerged = psVectorAlloc(total, PS_TYPE_F64); // Merged RAs
-    psVector *decMerged = psVectorAlloc(total, PS_TYPE_F64); // Merged Decs
-    psVector *sourceMerged = psVectorAlloc(total, PS_TYPE_U16); // Source image of merged sources
-    psVector *indexMerged = psVectorAlloc(total, PS_TYPE_U32); // Index of merged sources
-    long num = 0;                                                   // Number of merged sources
-
-    const char *raBoresight = NULL, *decBoresight = NULL; // Boresight coordinates
-    const char *filter = NULL;                    // Filter name
-    float airmass = NAN, exptime = NAN;           // Exposure details
-    double posangle = NAN, alt = NAN, az = NAN; // Telescope pointing
-    double mjd = NAN;                           // Time of exposure
-
-    for (int i = 0; i < numInputs; i++) {
+    ppMopsDetections *merged = NULL;    // Merged list
+    int num = 1;                                                         // Number of merged files
+    for (int i = 0; i < detections->n; i++) {
         ppMopsDetections *det = detections->data[i]; // Detections of interest
         if (!det) {
@@ -79,90 +45,69 @@
             continue;
         }
+        num++;
+        if (!merged) {
+            psTrace("ppMops.merge", 3, "Accepting %ld detections from input %d\n", det->num, i);
+            merged = psMemIncrRefCounter(det);
+            continue;
+        }
+        psTrace("ppMops.merge", 3, "Merging %ld detections from input %d\n", det->num, i);
 
-        // Check exposure characteristics
-        if (num == 0) {
-            raBoresight = det->raBoresight;
-            decBoresight = det->decBoresight;
-            filter = det->filter;
-            airmass = det->airmass;
-            exptime = det->exptime;
-            posangle = det->posangle;
-            alt = det->alt;
-            az = det->az;
-        } else {
-            if (strcmp(raBoresight, det->raBoresight) != 0) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure RA values differ: %s vs %s",
-                        raBoresight, det->raBoresight);
-                return false;
-            }
-            if (strcmp(decBoresight, det->decBoresight) != 0) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure Dec values differ: %s vs %s",
-                        decBoresight, det->decBoresight);
-                return false;
-            }
-            if (strcmp(filter, det->filter) != 0) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure filter values differ: %s vs %s",
-                        filter, det->filter);
-                return false;
-            }
-            if (fabsf(airmass - det->airmass) > AIRMASS_TOL) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure airmass values differ: %f vs %f",
-                        airmass, det->airmass);
-                return false;
-            }
-            if (fabsf(exptime - det->exptime) > EXPTIME_TOL) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure exposure time values differ: %f vs %f",
-                        exptime, det->exptime);
-                return false;
-            }
-            if (fabs(posangle - det->posangle) > POSANGLE_TOL) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure position angle values differ: %f vs %f",
-                        posangle, det->posangle);
-                return false;
-            }
-            if (fabs(alt - det->alt) > BORESIGHT_TOL) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure altitude values differ: %lf vs %lf",
-                        alt, det->alt);
-                return false;
-            }
-            if (fabs(az - det->az) > BORESIGHT_TOL) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure azimuth values differ: %lf vs %lf",
-                        az, det->az);
-                return false;
-            }
-            if (fabs(mjd - det->mjd) > MJD_TOL) {
-                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure MJD values differ: %lf vs %lf",
-                        mjd, det->mjd);
-                return false;
-            }
+        // XXX compare exposure properties
+        if (strcmp(merged->raBoresight, det->raBoresight) != 0) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure RA values differ: %s vs %s",
+                    merged->raBoresight, det->raBoresight);
+            return NULL;
+        }
+        if (strcmp(merged->decBoresight, det->decBoresight) != 0) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure Dec values differ: %s vs %s",
+                    merged->decBoresight, det->decBoresight);
+            return NULL;
+        }
+        if (strcmp(merged->filter, det->filter) != 0) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure filter values differ: %s vs %s",
+                    merged->filter, det->filter);
+            return NULL;
         }
 
-        psTrace("ppMops.merge", 3, "Accepting %ld detections from input %d\n", det->num, i);
-        memcpy(&raMerged->data.F64[num], det->ra->data.F64, det->num * PSELEMTYPE_SIZEOF(PS_TYPE_F64));
-        memcpy(&decMerged->data.F64[num], det->dec->data.F64, det->num * PSELEMTYPE_SIZEOF(PS_TYPE_F64));
-        for (long j = 0, k = num; j < det->num; j++, k++) {
-            sourceMerged->data.U16[k] = i;
-            indexMerged->data.U32[k] = j;
+        if (fabsf(merged->airmass - det->airmass) > AIRMASS_TOL) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure airmass values differ: %f vs %f",
+                    merged->airmass, det->airmass);
+            return NULL;
         }
-        num += det->num;
-    }
+        if (fabsf(merged->exptime - det->exptime) > EXPTIME_TOL) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure exposure time values differ: %f vs %f",
+                    merged->exptime, det->exptime);
+            return NULL;
+        }
+        if (fabs(merged->posangle - det->posangle) > POSANGLE_TOL) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure position angle values differ: %f vs %f",
+                    merged->posangle, det->posangle);
+            return NULL;
+        }
+        if (fabs(merged->alt - det->alt) > BORESIGHT_TOL) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure altitude values differ: %lf vs %lf",
+                    merged->alt, det->alt);
+            return NULL;
+        }
+        if (fabs(merged->az - det->az) > BORESIGHT_TOL) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure azimuth values differ: %lf vs %lf",
+                    merged->az, det->az);
+            return NULL;
+        }
+        if (fabs(merged->mjd - det->mjd) > MJD_TOL) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Exposure MJD values differ: %lf vs %lf",
+                    merged->mjd, det->mjd);
+            return NULL;
+        }
 
-    psTrace("ppMops.merge", 3, "Generating kd-tree from %ld sources\n", num);
-    psTree *tree = psTreePlant(2, LEAF_SIZE, PS_TREE_SPHERICAL, raMerged, decMerged); // kd tree
-    if (!tree) {
-        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate kd tree");
-        return false;
-    }
+        merged->seeing += det->seeing;  // Taking average
 
-    for (int i = 0; i < detections->n; i++) {
-        ppMopsDetections *det = detections->data[i]; // Detections of interest
-        if (!det) {
-            continue;
-        } else if (det->num == 0) {
-            continue;
+        psTree *tree = psTreePlant(2, LEAF_SIZE, PS_TREE_SPHERICAL, merged->ra, merged->dec); // kd tree
+        if (!tree) {
+            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate kd tree");
+            psFree(merged);
+            return NULL;
         }
-        psTrace("ppMops.merge", 3, "Checking %ld detections from input %d\n", det->num, i);
 
-        psVector *dupes = duplicates->data[i];            // Duplicates list
         psVector *coords = psVectorAlloc(2, PS_TYPE_F64); // Coordinates of interest
         for (int j = 0; j < det->num; j++) {
@@ -174,14 +119,14 @@
                 psFree(coords);
                 psFree(tree);
-                return false;
+                psFree(merged);
+                return NULL;
+            }
+            if (indices->n == 0) {
+                psTrace("ppMops.merge", 9, "No matches for source %d in input %d\n", j, i);
+                psFree(indices);
+                ppMopsDetectionsCopySingle(merged, det, j);
+                continue;
             }
             psTrace("ppMops.merge", 5, "%ld matches for source %d from input %d\n", indices->n, j, i);
-            psAssert(indices->n > 0, "Expect at least one match for source %d in input %d", j, i);
-
-            if (indices->n == 1 && sourceMerged->data.U16[indices->data.S64[0]] == i) {
-                // It's myself
-                psFree(indices);
-                continue;
-            }
 
             // Which one do we keep?
@@ -189,16 +134,6 @@
             long bestIndex = -1;           // Index with best distance
             for (int k = 0; k < indices->n; k++) {
-                long mergeIndex = indices->data.S64[k]; // Index of point in merged list
-                int source = sourceMerged->data.U16[mergeIndex]; // Source image
-                if (source == i) {
-                    continue;
-                }
-                long index = indexMerged->data.U32[mergeIndex];  // Index in source
-                psVector *dupes = duplicates->data[source];     // Duplicates list
-                if (dupes->data.U8[index]) {
-                    continue;
-                }
-
-                float distance = mergeDistance(detections->data[source], index); // Distance to centre of image
+                long index = indices->data.S64[k]; // Index of point
+                float distance = mergeDistance(merged, index); // Distance to centre of image
                 if (distance < bestDistance) {
                     bestDistance = distance;
@@ -207,84 +142,30 @@
             }
 
-            if (bestIndex >= 0) {
-                float distance = mergeDistance(det, j); // Distance to centre of image
-                if (bestIndex >= 0 && distance < bestDistance) {
-                    psTrace("ppMops.merge", 6, "New source clobbers old sources\n");
-                    for (int k = 0; k < indices->n; k++) {
-                        long mergeIndex = indices->data.S64[k]; // Index of point
-                        int source = sourceMerged->data.U16[mergeIndex]; // Source image
-                        if (source == i) {
-                            continue;
-                        }
-                        long index = indexMerged->data.U32[mergeIndex];  // Index in source
-                        psVector *dupes = duplicates->data[source];      // Duplicates list
-                        if (!dupes->data.U8[index]) {
-                            dupes->data.U8[index] = 0xFF;
-                            dupNum->data.U32[source]++;
-                        }
-                    }
-                } else {
-                    psTrace("ppMops.merge", 6, "Old sources clobber new source\n");
-                    dupes->data.U8[j] = 0xFF;
-                    dupNum->data.U32[i]++;
+            float distance = mergeDistance(det, j); // Distance to centre of image
+            if (distance < bestDistance) {
+                psTrace("ppMops.merge", 6, "New source clobbers old sources\n");
+                // Blow away existing sources
+                for (int k = 0; k < indices->n; k++) {
+                    long index = indices->data.S64[k]; // Index of point
+                    merged->mask->data.U8[index] = 0xFF;
                 }
+                ppMopsDetectionsCopySingle(merged, det, j);
+            } else {
+                psTrace("ppMops.merge", 6, "Old sources clobber new source\n");
             }
             psFree(indices);
         }
-        psFree(coords);
-    }
-    psFree(tree);
-    psFree(raMerged);
-    psFree(decMerged);
-    psFree(sourceMerged);
-    psFree(indexMerged);
 
-    // Remove duplicates
-    for (int i = 0; i < detections->n; i++) {
-        ppMopsDetections *det = detections->data[i]; // Detections of interest
-        if (!det) {
-            continue;
-        } else if (det->num == 0) {
-            continue;
-        }
-        psTrace("ppMops.merge", 3, "Purging %d duplicates from input %d\n", dupNum->data.U32[i], i);
+        psTrace("ppMops.merge", 3, "Done merging input %d, %ld merged sources\n", i, merged->num);
 
-#define VECTOR_PURGE_CASE(TYPE)                                      \
-        case PS_TYPE_##TYPE:                                         \
-          for (int i = 0, j = 0; i < vector->n; i++) {               \
-              if (!dupes->data.U8[i]) {                              \
-                  if (i == j) {                                      \
-                      j++;                                           \
-                      continue;                                      \
-                  }                                                  \
-                  vector->data.TYPE[j] = vector->data.TYPE[i];       \
-              }                                                      \
-          }                                                          \
-          break;
-
-        psVector *dupes = duplicates->data[i]; // Duplicates
-        psArray *table = psListToArray(det->table->list); // Table of data
-        for (int j = 0; j < table->n; j++) {
-            psMetadataItem *item = table->data[j]; // Table item
-            psAssert(item->type == PS_DATA_VECTOR, "Table column is not a vector: %x", item->type);
-            psVector *vector = item->data.V; // Vector to purge
-            switch (vector->type.type) {
-                VECTOR_PURGE_CASE(U8);
-                VECTOR_PURGE_CASE(U16);
-                VECTOR_PURGE_CASE(U32);
-                VECTOR_PURGE_CASE(U64);
-                VECTOR_PURGE_CASE(S8);
-                VECTOR_PURGE_CASE(S16);
-                VECTOR_PURGE_CASE(S32);
-                VECTOR_PURGE_CASE(S64);
-                VECTOR_PURGE_CASE(F32);
-                VECTOR_PURGE_CASE(F64);
-              default:
-                psAbort("Unrecognised vector type: %x", vector->type.type);
-            }
-        }
+        psFree(tree);
+        ppMopsDetectionsPurge(merged);
     }
 
-    return true;
+    psTrace("ppMops.merge", 2, "%ld sources in merged detections list\n", merged->num);
+
+    merged->seeing /= num;
+
+    return merged;
 }
 
Index: /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsRead.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsRead.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsRead.c	(revision 28794)
@@ -4,5 +4,4 @@
 
 #include <stdio.h>
-#include <string.h>
 #include <pslib.h>
 
@@ -17,6 +16,5 @@
     psArray *detections = psArrayAlloc(num); // Array of detections, to return
     for (int i = 0; i < num; i++) {
-        const char *name = inNames->data[i]; // File name
-        psFits *fits = psFitsOpen(name, "r"); // FITS file
+        psFits *fits = psFitsOpen(inNames->data[i], "r"); // FITS file
         if (!fits) {
             psError(PS_ERR_IO, false, "Unable to open input %d", i);
@@ -26,4 +24,10 @@
         if (!header) {
             psError(PS_ERR_IO, false, "Unable to read header %d", i);
+            return false;
+        }
+
+        psS64 diffSkyfileId = psMetadataLookupS64(NULL, header, "IMAGEID"); // Identifier for image
+        if (diffSkyfileId == 0) {
+            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find identifier for image %d", i);
             return false;
         }
@@ -43,8 +47,7 @@
             continue;
         }
-        ppMopsDetections *det = detections->data[i] = ppMopsDetectionsAlloc();
-        det->component = psStringNCopy(name, strrchr(name, '.') - name); // Strip off extension
-        det->header = header;
-        det->num = size;
+        ppMopsDetections *det = ppMopsDetectionsAlloc(size);
+
+        psTrace("ppMops.read", 3, "Reading %ld rows from %s\n", size, (const char*)inNames->data[i]);
 
         det->raBoresight = psMemIncrRefCounter(psMetadataLookupStr(NULL, header, "FPA.RA"));
@@ -61,55 +64,132 @@
                              psMetadataLookupF32(NULL, header, "FWHM_MIN"));
 
-        det->naxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1");
-        det->naxis2 = psMetadataLookupS32(NULL, header, "IMNAXIS2");
+        int naxis1 = psMetadataLookupS32(NULL, header, "IMNAXIS1"); // Number of columns
+        int naxis2 = psMetadataLookupS32(NULL, header, "IMNAXIS2"); // Number of rows
 
-        det->psfHeader = psFitsReadHeader(NULL, fits);
-        if (!det->psfHeader) {
-            psError(psErrorCodeLast(), false, "Unable to read header %d", i);
+        psFree(header);
+
+        psArray *table = psFitsReadTable(fits); // Table of interest
+        if (!table) {
+            psError(PS_ERR_IO, false, "Unable to read table %d", i);
             return false;
         }
-        psMetadata *table = det->table = psFitsReadTableAllColumns(fits); // Table of interest
-        if (!table) {
-            psError(psErrorCodeLast(), false, "Unable to read table %d", i);
-            return false;
+        psFitsClose(fits);
+
+        double plateScale = 0.0;        // Plate scale
+        long numGood = 0;               // Number of good rows
+        for (long j = 0; j < size; j++) {
+            psMetadata *row = table->data[j]; // Row of interest
+
+            psU32 flags = psMetadataLookupU32(NULL, row, "FLAGS");
+            if (flags & SOURCE_MASK) {
+                psTrace("ppMops.read", 10, "Discarding row %ld from input %d because of flags: %ud", j, i, flags);
+                continue;
+            }
+
+            det->x->data.F32[numGood] = psMetadataLookupF32(NULL, row, "X_PSF");
+            det->y->data.F32[numGood] = psMetadataLookupF32(NULL, row, "Y_PSF");
+            det->ra->data.F64[numGood] = DEG_TO_RAD(psMetadataLookupF64(NULL, row, "RA_PSF"));
+            det->dec->data.F64[numGood] = DEG_TO_RAD(psMetadataLookupF64(NULL, row, "DEC_PSF"));
+            det->mag->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_MAG");
+            det->magErr->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_INST_MAG_SIG");
+            det->chi2->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_CHISQ");
+            det->dof->data.S32[numGood] = psMetadataLookupS32(NULL, row, "PSF_NDOF");
+            det->cr->data.F32[numGood] = psMetadataLookupF32(NULL, row, "CR_NSIGMA");
+            det->extended->data.F32[numGood] = psMetadataLookupF32(NULL, row, "EXT_NSIGMA");
+            det->psfMajor->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_MAJOR");
+            det->psfMinor->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_MINOR");
+            det->psfTheta->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_THETA");
+            det->quality->data.F32[numGood] = psMetadataLookupF32(NULL, row, "PSF_QF");
+            det->numPix->data.S32[numGood] = psMetadataLookupS32(NULL, row, "PSF_NPIX");
+            det->xxMoment->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_XX");
+            det->xyMoment->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_XY");
+            det->yyMoment->data.F32[numGood] = psMetadataLookupF32(NULL, row, "MOMENTS_YY");
+            det->flags->data.U32[numGood] = psMetadataLookupU32(NULL, row, "FLAGS");
+            det->diffSkyfileId->data.S64[numGood] = diffSkyfileId;
+            det->naxis1->data.S32[numGood] = naxis1;
+            det->naxis2->data.S32[numGood] = naxis2;
+
+            det->nPos->data.S32[numGood] = psMetadataLookupS32(NULL, row, "DIFF_NPOS");
+            det->fPos->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_FRATIO");
+            det->nRatioBad->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_NRATIO_BAD");
+            det->nRatioMask->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_NRATIO_MASK");
+            det->nRatioAll->data.F32[numGood] = psMetadataLookupF32(NULL, row, "DIFF_NRATIO_ALL");
+
+            // Calculate error in RA, Dec
+            double xErr = psMetadataLookupF64(NULL, row, "X_PSF_SIG");
+            double yErr = psMetadataLookupF64(NULL, row, "Y_PSF_SIG");
+            double scale = psMetadataLookupF64(NULL, row, "PLTSCALE");
+            double angle = psMetadataLookupF64(NULL, row, "POSANGLE");
+
+            if (!isfinite(det->x->data.F32[numGood]) || !isfinite(det->y->data.F32[numGood]) ||
+                !isfinite(det->ra->data.F64[numGood]) || !isfinite(det->dec->data.F64[numGood]) ||
+                !isfinite(det->mag->data.F32[numGood]) || !isfinite(det->magErr->data.F32[numGood]) ||
+                !isfinite(xErr) || !isfinite(yErr) || !isfinite(scale) || !isfinite(angle)) {
+                psTrace("ppMops.read", 10,
+                        "Discarding row %ld from input %d because of non-finite values: "
+                        "%f %f %lf %lf %f %f %f %f %f %f",
+                        j, i,
+                        det->x->data.F32[numGood], det->y->data.F32[numGood],
+                        det->ra->data.F64[numGood], det->dec->data.F64[numGood],
+                        det->mag->data.F32[numGood], det->magErr->data.F32[numGood],
+                        xErr, yErr, scale, angle);
+                continue;
+            }
+
+            // XXX Not at all sure I've got the angles around the right way here...
+            double cosAngle = cos(angle), sinAngle = sin(angle);
+            double cosAngle2 = PS_SQR(cosAngle), sinAngle2 = PS_SQR(sinAngle);
+            double xErr2 = PS_SQR(xErr), yErr2 = PS_SQR(yErr);
+            double errScale = scale / 3600.0;
+            det->raErr->data.F64[numGood] = errScale * sqrt(cosAngle2 * xErr2 + sinAngle2 * yErr2);
+            det->decErr->data.F64[numGood] = errScale * sqrt(sinAngle2 * xErr2 + cosAngle2 * yErr2);
+
+            det->mask->data.U8[numGood] = 0;
+            plateScale += scale;
+            numGood++;
+        }
+        det->seeing *= plateScale / numGood;
+
+        det->x->n = numGood;
+        det->y->n = numGood;
+        det->ra->n = numGood;
+        det->dec->n = numGood;
+        det->raErr->n = numGood;
+        det->decErr->n = numGood;
+        det->mag->n = numGood;
+        det->magErr->n = numGood;
+        det->chi2->n = numGood;
+        det->dof->n = numGood;
+        det->cr->n = numGood;
+        det->extended->n = numGood;
+        det->psfMajor->n = numGood;
+        det->psfMinor->n = numGood;
+        det->psfTheta->n = numGood;
+        det->quality->n = numGood;
+        det->numPix->n = numGood;
+        det->xxMoment->n = numGood;
+        det->xyMoment->n = numGood;
+        det->yyMoment->n = numGood;
+        det->flags->n = numGood;
+        det->diffSkyfileId->n = numGood;
+        det->naxis1->n = numGood;
+        det->naxis2->n = numGood;
+        det->mask->n = numGood;
+        det->nPos->n = numGood;
+        det->fPos->n = numGood;
+        det->nRatioBad->n = numGood;
+        det->nRatioMask->n = numGood;
+        det->nRatioAll->n = numGood;
+
+        det->num = numGood;
+
+        if (isfinite(args->zp) && numGood > 0) {
+            psBinaryOp(det->mag, det->mag, "+", psScalarAlloc(args->zp, PS_TYPE_F32));
         }
 
-        psVector *ra = psMetadataLookupVector(NULL, table, "RA_PSF");
-        psVector *dec = psMetadataLookupVector(NULL, table, "DEC_PSF");
+        psTrace("ppMops.read", 2, "Read %ld good rows from %s\n", numGood, (const char*)inNames->data[i]);
 
-        det->ra = (psVector*)psBinaryOp(NULL, ra, "*", psScalarAlloc(DEG_TO_RAD(1.0), PS_TYPE_F64));
-        det->dec = (psVector*)psBinaryOp(NULL, dec, "*", psScalarAlloc(DEG_TO_RAD(1.0), PS_TYPE_F64));
-        det->x = psMemIncrRefCounter(psMetadataLookupVector(NULL, table, "X_PSF"));
-        det->y = psMemIncrRefCounter(psMetadataLookupVector(NULL, table, "Y_PSF"));
-        if (!det->ra || !det->dec || !det->x || !det->y) {
-            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find all of RA, Dec, X and Y columns");
-            return false;
-        }
-
-        psTrace("ppMops.read", 2, "Read %ld rows from %s\n", det->num, det->component);
-
-        if (!psFitsMoveExtName(fits, "SkyChip.deteff")) {
-            psWarning("No detection efficiencies included in %s", det->component);
-            psErrorStackPrint(stderr, "No detection efficiencies included in %s", det->component);
-            psErrorClear();
-            continue;
-        }
-
-        det->deteffHeader = psFitsReadHeader(NULL, fits);
-        if (!det->deteffHeader) {
-            psWarning("Unable to read detection efficiency header in %s", det->component);
-            psErrorClear();
-            continue;
-        }
-        det->deteffTable = psFitsReadTableAllColumns(fits);
-        if (!det->deteffTable) {
-            psFree(det->deteffHeader);
-            det->deteffHeader = NULL;
-            psWarning("Unable to read detection efficiency table in %s", det->component);
-            psErrorClear();
-            continue;
-        }
-        psTrace("ppMops.read", 2, "Read detection efficiency from %s\n", det->component);
-        psFitsClose(fits);
+        psFree(table);
+        detections->data[i] = det;
     }
 
Index: /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsWrite.c
===================================================================
--- /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsWrite.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/ppTranslate/src/ppMopsWrite.c	(revision 28794)
@@ -9,7 +9,7 @@
 #include "ppTranslateVersion.h"
 
-bool ppMopsWrite(const psArray *detections, const ppMopsArguments *args)
+bool ppMopsWrite(const ppMopsDetections *det, const ppMopsArguments *args)
 {
-    psTrace("ppMops.write", 1, "Writing %ld extensions to %s", detections->n, args->output);
+    psTrace("ppMops.write", 1, "Writing %ld rows to %s", det->num, args->output);
 
     psFits *fits = psFitsOpen(args->output, "w"); // FITS file
@@ -19,67 +19,134 @@
     }
 
-    // Primary header
-    {
-        psMetadata *header = psMetadataAlloc(); // Header to write
-        ppTranslateVersionHeader(header);
-        psMetadataAddStr(header, PS_LIST_TAIL, "EXP_NAME", 0, "Exposure name", args->exp_name);
-        psMetadataAddS64(header, PS_LIST_TAIL, "EXP_ID", 0, "Exposure identifier", args->exp_id);
-        psMetadataAddS64(header, PS_LIST_TAIL, "CHIP_ID", 0, "Chip stage identifier", args->chip_id);
-        psMetadataAddS64(header, PS_LIST_TAIL, "CAM_ID", 0, "Cam stage identifier", args->cam_id);
-        psMetadataAddS64(header, PS_LIST_TAIL, "FAKE_ID", 0, "Fake stage identifier", args->fake_id);
-        psMetadataAddS64(header, PS_LIST_TAIL, "WARP_ID", 0, "Warp stage identifier", args->warp_id);
-        psMetadataAddS64(header, PS_LIST_TAIL, "DIFF_ID", 0, "Diff stage identifier", args->diff_id);
-        psMetadataAddBool(header, PS_LIST_TAIL, "DIFF_POS", 0, "Positive subtraction?", args->positive);
-        psMetadataAddF32(header, PS_LIST_TAIL, "MAGZP", 0, "Magnitude zero point", args->zp);
-        psMetadataAddF32(header, PS_LIST_TAIL, "MAGZPERR", 0, "Error in magnitude zero point", args->zpErr);
-        psMetadataAddF32(header, PS_LIST_TAIL, "ASTRORMS", 0, "RMS of astrometric fit", args->rmsAstrom);
-        psMetadataAddStr(header, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", OBSERVATORY_CODE);
 
-        ppMopsDetections *det = detections->data[0]; // Representative set of detections
-        psMetadataAddStr(header, PS_LIST_TAIL, "RA", 0, "Right Ascension of boresight", det->raBoresight);
-        psMetadataAddStr(header, PS_LIST_TAIL, "DEC", 0, "Declination of boresight", det->decBoresight);
-        psMetadataAddStr(header, PS_LIST_TAIL, "FILTER", 0, "Filter name", det->filter);
-        psMetadataAddF32(header, PS_LIST_TAIL, "AIRMASS", 0, "Airmass at boresight", det->airmass);
-        psMetadataAddF32(header, PS_LIST_TAIL, "EXPTIME", 0, "Exposure time (sec)", det->exptime);
-        psMetadataAddF64(header, PS_LIST_TAIL, "POSANG", 0, "Position angle", det->posangle);
-        psMetadataAddF64(header, PS_LIST_TAIL, "ALT", 0, "Altitude of boresight", det->alt);
-        psMetadataAddF64(header, PS_LIST_TAIL, "AZ", 0, "Azimuth of boresight", det->az);
-        psMetadataAddF64(header, PS_LIST_TAIL, "MJD-OBS", 0, "MJD of observation", det->mjd);
+    psMetadata *header = psMetadataAlloc(); // Header to write
+    psString source = ppTranslateSource(), version = ppTranslateVersion();
+    psMetadataAddStr(header, PS_LIST_TAIL, "SWSOURCE", 0, "Software source", source);
+    psMetadataAddStr(header, PS_LIST_TAIL, "SWVERSN", 0, "Software version", version);
+    ppTranslateVersionHeader(header);
+    psFree(source);
+    psFree(version);
 
-        if (!psFitsWriteBlank(fits, header, NULL)) {
-            psError(psErrorCodeLast(), false, "Unable to write primary header");
+    psMetadataAddStr(header, PS_LIST_TAIL, "EXP_NAME", 0, "Exposure name", args->exp_name);
+    psMetadataAddS64(header, PS_LIST_TAIL, "EXP_ID", 0, "Exposure identifier", args->exp_id);
+    psMetadataAddS64(header, PS_LIST_TAIL, "CHIP_ID", 0, "Chip stage identifier", args->chip_id);
+    psMetadataAddS64(header, PS_LIST_TAIL, "CAM_ID", 0, "Cam stage identifier", args->cam_id);
+    psMetadataAddS64(header, PS_LIST_TAIL, "FAKE_ID", 0, "Fake stage identifier", args->fake_id);
+    psMetadataAddS64(header, PS_LIST_TAIL, "WARP_ID", 0, "Warp stage identifier", args->warp_id);
+    psMetadataAddS64(header, PS_LIST_TAIL, "DIFF_ID", 0, "Diff stage identifier", args->diff_id);
+    psMetadataAddBool(header, PS_LIST_TAIL, "DIFF_POS", 0, "Positive subtraction?", args->positive);
+
+    psMetadataAddF64(header, PS_LIST_TAIL, "MJD-OBS", 0, "MJD of exposure midpoint", det->mjd);
+    psMetadataAddStr(header, PS_LIST_TAIL, "RA", 0, "Right Ascension of boresight", det->raBoresight);
+    psMetadataAddStr(header, PS_LIST_TAIL, "DEC", 0, "Declination of boresight", det->decBoresight);
+    psMetadataAddF64(header, PS_LIST_TAIL, "TEL_ALT", 0, "Telescope altitude", det->alt);
+    psMetadataAddF64(header, PS_LIST_TAIL, "TEL_AZ", 0, "Telescope azimuth", det->az);
+    psMetadataAddF64(header, PS_LIST_TAIL, "EXPTIME", 0, "Exposure time (sec)", det->exptime);
+    psMetadataAddF64(header, PS_LIST_TAIL, "ROTANGLE", 0, "Rotator position angle", det->posangle);
+    psMetadataAddStr(header, PS_LIST_TAIL, "FILTER", 0, "Filter name", det->filter);
+    psMetadataAddF32(header, PS_LIST_TAIL, "AIRMASS", 0, "Airmass of exposure", det->airmass);
+    psMetadataAddStr(header, PS_LIST_TAIL, "OBSCODE", 0, "IAU Observatory code", OBSERVATORY_CODE);
+    psMetadataAddF32(header, PS_LIST_TAIL, "SEEING", 0, "Mean seeing", det->seeing);
+    psMetadataAddF32(header, PS_LIST_TAIL, "MAGZP", 0, "Magnitude zero point", args->zp);
+    psMetadataAddF32(header, PS_LIST_TAIL, "MAGZPERR", 0, "Error in magnitude zero point", args->zpErr);
+    psMetadataAddF32(header, PS_LIST_TAIL, "ASTRORMS", 0, "RMS of astrometric fit", args->rmsAstrom);
+
+    if (det->num == 0) {
+        // Write dummy table
+        psMetadata *row = psMetadataAlloc(); // Output row
+        psMetadataAddF64(row, PS_LIST_TAIL, "RA", 0, "Right ascension (degrees)", NAN);
+        psMetadataAddF64(row, PS_LIST_TAIL, "RA_ERR", 0, "Right ascension error (degrees)", NAN);
+        psMetadataAddF64(row, PS_LIST_TAIL, "DEC", 0, "Declination (degrees)", NAN);
+        psMetadataAddF64(row, PS_LIST_TAIL, "DEC_ERR", 0, "Declination error (degrees)", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "MAG", 0, "Magnitude", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "MAG_ERR", 0, "Magnitude error", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "PSF_CHI2", 0, "chi^2 of PSF fit", NAN);
+        psMetadataAddS32(row, PS_LIST_TAIL, "PSF_DOF", 0, "Degrees of freedom of PSF fit", 0);
+        psMetadataAddF32(row, PS_LIST_TAIL, "CR_SIGNIFICANCE", 0, "Significance of CR", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "EXT_SIGNIFICANCE", 0, "Significance of extendedness", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "PSF_MAJOR", 0, "PSF major axis (pixels)", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "PSF_MINOR", 0, "PSF minor axis (pixels)", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "PSF_THETA", 0, "PSF position angle (deg on chip)", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "PSF_QUALITY", 0, "PSF quality factor", NAN);
+        psMetadataAddS32(row, PS_LIST_TAIL, "PSF_NPIX", 0, "Number of pixels in PSF", 0);
+        psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_XX", 0, "xx moment", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_XY", 0, "xy moment", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_YY", 0, "yy moment", NAN);
+        psMetadataAddU32(row, PS_LIST_TAIL, "FLAGS", 0, "Detection bit flags", 0);
+        psMetadataAddS64(row, PS_LIST_TAIL, "DIFF_SKYFILE_ID", 0, "Identifier for diff skyfile", 0);
+
+        psMetadataAddS32(row, PS_LIST_TAIL, "N_POS", 0, "Number of positive pixels", 0);
+        psMetadataAddF32(row, PS_LIST_TAIL, "F_POS", 0, "Fraction of positive pixels", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_BAD", 0, "Ratio of positive pixels to negative", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_MASK", 0, "Ratio of positive pixels to masked", NAN);
+        psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_ALL", 0, "Ratio of positive pixels to all", NAN);
+        if (!psFitsWriteTableEmpty(fits, header, row, OUT_EXTNAME)) {
+            psErrorStackPrint(stderr, "Unable to write empty table.");
+            psFree(header);
+            psFree(row);
             return false;
         }
-        psFree(header);
+        psFree(row);
+    } else {
+        psArray *table = psArrayAlloc(det->num); // Table to write
+        for (long i = 0; i < det->num; i++) {
+            psMetadata *row = psMetadataAlloc(); // Output row
+            psMetadataAddF64(row, PS_LIST_TAIL, "RA", 0, "Right ascension (degrees)",
+                             RAD_TO_DEG(det->ra->data.F64[i]));
+            psMetadataAddF64(row, PS_LIST_TAIL, "RA_ERR", 0, "Right ascension error (degrees)",
+                             det->raErr->data.F64[i]);
+            psMetadataAddF64(row, PS_LIST_TAIL, "DEC", 0, "Declination (degrees)",
+                             RAD_TO_DEG(det->dec->data.F64[i]));
+            psMetadataAddF64(row, PS_LIST_TAIL, "DEC_ERR", 0, "Declination error (degrees)",
+                             det->decErr->data.F64[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "MAG", 0, "Magnitude", det->mag->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "MAG_ERR", 0, "Magnitude error", det->magErr->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "PSF_CHI2", 0, "chi^2 of PSF fit", det->chi2->data.F32[i]);
+            psMetadataAddS32(row, PS_LIST_TAIL, "PSF_DOF", 0, "Degrees of freedom of PSF fit",
+                             det->dof->data.S32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "CR_SIGNIFICANCE", 0, "Significance of CR",
+                             det->cr->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "EXT_SIGNIFICANCE", 0, "Significance of extendedness",
+                             det->extended->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "PSF_MAJOR", 0, "PSF major axis (pixels)", det->psfMajor->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "PSF_MINOR", 0, "PSF minor axis (pixels)", det->psfMinor->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "PSF_THETA", 0, "PSF position angle (deg on chip)",
+                             det->psfTheta->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "PSF_QUALITY", 0, "PSF quality factor",
+                             det->quality->data.F32[i]);
+            psMetadataAddS32(row, PS_LIST_TAIL, "PSF_NPIX", 0, "Number of pixels in PSF",
+                             det->numPix->data.S32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_XX", 0, "xx moment", det->xxMoment->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_XY", 0, "xy moment", det->xyMoment->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "MOMENTS_YY", 0, "yy moment", det->yyMoment->data.F32[i]);
+            psMetadataAddS32(row, PS_LIST_TAIL, "N_POS", 0, "Number of positive pixels",
+                             det->nPos->data.S32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "F_POS", 0, "Fraction of positive pixels",
+                             det->fPos->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_BAD", 0, "Ratio of positive pixels to negative",
+                             det->nRatioBad->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_MASK", 0, "Ratio of positive pixels to masked",
+                             det->nRatioMask->data.F32[i]);
+            psMetadataAddF32(row, PS_LIST_TAIL, "RATIO_ALL", 0, "Ratio of positive pixels to all",
+                             det->nRatioAll->data.F32[i]);
+            psMetadataAddU32(row, PS_LIST_TAIL, "FLAGS", 0, "Detection bit flags", det->flags->data.U32[i]);
+            psMetadataAddS64(row, PS_LIST_TAIL, "DIFF_SKYFILE_ID", 0, "Identifier for diff skyfile",
+                             det->diffSkyfileId->data.S64[i]);
+
+            table->data[i] = row;
+        }
+        if (!psFitsWriteTable(fits, header, table, OUT_EXTNAME)) {
+            psErrorStackPrint(stderr, "Unable to write table.");
+            psFree(header);
+            psFree(table);
+            return false;
+        }
+        psFree(table);
     }
 
-    for (int i = 0; i < detections->n; i++) {
-        ppMopsDetections *det = detections->data[i]; // Detections for extension
-        psTrace("ppMops.write", 1, "Writing extension %d to %s", i, args->output);
-        psString hdrName = NULL, psfName = NULL, deteffName = NULL;
-        psStringAppend(&hdrName, "%s.hdr", det->component);
-        psStringAppend(&psfName, "%s.psf", det->component);
-        psStringAppend(&deteffName, "%s.deteff", det->component);
-
-        if (!psFitsWriteBlank(fits, det->header, hdrName)) {
-            psError(psErrorCodeLast(), false, "Unable to write header %d", i);
-            return false;
-        }
-        if (!psFitsWriteTableAllColumns(fits, det->psfHeader, det->table, psfName)) {
-            psError(psErrorCodeLast(), false, "Unable to write table %d", i);
-            return false;
-        }
-        if (det->deteffHeader && det->deteffTable &&
-            !psFitsWriteTableAllColumns(fits, det->deteffHeader, det->deteffTable, deteffName)) {
-            psError(psErrorCodeLast(), false, "Unable to write detection efficiency %d", i);
-            return false;
-        }
-        psFree(hdrName);
-        psFree(psfName);
-        psFree(deteffName);
-    }
+    psFree(header);
     psFitsClose(fits);
 
-    psTrace("ppMops.write", 1, "Done writing to %s", args->output);
+    psTrace("ppMops.write", 1, "Done writing %ld rows to %s", det->num, args->output);
 
     return true;
Index: /branches/eam_branches/ipp-20100621/psconfig/tagsets/ipp-2.9.dist
===================================================================
--- /branches/eam_branches/ipp-20100621/psconfig/tagsets/ipp-2.9.dist	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/psconfig/tagsets/ipp-2.9.dist	(revision 28794)
@@ -73,4 +73,5 @@
   YYYYY  ppTranslate            ipp-2-9          -0
   YYYYY  ppViz                  ipp-2-9          -0
+  YYYYY  ppBackground		ipp-2-9		 -0
   YYYYY  ppSkycell              ipp-2-9          -0
 
Index: /branches/eam_branches/ipp-20100621/psconfig/tagsets/ipp-2.9.libs
===================================================================
--- /branches/eam_branches/ipp-20100621/psconfig/tagsets/ipp-2.9.libs	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/psconfig/tagsets/ipp-2.9.libs	(revision 28794)
@@ -34,5 +34,5 @@
 lib libpng               NONE           NONE   libpng-1.2.15.tar.gz     libpng-1.2.15    N NONE NONE NONE
 lib libjpeg              NONE           jpeg   jpegsrc.v6b-p1.tar.gz    jpeg-6b          N --enable-shared NONE install-lib
-lib libcfitsio           NONE           NONE   cfitsio3100-p2.tar.gz    cfitsio3100-p2   N --enable-shared shared,all NONE
+lib libcfitsio           NONE           NONE   cfitsio3100-p3.tar.gz    cfitsio3100-p3   N --enable-shared shared,all NONE
 lib libmysqlclient       NONE           mysql  mysql-5.0.51a.tar.gz     mysql-5.0.51a    N NONE NONE NONE
 lib libgsl               NONE           NONE   gsl-1.11.tar.gz          gsl-1.11         N NONE NONE NONE
@@ -54,5 +54,5 @@
 inc fcntl.h              NONE           NONE   NONE                     NONE             N NONE NONE NONE 
 inc fftw3.h              NONE           NONE   fftw-3.0.1.tar.gz        fftw-3.0.1       N --enable-float,--enable-shared,--disable-fortran NONE NONE
-inc fitsio.h             NONE           NONE   cfitsio3100-pap.tar.gz   cfitsio3100-pap  N --enable-shared shared NONE
+inc fitsio.h             NONE           NONE   cfitsio3100-p3.tar.gz   cfitsio3100-p3  N --enable-shared shared NONE
 inc glob.h               NONE           NONE   NONE                     NONE             N NONE NONE NONE 
 inc gsl/gsl_randist.h    NONE           NONE   gsl-1.11.tar.gz          gsl-1.11         N NONE NONE NONE 
Index: /branches/eam_branches/ipp-20100621/pstamp/Makefile.am
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/Makefile.am	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pstamp/Makefile.am	(revision 28794)
@@ -1,3 +1,9 @@
 SUBDIRS = src scripts
 
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA= ppstamp.pc
+
+EXTRA_DIST = \
+	ppstamp.pc.in
+
 CLEANFILES = *~ core core.*
Index: /branches/eam_branches/ipp-20100621/pstamp/configure.ac
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/configure.ac	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pstamp/configure.ac	(revision 28794)
@@ -33,4 +33,6 @@
 IPP_STDOPTS
 CFLAGS="${CFLAGS} -Wall -Werror"
+PPSTAMP_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=}"
+PPSTAMP_LIBS="${PSLIB_LIBS=} ${PSMODULES_LIBS=}"
 echo "PPSTAMP_CFLAGS: $PPSTAMP_CFLAGS"
 echo "PPSTAMP_LIBS: $PPSTAMP_LIBS"
@@ -43,4 +45,5 @@
   src/Makefile
   scripts/Makefile
+  ppstamp.pc
 ])
 
Index: /branches/eam_branches/ipp-20100621/pstamp/ppstamp.pc.in
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/ppstamp.pc.in	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/pstamp/ppstamp.pc.in	(revision 28794)
@@ -0,0 +1,11 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@/@PACKAGE_NAME@
+
+Name: @PACKAGE_NAME@
+Description: Pan-STARRS Postage Stamper
+Version: @VERSION@
+Requires: pslib psmodules
+Libs: -L${libdir} @PPSTAMP_LIBS@
+Cflags: -I${includedir} @PPSTAMP_CFLAGS@
Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read	(revision 28794)
@@ -26,5 +26,5 @@
 my $no_print_header = 0;    # omit the header keywords
 my $no_print_rows   = 0;    # omit the rows
-
+my $version = 0;
 
 my ( $input,			# Name of input text file
@@ -52,57 +52,107 @@
 my $status = 0;
 
-# The required keywords
-my $header = {
-        'QUERY_ID' => { name => 'QUERY_ID', 
-                        writetype => TSTRING, 
-                        comment => 'MOPS Query ID for this batch query',
-                        value => undef
-                      },
-        'FPA_ID'   => { name => 'FPA_ID', 
-                        writetype => TSTRING, 
-                        comment => 'orginal FPA_ID used at ingest',
-                        value => undef
-                      }, 
-        'MJD_OBS'  => { name => 'MJD-OBS', 
-                        writetype => TDOUBLE, 
-                        comment => 'starting time of the exposure, MJD',
-                        value => undef
-                      },
-        'FILTER'   => { name => 'FILTER', 
-                        writetype => TSTRING, 
-                        comment => 'effective filter use for the exposure',
-                        value => undef
-                      },
-        'OBSCODE'  => { name => 'OBSCODE', 
-                        writetype => TSTRING, 
-                        comment => 'site identifier (MPC observatory code)',
-                        value => undef
-                      },
- 	'STAGE'    => { 
-	                name => 'STAGE',
-			writetype => TSTRING,
-			comment => 'processing stage to examine',
-			value => undef
-	              }
-};
-
-# key_array insures that the order that the keywords is printed out is
-# the same as the ICD
-my @key_array = qw( QUERY_ID FPA_ID MJD_OBS FILTER OBSCODE STAGE);
+# Read the input file
+
+my $inFits = Astro::FITS::CFITSIO::open_file( $input, READONLY, $status ); # FITS file handle
+check_fitsio($status);
+
+$inFits->movnam_hdu(BINARY_TBL, EXTNAME, 0, $status) and check_fitsio($status);
+
+my $inHeader = $inFits->read_header(); # Header for input
+
+my $numRows;			# Number of rows in table
+$inFits->get_num_rows($numRows, $status) and check_fitsio($status);
+
+# The keywords found in the header
+# my $header = {
+#         'EXTVER'   => { name => 'EXTVER',
+# 		       writetype => TSTRING,
+# 		       comment => 'Extension version',
+# 		       value => undef
+#                       },
+#         'EXTNAME'  => { name => 'EXTNAME',
+# 		       writetype => TSTRING,
+# 		       comment => 'name of this binary table extension',
+# 	  	       value => undef
+# 		      },
+#         'QUERY_ID' => { name => 'QUERY_ID', 
+#                         writetype => TSTRING, 
+#                         comment => 'MOPS Query ID for this batch query',
+#                         value => undef
+#                       },
+#         'FPA_ID'   => { name => 'FPA_ID', 
+#                         writetype => TSTRING, 
+#                         comment => 'orginal FPA_ID used at ingest',
+#                         value => undef
+#                       }, 
+#         'MJD_OBS'  => { name => 'MJD-OBS', 
+#                         writetype => TDOUBLE, 
+#                         comment => 'starting time of the exposure, MJD',
+#                         value => undef
+#                       },
+#         'FILTER'   => { name => 'FILTER', 
+#                         writetype => TSTRING, 
+#                         comment => 'effective filter use for the exposure',
+#                         value => undef
+#                       },
+#         'OBSCODE'  => { name => 'OBSCODE', 
+#                         writetype => TSTRING, 
+#                         comment => 'site identifier (MPC observatory code)',
+#                         value => undef
+#                       },
+#  	'STAGE'    => { 
+# 	                name => 'STAGE',
+# 			writetype => TSTRING,
+# 			comment => 'processing stage to examine',
+# 			value => undef
+# 	              }
+# };
+
+my $parse_error = 0;
+# Parse the header to determine what we expect to find.
+foreach my $header_key (keys %{ $inHeader }) {
+    $inHeader->{$header_key} =~ s/\s+//g;
+    $inHeader->{$header_key} =~ s/\'//g;
+}
+my ($EXTVER,$headerEXTNAME,$QUERY_ID,$FPA_ID,$MJD_OBS,$FILTER,$OBSCODE,$STAGE) = 
+    ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},$inHeader->{FPA_ID},
+     $inHeader->{'MJD-OBS'},$inHeader->{FILTER},$inHeader->{OBSCODE},$inHeader->{STAGE});
+unless(defined($EXTVER) && defined($headerEXTNAME) &&
+       (($EXTVER == 1)||($EXTVER == 2)) && 
+       ($headerEXTNAME eq EXTNAME)) {
+    $parse_error = 1;
+}
+unless(($EXTVER == 2)||
+       ((defined($QUERY_ID))&&(defined($FPA_ID))&&(defined($MJD_OBS))&&
+	(defined($FILTER))&&(defined($OBSCODE)))) {
+    $parse_error = 2;
+}
 
 # Specification of columns
 my $column_defs = [ 
         # matching rownum from detectability original request
-        { name => 'ROWNUM',   type => '20A', writetype => TSTRING }, 
+        { name => 'ROWNUM',   type => '20A', writetype => TSTRING, version => 1 }, 
         # coordinate at start of exposure, in degrees
-        { name => 'RA1_DEG',  type => 'D',   writetype => TDOUBLE },
+        { name => 'RA1_DEG',  type => 'D',   writetype => TDOUBLE, version => 1 },
         # coordinate at start of exposure, in degrees
-        { name => 'DEC1_DEG', type => 'D',   writetype => TDOUBLE },
+        { name => 'DEC1_DEG', type => 'D',   writetype => TDOUBLE, version => 1 },
         # coordinate at end of exposure, in degrees
-        { name => 'RA2_DEG',  type => 'D',   writetype => TDOUBLE },
+        { name => 'RA2_DEG',  type => 'D',   writetype => TDOUBLE, version => 1 },
         # coordinate at end of exposure, in degrees
-        { name => 'DEC2_DEG', type => 'D',   writetype => TDOUBLE },
+        { name => 'DEC2_DEG', type => 'D',   writetype => TDOUBLE, version => 1 },
         # apparent magnitude
-        { name => 'MAG',      type => 'D',   writetype => TDOUBLE },
+        { name => 'MAG',      type => 'D',   writetype => TDOUBLE, version => 1 },
+    # v2 query_id: MOPS query ID for this batch query
+    { name => 'QUERY_ID', type => '20A', writetype => TSTRING, version => 2, default => $QUERY_ID },
+    # v2 fpa_id: original FPA_ID used at ingest
+    { name => 'FPA_ID', type => '20A', writetype => TSTRING, version => 2, default => $FPA_ID },
+    # v2 mjd obs: starting time of the exposure, MJD
+    { name => 'MJD-OBS', type => 'D', writetype => TDOUBLE, version => 2, default => $MJD_OBS },
+    # v2 filter: effective filter used for the exposure as a string, allowed values of g, r, i, z, y, B, V, w
+    { name => 'FILTER', type => '3A', writetype => TSTRING, version => 2, default => $FILTER },
+    # v2 obscode: site identifier (MPC observatory code)
+    { name => 'OBSCODE', type => '3A', writetype => TSTRING, version => 2, default => $OBSCODE },
+    # v2 stage: stage name to perform query on, allowed values of 'chip', 'warp', 'stack', and 'diff'
+    { name => 'STAGE', type => '20A', writetype => TSTRING, version => 2, default => $STAGE },
 ];
 
@@ -118,19 +168,11 @@
 }
 
-# Read the input file
-
-my $inFits = Astro::FITS::CFITSIO::open_file( $input, READONLY, $status ); # FITS file handle
-check_fitsio($status);
-
-$inFits->movnam_hdu(BINARY_TBL, EXTNAME, 0, $status) and check_fitsio($status);
-
-my $inHeader = $inFits->read_header(); # Header for input
-
-my $numRows;			# Number of rows in table
-$inFits->get_num_rows($numRows, $status) and check_fitsio($status);
-
 foreach my $col (@$column_defs) {
     my ($col_num, $col_type, $col_data);
 
+    if ($col->{version} > $EXTVER) {
+	@{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows);
+	next;
+    }
     $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status);
     $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
@@ -138,50 +180,42 @@
                                                                     and check_fitsio($status);
     $colData{$col->{name}} = $col_data;
-}
-
-# Now produce the output
-
-if (!$no_print_header) {
-    my $label;
-    my $data;
-    # I don't do this because I want the keys to be printed in a particular order
-    #foreach my $key (keys %$header) {
-    foreach my $key (@key_array) {
-        my $name = $header->{$key}->{name};
-        my $value = $inHeader->{$name};
-	if (($key eq 'STAGE')&& !(defined($value))) {
-	    $value = 'DIFF';
-	}
-        # get rid of quotes and whitespace
-        $value =~ s/\'//g;
-        if (defined $value) {
-            #print "$key\t\t\t$value\n";
-            $label .= sprintf "%-12s ", $key;
-            $data  .= sprintf "%-12s ", $value;
-        } else {
-            die "keyword $key not found in $input\n";
-        }
-    }
-    print "# " . $label . "\n" unless $no_print_label;
-    print $data  . "\n";
-}
-
-if (!$no_print_rows) {
-    if (!$no_print_label) {
-        print "# ";
-        foreach my $col (@$column_defs) {
-            printf "%-12s ", $col->{name};
-        }
-        print "\n";
-    }
-
-    for (my $i = 0; $i < $numRows; $i++) {
-        foreach my $col (@$column_defs) {
-            printf "%-12s ", $colData{$col->{name}}->[$i];
+    if ($col->{name} eq 'MJD-OBS') {
+	print @{ $col_data } . "\n";
+    }
+}
+
+# Verify uniqueness of important columns:
+my @unique_fields = ('QUERY_ID','OBSCODE','STAGE');
+foreach my $colName (@unique_fields) {
+    my %counter = ();
+    foreach my $row (@{ $colData{$colName} }) {
+	$counter{$row} = 1;
+    }
+    if (scalar(keys(%counter)) != 1) {
+	$parse_error = 3;
+    }
+}
+if ($parse_error) {
+    die "Unable to parse detectability query: $parse_error " . &error_message($parse_error) . "\n";;
+}
+
+# # Now produce the output
+
+
+if (!$no_print_label) {
+    print "# ";
+    foreach my $col (@$column_defs) {
+	printf "%-12s ", $col->{name};
+    }
+    print "\n";
+}
+
+for (my $i = 0; $i < $numRows; $i++) {
+    foreach my $col (@$column_defs) {
+	printf "%-12s ", $colData{$col->{name}}->[$i];
         #foreach my $aref (@col_arrays) {
-            #printf "%-12s ", $aref->[$i];
-        }
-        print "\n";
-    }
+	#printf "%-12s ", $aref->[$i];
+    }
+    print "\n";
 }
 
@@ -202,3 +236,21 @@
 }
 
+sub error_message {
+    my $error = shift;
+    if ($error == 1) {
+	return("Unknown EXTVER/EXTNAME");
+    }
+    if ($error == 2) {
+	return("Required header field not found");
+    }
+    if ($error == 3) {
+	return("Unique column not uniquely specified");
+    }
+    else {
+	return("Unknown fault.");
+    }
+}
+
+
+
 __END__
Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/detectability_respond.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/scripts/detectability_respond.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pstamp/scripts/detectability_respond.pl	(revision 28794)
@@ -42,5 +42,5 @@
 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
 my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project);
-my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps);
+my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom);
 GetOptions(
     'input=s'         =>      \$request_file,
@@ -51,4 +51,5 @@
     'verbose'         =>      \$verbose,
     'save-temps'      =>      \$save_temps,
+    'ignore-wisdom'   =>      \$ignore_wisdom,
     ) or pod2usage(2);
 
@@ -84,319 +85,371 @@
 }
 
+my %query = ();
+my %image_list_hash = ();
+my $wisdom_file = "${workdir}/wisdom.dat";
+if ((-e $wisdom_file)&&!($ignore_wisdom)) {
+    print "Reading wisdom file $wisdom_file instead of parsing...\n";
+    open(WISDOM,"$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
+    my $i = 0;
+    while(<WISDOM>) {
+	chomp;
+	my ($fpa_id,@key_values) = split /\s+/;
+	while ($#key_values > -1) {
+	    my $key = shift(@key_values);
+	    my $val = shift(@key_values);
+	    $query{$fpa_id}{$key}[$i] = $val;
+	}
+	$i++;
+    }
+    close(WISDOM);
+} # End reading wisdom.
+else {
 #
 # Parse input request file using detect_query_read (as it's already written).
 #
-my $dqr_command = "$detect_query_read --input $request_file";
-my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-    run(command => $dqr_command, verbose => $verbose);
-unless ($success) {
-    # This is a problem, because I'm not sure how we handle a failure to read something.
-    # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 
-    # we can't send that response back.
-    die("Unable to perform $dqr_command error code: $error_code");
-}
-my %query = ();
-my $Nrows = 0;
-{
-    my @column_names = ();
-    my $section = '';
-    foreach my $entry (split /\n/, (join "", @$stdout_buf)) {
-	if ($entry =~ /^#/) {
-	    @column_names = split /\s+/, $entry;
-	    shift(@column_names);  # Dump the hash sign.
-	    if ($section eq 'HEADER') {
-		$section = 'CONTENT';
+    my $dqr_command = "$detect_query_read --input $request_file";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	run(command => $dqr_command, verbose => $verbose);
+    unless ($success) {
+	# This is a problem, because I'm not sure how we handle a failure to read something.
+	# We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 
+	# we can't send that response back.
+	die("Unable to perform $dqr_command error code: $error_code");
+    }
+    my $Nrows = 0;
+    {
+	my @column_names = ();
+	foreach my $entry (split /\n/, (join "", @$stdout_buf)) {
+	    if ($entry =~ /^#/) {
+		@column_names = split /\s+/, $entry;
+		shift(@column_names);  # Dump the hash sign.
 	    }
 	    else {
-		$section = 'HEADER';
+		# ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE
+		my %row_data;
+		@row_data{@column_names} = (split /\s+/, $entry);
+		for (my $i = 0; $i <= $#column_names; $i++) {
+		push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]};
+		$Nrows = scalar(keys(%query));
+#		print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n";
 	    }
-	}
+	    }
+	}
+    }
+#
+# Identify target images.  This should properly collate targets on a single imfile.
+#
+    foreach my $fpa_id (keys %query) {
+	my %temp_hash;
+	my $query_style = 'byexp';
+	my $stage;
+	my $filter;
+	my $mjd;
+	# Determine the query style for this fpa_id
+	if ($fpa_id =~ /o.*g.*o/) {
+	$query_style = 'byexp';
+    }
+	elsif ($fpa_id =~ /\d+/) {
+	$query_style = 'byid';
+    }
 	else {
-	    # HEADER: 
-	    # QUERY_ID FPA_ID MJD_OBS FILTER OBSCODE STAGE
-	    # CONTENT:
-	    # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG
-	    my @columns = split /\s+/, $entry;
-	    for (my $i = 0; $i <= $#columns; $i++) {
-		$query{$section}{$column_names[$i]}[$Nrows] = $columns[$i];
-#		print "$section $column_names[$i] $Nrows $columns[$i]\n";
-	    }
-	    $Nrows++;
-	}
-    }
-}
-
-#
-# Identify target images.  This should properly collate targets on a single imfile.
-#
-my %image_list_hash;
-for (my $i = 1; $i < $Nrows; $i++) {
-    # This could use the fact that locate_images now accepts position arrays, but
-    # I'll save that for after I get the majority of things working.
-    my $image_set_tmp  = find_image_set($query{HEADER}{FPA_ID}[0],$query{HEADER}{STAGE}[0],
-					$query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],
-					$query{CONTENT}{RA1_DEG}[$i],$query{CONTENT}{DEC1_DEG}[$i],
-					$query{CONTENT}{ROWNUM}[$i],$verbose);
-    unless (%$image_set_tmp) {
-	# No images were returned, so create a dummy entry that 
-	$image_list_hash{'no_image'}{IMAGE}    = 'no_image';
-	$image_list_hash{'no_image'}{PSF}      = 'no_psf';
-	$image_list_hash{'no_image'}{MASK}     = 'no_mask';
-	$image_list_hash{'no_image'}{WEIGHT}   = 'no_weight';
-	$image_list_hash{'no_image'}{CATALOG}  = 'no_catalog';
-	$image_list_hash{'no_image'}{CLASS_ID} = 'no_class';
-	$image_list_hash{'no_image'}{ERROR}    = $PSTAMP_NO_IMAGE_MATCH;
-	push @{ $image_list_hash{'no_image'}{SKY_COORDINATES} }, "$query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]";
-	push @{ $image_list_hash{'no_image'}{ROWNUM} }, $query{CONTENT}{ROWNUM}[$i];
-	next;
-    }
-#     print "=== $image_set_tmp->{IMAGE}\n    $image_set_tmp->{PSF}\n";
-#     print "    $image_set_tmp->{MASK}\n    $image_set_tmp->{WEIGHT}\n";
-#     print "    $image_set_tmp->{SKY_COORDINATES}\n    $image_set_tmp->{ROWNUM}\n";
-
-    # This indexes the results for identical images into the same hash.
-    $image_list_hash{$image_set_tmp->{IMAGE}}{IMAGE}    = $image_set_tmp->{IMAGE};
-    $image_list_hash{$image_set_tmp->{IMAGE}}{PSF}      = $image_set_tmp->{PSF};
-    $image_list_hash{$image_set_tmp->{IMAGE}}{MASK}     = $image_set_tmp->{MASK};
-    $image_list_hash{$image_set_tmp->{IMAGE}}{WEIGHT}   = $image_set_tmp->{WEIGHT};
-    $image_list_hash{$image_set_tmp->{IMAGE}}{CATALOG}  = $image_set_tmp->{CATALOG};
-    $image_list_hash{$image_set_tmp->{IMAGE}}{CLASS_ID} = $image_set_tmp->{CLASS_ID};
-    $image_list_hash{$image_set_tmp->{IMAGE}}{ERROR}    = $image_set_tmp->{ERROR};
-    push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{SKY_COORDINATES} }, $image_set_tmp->{SKY_COORDINATES};
-    push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{ROWNUM} }, $image_set_tmp->{ROWNUM};
-}
-
-my $i = 0;
-foreach my $k (keys %image_list_hash) {
-    # If we errored out on finding an image, we need to not try to run psphot here.
-    if ($image_list_hash{$k}{ERROR} != 0) {
-	next;
-    }
-    # Write coordinates of the requested targets to a file.
-    my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$i.XXXX", 
-					    UNLINK => !$save_temps);
-    my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$i.XXXX", 
-					    UNLINK => !$save_temps);
-
-    for (my $j = 0; $j <= $#{ $image_list_hash{$k}{SKY_COORDINATES} }; $j++) {
-	print $coordfile "$image_list_hash{$k}{SKY_COORDINATES}[$j]\n";
-    }
-#    print "$k\n";
-    # Convert the sky coordinates to image coordinates with ppCoord.
-    my $command = "ppCoord -astrom $image_list_hash{$k}{CATALOG} -radec $coordname";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $command, verbose => $verbose);
-    unless ($success) {
-	my_die("Unable to perform $command. Error_code: $error_code",
-	       $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0],
-	       $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0],
-	       $query{HEADER}{STAGE}[0],$error_code);
-    }
-    my @response = split /\n/, (join "", @$stdout_buf);
-    foreach my $line (@response) {
-	my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line;
-	print $targetfile "$r_x $r_y\n";
-	$image_list_hash{$k}{EXTENSION_BASE} = $r_chip;
-    }
-
-#     print "psphot $image_list_hash{$k}{PSF}\n";
-    # Run psphotForced on the target list.
-    my $tmpdir  = tempdir("detect.$i.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps);
-    $image_list_hash{$k}{OUTROOT} = "$tmpdir/detectability.$query{HEADER}{STAGE}[0].$query{HEADER}{FPA_ID}[0]";
-    
-    my $psphot_cmd = "$psphotForced -psf $image_list_hash{$k}{PSF} ";
-    $psphot_cmd .= "-file $image_list_hash{$k}{IMAGE} ";
-    $psphot_cmd .= "-mask $image_list_hash{$k}{MASK} ";
-    $psphot_cmd .= "-variance $image_list_hash{$k}{WEIGHT} ";
-    $psphot_cmd .= "-srctext $targetname $image_list_hash{$k}{OUTROOT}";
-
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $psphot_cmd, verbose => $verbose);
-    unless ($success) {
-	$image_list_hash{$k}{ERROR} = $PSTAMP_SYSTEM_ERROR;
-    }
-}
-
-#
-# Convert psphot output to response
-#
-my @rownums = ();
-my @out_errors = ();
-my @psphot_Npix = ();
-my @psphot_Qfact= ();
-my @psphot_flux = ();
-my @psphot_error = ();
-
-foreach my $k (keys %image_list_hash) {
-    if ($image_list_hash{$k}{ERROR} == 0) {
-	my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf";
-	
-	my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE});
-	
-	push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
-	push @out_errors,     (map { $image_list_hash{$k}{ERROR} }  @{ $image_list_hash{$k}{ROWNUM} });
-	push @psphot_Npix,    @{ $tmp_Npix };
-	push @psphot_Qfact,   @{ $tmp_Qfact };
-	push @psphot_flux,    @{ $tmp_flux };
-	push @psphot_error,   @{ $tmp_flux_error };
-    }
-    else {
-	push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
-	push @out_errors,     (map { $image_list_hash{$k}{ERROR} }  @{ $image_list_hash{$k}{ROWNUM} });
-	push @psphot_Npix,    (map { 0 }  @{ $image_list_hash{$k}{ROWNUM} });
-	push @psphot_Qfact,   (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
-	push @psphot_flux,    (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
-	push @psphot_error,    (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
-    }	
-}
-
-write_response_file($output,
-		    $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0],
-		    $query{HEADER}{MJD_OBS}[0],$query{HEADER}{filter}[0],
-		    $query{HEADER}{obscode}[0],
-		    \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux, \@psphot_error);
-# print "Wrote response file $output\n";
-#
-# Add to datastore
-#
-# Files are added to the datastore by dquery_finish.pl
-#
-# Cleanup
-#
-# Since everything is written to temporary files, there should be nothing to cleanup.
-
-#
-# Utilities
-#
-sub find_image_set {
-    my $FPA_ID = shift;
-    my $stage  = lc(shift);
-    my $mjd    = shift;
-    my $filter = shift;
-    my $ra     = shift;
-    my $dec    = shift;
-    my $index  = shift;
-    my $verbose = shift;
-
-    # This is the set of things that we need in order to run psphotForced
-    my $option_mask |= 1;
-    $option_mask |= $PSTAMP_SELECT_IMAGE;
-    $option_mask |= $PSTAMP_SELECT_MASK;
-    $option_mask |= $PSTAMP_SELECT_VARIANCE;
-    $option_mask |= $PSTAMP_SELECT_PSF;
-    my $need_magic = 1;
-    my $mjd_min = $mjd;
-    my $mjd_max = $mjd + 1;
-
-    # Construct a row list. 
-    my @rowList;
-    $rowList[0]->{CENTER_X} = $ra;
-    $rowList[0]->{CENTER_Y} = $dec;
-    $rowList[0]->{ID} = 1;
-    $rowList[0]->{STAGE} = $stage;
-    $rowList[0]->{COORD_MASK} = 0;
-
-    #    print "$stage\n";
-    # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
-    my @images = locate_images($ipprc,$imagedb,
-			        \@rowList,
-			       "bycoord",$stage,
-			       undef,undef,undef,
-			       $option_mask,$need_magic,
-			       # $ra,$dec,
-			       $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose);  
-
-    my %image_info  = ();
-    foreach my $i (@images) {           # Scan over each result
-	foreach my $j (@{ $i }) {       # Scan over each image in the result
-	    # We only care about an image if it matches the FPA_ID in the request
-	    if ($stage eq 'diff') {
-		# Diffs match if either exposure name is defined and matches the FPA_ID
-		unless ((defined(${ $j }{exp_name_1}) && (${ $j }{exp_name_1} eq $FPA_ID))||
-			(defined(${ $j }{exp_name_2}) && (${ $j }{exp_name_2} eq $FPA_ID))||
-			(${ $j }{exp_id_1} eq $FPA_ID)||(${ $j }{exp_id_2} eq $FPA_ID)) {
-		    next;
+	exit_with_failure(21,"Parse error in request file");
+    }
+	# Confirm that we only have one stage/filter/mjd
+	for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+	$temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
+	$temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
+	$temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
+    }
+	if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
+	    $stage = (keys(%{ $temp_hash{STAGE} }))[0];
+	}
+	else {
+	exit_with_failure(21,"Too many STAGEs specified");
+    }
+	if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
+	$filter = (keys(%{ $temp_hash{FILTER} }))[0];
+    }
+	else {
+	exit_with_failure(21,"Too many FILTERs specified");
+    }
+	if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
+	$mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
+    }
+	else {
+	exit_with_failure(21,"Too many MJD-OBS specified");
+    }
+	# Set common request components
+	my $option_mask |= 1;
+	$option_mask |= $PSTAMP_SELECT_IMAGE;
+	$option_mask |= $PSTAMP_SELECT_MASK;
+	$option_mask |= $PSTAMP_SELECT_VARIANCE;
+	$option_mask |= $PSTAMP_SELECT_PSF;
+	my $need_magic = 1;
+	if ($stage eq 'stack') {
+	    $need_magic = 0;
+	}
+	my $mjd_min = $mjd;
+	my $mjd_max = $mjd + 1;
+	
+	# Construct a row list. 
+	my @rowList;
+	for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+	    $rowList[$i]->{CENTER_X} = $query{$fpa_id}{RA1_DEG}[$i];
+	    $rowList[$i]->{CENTER_Y} = $query{$fpa_id}{DEC1_DEG}[$i];
+	    $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i];
+	    $rowList[$i]->{COORD_MASK} = 0;
+	}
+	
+	# Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
+	my $pstamp_images_ref = locate_images($ipprc,$imagedb,
+					  \@rowList,
+					  $query_style,$stage,
+					  $fpa_id,undef,undef,
+					  $option_mask,$need_magic,
+					  # $ra,$dec,
+					  $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose);  
+	
+	foreach my $this_image_ref (@{ $pstamp_images_ref }) {
+	    foreach my $key (sort (keys %{ $this_image_ref } )) {
+		my $value = ${ $this_image_ref }{$key};
+		if ($key eq 'row_index') {
+		    $value = join ' ', @{ $this_image_ref->{$key} };
+		}
+#		print "$this_image_ref $key $value\n";
+		foreach my $valid_index (@{ $this_image_ref->{row_index} }) {
+		    $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image};
+		    $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask};
+		    $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight};
+		    $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf};
+		    $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id};
+		    $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb};
+		    $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic;
+		    
+		    if (exists($this_image_ref->{astrom})) {
+			$query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom};
+		    }
+		    else {
+		    $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf};
+		}
+		    if (exists($this_image_ref->{class_id})) {
+			$query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id};
+			$query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id};
+			
+		    }
+		    else {
+			$query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id};
+			$query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa';
+		    }
+		    
+		    $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state};
+		    if (exists($this_image_ref->{data_state})) {
+			$query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state};
+		    }
+		    else {
+			$query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state};
+		    }
+		    $query{$fpa_id}{FAULT}[$valid_index] = 0;
+		    $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked};
+		    if ($stage eq 'chip') {
+			$query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state};
+		    }
+		    
+		    # Determine if the data exists.
+		    if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or 
+			($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or
+			($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or 
+			($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or 
+			($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or 
+			($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) {
+			
+			# image is gone and it's not coming back
+			$query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
+		    }
+		    elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] = 0)) {
+			$query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
+		    }
+		    elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) {		      
+			$query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE;
+			
+			# updating stacks isn't implemented
+			if (($stage eq 'stack')) {
+			    $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+			}
+			# updating old burntool data isn't implemented
+			elsif ($stage eq 'chip') {
+			    if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and 
+				(abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) {
+				$query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
+			    }
+			}
+		    } # End determining error faults.
 		}
 	    }
-	    elsif ($stage eq 'stack') {
-		# Stacks hide the exposure name very well, so we can only match against stage_id
-		if (${ $j }{stage_id} ne $FPA_ID) {
-		    next;
-		}
+	}
+    }
+} # End calculating wisdom
+my %update_request;
+my %processing_request;
+
+open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
+foreach my $fpa_id (keys %query) {
+    for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
+	print WISDOM "$fpa_id\t";
+	foreach my $key (keys %{ $query{$fpa_id} }) {
+	    print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
+	}
+	print WISDOM "\n";
+	@{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 
+	    ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
+	     $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
+	push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
+    }
+}
+close(WISDOM);
+my $exit_code = 0;
+my $update_request_file = "${workdir}/update_request.dat";
+open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
+foreach my $images (keys %update_request) {
+    foreach my $fault (keys %{ $update_request{$images} }) {
+	if ($fault == 25) {
+	    $exit_code = 25;
+	}
+	my $update_request = join ' ', @{ $update_request{$images}{$fault} };
+	print UPDATE_REQUEST "$update_request\n";
+    }
+}
+close(UPDATE_REQUEST);
+# if ($exit_code != 0) {
+#     exit($exit_code);
+# }
+
+# This duplicates stuff returned by PSTAMP, but my thought is to convert that ---^ into a conditional, in which I read
+# from the wisdom.dat file.  This means only one pass on all that potentially slow stuff.  If that's the case, then 
+# I can recalculate the processing request.
+
+foreach my $fpa_id (keys %processing_request) {
+    foreach my $image (keys %{ $processing_request{$fpa_id} }) {
+	print "$fpa_id $image\t";
+	foreach my $i (@{ $processing_request{$fpa_id}{$image} }) {
+	    print "$i ";
+	}
+	print "\n";
+    }
+}
+#exit(10);
+
+# run ppCoord and psphotForced to calculate the required data.
+foreach my $fpa_id (keys %processing_request) {
+    foreach my $image (keys %{ $processing_request{$fpa_id} }) {
+	# Get this image specific data from the first entry. That entry is now the king of this set.
+	my $index = $processing_request{$fpa_id}{$image}[0];
+	my $fault = $query{$fpa_id}{FAULT}[$index];
+	my $catalog = $query{$fpa_id}{CATALOG}[$index];
+	my $psf   = $query{$fpa_id}{PSF}[$index];
+	my $mask  = $query{$fpa_id}{MASK}[$index];
+	my $weight= $query{$fpa_id}{WEIGHT}[$index];
+	my $stage = $query{$fpa_id}{STAGE}[$index];
+	# if there's a fault, then we can't process this image.
+	if ($fault != 0) {
+	    next;
+	}
+
+	# Create coordinate file to convert to positions.
+	my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$index.XXXX", 
+					      UNLINK => !$save_temps);
+	my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$index.XXXX", 
+						UNLINK => !$save_temps);
+	foreach my $i (@{ $processing_request{$fpa_id}{$image} }) {
+	    print $coordfile "$query{$fpa_id}{RA1_DEG}[$i] $query{$fpa_id}{DEC1_DEG}[$i]\n";
+	}
+	
+	# Convert the sky coordinates to image coordinates with ppCoord.
+	my $command = "ppCoord -astrom $catalog -radec $coordname";
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $command, verbose => $verbose);
+	unless ($success) {
+	    my_die("Unable to perform $command. Error_code: $error_code",
+		   $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index],
+		   $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index],
+		   $error_code);
+	}
+	my @response = split /\n/, (join "", @$stdout_buf);
+	my $i = 0;
+	foreach my $line (@response) {
+	    my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line;
+	    print $targetfile "$r_x $r_y\n";
+	    if (($r_ra == $query{$fpa_id}{RA1_DEG}[$processing_request{$fpa_id}{$image}[$i]])&&
+		($r_dec == $query{$fpa_id}{DEC1_DEG}[$processing_request{$fpa_id}{$image}[$i]])) {
+		$query{$fpa_id}{X_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_x;
+		$query{$fpa_id}{Y_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_y;
+		$query{$fpa_id}{EXTENSION_BASE}[$processing_request{$fpa_id}{$image}[$i]] = $r_chip;
 	    }
 	    else {
-		# For all the other stages (warp and chip are the ones I've tested), we can simply 
-		# directly match the exposure name to the FPA_ID
-		unless ((${ $j }{exp_name} eq $FPA_ID)||(${ $j }{exp_id} eq $FPA_ID)) {
-		    next;
-		}
+		$error_code = $PS_EXIT_PROG_ERROR;
+		my_die("Unable to match input RA/DEC with output RA/DEC: ($query{$fpa_id}{RA1_DEG}[$i],$query{$fpa_id}{DEC1_DEG}[$i]) -> ($r_ra,$r_dec) i. $error_code",
+		   $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index],
+		   $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index],
+		   $error_code);
 	    }
-	    # Debug prints of all the components of this image
-#   	    foreach my $k (keys %{ $j }) {
-# 		if ($k eq 'row_index') {
-# 		    print "$i $j $k @{${ $j }{$k} }\n";
-# 		}
-#  		print "$i $j $k ${ $j }{$k}\n";
-#  	    }
-
-	    # Check for existance of the images. Drawn mostly from pstampparse.pl
-	    my $run_state = ${ $j }{state};
-	    my $data_state = ${ $j }{data_state};
-	    $data_state = $run_state if $stage eq 'stack';
-	    my $fault = 0;
-	    if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
-		($run_state eq 'drop') or 
-		($run_state eq 'error_cleaned') or 
-		($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
-		# image is gone and it's not coming back
-		$fault = $PSTAMP_GONE;
+	    $i++;
+	}
+
+	# Run psphotForced on the target list.
+	my $tmpdir  = tempdir("detect.$index.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps);
+	my $outroot = "$tmpdir/detectability.${stage}.${fpa_id}.${index}";
+	$query{$fpa_id}{PROC_ERROR}[$index] = 0;
+	my $psphot_cmd = "$psphotForced -psf $psf ";
+	$psphot_cmd .= "-file $image ";
+	$psphot_cmd .= "-mask $mask ";
+	$psphot_cmd .= "-variance $weight ";
+	$psphot_cmd .= "-srctext $targetname $outroot ";
+	$psphot_cmd .= "-D OUTPUT.FORMAT PS1_DV1 ";
+	
+	( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $psphot_cmd, verbose => $verbose);
+	unless ($success) {
+	    $query{$fpa_id}{PROC_ERROR}[$index] = $PSTAMP_SYSTEM_ERROR;
+	}
+	
+	# Why not parse out results here?
+# Convert psphot output to response
+	if ($query{$fpa_id}{PROC_ERROR}[$index] == 0) {
+	    my $class_id = $query{$fpa_id}{CLASS_ID}[$index];
+	    my $cmf = "${outroot}.${class_id}.cmf";
+	    my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$query{$fpa_id}{EXTENSION_BASE}[$index]);
+	    foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) {
+		my $result_index = $processing_request{$fpa_id}{$image}[$i];
+		$query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index];
+
+		$query{$fpa_id}{NPIX}[$result_index] = ${ $tmp_Npix }[$i];
+		$query{$fpa_id}{QFACTOR}[$result_index] = ${ $tmp_Qfact }[$i];
+		$query{$fpa_id}{FLUX}[$result_index] = ${ $tmp_flux }[$i];
+		$query{$fpa_id}{FLUX_SIG}[$result_index] = ${ $tmp_flux_error }[$i];
+		
+#		print "$fpa_id $image $#{ $processing_request{$fpa_id}{$image} } $result_index $i ${ $tmp_Npix }[$i]\n";
 	    }
-	    elsif  (($data_state ne 'full') or ($need_magic and (${ $j }{magicked} < 0))) {
-		if (($stage eq 'stack')||($stage eq 'diff')) {
-		    # updating stacks and diffs isn't implemented
-		    $fault = $PSTAMP_NOT_IMPLEMENTED;
-		}
-		if ($stage eq 'chip') {
-		    my $burntool_state = ${ $j }{burntool_state};
-		    if ($burntool_state and (abs($burntool_state) < 14)) {
-			$fault = $PSTAMP_NOT_AVAILABLE;
-		    }
-		}
-		
-		if ($fault == 0) {
-		    # This bombs us out to dqueryparse, which will then flag a job for this run to be updated.
-		    my_die_for_update($data_state,$query{HEADER}{STAGE}[0],
-				      ${ $j }{stage_id},${ $j }{class_id} || ${ $j }{skycell_id},
-				      $need_magic,$imagedb,$PSTAMP_NOT_AVAILABLE);
-		}
+	}
+	else {
+	    foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) {
+		my $result_index = $processing_request{$fpa_id}{$image}[$i];
+		$query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index];
+
+		$query{$fpa_id}{NPIX}[$result_index] = 0;
+		$query{$fpa_id}{QFACTOR}[$result_index] = 0.0;
+		$query{$fpa_id}{FLUX}[$result_index] = 0.0;
+		$query{$fpa_id}{FLUX_SIG}[$result_index] = 0.0;
 	    }
-
-	    # This image matches, so we want to save the information into our output structure
-	    $image_info{ROWNUM} = $index;
-	    $image_info{IMAGE}  = ${ $j }{image};
-	    $image_info{PSF}    = ${ $j }{psf};
-	    $image_info{MASK}   = ${ $j }{mask};
-	    $image_info{WEIGHT} = ${ $j }{weight};
-	    $image_info{ERROR}  = $fault;
-	    $image_info{SKY_COORDINATES} = "$ra $dec";
-	    # To do sky->image coordinate transformations, we need to use the cmf/smf file. If 
-	    # an astrom reference (the camera stage smf file) exists, then use that, as we're dealing with
-	    # with the chip stage. Otherwise, use the stage-dependent cmf (and set the class_id to fpa).
-	    # The EXTENSION_BASE stores the basename of the extension that will be generated by psphotForced.
-	    if (exists(${ $j }{astrom})) {
-		$image_info{CATALOG} = ${ $j }{astrom};
-		$image_info{CLASS_ID} = ${ $j }{class_id};
-
-	    }
-	    else {
-		$image_info{CATALOG} = ${ $j }{cmf};
-		$image_info{CLASS_ID} = 'fpa';
-
-	    }
-
-	}
-    }
-    return(\%image_info);
-}
+	}
+
+	    	
+    }
+}
+
+write_response_file($output,\%query);
+
+#
+# Utilities
+#
 
 # Taken largely from detect_query_read
@@ -435,5 +488,5 @@
 	    ];
     }
-    elsif ($CMFversion eq 'PS1_DV2') {
+    elsif ($CMFversion eq 'PS1_DV1') {
 	$column_defs = [
 	    { name => 'PSF_QF',   type => '1E', writetype => TDOUBLE },
@@ -457,4 +510,5 @@
 
     my $correct_error = 0;
+#    print STDERR "Ncols:" .  $#{ $column_defs } . "\n";
     foreach my $col (@$column_defs) {
 	my ($col_num,$col_type,$col_data);
@@ -466,5 +520,5 @@
 	$inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
 	$inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status) and check_fitsio($status);
-	
+#	print STDERR "$col\t>>" . $col->{name} . "<<\t>>" . @{ $col_data } . "<<\n";
 	if ($col->{name} eq 'PSF_QF') {
 	    @tmp_Qfact = @{ $col_data };
@@ -477,5 +531,5 @@
 	}
 	elsif ($col->{name} eq 'PSF_INST_MAG_SIG') {
-	    @tmp_flux_err = map { $_ = $_ / (2.5 * log10(exp(1))) } @{ $col_data };
+	    @tmp_flux_err = map { $_ = $_ / (2.5 * (log(exp(1)) / log(10))) } @{ $col_data };
 	    $correct_error = 1;
 	}
@@ -493,4 +547,6 @@
     }
     $inFits->close_file( $status ) and check_fitsio($status);    
+#    print STDERR "$CMFversion\n";
+#    print STDERR "Q: $#tmp_Qfact\t@tmp_Qfact\nN: $#tmp_Npix\t@tmp_Npix\n";
     return(\@tmp_Npix, \@tmp_Qfact, \@tmp_flux, \@tmp_flux_err);
 }
@@ -499,40 +555,70 @@
 sub write_response_file {
     my $outfile = shift;
-    my $query_id = shift;
-    my $FPA_ID = shift;
-    my $MJD_OBS = shift;
-    my $filter = shift;
-    my $obscode = shift;
-    my $rownum_ref = shift;
-    my $out_err_ref = shift;
-    my $psphot_Npix_ref = shift;
-    my $psphot_Qfact_ref = shift;
-    my $psphot_flux_ref = shift;
-    my $psphot_error_ref = shift;
-    my $status = 0;
-
-    # Specification of columns to write
-    my $columns = [
-	# matching rownum from detectability original request
-        { name => 'ROWNUM',   type => 'V', writetype => TULONG }, 
-	# any errors that occurred during processing
-        { name => 'ERROR_CODE',   type => 'V', writetype => TULONG }, 
-        # number of pixels used in hypothetical PSF for the query detection
-        { name => 'DETECT_N', type => 'V',   writetype => TULONG },
-        # detectibility, indicating the fraction of PSF pixels detetable by IPP
-        { name => 'DETECT_F', type => 'D',   writetype => TDOUBLE },
-	# flux of the target source
-	{ name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
-	# error in the flux of the target source
-	{ name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
-	];
-    
-    # Header translation table
-    my $headers = {
-	'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, 
-                        comment => 'MOPS Query ID for this batch query' },
-	'FPA_ID' => { name => 'FPA_ID',   type => TSTRING, 
-		      comment => 'original FPA_ID used at ingest' },
-    };
+    my $query_ref = shift;
+
+    my %query = %{ $query_ref };
+
+    my $columns;
+    my $headers;
+
+    my $EXTVER_IS_1 = (scalar(keys(%query)) == 1);
+#    print "EXTVER: $EXTVER_IS_1\n";
+    my ($query_id,$FPA_ID,$MJD_OBS,$filter,$obscode,$status);
+    if ($EXTVER_IS_1 == 1) {
+	# Specification of columns to write
+	$columns = [
+	    # matching rownum from detectability original request
+	    { name => 'ROWNUM',   type => 'V', writetype => TULONG }, 
+	    # any errors that occurred during processing
+	    { name => 'ERROR_CODE',   type => 'V', writetype => TULONG }, 
+	    # number of pixels used in hypothetical PSF for the query detection
+	    { name => 'DETECT_N', type => 'V',   writetype => TULONG },
+	    # detectibility, indicating the fraction of PSF pixels detetable by IPP
+	    { name => 'DETECT_F', type => 'D',   writetype => TDOUBLE },
+	    # flux of the target source
+	    { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
+	    # error in the flux of the target source
+	    { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
+	    ];
+	
+	# Header translation table
+	$headers = {
+	    'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, 
+			    comment => 'MOPS Query ID for this batch query' },
+	    'FPA_ID' => { name => 'FPA_ID',   type => TSTRING, 
+			  comment => 'original FPA_ID used at ingest' },
+# 	    'MJD-OBS' => { name => 'FPA_ID',   type => TSTRING, 
+# 			  comment => 'original FPA_ID used at ingest' },
+# 	    'FILTER' => { name => 'FPA_ID',   type => TSTRING, 
+# 			  comment => 'original FPA_ID used at ingest' },
+# 	    'OBSCODE' => { name => 'FPA_ID',   type => TSTRING, 
+# 			  comment => 'original FPA_ID used at ingest' },
+	};
+    }
+    else {
+	# Specification of columns to write
+	$columns = [
+	    # matching rownum from detectability original request
+	    { name => 'ROWNUM',   type => 'V', writetype => TULONG }, 
+	    # any errors that occurred during processing
+	    { name => 'ERROR_CODE',   type => 'V', writetype => TULONG }, 
+	    # number of pixels used in hypothetical PSF for the query detection
+	    { name => 'DETECT_N', type => 'V',   writetype => TULONG },
+	    # detectibility, indicating the fraction of PSF pixels detetable by IPP
+	    { name => 'DETECT_F', type => 'D',   writetype => TDOUBLE },
+	    # flux of the target source
+	    { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
+	    # error in the flux of the target source
+	    { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
+	    # The FPA That would be in the header if it were to be there.
+	    { name => 'FPA_ID',   type => '20A', writetype => TSTRING },
+	    ];
+	
+	# Header translation table
+	$headers = {
+	    'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING, 
+			    comment => 'MOPS Query ID for this batch query' },
+	};
+    }	
 
     # Parse the list of columns
@@ -548,31 +634,36 @@
     }
 
-    my $numRows = $#{ $rownum_ref } + 1;
     my $inHeader = { };
 
     # Hack to force the data to match the detect_response_create formats
+
     $inHeader->{QUERY_ID}->{value} = $query_id;
-    $inHeader->{FPA_ID}->{value} = $FPA_ID;
-    $inHeader->{MJD_OBS}->{value} = $MJD_OBS;
-    $inHeader->{FILTER}->{value} = $filter;
-    $inHeader->{OBSCODE}->{value} = $obscode;
+    if ($EXTVER_IS_1 == 1) {
+	my $fpa_id = (keys(%query))[0];
+	$inHeader->{FPA_ID}->{value} = $fpa_id;
+    }
     
     # Fill the table columns with the data, making sure the flux is defined
-    for (my $i = 0; $i < $numRows; $i++) {
-	push @{$colData{'ROWNUM'}},      ${ $rownum_ref }[$i];
-	push @{$colData{'ERROR_CODE'}},  ${ $out_err_ref }[$i];
-	push @{$colData{'DETECT_N'}},    ${ $psphot_Npix_ref }[$i];
-	push @{$colData{'DETECT_F'}},    ${ $psphot_Qfact_ref }[$i];
-	push @{$colData{'TARGET_FLUX'}}, ${ $psphot_flux_ref }[$i];
-	push @{$colData{'TARGET_FLUX_SIG'}}, ${ $psphot_error_ref }[$i];
-	unless (defined(${ $colData{'TARGET_FLUX'}}[-1])) {
-	    $colData{'TARGET_FLUX'}[-1] = 0.0;
-	}
-	unless (defined(${ $colData{'TARGET_FLUX_SIG'}}[-1])) {
-	    $colData{'TARGET_FLUX_SIG'}[-1] = 0.0;
-	}
-    }
-
+    foreach my $fpa_id (keys %query) {
+	$inHeader->{QUERY_ID}->{value} = $query{$fpa_id}{QUERY_ID}[0];
+	if ($EXTVER_IS_1 == 1) {
+	    my $fpa_id = (keys(%query))[0];
+	    $inHeader->{FPA_ID}->{value} = $fpa_id;
+	}
+
+	push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} };
+	push @{$colData{'ERROR_CODE'}}, @{ $query{$fpa_id}{PROC_ERROR} };
+	push @{$colData{'DETECT_N'}}, @{ $query{$fpa_id}{NPIX} };
+	push @{$colData{'DETECT_F'}}, @{ $query{$fpa_id}{QFACTOR} };
+	push @{$colData{'TARGET_FLUX'}}, @{ $query{$fpa_id}{FLUX} };
+	push @{$colData{'TARGET_FLUX_SIG'}}, @{ $query{$fpa_id}{FLUX_SIG} };
+	if ($EXTVER_IS_1 != 1) {
+	    push @{$colData{'FPA_ID'}}, (map {$fpa_id} @{ $query{$fpa_id}{ROWNUM} });
+	}
+    }
+    my $numRows = 0;
     # Back to detect_response_create:
+    $status = 0;
+#    print "$output\n";
     my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
     check_fitsio( $status );
@@ -580,4 +671,5 @@
     check_fitsio( $status );
     my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
+    
     $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, $EXTNAME, $status);
     check_fitsio( $status );
@@ -601,9 +693,28 @@
 	check_fitsio( $status );
     }
+
     for (my $i = 0; $i < scalar @colNames; $i++) {
 	my $colName = $colNames[$i];# Column name
 	my $writeType = $colWriteType[$i];
+	my $numRows = scalar(@{$colData{$colName}});
+	unless(defined($writeType)) {
+	    print "write type undefined for $colName\n";
+	}
+	unless(defined($numRows)) {
+	    print "num Rows undefined for $colName\n";
+	}
+	unless(defined($status)) {
+	    print "status undefined for $colName\n";
+	}
+	unless(defined($colData{$colName})) {
+	    print "col data undefined for $colName\n";
+	}
+	unless(defined($colName)) {
+	    print "column name undefined for $i\n";
+	}
+#	print STDERR "$writeType $i $numRows $colName $status @{ $colData{$colName} }\n";
 	$outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData{$colName}, $status );
 	check_fitsio( $status );
+	
     }
     $outFits->close_file( $status );
@@ -641,15 +752,9 @@
 }
 
-sub my_die_for_update {
-    my $state = shift;
-    my $stage = shift;
-    my $stage_id = shift;
-    my $component = shift;
-    my $need_magic = shift;
-    my $imagedb = shift;
-    my $exit_code = shift;
-
-    print "$state $stage $stage_id $component $need_magic $imagedb\n";
-    print STDERR "$state $stage $stage_id $component $need_magic $imagedb\n";
-    exit($exit_code);
-}
+sub exit_with_failure {
+    my $status = shift;
+    my $message = shift;
+    carp("$message");
+    exit($status);
+}
+
Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/dqueryparse.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/scripts/dqueryparse.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pstamp/scripts/dqueryparse.pl	(revision 28794)
@@ -102,5 +102,5 @@
     if $extname ne "MOPS_DETECTABILITY_QUERY";
 my_die("$req_file is version $extver expecting 1", $PS_EXIT_PROG_ERROR)
-    if $extver ne 1;
+    if ($extver ne 1) and ($extver ne 2);
 
 # Set up the workdir for this query.
@@ -129,6 +129,9 @@
     $fault = $error_code >> 8;
     if ($fault == $PSTAMP_NOT_AVAILABLE) {
-	$data_to_update = (split /\n/, (join "", @$stdout_buf))[-1];	
+	unless (-e "$outdir/update_request.dat") {
+	    my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
+	}
     }	
+    
 }
 
@@ -158,27 +161,33 @@
     # Failed to run correctly, which means that we need to queue a job and flag data for updating.
     # Get the dependency id for the data we're requesting be updated.
-    my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
-
-    # Link this request to a job and link that job to any dependency
-    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
-    $command .= " -job_type detect_query -state run -fault 0";
-    $command .= " -rownum 1";
-    $command .= " -dep_id $dep_id" if $dep_id;
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
-    if ($success) {
-        $job_id = join "", @$stdout_buf;
-        chomp $job_id;
-        if ($job_id && -e $response_file) {
-	    # We shouldn't have a response file at this stage.
-            rename $response_file, "$outdir/response${job_id}.fits";
-        }
-        $result = 0;
-    } else {
-        warn("Unable to perform $command error code: $error_code");
-        $result = $error_code >> 8;
-    }
+    open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
+    while (<UPDATE_REQUEST>) {
+	my $data_to_update = $_;
+	chomp($data_to_update);
+	my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
+
+	# Link this request to a job and link that job to any dependency
+	my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
+	$command .= " -job_type detect_query -state run -fault 0";
+	$command .= " -rownum 1";
+	$command .= " -dep_id $dep_id" if $dep_id;
+
+	my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+	    run(command => $command, verbose => $verbose);
+
+	if ($success) {
+	    $job_id = join "", @$stdout_buf;
+	    chomp $job_id;
+	    if ($job_id && -e $response_file) {
+		# We shouldn't have a response file at this stage.
+		rename $response_file, "$outdir/response${job_id}.fits";
+	    }
+	    $result = 0;
+	} else {
+	    warn("Unable to perform $command error code: $error_code");
+	    $result = $error_code >> 8;
+	}
+    }
+    close(UPDATE_REQUEST);
 }
 
Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_checkdependent.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_checkdependent.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_checkdependent.pl	(revision 28794)
@@ -225,5 +225,5 @@
                 print "chipRun state is $chip->{chip_id} is in state $chip->{state} cannot update\n";
                 faultJobs('stop', undef, undef, $PSTAMP_GONE);
-                return;
+                return 0;
             } elsif (($chip->{data_state} ne 'update') and ($chip->{data_state} ne 'full')) {
                 my $command = "$chiptool -setimfiletoupdate -chip_id $chip_id -class_id $chip->{class_id}";
@@ -249,5 +249,5 @@
             print "chipRun state is $run->{chip_id} is in state $state cannot update\n";
             faultJobs('stop', undef, undef, $PSTAMP_GONE);
-            return;
+            return 0;
         }
 
@@ -277,5 +277,4 @@
     # if chipProcessedImfile.state is cleaned call check_states_chip
 
-    # need to code warptool -setskyfiletoupdate
     my $metadata = shift;
     my $whole_run = shift;  # if true queue entire run for update
@@ -287,5 +286,5 @@
     my $warp_id = $metadata->{warp_id};
     my $state = $metadata->{state};
-    if ($state =~ /error/) {
+    if (($state =~ /error/) or ($state =~ /purged/) or ($state =~ /scrubbed/) or ($state eq 'drop')) {
         print STDERR "warpRun $warp_id has state $state faulting jobs\n";
         faultJobs('stop', undef, undef, $PSTAMP_GONE);
Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_job_run.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_job_run.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pstamp/scripts/pstamp_job_run.pl	(revision 28794)
@@ -92,11 +92,18 @@
     my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
 
-    # XXX: remove -astrom from argString and add it here
-
     $argString .= " -file $params->{image}";
     $argString .= " -mask $params->{mask}";
+    my @file_list = ($params->{image}, $params->{mask});
     if ($options & $PSTAMP_SELECT_VARIANCE) {
         $argString .= " -variance $params->{weight}";
-    }
+        push @file_list, $params->{weight};
+    }
+
+    if ($params->{astrom}) {
+        $argString .= " -astrom $params->{astrom}";
+        push @file_list, $params->{astrom};
+    }
+
+    check_files(@file_list);
 
     my $command = "$ppstamp $outputBase $argString";
@@ -341,4 +348,12 @@
 }
 
+sub check_files {
+    foreach my $f (@_) {
+        if (!$ipprc->file_exists($f)) {
+            my_die( "file $f does not exist:", $job_id, $PS_EXIT_SYS_ERROR);
+        }
+    }
+}
+
 sub my_die
 {
Index: /branches/eam_branches/ipp-20100621/pstamp/scripts/pstampparse.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/scripts/pstampparse.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pstamp/scripts/pstampparse.pl	(revision 28794)
@@ -500,8 +500,10 @@
     }
 
+if (0) {
     # add astrometry file for raw and chip images if one is available
     if (($stage eq "chip") || ($stage eq "raw")) {
         $args .= " -astrom $image->{astrom}" if $image->{astrom};
     }
+}
 
     $image->{job_args} = $args;
@@ -812,4 +814,12 @@
         ($stage ne 'chip' and $state eq 'full')) {
         my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR);
+    }
+
+    if (($stage eq 'diff') and ($stage_id <= 22778)) {
+    	print STDERR "diff_id $stage_id cannot be updated\n";
+	$$r_dep_id = 0;
+	$$r_fault = $PSTAMP_GONE;
+	$$r_jobState = 'stop';
+	return;
     }
 
@@ -877,5 +887,5 @@
             $$r_newState = 'stop';
             $$r_fault = $PSTAMP_GONE;
-        } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) {
+        } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0)) or ($run_state eq 'goto_cleaned')) {
    
             if ($stage eq 'stack') {
Index: /branches/eam_branches/ipp-20100621/pstamp/src/pstampGetROI.c
===================================================================
--- /branches/eam_branches/ipp-20100621/pstamp/src/pstampGetROI.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pstamp/src/pstampGetROI.c	(revision 28794)
@@ -175,5 +175,5 @@
         rval   = ohana_dms_to_ddd(p1, argv[argnum]);
         if (rval) {
-            rval  = ohana_dms_to_ddd(p1, argv[argnum+1]);
+            rval  = ohana_dms_to_ddd(p2, argv[argnum+1]);
         }
     } else {
Index: /branches/eam_branches/ipp-20100621/pswarp/src/pswarpParseCamera.c
===================================================================
--- /branches/eam_branches/ipp-20100621/pswarp/src/pswarpParseCamera.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pswarp/src/pswarpParseCamera.c	(revision 28794)
@@ -61,4 +61,5 @@
 {
     psAssert(config, "Require configuration");
+    bool mdok;                          // Status of MD lookup
 
     // The input image(s) is required: it defines the camera
@@ -94,4 +95,10 @@
     psFree(skyConfig);
 
+    psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSWARP_RECIPE);
+    if (!recipe) {
+        psError(PSWARP_ERR_CONFIG, false, "missing recipe %s", PSWARP_RECIPE);
+        return false;
+    }
+
     // The output skycell
     pmFPAfile *output = pmFPAfileDefineSkycell(config, NULL, "PSWARP.OUTPUT");
@@ -118,5 +125,5 @@
     }
 
-    if (astrom) {
+    if (astrom && psMetadataLookupBool(&mdok, recipe, "SOURCES")) {
         pmFPAfile *outSources = pmFPAfileDefineSkycell(config, output->fpa, "PSWARP.OUTPUT.SOURCES");
         if (!outSources) {
@@ -127,11 +134,4 @@
     }
 
-    psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSWARP_RECIPE);
-    if (!recipe) {
-        psError(PSWARP_ERR_CONFIG, false, "missing recipe %s", PSWARP_RECIPE);
-        return false;
-    }
-
-    bool mdok;                          // Status of MD lookup
     if (psMetadataLookupBool(&mdok, recipe, "PSF")) {
         // This file, PSPHOT.INPUT, is just used as a carrier; output files (eg, PSPHOT.RESID) are defined by
Index: /branches/eam_branches/ipp-20100621/pswarp/src/pswarpTransformReadout.c
===================================================================
--- /branches/eam_branches/ipp-20100621/pswarp/src/pswarpTransformReadout.c	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/pswarp/src/pswarpTransformReadout.c	(revision 28794)
@@ -195,10 +195,12 @@
     psFree(interp);
 
-    if (goodPixels > 0) {
+    if (goodPixels > 0 && psMetadataLookupBool(&mdok, recipe, "SOURCES")) {
         if (!pswarpTransformSources(output, input, config)) {
             psError(psErrorCodeLast(), false, "Unable to interpolate image.");
             return false;
         }
-
+    }
+
+    if (goodPixels > 0) {
         // Data is only written out if there are good pixels
         output->data_exists = true;
Index: /branches/eam_branches/ipp-20100621/tools/czarplot.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/czarplot.pl	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/tools/czarplot.pl	(revision 28794)
@@ -0,0 +1,82 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+
+use czartool::CzarDb;
+use czartool::Czarplot;
+
+my $czarDbName = "czardb";
+my $label = undef;
+my $stage = undef;
+my $save_temps = undef;
+my $interval = undef;
+my $begin = undef;
+my $end = undef;
+my $path = undef;
+my $verbose = undef;
+my $histogram = undef;
+my $timeSeries = undef;
+my $savingToFile = undef;
+
+GetOptions (
+        "dbname|d=s" => \$czarDbName,
+        "label|l=s" => \$label,
+        "stage|s=s" => \$stage,
+        "interval|i=s" => \$interval,
+        "begin|b=s" => \$begin,
+        "end|e=s" => \$end,
+        "output|o=s" => \$path,
+        "histogram|h" => \$histogram,
+        "timeseries|t" => \$timeSeries,
+        "verbose|v" => \$verbose,
+        );
+
+print "\n";
+if (!$histogram) {
+    print "* OPTIONAL: plot histogram                  -h                          (default=off)\n";}
+if (!$timeSeries) {
+    print "* OPTIONAL: plot timeseries                 -t                          (default=on)\n";} 
+if (!$label) {
+    print "* OPTIONAL: choose a label                  -l <labellName>             (default='all_labels')\n";}
+if (!$stage) {
+    print "* OPTIONAL: choose a stage                  -s <chip|cam|warp|etc>      (default=none)\n";}
+if (!$interval) {
+    print "* OPTIONAL: choose time interval in past    -i <'1 hour'|'1 day'|etc>   (default=interval since 7am this morning)\n";} 
+if (!$begin) {
+    print "* OPTIONAL: choose a begin time             -b <datetime>               (default=7am this morning)\n";} 
+if (!$end) {
+    print "* OPTIONAL: choose an end time              -e <datetime>               (default=now)\n";} 
+if (!$path) {
+    print "* OPTIONAL: choose an output location       -o <path>                   (default=none)\n";} 
+
+print "\n";
+
+# default values
+if (!$label) {$label = "all_labels";}
+if (!$histogram && !$timeSeries) {$timeSeries = 1; $histogram = 1;}
+if (!$verbose) {$verbose = 0;}
+if (!$save_temps) {$save_temps = 0;}
+if (!$path) {$savingToFile = 0; $path = ".";}
+else {$savingToFile = 1;}
+
+my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", $verbose, $save_temps);
+$czarDb->setDateFormat("%Y%m%d-%H%i%s");
+
+my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", $savingToFile ? "png" : "X11", $path, $save_temps);
+
+my @allStages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+
+# sort out times
+if (!$end) {$end = $czarDb->getNowTimestamp();}
+if (!$begin) {
+    if ($interval) {$begin = $czarDb->subtractInterval($end, $interval);}
+    else {$begin =  strftime('%Y-%m-%d 07:00',localtime);}
+}
+
+if ($timeSeries) {$czarplot->createTimeSeries($label, $stage, $begin, $end);}
+if ($histogram) {$czarplot->createHistogram($label, $begin, $end);}
+
Index: /branches/eam_branches/ipp-20100621/tools/czartool.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/czartool.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/tools/czartool.pl	(revision 28794)
@@ -3,26 +3,35 @@
 use warnings;
 use strict;
-use PS::IPP::Config 1.01 qw( :standard );
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
-use Pod::Usage qw( pod2usage );
-use IPC::Cmd 0.36 qw( can_run run );
-use DBI;
-use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
-use File::Temp qw(tempfile);
+
+# local classes
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Pantasks;
+
+
+my $czarDbName = "czardb";
+my $save_temps = 1;
+my $interval = undef;
+
+GetOptions ( 
+        "dbname|d=s" => \$czarDbName,
+        "interval|i=s" => \$interval,
+        );
+
 
 my @states = ("full", "new", "drop", "wait");
-my $db = connectToDb();
-
-if (!$db) {die;}
-
-my @stdscienceLabels = `czartool_getLabels.pl -s stdscience`;
-my @distributionLabels = `czartool_getLabels.pl -s distribution`;
-my @publishingLabels = `czartool_getLabels.pl -s publishing`;
-
-checkAllLabels("new");
-printInstructions();
-
-poll();
-$db->disconnect();
+my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp");
+my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
+my $pantasks = new czartool::Pantasks();
+$czarDb->setDateFormat("%Y%m%d-%H%i%s");
+
+my @stdscienceLabels = undef;
+my @distributionLabels = undef;
+my @publishingLabels = undef;
+
+my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); 
+promptPoll();
+
 
 ###########################################################################
@@ -41,5 +50,11 @@
 #
 ###########################################################################
-sub poll {
+sub promptPoll {
+
+    @stdscienceLabels = @{$pantasks->getLabels("stdscience")};
+    @distributionLabels = @{$pantasks->getLabels("distribution")};
+    @publishingLabels = @{$pantasks->getLabels("publishing")};
+    checkAllLabels("new");
+    printInstructions();
 
     my $key;
@@ -60,25 +75,4 @@
 ###########################################################################
 #
-# Connects to the database
-#
-###########################################################################
-sub connectToDb {
-
-    my $dbname = 'gpc1';
-    my $dbserver = 'ippdb01';
-    my $dbuser = 'ippuser';
-    my $dbpass = 'ippuser';
-    my $db = DBI->connect(
-            "DBI:mysql:database=${dbname};host=${dbserver};mysql_socket=" .
-            DB_SOCKET(),${dbuser},${dbpass},
-            {
-            RaiseError => 1, AutoCommit => 1}
-            ) or die "Unable to connect to database $DBI::errstr\n";
-
-    return $db;
-}
-
-###########################################################################
-#
 # Compares two arrays and stores the differences
 #
@@ -120,5 +114,5 @@
 
         chomp($line);
-    
+
         print "$line\n";
     }
@@ -163,5 +157,5 @@
     chomp($input);
     if ($input =~ m/^([0-9]|[1-9][0-9]|[1-9][0-9][0-9])$/) {
-       return checkOneLabel($stdscienceLabels[$input]);
+        return checkOneLabel($stdscienceLabels[$input]);
     }
 
@@ -179,11 +173,12 @@
     chomp($label);
 
-    printf("\n+-------------------------------------------------------------------------------------+\n");
-    printf("|                        %32s                             |\n", $label);
-    printf("+------------+-------+-------+-------+-------+-------+-------+-------+--------+-------+\n");
-    printf("|   state    |  chip |  cam  |  fake | warp  | stack |  diff | magic |destreak|  dist |\n");
-    printf("+------------+-------+-------+-------+-------+-------+-------+-------+--------+-------+\n");
+    printf("\n+---------------------------------------------------------------------------------------------------------------------------------+\n");
+    printf("|                                                  %32s                                               |\n", $label);
+    printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
+    printf("|   state    |    chip    |    cam     |    fake    |    warp    |   stack    |    diff    |   magic    |  destreak  |    dist    |\n");
+    printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
 
     my $state;
+    my $stage;
     my $i=0;
     foreach $state (@states) {
@@ -191,18 +186,15 @@
         chomp($label);
         printf("| %10s ", $state);
-        printf("| %5s ", getStateAndFaults($label, "chipRun", $state, "chip"));
-        printf("| %5s ", getStateAndFaults($label, "camRun", $state, "cam"));
-        printf("| %5s ", getStateAndFaults($label, "fakeRun", $state, "fake"));
-        printf("| %5s ", getStateAndFaults($label, "warpRun", $state, "warp"));
-        printf("| %5s ", getStateAndFaults($label, "stackRun", $state, "stack"));
-        printf("| %5s ", getStateAndFaults($label, "diffRun", $state, "diff"));
-        printf("| %5s ", getStateAndFaults($label, "magicRun", $state, "magic"));
-        printf("| %6s ", getStateAndFaults($label, "magicDSRun", $state, "magicDS"));
-        printf("| %5s ", getStateAndFaults($label, "distRun", $state, "dist"));
+
+        foreach $stage (@stages) {
+
+            printf("| %10s ", getStateAndFaultsString($label, $state, $stage));
+        }
+
         printf("|\n");
         $i++;
     }
 
-    printf("+------------+-------+-------+-------+-------+-------+-------+-------+--------+-------+\n");
+    printf("+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+\n");
 
 }
@@ -225,7 +217,8 @@
     my $distLabel;
     my $pubLabel;
-    my $i=1;
     my $distributing;
     my $publishing;
+    my $stage;
+    my $i=1;
     foreach $stdsLabel (@stdscienceLabels) {
 
@@ -246,13 +239,10 @@
         printf("| %10s    ", $distributing ? "yes" : "NO" );
         printf("| %10s    ", $publishing ? "yes" : "NO" );
-        printf("| %10s ", getStateAndFaults($stdsLabel, "chipRun", $state, "chip"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "camRun", $state, "cam"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "fakeRun", $state, "fake"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "warpRun", $state, "warp"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "stackRun", $state, "stack"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "diffRun", $state, "diff"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "magicRun", $state, "magic"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "magicDSRun", $state, "magicDS"));
-        printf("| %10s ", getStateAndFaults($stdsLabel, "distRun", $state, "dist"));
+
+        foreach $stage (@stages) {
+
+            printf("| %10s ", getStateAndFaultsString($stdsLabel, $state, $stage));
+        }
+
         printf("|\n");
         $i++;
@@ -265,15 +255,15 @@
 ###########################################################################
 #
-# Returns state and fault-count (if new) as a string
-#
-###########################################################################
-sub getStateAndFaults {
-    my ($label, $table, $state, $stage) = @_;
-
-    my $new = checkLabel($label, $table, $state, $stage);
+# Returns state and fault-count (if new) as a string TODO untidy
+#
+###########################################################################
+sub getStateAndFaultsString {
+    my ($label, $state, $stage) = @_;
+
+    my $new = $gpc1Db->countExposures($label, $stage, $state);
 
     if ($state ne "new") {return $new;}
 
-    my $faults = countFaults($label,$table,$stage);
+    my $faults = $gpc1Db->countFaults($label, $stage);
 
     if ($faults < 1) {return $new;}
@@ -282,63 +272,3 @@
 }
 
-###########################################################################
-#
-# Returns count of exposures with this state for this label
-#
-###########################################################################
-sub checkLabel {
-    my ($label, $table, $state, $stage) = @_;
-
-    if ($state eq "fault") {return countFaults($label, $table, $stage);}
-
-    my $query = $db->prepare(<<SQL);
-    SELECT count(state)  
-        FROM $table 
-        WHERE label LIKE '$label' 
-        AND state = '$state'
-
-SQL
-
-        $query->execute;
-    return scalar $query->fetchrow_array();
-}
-
-###########################################################################
-#
-# Returns count of faults for this stage and label
-#
-###########################################################################
-sub countFaults {
-    my ($label, $table, $stage) = @_;
-
-    my $joinTable;
-    my $id = $stage."_id";
-
-    if ($stage eq "chip") {$joinTable="chipProcessedImfile";}
-    elsif ($stage eq "cam") {$joinTable="camProcessedExp";}
-    elsif ($stage eq "fake") {$joinTable="fakeProcessedImfile";}
-    elsif ($stage eq "warp") {$joinTable="warpSkyfile";}
-    elsif ($stage eq "stack") {$joinTable="stackSumSkyfile";}
-    elsif ($stage eq "diff") {$joinTable="diffSkyfile";}
-    elsif ($stage eq "magic") {$joinTable="magicNodeResult";}
-    elsif ($stage eq "magicDS") {$id = "magic_ds_id"; $joinTable="magicDSFile";}
-    elsif ($stage eq "dist") {$joinTable="distComponent";}
-    else {return -1;}
-
-    my $faultCol =  $joinTable.".fault";
-    my $stateCol =  $table.".state";
-
-    my $query = $db->prepare(<<SQL);
-    SELECT COUNT(DISTINCT $id) 
-        FROM $table
-        JOIN $joinTable USING ($id)
-        WHERE label LIKE '$label'
-        AND $faultCol != 0
-        AND $stateCol = 'new'
-
-SQL
-
-        $query->execute;
-    return scalar $query->fetchrow_array();
-}
-
+
Index: /branches/eam_branches/ipp-20100621/tools/czartool/CzarDb.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/czartool/CzarDb.pm	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/tools/czartool/CzarDb.pm	(revision 28794)
@@ -0,0 +1,537 @@
+#!/usr/bin/perl i-w
+
+package czartool::CzarDb;
+
+use warnings;
+use strict;
+
+my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist"); # TODO put elsewhere
+
+use base 'czartool::MySQLDb';
+our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+# Override constructor
+sub new {
+    my ($class) = @_;
+
+    # Call the constructor of the parent class, Person.
+    my $self = $class->SUPER::new( $_[1], $_[2], $_[3], $_[4],  $_[5], $_[6]);
+
+    bless $self, $class;
+
+    $self->update();    
+    return $self;
+}
+
+###########################################################################
+#
+# Gets current_labels table
+#
+###########################################################################
+sub getCurrentLabels {
+    my ($self, $server, $labels) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+
+    SELECT label
+        FROM current_labels
+        WHERE server LIKE '$server';
+SQL
+
+
+    if (!$query->execute) {
+
+        return 0;
+    }
+
+    ${$labels} = $query->fetchall_arrayref();
+
+    return 1;
+}
+
+###########################################################################
+#
+# Updates server table
+#
+###########################################################################
+sub updateServerStatus {
+    my ($self, $server, $alive, $running) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO servers
+        (server, alive, running)
+        VALUES
+        ('$server', $alive, $running);
+SQL
+
+       $query->execute;
+}
+
+###########################################################################
+#
+# Sets priority for this label
+#
+###########################################################################
+sub setLabelPriority {
+    my ($self, $label, $priority) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+     UPDATE current_labels 
+         SET priority = $priority 
+         WHERE label LIKE '$label' 
+         AND server LIKE 'stdscience';
+SQL
+
+    $query->execute;
+}
+
+###########################################################################
+#
+# Updates current_labels table
+#
+###########################################################################
+sub updateCurrentLabels {
+    my ($self, $server, $labels) = @_;
+
+    my $size = scalar @{$labels};
+    if ($size < 1) { return; }
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    DELETE FROM current_labels WHERE server LIKE '$server';
+SQL
+
+    $query->execute;
+
+    my $label = undef;
+
+    foreach $label (@{$labels}) {
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        INSERT INTO current_labels
+            (server, label)
+            VALUES
+            ('$server', '$label');
+SQL
+
+       $query->execute;
+    }
+}
+###########################################################################
+#
+# Inserts new time data into relevant table for a given label
+#
+###########################################################################
+sub insertNewTimeData {
+    my ($self, $stage, $label, $pending, $processed, $faults, $reverting) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO $stage
+        (label, pending, processed, faults, reverting)
+        VALUES
+        ('$label', $pending, $processed, $faults, $reverting);
+SQL
+
+        $query->execute;
+}
+
+###########################################################################
+#
+# Gets time series data and stores it to temp file
+#
+###########################################################################
+sub createTimeSeriesData {
+    my ($self, $label, $stage, $fromTime, $toTime, $minX, $maxX, $minY, $maxY, $timeDiff) = @_;
+
+    my $dataFile = "/tmp/czarplot_gnuplot_".$label."_".$stage."_t.dat";
+    open (GNUDAT, ">$dataFile");
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        MIN(processed), GREATEST(MAX(pending), MAX(faults), max(processed)-min(processed)) AS maxY, 
+        LEAST(MIN(pending), MIN(faults), max(processed)-min(processed)) AS minY,
+        DATE_FORMAT(MAX(timestamp),'$self->{_dateFormat}'), 
+        DATE_FORMAT(MIN(timestamp),'$self->{_dateFormat}'),
+        TIME_TO_SEC(TIMEDIFF(max(timestamp ), min(timestamp)))
+            FROM $stage 
+            WHERE label LIKE '$label'
+            AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+SQL
+
+
+
+    if (!$query->execute) {return undef;}
+
+    my $minProcessed;
+
+    ($minProcessed, ${$maxY}, ${$minY}, ${$maxX}, ${$minX}, ${$timeDiff}) = $query->fetchrow_array();
+
+    if (!$minProcessed) {return undef;}
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        DATE_FORMAT(timestamp, '$self->{_dateFormat}'), pending, faults, processed-$minProcessed 
+        FROM $stage 
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime' 
+        ORDER BY timestamp;
+SQL
+
+    $query->execute;
+
+    # loop round results
+    while (my @row = $query->fetchrow_array()) {
+
+        my ($timestamp, $pending, $faults, $processed) = @row;
+    #    print  "'$timestamp' '$pending' '$faults' '$processed'\n";
+        print GNUDAT "$timestamp $pending $faults $processed\n";
+    }
+    close(GNUDAT);
+
+    return $dataFile;
+}
+
+###########################################################################
+#
+# Returns number of faults at a particular time in the past for a given label and stage
+#
+###########################################################################
+sub countFaultsInPast {
+    my ($self, $label, $stage, $interval) = @_; # TODO use time not interval
+
+    # get earliest time for this label
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT GREATEST(MIN(timestamp), now() - INTERVAL $interval) 
+        FROM $stage
+        WHERE label LIKE '$label'; 
+SQL
+    $query->execute;
+
+my $timestamp = $query->fetchrow_array(); 
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT faults 
+        FROM $stage
+        WHERE label LIKE '$label' 
+        AND timestamp <= '$timestamp' LIMIT 1;
+SQL
+    $query->execute;
+
+return $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns the number of exposures that are pending right now for a given stage and label
+#
+###########################################################################
+sub countPendingNow {
+    my ($self, $label, $stage) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT pending - faults 
+        FROM $stage 
+        WHERE label LIKE '$label' 
+        ORDER BY timestamp DESC LIMIT 1;
+SQL
+        $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Creates histogram data
+#
+###########################################################################
+sub countProcessedPendingAndFaults {
+    my ($self, $label, $stage, $fromTime, $toTime, $processed, $pending, $faults) = @_;
+
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        pending, faults 
+        FROM $stage
+        WHERE label LIKE '$label'
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'
+        ORDER BY timestamp DESC LIMIT 1;
+SQL
+
+    $query->execute;
+    (${$pending}, ${$faults}) = $query->fetchrow_array();    
+
+    $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        MAX(processed) - MIN(processed) 
+        FROM $stage 
+        WHERE label LIKE '$label' 
+        AND timestamp >= '$fromTime' AND timestamp <= '$toTime'; 
+SQL
+    $query->execute;
+    (${$processed}) = $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Determines how much has been processed in the provided interval of time 
+# (format: 1 HOUR, 1 MINUTE 1 DAY etc)
+#
+###########################################################################
+sub countProcessed { # TODO use time not interval
+    my ($self, $label, $stage, $interval) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT 
+        MAX(processed) - MIN(processed) 
+        FROM $stage 
+        WHERE label LIKE '$label' 
+        AND timestamp > (now() - INTERVAL $interval); 
+SQL
+        $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Update Db to newest version 
+#
+###########################################################################
+sub update { 
+    my ($self) = @_;
+
+    my $currentRevision = -1;
+    my $latestRevision = 7;
+
+    while ($currentRevision != $latestRevision) {
+
+        $currentRevision = $self->getRevision();
+        if ($self->{_verbose}) {print "* Current Db revision = $currentRevision\n";}
+
+        if ($currentRevision == 0) {$self->createRevision_1();}
+        elsif ($currentRevision == 1) {$self->createRevision_2();}
+        elsif ($currentRevision == 2) {$self->createRevision_3();}
+        elsif ($currentRevision == 3) {$self->createRevision_4();}
+        elsif ($currentRevision == 4) {$self->createRevision_5();}
+        elsif ($currentRevision == 5) {$self->createRevision_6();}
+        elsif ($currentRevision == 6) {$self->createRevision_7();}
+    }
+}
+
+#######################################################################################
+# 
+# Create revision 1 of the database 
+#
+#######################################################################################
+sub createRevision_1 {
+    my ($self) = @_;
+
+    print "* Creating revision 1 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE revision (
+            revision INT, 
+            created TIMESTAMP DEFAULT NOW(),
+            primary key (revision)
+            );
+SQL
+        $query->execute;
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+    my $query = $self->{_db}->prepare(<<SQL);
+        CREATE TABLE $stage (
+                timestamp TIMESTAMP DEFAULT NOW(),
+                label VARCHAR(128) DEFAULT "NONE", 
+                pending BIGINT NOT NULL,
+                faults BIGINT NOT NULL
+                );
+SQL
+            $query->execute;
+
+    }
+
+    $self->setRevision(1);
+}
+
+#######################################################################################
+# 
+# Create revision 2 of the database 
+#
+#######################################################################################
+sub createRevision_2 {
+    my ($self) = @_;
+
+    print "* Creating revision 2 of '$self->{_dbName}'\n";
+
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+    my $query = $self->{_db}->prepare(<<SQL);
+        ALTER TABLE $stage 
+            ADD COLUMN reverting TINYINT NOT NULL DEFAULT 0;
+SQL
+            $query->execute;
+
+    }
+
+    $self->setRevision(2);
+}
+
+#######################################################################################
+# 
+# Create revision 3 of the database 
+#
+#######################################################################################
+sub createRevision_3 {
+    my ($self) = @_;
+
+    print "* Creating revision 3 of '$self->{_dbName}'\n";
+
+
+    my $stage = undef;
+    foreach $stage (@stages) {
+
+    my $query = $self->{_db}->prepare(<<SQL);
+        ALTER TABLE $stage 
+            ADD COLUMN processed BIGINT NOT NULL DEFAULT 0;
+SQL
+
+      $query->execute;
+    }
+
+    $self->setRevision(3);
+}
+
+#######################################################################################
+# 
+# Create revision 4 of the database 
+#
+#######################################################################################
+sub createRevision_4 {
+    my ($self) = @_;
+
+    print "* Creating revision 4 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE current_labels (server VARCHAR(128), label VARCHAR(128));
+SQL
+
+      $query->execute;
+
+    $self->setRevision(4);
+}
+
+#######################################################################################
+# 
+# Create revision 5 of the database 
+#
+#######################################################################################
+sub createRevision_5 {
+    my ($self) = @_;
+
+    print "* Creating revision 5 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    CREATE TABLE servers (
+            timestamp TIMESTAMP DEFAULT NOW(),
+            server VARCHAR(128), 
+            alive TINYINT, running TINYINT);
+SQL
+
+      $query->execute;
+
+    $self->setRevision(5);
+}
+
+#######################################################################################
+# 
+# Create revision 6 of the database 
+#
+#######################################################################################
+sub createRevision_6 {
+    my ($self) = @_;
+
+    print "* Creating revision 6 of '$self->{_dbName}'\n";
+
+    my $stage = undef;
+    my $index = undef;
+    foreach $stage (@stages) {
+
+        $index = $stage . "Index";
+        my $query = $self->{_db}->prepare(<<SQL);
+        CREATE INDEX $index ON $stage (timestamp, label);
+SQL
+
+      $query->execute;
+    }
+
+        my $query = $self->{_db}->prepare(<<SQL);
+        CREATE INDEX serverIndex ON servers (timestamp, server);
+SQL
+
+      $query->execute;
+
+
+    $self->setRevision(6);
+}
+
+#######################################################################################
+# 
+# Create revision 7 of the database 
+#
+#######################################################################################
+sub createRevision_7 {
+    my ($self) = @_;
+
+    print "* Creating revision 7 of '$self->{_dbName}'\n";
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    ALTER TABLE current_labels 
+        ADD COLUMN priority INT NOT NULL DEFAULT 0;
+SQL
+
+    $query->execute;
+
+    $self->setRevision(7);
+}
+
+#######################################################################################
+# 
+# Sets current revision of ippToPsps database
+#
+#######################################################################################
+sub setRevision {
+    my ($self, $revision) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    INSERT INTO revision (revision) VALUES ($revision);
+SQL
+    $query->execute;
+}
+
+#######################################################################################
+# 
+# Gets current revision of ippToPsps database
+#
+#######################################################################################
+sub getRevision {
+    my ($self) = @_;
+
+    if (!$self->SUPER::doesTableExist("revision")) {return 0;}
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT revision
+        FROM revision
+        ORDER BY revision DESC LIMIT 1;
+SQL
+
+    $query->execute;
+    my @row = $query->fetchrow_array();
+
+    return $row[0];
+}
+
+1;
+
Index: /branches/eam_branches/ipp-20100621/tools/czartool/Czarplot.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/czartool/Czarplot.pm	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/tools/czartool/Czarplot.pm	(revision 28794)
@@ -0,0 +1,269 @@
+#!/usr/bin/perl -w
+package czartool::Czarplot;
+
+use warnings;
+use strict;
+
+my @allStages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {
+        _czarDb => shift,
+        _dateFormat => shift,
+        _outputFormat => shift,
+        _outputPath => shift,
+        _save_temps => shift,
+    };
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Generates a suitable filename for this plot
+#
+###########################################################################
+sub createImageFileName {
+    my ($self, $label, $stage, $suffix) = @_;
+
+    my $prefix = $self->{_outputPath} ? $self->{_outputPath} : ".";
+    my $stagePart = $stage ? $stage : "all_stages";
+    return $prefix . "/czarplot_" . $label . "_" . $stagePart . "_$suffix.png";
+}
+
+###########################################################################
+#
+# Plots a time series for all stages for this label
+#
+###########################################################################
+sub createTimeSeries {
+    my ($self, $label, $selectedStage, $beginTime, $endTime) = @_;
+
+    my $outputFile = createImageFileName($self, $label, $selectedStage, "t");
+
+    my ($minX, $maxX, $minY, $maxY, $timeDiff);
+    $minX = 999999999;      
+    $maxX = -9999999999;
+    $minY = 999999999;          
+    $maxY = -99999999;
+
+    my $stages = undef;                 
+
+    if (!$selectedStage) {$stages = \@allStages;}
+    else {$stages = ["$selectedStage"]};        
+
+    my %gnuplotFiles;
+    my $stage = undef;
+    my $gnuplotFile = undef;
+    foreach $stage (@{$stages}) {
+
+        $gnuplotFile = $self->{_czarDb}->createTimeSeriesData($label, $stage, $beginTime, $endTime, \$minX, \$maxX, \$minY, \$maxY, \$timeDiff);
+        if ($gnuplotFile) {$gnuplotFiles{$stage} = $gnuplotFile;}
+
+    }                                                           
+
+    my $numOfPlots =  keys(%gnuplotFiles);
+
+    if ($numOfPlots == 0 ) {
+
+        print "Warning: No plots could be generated for '$label' during time period '$beginTime', '$endTime'\n";
+        return;
+    } 
+
+    if ($timeDiff == 0) {
+
+        print "WARNING: Zero time difference for '$label' during time period '$beginTime', '$endTime'\n";
+        return;
+    }
+
+    plotTimeSeries($self, \%gnuplotFiles, $outputFile, $label, $beginTime, $endTime, $maxX, $minX, $maxY, $minY, $timeDiff);
+}                                                    
+
+###########################################################################
+#
+# Plots a histogram of stuff processed in provided interval and label for all stages
+#
+###########################################################################
+sub createHistogram {
+    my ($self, $label, $beginTime, $endTime) = @_;
+
+    my $outputFile = createImageFileName($self, $label, undef, "h");
+
+    my $inputFile = "/tmp/czarplot_gnuplot_".$label."_h.dat"; # TODO use stage not table
+    open (GNUDAT, ">$inputFile");
+
+    my ($processed, $pending, $faults);
+    my $stage = undef;
+    foreach $stage (@allStages) {
+
+        $self->{_czarDb}->countProcessedPendingAndFaults($label, $stage, $beginTime, $endTime, \$processed, \$pending, \$faults);
+        print GNUDAT "$stage $processed, $pending, $faults\n";
+    }
+
+    close(GNUDAT);
+
+    plotHistogram($self, $inputFile, $outputFile, $label, $beginTime, $endTime);
+}
+
+
+###########################################################################
+#
+# Sets the output path 
+#
+###########################################################################
+sub setOutputFormat {
+    my ($self, $outputFormat) = @_;
+    $self->{_outputFormat} = $outputFormat if defined($outputFormat);
+}           
+
+###########################################################################
+#
+# Sets the output type 
+#
+###########################################################################
+sub setOutputPath {
+    my ($self, $outputPath) = @_;
+    $self->{_outputPath} = $outputPath if defined($outputPath);
+}           
+
+###########################################################################
+#
+# Sets the date format to be used
+#
+###########################################################################
+sub setDateFormat {
+    my ($self, $dateFormat) = @_;
+    $self->{_dateFormat} = $dateFormat if defined($dateFormat);
+    return $self->{_dateFormat};
+}       
+
+###########################################################################
+#
+# Sorts out plotting max/min x/y
+#
+###########################################################################
+sub sortOutMaxMinY {
+    my ($self, $minY, $maxY) = @_;
+
+    ${$maxY} = ${$maxY} + (${$maxY}/10);
+    ${$minY} = ${$minY} - (${$minY}/10);
+    if (${$maxY} == 0) {${$maxY} = 1;}
+    if (${$minY} == 0) {${$minY} = -1;}
+}
+
+###########################################################################
+#
+# Plots a time-series of pending/faults
+#
+###########################################################################
+sub plotTimeSeries {
+    my ($self, $gnuplotFiles, $outputFile, $label, $fromTime, $toTime, $maxX, $minX, $maxY, $minY, $timeDiff) = @_;
+
+    $self->sortOutMaxMinY(\$minY, \$maxY);
+
+    my $timeFormat = undef;
+
+    my $divX = $timeDiff/4;
+
+    # if less than a couple of hour's data plotted, show 30 mins tics
+    if ($timeDiff < 7200) {
+        $timeFormat = "%H:%M";
+        $divX = 1800;
+    }
+    # if less than half a day's data plotted, show hourly tics
+    elsif ($timeDiff < 43200) {
+        $timeFormat = "%H:%M";
+        $divX = 3600;
+    }
+    # if less than one day's data plotted, show 2 hourly tics
+    elsif ($timeDiff < 86400) {
+        $timeFormat = "%H:%M";
+        $divX = 7200;
+    }
+    # if more than one day's data plotted, show daily tics
+    else {
+        
+        $timeFormat = "%m/%d %H:%M";
+        $divX = 86400;
+    }
+
+    my $numOfPlots = keys %$gnuplotFiles;
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set output \"$outputFile\";" .
+        "set title \"All stages for '$label' since $fromTime\";" .
+        "set key left top;" .
+        "set xdata time;" .
+        "set timefmt \"$self->{_dateFormat}\";" .
+        "set xrange [\"$minX\":\"$maxX\"];" .
+        "set format x \"$timeFormat\";" .
+        "set xtics \"$minX\", $divX, \"$maxX\";" .
+        "set grid xtics;" .
+        "set xlabel \"Time\";" .
+        "set ylabel \"Exposures\";" .
+        "plot ";
+
+    my $firstIn = 1;
+    foreach my $stage (keys %$gnuplotFiles) {
+        if (!$firstIn) {print GP ",";}
+
+        # for single stage plots, show pending/processed/faults
+        if ($numOfPlots == 1) {
+
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"processed\" with lines lt 2,";
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:2 title \"pending\" with lines lt 3,";
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:3 title \"faults\" with lines lt 1";
+        }
+        # when plotting multiple stages, show only processed
+        else {
+            print GP "'" . $gnuplotFiles->{$stage} . "' using 1:4 title \"$stage\" with lines";
+        }
+        $firstIn = 0;
+    }
+
+    print GP "\n";
+    close GP;
+}
+
+###########################################################################
+#
+# Plots a histogram of processed stuff
+#
+###########################################################################
+sub plotHistogram {
+    my ($self, $inputFile, $outputFile, $label, $fromTime, $toTime) = @_;
+
+    open (GP, "|/usr/bin/gnuplot -persist") or die "no gnuplot";
+    use FileHandle;
+    GP->autoflush(1);
+
+    print GP
+        "set term $self->{_outputFormat};" .
+        "set output \"$outputFile\";" .
+        "set title \"Processing status for '$label' since $fromTime\";" .
+        "set grid;" .
+        "set boxwidth;" .
+        "set style data histogram;" .
+        "set style histogram cluster gap 3;" .
+        "set style fill solid border -1;" .
+        "set bmargin 5;" .
+        "set ylabel \"Exposures\";" .
+        "plot '$inputFile' using 2:xtic(1) title \"Processed\" lt 2, '' using 3 title \"Pending\" lt 3, '' using 4 title \"Faults\" lt 1\n";
+
+    close GP;
+}
+
+1;
Index: /branches/eam_branches/ipp-20100621/tools/czartool/Gpc1Db.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/czartool/Gpc1Db.pm	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/tools/czartool/Gpc1Db.pm	(revision 28794)
@@ -0,0 +1,115 @@
+#!/usr/bin/perl i-w
+
+package czartool::Gpc1Db;
+
+use warnings;
+use strict;
+
+use czartool::MySQLDb;
+our @ISA = qw(czartool::MySQLDb);    # inherits from MySQLDb
+
+###########################################################################
+#
+# Returns the right table for a given stage TODO make private
+#
+###########################################################################
+sub getTableForStage {
+    my ($self, $stage) = @_;
+
+    if ($stage eq "chip" ) {return "chipRun";}
+    elsif ($stage eq "cam" ) {return "camRun";}
+    elsif ($stage eq "fake" ) {return "fakeRun";}
+    elsif ($stage eq "warp" ) {return "warpRun";}
+    elsif ($stage eq "stack" ) {return "stackRun";}
+    elsif ($stage eq "diff" ) {return "diffRun";}
+    elsif ($stage eq "magic" ) {return "magicRun";}
+    elsif ($stage eq "magicDS" ) {return "magicDSRun";}
+    elsif ($stage eq "dist" ) {return "distRun";}
+
+    return "ERROR";
+}
+
+
+###########################################################################
+#
+# Returns count of faults for this stage and label
+#
+###########################################################################
+sub countFaults {
+    my ($self, $label, $stage) = @_;
+
+
+    my $table = getTableForStage($self, $stage);
+    my $joinTable;
+    my $id = $stage."_id";
+
+    if ($stage eq "chip") {$joinTable="chipProcessedImfile";}
+    elsif ($stage eq "cam") {$joinTable="camProcessedExp";}
+    elsif ($stage eq "fake") {$joinTable="fakeProcessedImfile";}
+    elsif ($stage eq "warp") {$joinTable="warpSkyfile";}
+    elsif ($stage eq "stack") {$joinTable="stackSumSkyfile";}
+    elsif ($stage eq "diff") {$joinTable="diffSkyfile";}
+    elsif ($stage eq "magic") {$joinTable="magicNodeResult";}
+    elsif ($stage eq "magicDS") {$id = "magic_ds_id"; $joinTable="magicDSFile";}
+    elsif ($stage eq "dist") {$joinTable="distComponent";}
+    else {return -1;}
+
+    my $faultCol =  $joinTable.".fault";
+    my $stateCol =  $table.".state";
+
+    my $query =  $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(DISTINCT $id) 
+        FROM $table
+        JOIN $joinTable USING ($id)
+        WHERE label LIKE '$label'
+        AND $faultCol != 0
+        AND $stateCol = 'new'
+SQL
+
+        $query->execute;
+    return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns priority for this label
+#
+###########################################################################
+sub getPriority {
+    my ($self, $label) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT priority 
+        FROM Label 
+        WHERE label LIKE '$label' 
+SQL
+
+        $query->execute;
+    my $priority = scalar $query->fetchrow_array();
+    if (!$priority) {return 50000;} # assume labels not given priority in gpc1 Db have highest priority
+    return $priority;
+}
+
+###########################################################################
+#
+# Returns count of exposures with this state for this label
+#
+###########################################################################
+sub countExposures {
+    my ($self, $label, $stage, $state) = @_;
+
+    my $table = getTableForStage($self, $stage);
+    if ($state eq "fault") {return countFaults($stage);}
+
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT count(state)  
+        FROM $table 
+        WHERE label LIKE '$label' 
+        AND state = '$state'
+SQL
+
+        $query->execute;
+    return scalar $query->fetchrow_array();
+}
+1;
Index: /branches/eam_branches/ipp-20100621/tools/czartool/MySQLDb.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/czartool/MySQLDb.pm	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/tools/czartool/MySQLDb.pm	(revision 28794)
@@ -0,0 +1,149 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use DBI;
+
+package czartool::MySQLDb;
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+
+    my $class = shift;
+    my $self = { 
+        _dbName => shift,
+        _dbHost => shift,
+        _dbUser => shift,
+        _dbPass => shift,
+        _verbose => shift,
+        _save_temps => shift,
+    };                              
+
+    my $dbname = $self->{_dbName};
+    my $dbhost = $self->{_dbHost};
+    my $dbuser = $self->{_dbUser};
+    my $dbpass = $self->{_dbPass};
+
+    use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+
+     $self->{_db} = DBI->connect("DBI:mysql:database=${dbname};host=${dbhost};" .
+            "mysql_socket=" . DB_SOCKET(),
+            ${dbuser},${dbpass},
+            { RaiseError => 1, AutoCommit => 1}
+            );
+
+    if ($self->{_verbose}) {printf("* Connection to " . $dbname . "@" . $dbhost . ": %s\n", $self->{_db} ? "success" : "FAILED ($DBI::errstr)");}
+
+    bless $self, $class;                                
+    return $self;                                           
+}                     
+
+###########################################################################
+#
+# Sets the date format to be used
+#
+###########################################################################
+sub setDateFormat {
+    my ($self, $dateFormat) = @_;
+    $self->{_dateFormat} = $dateFormat if defined($dateFormat);
+    return $self->{_dateFormat};
+}
+
+sub setDbHost {
+    my ( $self, $dbHost ) = @_;                              
+    $self->{_dbHost} = $dbHost if defined($dbHost);        
+    return $self->{_dbHost};                            
+}                                                                       
+
+sub getDbHost {
+    my( $self ) = @_;                                                       
+    return $self->{_dbHost};                                                 
+}                                                                               
+
+###########################################################################
+#
+# Subtracts the provided interval from the provided time
+#
+###########################################################################
+sub subtractInterval {
+    my ($self, $time, $interval) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT '$time' - INTERVAL $interval; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns whether time1 is before time2
+#
+###########################################################################
+sub isBefore {
+    my ($self, $time1, $time2) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT '$time1' < '$time2'; 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+###########################################################################
+#
+# Returns 'now' as a timestamp
+#
+###########################################################################
+sub getNowTimestamp {
+    my ($self) = @_;
+
+      my $query = $self->{_db}->prepare(<<SQL);
+          SELECT now(); 
+SQL
+    $query->execute;
+
+return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Checks whether a certain table exists 
+#
+#######################################################################################
+sub doesTableExist {
+    my ($self, $table) = @_;
+
+    my $query = $self->{_db}->prepare(<<SQL);
+    SELECT COUNT(*) 
+        FROM information_schema.tables 
+        WHERE table_schema = '$self->{_dbName}' 
+        AND table_name = '$table';
+SQL
+
+    $query->execute;
+
+    return scalar $query->fetchrow_array();
+}
+
+#######################################################################################
+# 
+# Destructor 
+#
+#######################################################################################
+sub DESTROY {
+    my($self) = @_;                                                       
+
+    if ($self->{_verbose}) {
+        print "* " . ref($self) . " is disconnecting from " . $self->{_dbName} . "@" . $self->{_dbHost} . "\n";
+    }
+    #$self->{_db}->disconnect();
+}
+1;                                        
+
Index: /branches/eam_branches/ipp-20100621/tools/czartool/Pantasks.pm
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/czartool/Pantasks.pm	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/tools/czartool/Pantasks.pm	(revision 28794)
@@ -0,0 +1,139 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+
+package czartool::Pantasks;
+
+
+my @servers = (
+        "addstar", 
+        "cleanup", 
+        "detrend", 
+        "distribution", 
+        "pstamp", 
+        "update", 
+        "publishing", 
+        "registration", 
+        "replication", 
+        "stdscience", 
+        "summitcopy"
+        );
+
+
+###########################################################################
+#
+# Returns the server list as an array 
+#
+###########################################################################
+sub getServerList {
+    my ($self) = @_;
+
+    return \@servers;
+}
+
+###########################################################################
+#
+# Constructor
+#
+###########################################################################
+sub new {
+    my $class = shift;
+    my $self = {};
+
+    bless $self, $class;
+    return $self;
+}
+
+###########################################################################
+#
+# Returns the correct server for this processing stage 
+#
+###########################################################################
+sub getServerForThisStage {
+    my ($self, $stage) = @_;
+
+    if ($stage eq "destreak" or $stage eq "dist") {return "distribution";}
+    return "stdscience";
+}
+
+
+###########################################################################
+#
+# Gets labels for provided server 
+#
+###########################################################################
+sub getLabels {
+    my ($self, $server) = @_;
+
+    my @labels;
+
+    my @cmdOut = `echo "show.labels;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    my $passedHeader=0;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/pantasks:\s+(.*)/) {
+
+            # HACK quit if we get 'show.labels: Command not found.'
+            if ($1 =~ m/.*show\.labels.*/i) {return \@labels;};
+            # HACK quit if we get 'Connection reset by peer'
+            if ($1 =~ m/.*Connection reset by peer.*/i) {return \@labels;};
+            # HACK quit if we get 'server is busy' message
+            if ($1 =~ m/.*server is busy.*/i) {return \@labels;};
+            # HACK to get around 'dummy' label in stdscience
+            if ($1 !~ m/.*dummy.*/) {push(@labels, $1);};
+            $passedHeader=1; 
+            next;
+        }
+
+        if ($passedHeader) {push(@labels, $line);}
+    }
+
+    return \@labels;
+}
+
+###########################################################################
+#
+# Returns whether this server is alive and running 
+#
+###########################################################################
+sub getServerStatus {
+    my ($self, $server, $alive, $running) = @_;
+
+    my @cmdOut = `echo "status ;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    ${$alive} = 0;
+    ${$running} = 0;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/Scheduler is stopped/) {${$running} = 0; ${$alive}=1;last;}
+        if ($line =~ m/Scheduler is running/) {${$running} = 1; ${$alive}=1;last;}
+        if ($line =~ m/Task Status/) {last;}
+
+    }
+}
+
+###########################################################################
+#
+# Gets server status 
+#
+###########################################################################
+sub getServerCurrentStatus {
+    my ($self, $server) = @_;
+
+
+    my @cmdOut = `echo "status;quit" | pantasks_client -c ~ipp/$server/ptolemy.rc 2>&1`;
+    my $line;
+    my $passedHeader=0;
+    foreach $line (@cmdOut) {
+
+        chomp($line);
+        if ($line =~ m/.*Task Status.*/) {$passedHeader=1;next;}
+        if ($passedHeader){print "$line<br>";}
+    }
+}
+
+
Index: /branches/eam_branches/ipp-20100621/tools/delete_convolved_diff.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/delete_convolved_diff.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/tools/delete_convolved_diff.pl	(revision 28794)
@@ -52,5 +52,5 @@
 JOIN diffSkyfile USING(diff_id)
 JOIN magicRun USING(diff_id)
-WHERE diffRun.state IN ('full', 'drop')
+WHERE diffRun.state IN ('full', 'drop', 'update')
     AND magicRun.state IN ('full', 'drop', 'censored')
     AND diffRun.bothways = 0
@@ -66,5 +66,5 @@
     ON magicNegative.diff_id = diffRun.diff_id
     AND magicNegative.inverse = 1
-WHERE diffRun.state IN ('full', 'drop')
+WHERE diffRun.state IN ('full', 'drop', 'update')
     AND magicPositive.state IN ('full', 'drop', 'censored')
     AND magicNegative.state IN ('full', 'drop', 'censored')
@@ -83,5 +83,5 @@
 #        foreach my $ext ( @extensions ) {
             my $file = "$path.$ext";
-            print "$file\n";
+#            print "$file\n";
             eval { $neb->delete($file); };
 #            $file = eval { $neb->find($file); };
Index: /branches/eam_branches/ipp-20100621/tools/errors.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/errors.pl	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/tools/errors.pl	(revision 28794)
@@ -82,4 +82,16 @@
     $label_table = "publishRun";
     $fault_table = "publishDone";
+} elsif ($stage eq "diffphot") {
+    $sql = "SELECT diff_phot_id, skycell_id, hostname, path_base FROM diffPhotRun JOIN diffPhotSkyfile USING(diff_phot_id) WHERE fault != 0 AND state = 'new'";
+    $label_table = "diffPhotRun";
+    $fault_table = "diffPhotSkycell";
+} elsif ($stage eq "chip_bg") {
+    $sql = "SELECT chip_bg_id, class_id, hostname, path_base FROM chipBackgroundRun JOIN chipBackgroundImfile USING(chip_bg_id) WHERE fault != 0 AND state = 'new'";
+    $label_table = "chipBackgroundRun";
+    $fault_table = "chipBackgroundImfile";
+} elsif ($stage eq "warp_bg") {
+    $sql = "SELECT warp_bg_id, skycell_id, hostname, path_base FROM warpBackgroundRun JOIN warpBackgroundSkyfile USING(warp_bg_id) WHERE fault != 0 AND state = 'new'";
+    $label_table = "warpBackgroundRun";
+    $fault_table = "warpBackgroundSkyfile";
 } else {
     die "Unsupported stage: $stage\n";
Index: /branches/eam_branches/ipp-20100621/tools/probe_burntool_states.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/probe_burntool_states.pl	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/tools/probe_burntool_states.pl	(revision 28794)
@@ -0,0 +1,245 @@
+#! /usr/bin/env perl
+
+use warnings;
+use strict;
+use Carp;
+use IPC::Cmd 0.36 qw( can_run run);
+use PS::IPP::Metadata::List qw( parse_md_list );
+use PS::IPP::Config 1.01 qw( :standard );
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use Pod::Usage qw( pod2usage );
+use DateTime;
+
+my $debug = 0;
+my $missing_tools = 0;
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+
+my $mdcParser = PS::IPP::Metadata::Config->new;
+
+my ($help,$date,$camera,$dbname,$verbose,$bt_state);
+
+GetOptions(
+    'help|h'           => \$help,
+    'date=s'           => \$date,
+    'camera=s'         => \$camera,
+    'dbname=s'         => \$dbname,
+    'verbose'          => \$verbose,
+    'bt_state=s'       => \$bt_state,
+    ) or pod2usage ( 2 );
+pod2usage( -msg =>
+"USAGE: probe_burntool_state.pl <options>
+       Options:
+          --help                 This help
+          --date YYYY-MM-DD      Check only this date.
+          --camera <camera>      Default GPC1.                                                
+          --dbname <db>          Default gpc1.
+          --verbose              
+          --bt_state <value>     Pretend this value is good.\n",
+	   -exitval => 2, ) if (defined($help));
+pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
+
+
+unless(defined($camera)) {
+    $camera = 'GPC1';
+}
+unless(defined($dbname)) {
+    $dbname = 'gpc1';
+}
+unless(defined($bt_state)) {
+    $bt_state = get_goodBTvalue();
+}
+# Read the config file for nightly science, and construct a list of targets.
+my @target_list;
+my %object_list;
+my %obsmode_list;
+my %comment_list;
+
+my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -";
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    run(command => $conf_cmd, verbose => $debug);
+unless ($success) {
+    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+    &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR);
+}
+
+my $metadata = $mdcParser->parse(join "", @$stdout_buf);
+foreach my $entry (@{ $metadata }) {
+    if (${ $entry }{name} eq 'TARGETS') {
+        my @target_data = @{ ${ $entry }{value} };
+        my $this_target = '';
+        
+        foreach my $tentry (@target_data) {
+            if (${ $tentry }{name} eq 'NAME') {
+                $this_target = ${ $tentry }{value};
+                push @target_list, $this_target;
+                $obsmode_list{$this_target} = '.*';
+                $object_list{$this_target} = '.*';
+                $comment_list{$this_target} = '.*';
+            }
+            elsif (${ $tentry }{name} eq 'OBSMODE') {
+                $obsmode_list{$this_target} = ${ $tentry }{value};
+                $obsmode_list{$this_target} =~ s/%//;
+            }
+            elsif (${ $tentry }{name} eq 'OBJECT') {
+                $object_list{$this_target} = ${ $tentry }{value};
+                $object_list{$this_target} =~ s/%//;
+            }
+            elsif (${ $tentry }{name} eq 'COMMENT') {
+                $comment_list{$this_target} = ${ $tentry }{value};
+                $comment_list{$this_target} =~ s/%//;
+            }
+        }
+    }
+}
+
+my $db = init_gpc_db();
+
+my ($start_date,$end_date);
+
+if (defined($date)) {
+    my ($year,$month,$day) = split /-/,$date;
+    $start_date = DateTime->new(year => $year, month => $month, day => $day,
+				hour => 0, minute => 0, second => 0, nanosecond => 0,
+				time_zone => 'Pacific/Honolulu');
+    $end_date   = DateTime->new(year => $year, month => $month, day => $day,
+				hour => 0, minute => 0, second => 0, nanosecond => 0,
+				time_zone => 'Pacific/Honolulu');
+    $end_date->add(days => 1);
+}
+else {
+    my @now = localtime();
+    $start_date = DateTime->new(year => 2009, month => 5, day => 1,
+				hour => 0, minute => 0, second => 0, nanosecond => 0,
+				time_zone => 'Pacific/Honolulu');
+    $end_date   = DateTime->new(year => 1900 + $now[5], month => $now[4] + 1, day => $now[3],
+				hour => 0, minute => 0, second => 0, nanosecond => 0,
+				time_zone => 'Pacific/Honolulu');
+    $end_date->add(days => 1);
+}
+
+my %to_process = ();
+while (DateTime->compare($start_date,$end_date) != 0) {
+    my $search_date = $start_date->ymd;
+#    print "$search_date " . $start_date->ymd . " " . $end_date->ymd . ">>$bt_state<<\n";
+    my $sth = "SELECT DISTINCT SUBSTR(rawExp.dateobs,1,10),rawExp.exp_type,rawExp.obs_mode,rawExp.object,rawExp.comment,burntool_state ";
+    $sth .=   " FROM rawExp JOIN rawImfile USING(exp_id) WHERE ";
+    $sth .=   "     date(rawExp.dateobs) >= '${search_date}T00:00:00' ";
+    $sth .=   " AND date(rawExp.dateobs) <= '${search_date}T23:59:59' ";
+    $sth .=   " ORDER BY SUBSTR(rawExp.dateobs,1,10),burntool_state ";
+#    print "$sth\n";
+    my $data_ref = $db->selectall_arrayref( $sth );
+    
+    my %count = ();
+    my $date = $search_date;
+    foreach my $row_ref (@{ $data_ref }) {
+	my ($date_str,$exp_type,$obs_mode,$object,$comment,$burntool_state) = @{ $row_ref };
+	unless(defined($burntool_state)) {
+	    $burntool_state = 0;
+	}
+	unless(defined($exp_type)) {
+	    $exp_type = '';
+	}
+	unless(defined($obs_mode)) {
+	    $obs_mode = '';
+	}
+	unless(defined($object)) {
+	    $object = '';
+	}
+	unless(defined($comment)) {
+	    $comment = '';
+	}
+#	print ">$date_str><$exp_type><$obs_mode><$object><$comment><$burntool_state>" . is_science($exp_type,$obs_mode,$object,$comment) . "<\n";
+	if (is_science($exp_type,$obs_mode,$object,$comment) == 1) {
+	    $count{$burntool_state} ++;
+	}
+    }
+#     if ($verbose) {
+# 	print "$date\n";
+#     }
+    foreach my $burntool_state (sort {$a <=> $b} (keys %count)) {
+	if ($verbose) {
+	    print "$date\t$count{$burntool_state}\t$burntool_state\t$bt_state\n";
+	}
+	unless(exists($to_process{$date})) {
+	    if ((abs($burntool_state) != $bt_state)
+		&&($count{$burntool_state} != 0)) {
+		print "\tbt.add.date $date\n";
+		$to_process{$date} = 1;
+	    }
+	}
+    }
+    $start_date->add(days => 1);
+}
+		
+
+
+
+sub get_goodBTvalue {
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+
+    my $config_cmd = "$ppConfigDump -camera $camera -dump-camera - | grep BURNTOOL.STATE.GOOD | uniq";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run ( command => $config_cmd, verbose => $debug);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", 0, 0, 0, $PS_EXIT_SYS_ERROR);
+    }
+    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", 0, 0, 0, $PS_EXIT_SYS_ERROR);
+    my $burntoolStateGood = 999;
+    foreach my $cfg (@$recipeData) {
+        if ($cfg->{name} eq 'BURNTOOL.STATE.GOOD') {
+            $burntoolStateGood = $cfg->{value};
+        }
+    }
+    if ($burntoolStateGood == 999) {
+        &my_die("Failed to determine BURNTOOL.STATE.GOOD", $burntoolStateGood, 0, 0, $PS_EXIT_SYS_ERROR);
+    }
+    return($burntoolStateGood);
+}
+
+
+sub init_gpc_db {
+    use constant DB_SOCKET => '/var/run/mysqld/mysqld.sock';
+    my $dbserver = 'ippdb01';
+    my $dbuser = 'ippuser';
+    my $dbpass = 'ippuser';
+    $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" .
+                       "mysql_socket=" . DB_SOCKET(),
+                       ${dbuser},${dbpass},
+		       { RaiseError => 1, AutoCommit => 1}
+        ) or die "Unable to connect to database $DBI::errstr\n";
+    return($db);
+}
+
+sub is_science {
+#    if (is_science($exp_type,$obs_mode,$object,$comment)) {
+    my ($exp_type,$obs_mode,$object,$comment) = @_;
+    my $return_value = 0;
+    unless($object) {
+        $object = '';
+    }
+    unless($exp_type) {
+	return(0);
+    }
+    if ($exp_type eq 'OBJECT') {
+        foreach my $target (@target_list) {
+#           print STDERR "$target $obsmode_list{$target} $exp_type $obs_mode $object $comment\n";
+            if (($obs_mode =~ /$obsmode_list{$target}/i)&&
+                ($object   =~ /$object_list{$target}/i)&&
+                ($comment  =~ /$comment_list{$target}/i)) {
+                $return_value = 1;
+            }
+#	    print ">$return_value $exp_type $obs_mode $object $comment <> $target $obsmode_list{$target} $object_list{$target} $comment_list{$target}\n";
+            if ($return_value) {
+                return($return_value);
+            }
+        }
+    }
+    return(0);
+}
Index: /branches/eam_branches/ipp-20100621/tools/roboczar.pl
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/roboczar.pl	(revision 28794)
+++ /branches/eam_branches/ipp-20100621/tools/roboczar.pl	(revision 28794)
@@ -0,0 +1,240 @@
+#!/usr/bin/perl -w
+
+use warnings;
+use strict;
+use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
+use POSIX qw/strftime/;
+
+# local classes
+use czartool::CzarDb;
+use czartool::Gpc1Db;
+use czartool::Pantasks;
+use czartool::Czarplot;
+
+
+my $period = 60;
+my $czarDbName = "czardb"; # TODO variables for other Db stuff, host etc
+my $save_temps = 0;
+
+GetOptions (
+        "period|p=s" => \$period, # TODO more Db args
+        "dbname|d=s" => \$czarDbName,
+        );
+
+my $czarDb = new czartool::CzarDb($czarDbName, "ippdb01", "ipp", "ipp", 0, $save_temps); # TODO last arg here is save_temps, should get as arg
+my $gpc1Db = new czartool::Gpc1Db("gpc1", "ippdb01", "ippuser", "ippuser");
+my $pantasks = new czartool::Pantasks();
+my $czarplot = new czartool::Czarplot($czarDb, "%Y%m%d-%H%M%S", "png", "/tmp", $save_temps);
+$czarDb->setDateFormat("%Y%m%d-%H%i%s");
+
+my @stages = ("chip", "cam", "fake", "warp", "stack", "diff", "magic", "magicDS", "dist");
+
+
+timePoll($period);
+
+###########################################################################
+#
+# Updates the labels from pantasks for all interested servers 
+#
+###########################################################################
+sub updateLabels {
+
+    print "* Updating labels\n";
+    my @servers = ("stdscience", "distribution", "publishing");
+
+    my $server = undef;
+    foreach $server (@servers) {
+
+        my @labels = @{$pantasks->getLabels($server)};
+        if (@labels) { 
+        
+            $czarDb->updateCurrentLabels($server, \@labels);
+        }
+        else {
+        
+             print "WARNING: No labels to update for '$server'\n";
+        }
+    }
+}
+
+###########################################################################
+#
+# Updates pantasks server status TODO should really get info for all servers at once 
+#
+###########################################################################
+sub updateServerStatus {
+    print "* Checking all pantasks servers\n";
+
+    my $servers = $pantasks->getServerList();
+
+    my $server = undef;
+    my $alive = undef;
+    my $running = undef;
+    foreach $server (@{$servers}) {
+
+        $pantasks->getServerStatus($server, \$alive, \$running);
+        $czarDb->updateServerStatus($server, $alive, $running);
+    }
+}
+
+###########################################################################
+#
+# Polls with provided period (seconds) 
+#
+###########################################################################
+sub timePoll {
+    my ($period) = @_;
+
+    my $label;
+    my $new;
+    my $full;
+    my $faults;
+    my $stage;
+    my ($totalNew,$totalFull,$totalFaults);
+    my $query = undef;
+    my $str = undef;
+    my $reverting = 0;
+    my $server = undef;
+    my $rows = undef;
+    my $row = undef;
+    my $begin = undef;
+    my $end = undef;
+    my $priority = undef;
+
+    while (1) {
+
+        updateServerStatus();
+        updateLabels();
+        if (!$czarDb->getCurrentLabels("stdscience", \$rows)) {next;}
+
+        my $size = @{$rows};
+        if($size < 1) {
+            
+            print "* WARNING: no stdscience labels found in Db\n";
+            next;
+        }
+
+        # get priority
+        foreach $row ( @{$rows} ) {
+            my ($label) = @{$row};
+            $priority = $gpc1Db->getPriority($label);
+            $czarDb->setLabelPriority($label, $priority);
+        }
+
+        # sort out times
+        $begin =  strftime('%Y-%m-%d 07:00',localtime);
+        $end = $czarDb->getNowTimestamp();
+
+        if ($czarDb->isBefore($end, $begin)) {
+
+            $begin = $czarDb->subtractInterval($begin, "1 DAY");
+        }
+
+        foreach $stage (@stages) {
+
+            $server = $pantasks->getServerForThisStage($stage);
+            $str = `czartool_revert.pl -s $server -t $stage`;
+            if ($str =~ m/on/) {$reverting = 1;}
+            else {$reverting = 0;}
+            print "* Checking labels for $stage stage\n";
+
+            $totalNew=$totalFaults=$totalFull=0;
+            foreach $row ( @{$rows} ) {
+                my ($label) = @{$row};
+
+                chomp($label);
+
+                $new = $gpc1Db->countExposures($label, $stage, "new");
+                $full = $gpc1Db->countExposures($label, $stage, "full");
+                $faults = $gpc1Db->countFaults($label, $stage);
+
+                #printf("%ld, %s, %s, %d, %d\n", $currentTime, $label, $stage, $new, $faults);
+                $totalNew += $new;
+                $totalFull += $full;
+                $totalFaults += $faults;
+
+                $czarDb->insertNewTimeData($stage, $label, $new, $full, $faults, $reverting);
+                $czarplot->createTimeSeries($label,  $stage, $begin, $end);
+            }
+
+            $czarDb->insertNewTimeData($stage, "all_labels", $totalNew, $totalFull, $totalFaults, $reverting);
+        }
+
+        print "* Generating plots\n";
+        foreach $row ( @{$rows} ) {
+            my ($label) = @{$row};
+
+            $czarplot->createTimeSeries($label, undef, $begin, $end);
+            $czarplot->createHistogram($label, $begin, $end);
+
+            #routineChecks($label, "1 HOUR");
+        }
+        $czarplot->createTimeSeries("all_labels", undef, $begin, $end);
+        $czarplot->createHistogram("all_labels", $begin, $end);
+        foreach $stage (@stages) {
+
+            $czarplot->createTimeSeries("all_labels",  $stage, $begin, $end); # TODO must be a neater way...
+        }
+        print "--------------------------------------------------------------------------\n";
+        print "* Going to sleep\n";
+        sleep($period);
+        print "* Waking up\n";
+
+        #sendEmail("roydhenderson\@gmail.com", "roboczar\@ipp.com", "Roboczar update", "Some content");
+    };
+}
+
+###########################################################################
+#
+# Performs some routine checks on processing status and sends alerts if it needs to 
+#
+###########################################################################
+sub routineChecks {
+    my ($label, $interval) = @_;
+
+    my $faultsNow;
+    my $faultsInPast;
+    my $newFaults;
+    my $pendingNow;
+    my $processedRecently;
+    my $stage = undef;
+
+    print "* Checking all stages for label $label\n";
+
+    foreach $stage (@stages) {
+
+        # check for increasing faults
+        $faultsNow = $czarDb->countFaultsInPast($label, $stage, "0 MINUTE");
+        $faultsInPast = $czarDb->countFaultsInPast($label, $stage, $interval);
+        if ($faultsNow > $faultsInPast) {
+            $newFaults = $faultsNow - $faultsInPast;
+            print "There have been $newFaults new faults in the last $interval (label='$label', stage='$stage')\n";
+        }
+
+        # check for lack of processing
+        $pendingNow =  $czarDb->countPendingNow($label, $stage);
+        $processedRecently = $czarDb->countProcessed($label, $stage, $interval);
+        if ($pendingNow > 0 && $processedRecently < 1) {
+
+            print "Only $processedRecently exposures have processed out of $pendingNow($faultsNow) pending in the last $interval (label='$label', stage='$stage')\n";
+
+        }
+    }
+}
+
+###########################################################################
+#
+# Sends an email 
+#
+###########################################################################
+sub sendEmail {
+    my ($to, $from, $subject, $message) = @_;
+
+    my $sendmail = '/usr/lib/sendmail';
+    open(MAIL, "|$sendmail -oi -t");
+    print MAIL "From: $from\n";
+    print MAIL "To: $to\n";
+    print MAIL "Subject: $subject\n\n";
+    print MAIL "$message\n";
+    close(MAIL);
+}
Index: /branches/eam_branches/ipp-20100621/tools/test_destreak
===================================================================
--- /branches/eam_branches/ipp-20100621/tools/test_destreak	(revision 28793)
+++ /branches/eam_branches/ipp-20100621/tools/test_destreak	(revision 28794)
@@ -116,4 +116,5 @@
             . " --uri $uri"
             . " --no-update"
+            . " --magicked 0"
 
             ;
@@ -184,4 +185,5 @@
             . " --uri $uri"
             . " --no-update"
+            . " --magicked 0"
         ;