Changeset 23594

branches/cnb_branches/cnb_branch_20090301

Property svn:mergeinfo changed
/trunk merged: 23354-23405,23409-23419,23426-23445,23447-23480,23485-23507,23524-23527,23530-23531,23533-23535,23538,23540-23567,23573-23577,23582-23593

branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/Changes

r20155	r23594
1	1	Revision history for Nebulous
	2
	3	0.17
	4	- retry database transactions when a deadlock is detected
2	5
3	6	0.16

branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/bin/neb-admin

-              r23352
+              r23594
     exit unless scalar @rows;
+    # compare number of responses to limit below
+    my $Npending = @rows;
     print "replicatePending MULTI\n\n";
 …
+    }
+    return 1;
+    # use a different exit status if we hit the limit (likely more files pending)
+    if ($Npending == $limit) {
+        exit 1;
+    }
+    exit 0;
+}
 sub removal

branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/lib/Nebulous/Server.pm

-              r20990
+              r23594
     my $uri;
+    eval {
+        {
+            # create storage_object
+            my $query = $db->prepare_cached( $sql->new_object );
+            $query->execute('NULL', $key->path);
+        }
+        my $so_id;
+        {
+            # get object ID
+            my $query = $db->prepare_cached( $sql->last_insert_id );
+            $query->execute;
+            ($so_id) = $query->fetchrow_array;
+            # XXX finish seems to be required when using LAST_INSERT_ID() or we
+            # get a warning about the stmt handling still be active the next
+            # time LAST_INSERT_ID() is invoked
+            $query->finish;
+        }
+        {
+            # create storage_object_attr
+            my $query = $db->prepare_cached( $sql->new_object_attr );
+            $query->execute($so_id);
+        }
+        {
+            # create instance with no URI
+TRANS: while (1) {
+        eval {
+            {
+                # create storage_object
+                my $query = $db->prepare_cached( $sql->new_object );
+                $query->execute('NULL', $key->path);
+            }
+            my $so_id;
+            {
+                # get object ID
+                my $query = $db->prepare_cached( $sql->last_insert_id );
+                $query->execute;
+                ($so_id) = $query->fetchrow_array;
+                # XXX finish seems to be required when using LAST_INSERT_ID() or we
+                # get a warning about the stmt handling still be active the next
+                # time LAST_INSERT_ID() is invoked
+                $query->finish;
+            }
+            {
+                # create storage_object_attr
+                my $query = $db->prepare_cached( $sql->new_object_attr );
+                $query->execute($so_id);
+            }
+            {
+                # create instance with no URI
 #            my $query = $db->prepare_cached( $sql->new_instance );
             my $query = $db->prepare_cached( $sql->new_object_instance );
             $query->execute($vol_id);
+        }
         my $ins_id;
+        {
             # get instance ID
             my $query = $db->prepare_cached( $sql->last_insert_id );
             $query->execute;
             ($ins_id) = $query->fetchrow_array;
             # XXX finish seems to be required when using LAST_INSERT_ID() or we
             # get a warning about the stmt handling still be active the next
             # time LAST_INSERT_ID() is invoked
             $query->finish;
+        }
         # Unfortunately, since we want to use the instance row's ID as part of the
         # actual on disk file name we can't try to create the file until after
         # we've create both a new storage_storage object and instance.
         # TODO add some stuff here to retry if unsucessful
         $uri = $self->_create_empty_instance_file($key->path, $so_id, $ins_id, $vol_path, $vol_xattr);
         $log->debug("created $uri on volume ID: $vol_id");
+        {
             # update the instance with URI & vol_id that the file is on
             my $query = $db->prepare_cached( $sql->update_instance_uri );
             # vol_id, uri, ins_id
             $query->execute($vol_id, "$uri", $ins_id);
+        }
         $db->commit;
         $log->debug("commit");
     };
     if ($@) {
+                my $query = $db->prepare_cached( $sql->new_object_instance );
+                $query->execute($vol_id);
+            }
+            my $ins_id;
+            {
+                # get instance ID
+                my $query = $db->prepare_cached( $sql->last_insert_id );
+                $query->execute;
+                ($ins_id) = $query->fetchrow_array;
+                # XXX finish seems to be required when using LAST_INSERT_ID() or we
+                # get a warning about the stmt handling still be active the next
+                # time LAST_INSERT_ID() is invoked
+                $query->finish;
+            }
+            # Unfortunately, since we want to use the instance row's ID as part of the
+            # actual on disk file name we can't try to create the file until after
+            # we've create both a new storage_storage object and instance.
+            # TODO add some stuff here to retry if unsucessful
+            $uri = $self->_create_empty_instance_file($key->path, $so_id, $ins_id, $vol_path, $vol_xattr);
+            $log->debug("created $uri on volume ID: $vol_id");
+            {
+                # update the instance with URI & vol_id that the file is on
+                my $query = $db->prepare_cached( $sql->update_instance_uri );
+                # vol_id, uri, ins_id
+                $query->execute($vol_id, "$uri", $ins_id);
+            }
+            $db->commit;
+            $log->debug("commit");
+        };
+        if ($@) {
 #        and not $key->soft_volume
+        $db->rollback;
+        $log->debug("rollback");
+        $log->logdie("error: $@");
+            $db->rollback;
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie("error: $@");
+        }
+        last;
+    }
 …
     $newkey = parse_neb_key($newkey);
+    eval {
+        # rename storage_object
+        my $query = $db->prepare_cached($sql->rename_object);
+        # this SQL statment takes the new key name as the first param
+        my $rows = $query->execute($newkey->path, $key->path);
+        # if we affected more then one row something very bad has happened.
+        unless ($rows == 1) {
+            $query->finish;
+            $log->logdie("affected row count is $rows instead of 1");
+        }
+        $db->commit;
+        $log->debug("commit");
+    };
+    if ($@) {
+        $db->rollback;
+        $log->debug("rollback");
+        $log->logdie("database error: $@");
+    }
+TRANS: while (1) {
+        eval {
+            # rename storage_object
+            my $query = $db->prepare_cached($sql->rename_object);
+            # this SQL statment takes the new key name as the first param
+            my $rows = $query->execute($newkey->path, $key->path);
+            # if we affected more then one row something very bad has happened.
+            unless ($rows == 1) {
+                $query->finish;
+                $log->logdie("affected row count is $rows instead of 1");
+            }
+            $db->commit;
+            $log->debug("commit");
+        };
+        if ($@) {
+            $db->rollback;
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie("database error: $@");
+        }
+        last;
+    }
     $log->debug("leaving");
 …
     # key1.swap -> key2
+    eval {
+        {
+            # key1 -> key1.swap
+            my $query = $db->prepare_cached($sql->rename_object);
+            # this SQL statment takes the new key name as the first param
+            my $rows = $query->execute($key1->path . ".swap", $key1->path);
+            # if we affected more then one row something very bad has happened.
+            unless ($rows == 1) {
+                $query->finish;
+                $log->logdie("affected row count is $rows instead of 1");
+            }
+        }
+        {
+            # key2 -> key1
+            my $query = $db->prepare_cached($sql->rename_object);
+            # this SQL statment takes the new key name as the first param
+            my $rows = $query->execute($key1->path, $key2->path);
+            # if we affected more then one row something very bad has happened.
+            unless ($rows == 1) {
+                $query->finish;
+                $log->logdie("affected row count is $rows instead of 1");
+            }
+        }
+        {
+            # key1.swap -> key2
+            my $query = $db->prepare_cached($sql->rename_object);
+            # this SQL statment takes the new key name as the first param
+            my $rows = $query->execute($key2->path, $key1->path . ".swap");
+            # if we affected more then one row something very bad has happened.
+            unless ($rows == 1) {
+                $query->finish;
+                $log->logdie("affected row count is $rows instead of 1");
+            }
+        }
+        $db->commit;
+        $log->debug("commit");
+    };
+    if ($@) {
+        $db->rollback;
+        $log->debug("rollback");
+        $log->logdie("database error: $@");
+TRANS: while (1) {
+        eval {
+            {
+                # key1 -> key1.swap
+                my $query = $db->prepare_cached($sql->rename_object);
+                # this SQL statment takes the new key name as the first param
+                my $rows = $query->execute($key1->path . ".swap", $key1->path);
+                # if we affected more then one row something very bad has happened.
+                unless ($rows == 1) {
+                    $query->finish;
+                    $log->logdie("affected row count is $rows instead of 1");
+                }
+            }
+            {
+                # key2 -> key1
+                my $query = $db->prepare_cached($sql->rename_object);
+                # this SQL statment takes the new key name as the first param
+                my $rows = $query->execute($key1->path, $key2->path);
+                # if we affected more then one row something very bad has happened.
+                unless ($rows == 1) {
+                    $query->finish;
+                    $log->logdie("affected row count is $rows instead of 1");
+                }
+            }
+            {
+                # key1.swap -> key2
+                my $query = $db->prepare_cached($sql->rename_object);
+                # this SQL statment takes the new key name as the first param
+                my $rows = $query->execute($key2->path, $key1->path . ".swap");
+                # if we affected more then one row something very bad has happened.
+                unless ($rows == 1) {
+                    $query->finish;
+                    $log->logdie("affected row count is $rows instead of 1");
+                }
+            }
+            $db->commit;
+            $log->debug("commit");
+        };
+        if ($@) {
+            $db->rollback;
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie("database error: $@");
+        }
+        last;
+    }
 …
     my $uri;
+    eval {
+        my $so_id;
+        {
+            # verify that at least one instance is currently available
+            my $query = $db->prepare_cached( $sql->get_object_instances );
+            my $rows = $query->execute($key->path, 1);
+            unless ( $rows > 0 ) {
+TRANS: while (1) {
+        eval {
+            my $so_id;
+            {
+                # verify that at least one instance is currently available
+                my $query = $db->prepare_cached( $sql->get_object_instances );
+                my $rows = $query->execute($key->path, 1);
+                unless ( $rows > 0 ) {
+                    $query->finish;
+                    $log->logdie( "storage object does not exist" );
+                }
+                $so_id = $query->fetchrow_hashref->{ 'so_id' };
                 $query->finish;
+                $log->logdie( "storage object does not exist" );
+            }
+            $so_id = $query->fetchrow_hashref->{ 'so_id' };
+            $query->finish;
+        }
+        {
+            my $query = $db->prepare_cached( $sql->new_instance );
+            $query->execute($so_id, $vol_id);
+        }
+        my $ins_id;
+        {
+            my $query = $db->prepare_cached( $sql->last_insert_id );
+            $query->execute();
+            ($ins_id) = $query->fetchrow_array;
+            # XXX finish seems to be required when using LAST_INSERT_ID() or we
+            # get a warning about the stmt handling still being active the next
+            # time LAST_INSERT_ID() is invoked
+            $query->finish;
+        }
+        # Unfortunately, since we want to use the instance row's ID as part of
+        # the actual on disk file name we can't try to create the file until
+        # after we've create both a new storage_storage object and instance.
+        # TODO add some stuff here to retry if unsucessful
+        $uri = $self->_create_empty_instance_file($key->path, $so_id, $ins_id, $vol_path, $vol_xattr);
+        {
+            my $query = $db->prepare_cached( $sql->update_instance_uri );
+            # vol_id, uri, ins_id
+            $query->execute($vol_id, $uri, $ins_id);
+        }
+        $db->commit;
+        $log->debug("commit");
+    };
+    if ($@) {
+        $db->rollback;
+        # handle soft volumes
+        if (defined $vol_name and defined $key->soft_volume) {
+            $log->debug("retrying with 'any' volume");
+            return $self->replicate_object($key->path, 'any');
+        }
+        $log->debug("rollback");
+        $log->logdie("error: $@");
+            }
+            {
+                my $query = $db->prepare_cached( $sql->new_instance );
+                $query->execute($so_id, $vol_id);
+            }
+            my $ins_id;
+            {
+                my $query = $db->prepare_cached( $sql->last_insert_id );
+                $query->execute();
+                ($ins_id) = $query->fetchrow_array;
+                # XXX finish seems to be required when using LAST_INSERT_ID() or we
+                # get a warning about the stmt handling still being active the next
+                # time LAST_INSERT_ID() is invoked
+                $query->finish;
+            }
+            # Unfortunately, since we want to use the instance row's ID as part of
+            # the actual on disk file name we can't try to create the file until
+            # after we've create both a new storage_storage object and instance.
+            # TODO add some stuff here to retry if unsucessful
+            $uri = $self->_create_empty_instance_file($key->path, $so_id, $ins_id, $vol_path, $vol_xattr);
+            {
+                my $query = $db->prepare_cached( $sql->update_instance_uri );
+                # vol_id, uri, ins_id
+                $query->execute($vol_id, $uri, $ins_id);
+            }
+            $db->commit;
+            $log->debug("commit");
+        };
+        if ($@) {
+            $db->rollback;
+            # handle soft volumes
+            if (defined $vol_name and defined $key->soft_volume) {
+                $log->debug("retrying with 'any' volume");
+                return $self->replicate_object($key->path, 'any');
+            }
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie("error: $@");
+        }
+        last;
+    }
 …
     my $write_lock;
+    eval {
+        {
+            # this will set update locks
+            my $query = $db->prepare_cached( $sql->get_object_locks );
+            my $rows = $query->execute( $key->path );
+            unless ( $rows == 1 ) {
+TRANS: while (1) {
+        eval {
+            {
+                # this will set update locks
+                my $query = $db->prepare_cached( $sql->get_object_locks );
+                my $rows = $query->execute( $key->path );
+                unless ( $rows == 1 ) {
+                    $query->finish;
+                    $log->logdie( "storage object does not exist" );
+                }
+                my $row = $query->fetchrow_hashref;
                 $query->finish;
+                $log->logdie( "storage object does not exist" );
+            }
+            my $row = $query->fetchrow_hashref;
+            $query->finish;
+            $so_id      = $row->{ 'so_id' };
+            $read_lock  = $row->{ 'read_lock' };
+            $write_lock = $row->{ 'write_lock' };
+        }
+        if ($type eq 'write') {
+            # can't set a write lock twice and
+            # can't set a write lock if there are read locks
+            if ($write_lock) {
+                $log->logdie("can not write lock twice -- retry");
+            }
+            if ($read_lock > 0) {
+                $log->logdie("can not write lock after read lock -- retry");
+            }
+            {
+                my $query = $db->prepare_cached( $sql->set_write_lock );
+                my $rows = $query->execute($key->path);
+                # if we affected more then one row something very bad has happened.
+                unless ($rows == 1) {
+                    $log->logdie("affected row count is $rows instead of 1");
+                }
+            }
+        } elsif ($type eq 'read') {
+            # can't set a read lock if there's a write lock
+            if ($write_lock) {
+                $log->logdie("can not read lock after write lock -- retry");
+            }
+            {
+                my $query = $db->prepare_cached( $sql->increment_read_lock );
+                my $rows = $query->execute($key->path);
+                # if we affected more then one row something very bad has happened.
+                unless ($rows == 1) {
+                    $log->logdie("affected row count is $rows instead of 1");
+                }
+            }
+        }
+        $db->commit;
+        $log->debug("commit");
+    };
+    if ($@) {
+        $db->rollback;
+        $log->debug("rollback");
+        $log->logdie("error: $@");
+                $so_id      = $row->{ 'so_id' };
+                $read_lock  = $row->{ 'read_lock' };
+                $write_lock = $row->{ 'write_lock' };
+            }
+            if ($type eq 'write') {
+                # can't set a write lock twice and
+                # can't set a write lock if there are read locks
+                if ($write_lock) {
+                    $log->logdie("can not write lock twice -- retry");
+                }
+                if ($read_lock > 0) {
+                    $log->logdie("can not write lock after read lock -- retry");
+                }
+                {
+                    my $query = $db->prepare_cached( $sql->set_write_lock );
+                    my $rows = $query->execute($key->path);
+                    # if we affected more then one row something very bad has happened.
+                    unless ($rows == 1) {
+                        $log->logdie("affected row count is $rows instead of 1");
+                    }
+                }
+            } elsif ($type eq 'read') {
+                # can't set a read lock if there's a write lock
+                if ($write_lock) {
+                    $log->logdie("can not read lock after write lock -- retry");
+                }
+                {
+                    my $query = $db->prepare_cached( $sql->increment_read_lock );
+                    my $rows = $query->execute($key->path);
+                    # if we affected more then one row something very bad has happened.
+                    unless ($rows == 1) {
+                        $log->logdie("affected row count is $rows instead of 1");
+                    }
+                }
+            }
+            $db->commit;
+            $log->debug("commit");
+        };
+        if ($@) {
+            $db->rollback;
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie("error: $@");
+        }
+        last;
+    }
 …
     my $write_lock;
+    eval {
+        {
+            # this will set update locks
+            my $query = $db->prepare_cached( $sql->get_object_locks );
+            my $rows = $query->execute($key->path);
+            unless ($rows == 1) {
+TRANS: while (1) {
+        eval {
+            {
+                # this will set update locks
+                my $query = $db->prepare_cached( $sql->get_object_locks );
+                my $rows = $query->execute($key->path);
+                unless ($rows == 1) {
+                    $query->finish;
+                    $log->logdie("storage object does not exist");
+                }
+                my $row = $query->fetchrow_hashref;
                 $query->finish;
+                $log->logdie("storage object does not exist");
+            }
+            my $row = $query->fetchrow_hashref;
+            $query->finish;
+            $so_id      = $row->{ 'so_id' };
+            $read_lock  = $row->{ 'read_lock' };
+            $write_lock = $row->{ 'write_lock' };
+        }
+        if ($type eq 'write') {
+            # can't remove a write lock if it doesn't exist
+            if ($read_lock) {
+                $log->logdie("can not have a write lock under a read lock");
+            }
+            unless ($write_lock) {
+                $log->logdie("can not remove non-existant write lock");
+            }
+            {
+                my $query = $db->prepare_cached( $sql->delete_write_lock );
+                my $rows = $query->execute($key->path);
+                # if we affected more then one row something very bad has happened.
+                unless ($rows == 1) {
+                    $log->logdie("affected row count is $rows instead of 1");
+                }
+            }
+        } elsif ($type eq 'read') {
+            # can't remove a read lock if there's a write lock and
+            # can't remove a read lock if there aren't any
+            if ($write_lock) {
+                $log->logdie("can not have a read lock under a write lock");
+            }
+            if ($read_lock == 0) {
+                $log->logdie("can not remove non-existant read lock");
+            }
+            {
+                my $query = $db->prepare_cached( $sql->decrement_read_lock );
+                my $rows = $query->execute($key->path);
+                # if we affected more then one row something very bad has happened.
+                unless ($rows == 1) {
+                    $log->logdie("affected row count is $rows instead of 1");
+                }
+            }
+        }
+        $db->commit;
+        $log->debug("commit");
+    };
+    if ($@) {
+        $db->rollback;
+        $log->debug("rollback");
+        $log->logdie("error: $@");
+                $so_id      = $row->{ 'so_id' };
+                $read_lock  = $row->{ 'read_lock' };
+                $write_lock = $row->{ 'write_lock' };
+            }
+            if ($type eq 'write') {
+                # can't remove a write lock if it doesn't exist
+                if ($read_lock) {
+                    $log->logdie("can not have a write lock under a read lock");
+                }
+                unless ($write_lock) {
+                    $log->logdie("can not remove non-existant write lock");
+                }
+                {
+                    my $query = $db->prepare_cached( $sql->delete_write_lock );
+                    my $rows = $query->execute($key->path);
+                    # if we affected more then one row something very bad has happened.
+                    unless ($rows == 1) {
+                        $log->logdie("affected row count is $rows instead of 1");
+                    }
+                }
+            } elsif ($type eq 'read') {
+                # can't remove a read lock if there's a write lock and
+                # can't remove a read lock if there aren't any
+                if ($write_lock) {
+                    $log->logdie("can not have a read lock under a write lock");
+                }
+                if ($read_lock == 0) {
+                    $log->logdie("can not remove non-existant read lock");
+                }
+                {
+                    my $query = $db->prepare_cached( $sql->decrement_read_lock );
+                    my $rows = $query->execute($key->path);
+                    # if we affected more then one row something very bad has happened.
+                    unless ($rows == 1) {
+                        $log->logdie("affected row count is $rows instead of 1");
+                    }
+                }
+            }
+            $db->commit;
+            $log->debug("commit");
+        };
+        if ($@) {
+            $db->rollback;
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie("error: $@");
+        }
+        last;
+    }
 …
     $key = parse_neb_key($key);
+    eval {
+        my $query;
+        if ($flags eq 'create') {
+            $query = $db->prepare_cached( $sql->new_object_xattr );
+        } else {
+            # replace
+            $query = $db->prepare_cached( $sql->replace_object_xattr );
+        }
+        # name, value, ext_id
+        my $rows = $query->execute($name, $value, $key->path);
+        $query->finish;
+        # if we affected more then one row something very bad has happened.
+        if ($flags eq 'create') {
+            unless ($rows == 1) {
+                $log->logdie( "affected row count is $rows instead of 1" );
+            }
+        } else {
+            # replace_object_xattr can effect either 1 or 2 rows.  2 rows in
+            # the case of a replace and 1 if the xattr didn't already exist.
+            unless ($rows == 1 or $rows == 2) {
+                $log->logdie( "affected row count is $rows instead of 2" );
+            }
+        }
+        $db->commit;
+        $log->debug("commit");
+    };
+    if ($@) {
+        $db->rollback;
+        $log->debug("rollback");
+        $log->logdie("database error: $@");
+TRANS: while (1) {
+        eval {
+            my $query;
+            if ($flags eq 'create') {
+                $query = $db->prepare_cached( $sql->new_object_xattr );
+            } else {
+                # replace
+                $query = $db->prepare_cached( $sql->replace_object_xattr );
+            }
+            # name, value, ext_id
+            my $rows = $query->execute($name, $value, $key->path);
+            $query->finish;
+            # if we affected more then one row something very bad has happened.
+            if ($flags eq 'create') {
+                unless ($rows == 1) {
+                    $log->logdie( "affected row count is $rows instead of 1" );
+                }
+            } else {
+                # replace_object_xattr can effect either 1 or 2 rows.  2 rows in
+                # the case of a replace and 1 if the xattr didn't already exist.
+                unless ($rows == 1 or $rows == 2) {
+                    $log->logdie( "affected row count is $rows instead of 2" );
+                }
+            }
+            $db->commit;
+            $log->debug("commit");
+        };
+        if ($@) {
+            $db->rollback;
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie("database error: $@");
+        }
+        last;
+    }
 …
     $key = parse_neb_key($key);
+    eval {
+        my $query = $db->prepare_cached( $sql->remove_object_xattr );
+        # ext_id, name
+        my $rows = $query->execute($key->path, $name);
+        $query->finish;
+        # if we affected more then one row something very bad has happened.
+        unless ($rows == 1) {
+            $log->logdie( "affected row count is $rows instead of 1" );
+        }
+        $db->commit;
+        $log->debug("commit");
+    };
+    if ($@) {
+        $db->rollback;
+        $log->debug("rollback");
+        $log->logdie("database error: $@");
+    }
+TRANS: while (1) {
+        eval {
+            my $query = $db->prepare_cached( $sql->remove_object_xattr );
+            # ext_id, name
+            my $rows = $query->execute($key->path, $name);
+            $query->finish;
+            # if we affected more then one row something very bad has happened.
+            unless ($rows == 1) {
+                $log->logdie( "affected row count is $rows instead of 1" );
+            }
+            $db->commit;
+            $log->debug("commit");
+        };
+        if ($@) {
+            $db->rollback;
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie("database error: $@");
+        }
+        last;
+    }
     $log->debug("leaving");
 …
     $log->debug( "entered - @_" );
+    eval {
+        my $so_id;
+        my $instances;
+        # get so_id
+        {
+            my $query = $db->prepare_cached( $sql->get_object_from_uri );
+            my $rows = $query->execute( $uri );
+            unless ( $rows > 0 ) {
+TRANS: while (1) {
+        eval {
+            my $so_id;
+            my $instances;
+            # get so_id
+            {
+                my $query = $db->prepare_cached( $sql->get_object_from_uri );
+                my $rows = $query->execute( $uri );
+                unless ( $rows > 0 ) {
+                    $query->finish;
+                    $log->logdie( "no instance is associated with uri" );
+                }
+                $so_id = $query->fetchrow_hashref->{ 'so_id' };
                 $query->finish;
+                $log->logdie( "no instance is associated with uri" );
+            }
+            $so_id = $query->fetchrow_hashref->{ 'so_id' };
+            $query->finish;
+        }
+        {
+            my $query = $db->prepare_cached( $sql->get_instance_count );
+            $query->execute( $so_id );
+            $instances = $query->fetchrow_hashref->{ 'count(ins_id)' };
+            $query->finish;
+        }
+        # remove instance
+        {
+            my $query = $db->prepare_cached( $sql->delete_instance );
+            my $rows = $query->execute( $uri );
+            $query->finish;
+            # if we affected something other then two rows something very bad
+            # has happened
+            unless ( $rows == 1 ) {
+                $log->logdie( "affected row count is $rows instead of 1" );
+            }
+        }
+        # if we just deleted the last instance associated with a storage object
+        # remove it too
+        if ( $instances == 1 ) {
+            # we just removed the last instance
+            my $query = $db->prepare_cached( $sql->delete_object );
+            my $rows = $query->execute( $so_id );
+            $query->finish;
+            # TODO: this will have to be changed in order to support hardlinks
+            unless ( $rows == 1 ) {
+                $log->logdie( "affected row count is $rows instead of 2" );
+            }
+        }
+        $db->commit;
+        $log->debug("commit");
+    };
+    if ( $@ ) {
+        $db->rollback;
+        $log->debug("rollback");
+        $log->logdie( "database error: $@" );
+            }
+            {
+                my $query = $db->prepare_cached( $sql->get_instance_count );
+                $query->execute( $so_id );
+                $instances = $query->fetchrow_hashref->{ 'count(ins_id)' };
+                $query->finish;
+            }
+            # remove instance
+            {
+                my $query = $db->prepare_cached( $sql->delete_instance );
+                my $rows = $query->execute( $uri );
+                $query->finish;
+                # if we affected something other then two rows something very bad
+                # has happened
+                unless ( $rows == 1 ) {
+                    $log->logdie( "affected row count is $rows instead of 1" );
+                }
+            }
+            # if we just deleted the last instance associated with a storage object
+            # remove it too
+            if ( $instances == 1 ) {
+                # we just removed the last instance
+                my $query = $db->prepare_cached( $sql->delete_object );
+                my $rows = $query->execute( $so_id );
+                $query->finish;
+                # TODO: this will have to be changed in order to support hardlinks
+                unless ( $rows == 1 ) {
+                    $log->logdie( "affected row count is $rows instead of 2" );
+                }
+            }
+            $db->commit;
+            $log->debug("commit");
+        };
+        if ( $@ ) {
+            $db->rollback;
+            $log->debug("rollback");
+            if ($@ =~ /Deadlock found/) {
+                $log->warn("database deadlock retrying transaction: $@");
+                redo TRANS;
+            }
+            $log->logdie( "database error: $@" );
+        }
+        last;
+    }

branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/stats.pl

-              r23352
+              r23594
         my $stat = Statistics::Descriptive::Sparse->new();
         $stat->add_data(@$duration);
         printf("%-40s %f\n", "$method: mean duration: ", $stat->mean());
+        printf("%-40s : %f\n", "$method - mean duration", $stat->mean());
+    }
+    {
 …
         $stat->add_data(@$times);
         $total_stat->add_data(@$times);
         printf("%-40s %f\n","$method: calls/s: ", $stat->count / ($stat->max - $stat->min));
+        printf("%-40s : %f\n","$method - calls/s", $stat->count / ($stat->max - $stat->min));
+    }
+}
+print "max : ", $total_stat->max, "\n";
+print "min : ", $total_stat->min, "\n";
+print "total s : ", ($total_stat->max - $total_stat->min), "\n";
+print "count : ", $total_stat->count, "\n";
+print "total : calls/s: ", $total_stat->count / ($total_stat->max - $total_stat->min), "\n";
+print "\n";
+printf("%-40s : %f\n", "first timestamp", $total_stat->min);
+printf("%-40s : %f\n", "last timestamp", $total_stat->max);
+printf("%-40s : %f\n", "total wallclock seconds", ($total_stat->max - $total_stat->min));
+printf("%-40s : %f\n", "total call count", $total_stat->count);
+printf("%-40s : %f\n", "total calls/s:", $total_stat->count / ($total_stat->max - $total_stat->min));

branches/cnb_branches/cnb_branch_20090301/Ohana

Property svn:mergeinfo changed

/branches/eam_branches/eam_branch_20090303/Ohana (added)	merged: 23220-23221,23225
/branches/eam_branches/eam_branch_20090322/Ohana (added)	merged: 23482-23484,23512-23519,23522-23523,23528-23529
/trunk/Ohana	merged: 23530-23531,23541,23553-23558,23563,23582-23583,23586,23593

branches/cnb_branches/cnb_branch_20090301/Ohana/src/libohana/src/IOBufferOps.c

-              r17483
+              r23594
 int ReadtoIOBuffer (IOBuffer *buffer, int fd) {
   int Nread, Nfree;
+  int Nread, Nfree, Nwant;
   if (fd == 0) {
 …
+  }
+  // if we run out of space, double Nblock
   Nfree = buffer[0].Nalloc - buffer[0].Nbuffer;
   if (Nfree < buffer[0].Nblock) {
 …
+  }
+  Nread = read (fd, &buffer[0].buffer[buffer[0].Nbuffer], buffer[0].Nblock);
+  if (DEBUG) fprintf (stderr, "read IO buffer: (%lx) %d from %d\n", (unsigned long) buffer, Nread, buffer[0].Nblock);
+  // ensure we never read more than space available
+  Nwant = MIN (Nfree, buffer[0].Nblock);
+  Nread = read (fd, &buffer[0].buffer[buffer[0].Nbuffer], Nwant);
+  if (DEBUG) fprintf (stderr, "read IO buffer: (%lx) %d from %d\n", (unsigned long) buffer, Nread, Nwant);
   /* on success, increase the block size for the next read */

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.basic/break.c

-              r7917
+              r23594
     remove_argument (N, &argc, argv);
     value = -1;
     if (!strcasecmp (argv[N], "on")) value = TRUE;
     if (!strcasecmp (argv[N], "off")) value = FALSE;
+    if (!strcasecmp (argv[N], "on")) value = 1;
+    if (!strcasecmp (argv[N], "off")) value = 0;
     if (value == -1) {
       gprint (GP_ERR, "USAGE: break -auto [on / off]\n");

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/cmd.basic/run_if.c

-              r16462
+              r23594
         status = multicommand (input);
         if (ThisList == 0) add_history (input);
+        if (auto_break && !status) return (FALSE);
+        if (auto_break && !status) {
+          free (input);
+          return (FALSE);
+        }
+      }
+    }

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dimm/camera_cmds.c

Property svn:mergeinfo changed
/trunk/Ohana/src/opihi/dimm/camera_cmds.c merged: 23530

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dimm/telescope_cmds.c

Property svn:mergeinfo changed
/trunk/Ohana/src/opihi/dimm/telescope_cmds.c merged: 23530

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/include/pantasks.h

-              r21153
+              r23594
 int CheckControllerOutput (void);
 int PrintControllerOutput (void);
+void PrintControllerBusyJobs ();
 int AddHost (char *hostname, int max_threads);
 …
 // functions related to the server threads
+void *CheckJobsAndTasksThread (void *data);
 void CheckTasksSetState (int state);
 int CheckTasksGetState (void);
-void *CheckTasksThread (void *data);
 void CheckJobsSetState (int state);
 int CheckJobsGetState (void);
+void *CheckJobsThread (void *data);
+// void *CheckTasksThread (void *data);
+// void *CheckJobsThread (void *data);
 void CheckControllerSetState (int state);
 …
 void CheckInputs (void);
+void SerialThreadLock (void);
+void SerialThreadUnlock (void);
+void ClientThreadLock (void);
+void ClientThreadUnlock (void);
+void CommandThreadLock (void);
+void CommandThreadUnlock (void);
+void ControlThreadLock (void);
+void ControlThreadUnlock (void);
+void JobTaskThreadLock (void);
+void JobTaskThreadUnlock (void);
 int InitPassword (void);

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/include/shell.h

r21153	r23594
166	166	int gprint PROTO((gpDest dest, char *format, ...));
167	167	int gwrite PROTO((char *buffer, int size, int N, gpDest dest));
	168	int gprint_syserror PROTO((gpDest dest, int myError, char *format, ...));
	169	int gprintv PROTO((gpDest dest, char *format, va_list argp));
168	170
169	171	/* socket functions */

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/lib.shell/SocketOps.c

r21041	r23594
5	5	# define DEBUG 0
6	6
	7	// these three static variables are only modified in the setup command before
	8	// the threads are started. Thread-safety is not a problem for these.
7	9	static int NVALID;
8	10	static int Nvalid;

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/lib.shell/gprint.c

-              r22423
+              r23594
   int status;
-  gpStream *stream;
   va_list argp;
+  va_start (argp, format);
+  status = gprintv (dest, format, argp);
+  va_end (argp);
+  return (status);
+}
+int gprintv (gpDest dest, char *format, va_list argp) {
+  int status;
+  gpStream *stream;
   // this thread only writes to its own stream
   stream = gprintGetStream (dest);
-  va_start (argp, format);
   if (stream[0].mode == GP_FILE) {
 …
     vPrintIOBuffer (stream[0].buffer, format, argp);
+  }
-  va_end (argp);
   return (TRUE);
+}
 …
+}
+/* I'm going to need to have different output targets for different threads
+   we can do this with these functions:
+   pthread_t pthread_self(void);
+   int pthread_equal(pthread_t thread1, pthread_t thread2);
+   // returns TRUE if equal, FALSE if not
+*/
+# define MAX_ERROR_LENGTH 256           // Maximum length string for error messages
+// print an error (based on errno values) to gprint destination
+int gprint_syserror (gpDest dest, int myError, char *format, ...) {
+  char errorBuf[MAX_ERROR_LENGTH];
+  char *errorMsg;
+  va_list argp;
+  // there are two strerror_r implementations; choose the right one:
+#if ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && ! _GNU_SOURCE)
+  errorMsg = strerror_r (myError, errorBuf, MAX_ERROR_LENGTH);
+#else
+  strerror_r (myError, errorBuf, MAX_ERROR_LENGTH);
+  errorMsg = errorBuf;
+#endif
+  va_start (argp, format);
+  gprintv (dest, format, argp);
+  va_end (argp);
+  gprintv (dest, "%s\n", errorMsg);
+  return TRUE;
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/CheckController.c

-              r14590
+              r23594
 # include "pantasks.h"
+static struct timeval start;
+void TimerMark ();
+float TimerElapsed (int reset);
+void TimerMark (struct timeval *start);
+float TimerElapsed (struct timeval *start, int reset);
 int CheckController () {
 …
   Job *job;
   IOBuffer buffer;
+  struct timeval start;
   /* get the list of completed jobs (exit / crash), update the job status */
 …
   /*** check EXIT jobs ***/
   InitIOBuffer (&buffer, 0x100);
-  // TimerMark ();
-  // status = ControllerCommand ("stop", CONTROLLER_PROMPT, &buffer);
-  // if (VerboseMode()) gprint (GP_ERR, "stop controller %f\n", TimerElapsed(TRUE));
   TimerMark ();
+  TimerMark (&start);
   FlushIOBuffer (&buffer);
   status = ControllerCommand ("jobstack exit", CONTROLLER_PROMPT, &buffer);
   if (VerboseMode()) gprint (GP_ERR, "check exit stack %f\n", TimerElapsed(TRUE));
+  if (VerboseMode()) gprint (GP_ERR, "check exit stack %f\n", TimerElapsed(&start, TRUE));
   if (!status) goto escape;
 …
   status = sscanf (buffer.buffer, "%*s %d", &Njobs);
   if (status != 1) goto escape;
   if (VerboseMode()) gprint (GP_ERR, "parse %d jobs on stack %f\n", Njobs, TimerElapsed(TRUE));
+  if (VerboseMode()) gprint (GP_ERR, "parse %d jobs on stack %f\n", Njobs, TimerElapsed(&start, TRUE));
   p = buffer.buffer;
 …
     status = sscanf (p, "%d", &JobID);
+    // the operations within this locked block only interact with the controller or
+    // modify the properties of the selected job
+    JobTaskLock();
     job = FindControllerJob (JobID);
     if (job == NULL) {
       gprint (GP_ERR, "misplaced job? %d not in EXIT job list\n", JobID);
+      JobTaskUnlock();
       continue;
+    }
     /* this checks the individual job status, grabs stdout/stderr */
     CheckControllerJob (job);
+    JobTaskUnlock();
+  }
   if (VerboseMode()) gprint (GP_ERR, "clear %d exit jobs %f\n", i, TimerElapsed(TRUE));
+  if (VerboseMode()) gprint (GP_ERR, "clear %d exit jobs %f\n", i, TimerElapsed(&start, TRUE));
   /*** check CRASH jobs ***/
 …
   status = sscanf (buffer.buffer, "%*s %d", &Njobs);
   if (status != 1) goto escape;
   if (VerboseMode()) gprint (GP_ERR, "check crash stack %f\n", TimerElapsed(TRUE));
+  if (VerboseMode()) gprint (GP_ERR, "check crash stack %f\n", TimerElapsed(&start, TRUE));
   p = buffer.buffer;
 …
+    }
     p = q + 1;
     status = sscanf (p, "%d", &JobID);
+    // the operations within this locked block only interact with the controller or
+    // modify the properties of the selected job
+    JobTaskLock();
     job = FindControllerJob (JobID);
     if (job == NULL) {
       gprint (GP_ERR, "misplaced job? %d not in CRASH job list\n", JobID);
+      JobTaskUnlock();
       continue;
+    }
     /* this checks the individual job status, grabs stdout/stderr */
     CheckControllerJob (job);
+    JobTaskUnlock();
+  }
   if (VerboseMode()) gprint (GP_ERR, "clear %d crash jobs %f\n", i, TimerElapsed(TRUE));
+  if (VerboseMode()) gprint (GP_ERR, "clear %d crash jobs %f\n", i, TimerElapsed(&start, TRUE));
   FlushIOBuffer (&buffer);
-  // status = ControllerCommand ("run", CONTROLLER_PROMPT, &buffer);
   FreeIOBuffer (&buffer);
   return (TRUE);
 …
  escape:
   FlushIOBuffer (&buffer);
-  // status = ControllerCommand ("run", CONTROLLER_PROMPT, &buffer);
   FreeIOBuffer (&buffer);
   return (FALSE);
+}
 void TimerMark () {
     gettimeofday (&start, (void *) NULL);
+void TimerMark (struct timeval *start) {
+    gettimeofday (start, (void *) NULL);
+}
 float TimerElapsed (int reset) {
+float TimerElapsed (struct timeval *start, int reset) {
   float dtime;
 …
   gettimeofday (&stop, (void *) NULL);
   dtime = DTIME (stop, start);
   if (reset) gettimeofday (&start, (void *) NULL);
+  dtime = DTIME (stop, start[0]);
+  if (reset) gettimeofday (start, (void *) NULL);
   return (dtime);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/CheckJobs.c

-              r15871
+              r23594
   next_timeout = 1.0;
+  JobTaskLock();
   /** test all jobs: ready to test?  finished? **/
   while ((job = NextJob ()) != NULL) {
 …
         /* set taskarg variables */
         for (i = 0; i < job[0].argc; i++) {
             sprintf (varname, "taskarg:%d", i);
             set_str_variable (varname, job[0].argv[i]);
+          sprintf (varname, "taskarg:%d", i);
+          set_str_variable (varname, job[0].argv[i]);
+        }
         set_int_variable ("taskarg:n", job[0].argc);
 …
         /* set options variables */
         for (i = 0; i < job[0].optc; i++) {
             sprintf (varname, "options:%d", i);
             set_str_variable (varname, job[0].optv[i]);
+          sprintf (varname, "options:%d", i);
+          set_str_variable (varname, job[0].optv[i]);
+        }
         set_int_variable ("options:n", job[0].optc);
 …
           if (VerboseMode()) gprint (GP_LOG, "job %s (%d) crash\n", task[0].name, job[0].JobID);
           if (task[0].crash != NULL) {
+            // we need to unlock JobTask since JobTaskLock is called inside CommandLock in the client thread
+            JobTaskUnlock();
+            CommandLock();
             exec_loop (task[0].crash);
+            CommandUnlock();
+            JobTaskLock();
+          }
+        }
 …
+            }
+          }
+          if (macro != NULL) exec_loop (macro);
+          if (macro != NULL) {
+            // we need to unlock JobTask since JobTaskLock is called inside CommandLock in the client thread
+            JobTaskUnlock();
+            CommandLock();
+            exec_loop (macro);
+            CommandUnlock();
+            JobTaskLock();
+          }
+        }
 …
         DeleteJob (job);
         continue;
-        break;
       default:
 …
     /* check for timeout - (local jobs only)
        we only check timeout after a poll (forces at least one poll)
      */
+    */
     if (job[0].mode == JOB_LOCAL) {
       if (GetTaskTimer(job[0].start, FALSE) < task[0].timeout_period) {
 …
       /* set taskarg variables */
       for (i = 0; i < job[0].argc; i++) {
           sprintf (varname, "taskarg:%d", i);
           set_str_variable (varname, job[0].argv[i]);
+        sprintf (varname, "taskarg:%d", i);
+        set_str_variable (varname, job[0].argv[i]);
+      }
       set_int_variable ("taskarg:n", job[0].argc);
 …
       /* run task[0].timeout macro, if it exists */
       if (task[0].timeout != NULL) {
+        // we need to unlock JobTask since JobTaskLock is called inside CommandLock in the client thread
+        JobTaskUnlock();
+        CommandLock();
         exec_loop (task[0].timeout);
+        CommandUnlock();
+        JobTaskLock();
+      }
 …
+  }
   // fprintf (stderr, "check %d jobs\n", Ncheck);
+  JobTaskUnlock();
   return (next_timeout);
+}
 …
 /*
   job / task timeline:
   task:
            exec
   start       create
   task clock  new job
   job:
            1xpoll     2xpoll     3xpoll
   start       check      check      check
   job clock   status     status     status
   .           .          .          timeout
                                     run
                                     timeout
   must be at least one poll before timeout
   (timeout >= poll)
+   job / task timeline:
+   task:
+           exec
+   start       create
+   task clock  new job
+   job:
+           1xpoll     2xpoll     3xpoll
+   start       check      check      check
+   job clock   status     status     status
+   .           .          .          timeout
+   run
+   timeout
+   must be at least one poll before timeout
+   (timeout >= poll)
 */

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/CheckPassword.c

r8548	r23594
2	2	# define DEBUG 0
3	3
	4	// this static var is only used by InitPassword and CheckPassword below.
	5	// Both functions are only called by the main thread.
4	6	static char PASSWORD[256];
5	7

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/CheckTasks.c

-              r23352
+              r23594
   int status;
   float time_running, next_timeout, fuzz;
-  // struct timeval now;
   // actual maximum delay is controlled in job_threads.c
   next_timeout = 1.0;
+  JobTaskLock();
   /** test all tasks: ready to test? ready to run? **/
   while ((task = NextTask ()) != NULL) {
 …
     // add random offset between 0 and 5% of exec_period
     // XXX this should be optional
     fuzz = 0.1*task[0].exec_period*drand48();
+    fuzz = task[0].exec_period*(0.5*drand48() - 0.25);
     task[0].last.tv_usec = 1e6*(fuzz - (int)fuzz);
     task[0].last.tv_sec += (int) fuzz;
 …
     /* ready to run? : run task.exec macro */
     if (task[0].exec != NULL) {
+      // we need to unlock JobTask since JobTaskLock is called inside CommandLock in the client thread
+      JobTaskUnlock();
+      CommandLock();
       status = exec_loop (task[0].exec);
+      CommandUnlock();
+      JobTaskLock();
       if (!status) {
         continue;
+      }
+    }
-    // gettimeofday (&now, (void *) NULL);
-    // fprintf (stderr, "t1: %d %6d  - \n", now.tv_sec, now.tv_usec);
     /* check if there are errors with this task */
 …
+    }
-    // gettimeofday (&now, (void *) NULL);
-    // fprintf (stderr, "t2: %d %6d  - \n", now.tv_sec, now.tv_usec);
     /* construct job from task */
     job = CreateJob (task);
+    if (!job) {
+      continue;
+    }
     if (DEBUG) fprintf (stderr, "create job: (%zx) %d of %d\n", (size_t) job[0].stdout_buff.buffer, job[0].stdout_buff.Nbuffer, job[0].stdout_buff.Nalloc);
+    // gettimeofday (&now, (void *) NULL);
+    // fprintf (stderr, "t3: %d %6d  - \n", now.tv_sec, now.tv_usec);
+    /* execute job - XXX add status test */
+    SubmitJob (job);
+    // fprintf (stderr, "nl: %d %6d  - ",
+    // task[0].last.tv_sec, task[0].last.tv_usec);
+    /* increment job counters */
+    task[0].Njobs ++;
+    task[0].Npending ++;
+    // fprintf (stderr, "%d %6d\n",
+    // task[0].last.tv_sec, task[0].last.tv_usec);
+    /* execute job */
+    if (!SubmitJob (job)) {
+      DeleteJob (job);
+      continue;
+    }
+    task[0].Njobs ++; // number of jobs successfully submitted
     /* increment Nrun for inclusive ranges with Nmax */
     BumpTimeRanges (task[0].ranges, task[0].Nranges);
+  }
+  JobTaskUnlock();
   return (next_timeout);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/ControllerOps.c

-              r20032
+              r23594
 /* local static variables to track the controller host properties */
+/* these are used by AddHost and DeleteHost, and are only called by controller_host.c (clientThread) */
+/* or by RestartController : lock between these two? */
 static Host *hosts = NULL;
 static int Nhosts = 0;
 …
+}
+// XXX possible race condition problem: if we delete a host while the controller
+// is being restarted.  to fix this, we need to keep the deletion in controller_thread,
+// but perhaps mark it in the client_thread?
 int DeleteHost (char *hostname) {
 …
+  }
+  // This function is called by the JobTaskThread via SubmitJob.  We need to unlock the
+  // JobTaskLock to avoid a dead lock with the JobTaskLock called in CheckController
+  JobTaskUnlock();
+  ControlLock(__func__);
   InitIOBuffer (&buffer, 0x100);
   status = ControllerCommand (cmd, CONTROLLER_PROMPT, &buffer);
   free (cmd);
+  ControlUnlock(__func__);
+  JobTaskLock();
   /* extract the job PID from the controller response */
 …
+  }
   sscanf (p, "%*s %s", string);
+  FreeIOBuffer (&buffer);
   job[0].pid = atoi (string);
+  FreeIOBuffer (&buffer);
+  if (job[0].pid < 0) {
+    return (FALSE);
+  }
   return (TRUE);
+}
 …
   if ((status == -1) && (errno == EPIPE)) {
     StopController ();
     gprint (GP_ERR, "controller is down (pipe closed), restarting\n");
+    fprintf (stderr, "controller is down (pipe closed), restarting\n");
     if (!RestartController ()) {
       return (FALSE);
 …
     if (status ==  0) {
       StopController ();
       gprint (GP_ERR, "controller is down (EOF), restarting\n");
+      fprintf (stderr, "controller is down (EOF), restarting\n");
       if (!RestartController ()) {
         return (FALSE);
 …
+    }
     if (status == -1) {
       gprint (GP_ERR, "controller is not responding (%d tries)\n", j);
+      fprintf (stderr, "controller is not responding (%d tries)\n", j);
       gwrite (buffer[0].buffer, 1, buffer[0].Nbuffer, GP_ERR);
+    }
+  }
   if (status == -1) {
     gprint (GP_ERR, "controller still not responding, giving up\n");
+    fprintf (stderr, "controller still not responding, giving up\n");
     return (FALSE);
+  }
 …
     bzero (buffer[0].buffer + buffer[0].Nbuffer, buffer[0].Nalloc - buffer[0].Nbuffer);
+  }
   /* if (VerboseMode()) gprint (GP_ERR, "message received, %d cycles\n", i); */
+  if (VerboseMode()) fprintf (stderr, "message received, %d cycles\n", i);
   return (TRUE);
+}
 …
+}
+void PrintControllerBusyJobs () {
+  int status;
+  char command[1024];
+  IOBuffer buffer;
+  /* check if controller is running */
+  status = CheckControllerStatus ();
+  if (!status) {
+    return;
+  }
+  sprintf (command, "jobstack busy");
+  InitIOBuffer (&buffer, 0x100);
+  status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
+  if (status) {
+    gprint (GP_LOG, " jobs currently running remotely:\n");
+    gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
+  } else {
+    gprint (GP_LOG, "controller is not responding\n");
+  }
+  FreeIOBuffer (&buffer);
+  return;
+}
 int PrintControllerOutput () {
 …
   InitIOBuffer (&buffer, 0x100);
   // XXX lock the host table? SerialThreadLock ();
+  // XXX lock the host table? no: that would risk a dead lock between client and controller threads:
   gprint (GP_ERR, "pcontrol restarted, reloading hosts\n");
   for (i = 0; i < Nhosts; i++) {
 …
     status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
+  }
-  // SerialThreadUnlock ();
   if (status) gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/InputQueue.c

-              r16449
+              r23594
 # include "pantasks.h"
+// we use fprintf for DEBUG statements to avoid deadlocking issues
 static int Ninputs = 0;
 …
 void AddNewInput (char *input) {
+  // XXX define the InputMutex
   SerialThreadLock ();
+  if (DEBUG) gprint (GP_LOG, "adding a new input (%s)\n", input);
+  if (DEBUG) fprintf (stderr, "adding a new input (%s)\n", input);
   inputs[Ninputs] = input;
   Ninputs ++;
 …
     REALLOCATE (inputs, char *, NINPUTS);
+  }
   if (DEBUG) gprint (GP_LOG, "done new input (%s)\n", input);
+  if (DEBUG) fprintf (stderr, "done new input (%s)\n", input);
   SerialThreadUnlock ();
+}
 …
   int i, j;
+  if (DEBUG) gprint (GP_LOG, "deleting an input (%s)\n", input);
+  if (DEBUG) fprintf (stderr, "deleting an input (%s)\n", input);
+  // XXX lock here
   for (i = 0; i < Ninputs; i++) {
     if (inputs[i] == input) {
 …
         REALLOCATE (inputs, char *, NINPUTS);
+      }
+      if (DEBUG) gprint (GP_LOG, "deleted an input\n");
+      // XXX unlock here
+      if (DEBUG) fprintf (stderr, "deleted an input\n");
       return TRUE;
+    }
+  }
   // did not find the input
+  // XXX unlock here
   return FALSE;
+}
 …
   char *input, *line, *outline, tmp;
+  if (Ninputs < 1) return;
+  // XXX lock here
+  if (Ninputs < 1) {
+    // XXX unlock here
+    return;
+  }
   input = inputs[0];
   if (DEBUG) gprint (GP_LOG, "got an input (%s)\n", input);
+  if (DEBUG) fprintf (stderr, "got an input (%s)\n", input);
   Nbytes = snprintf (&tmp, 0, "input %s", input);
   ALLOCATE (line, char, Nbytes + 1);
   snprintf (line, Nbytes + 1, "input %s", input);
+  // XXX unlock here
   status = command (line, &outline, TRUE);

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/JobIDOps.c

r17419	r23594
1	1	# include "pantasks.h"
2	2
3		# define MAX_N_JOBS 1000
	3	# define MAX_N_JOBS 10000
4	4	static char *JobIDList;
5	5	static int JobIDPtr;

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/JobOps.c

-              r18160
+              r23594
+  }
-  /* check if controller is running */
-  status = CheckControllerStatus ();
-  if (!status) {
-    return;
+  }
-  sprintf (command, "jobstack busy");
-  InitIOBuffer (&buffer, 0x100);
-  SerialThreadLock ();
-  status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
-  if (status) {
-    gprint (GP_LOG, " jobs currently running remotely:\n");
-    gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
-  } else {
-    gprint (GP_LOG, "controller is not responding\n");
+  }
-  FreeIOBuffer (&buffer);
   return;
+}
 …
   job[0].JobID = NextJobID ();
+  if (job[0].JobID < 0) {
+    free (job);
+    return NULL;
+  }
   job[0].pid = 0;
   job[0].mode = JOB_LOCAL;
 …
     REALLOCATE (jobs, Job *, NJOBS);
+  }
+  /* increment job counters */
+  task[0].Npending ++;
   return (jobs[Njobs-1]);
+}
 …
 int SubmitJob (Job *job) {
+  int status;
   if (job[0].mode == JOB_LOCAL) {
     if (DEBUG) fprintf (stderr, "submit job: (%zx) %d of %d\n", (size_t) job[0].stdout_buff.buffer, job[0].stdout_buff.Nbuffer, job[0].stdout_buff.Nalloc);
     SubmitLocalJob (job);
+    status = SubmitLocalJob (job);
   } else {
+    SubmitControllerJob (job);
+    status = SubmitControllerJob (job);
+  }
+  if (!status) {
+    return FALSE;
+  }
 …
   if (job[0].mode == JOB_LOCAL) {
     CheckLocalJob (job);
+  } else {
+    /* controller jobs are now checked en masse by CheckController */
+    /* CheckControllerJob (job); */
+    /* controller jobs are checked en masse by CheckController */
+  }
   return (job[0].state);

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/ListenClients.c

-              r16905
+              r23594
 # define DEBUG 0
+// XXX make the calling functions thread-safe
 static int NCLIENTS;
 static int Nclients;
 …
 void InitClients () {
+  ClientLock();
   Nclients = 0;
   NCLIENTS = 10;
   ALLOCATE (clients, int, NCLIENTS);
   ALLOCATE (buffers, IOBuffer *, NCLIENTS);
+  ClientUnlock();
+}
 …
 void AddNewClient (int client) {
+  if (DEBUG) gprint (GP_LOG, "adding a new client (%d)\n", client);
+  ClientLock();
+  if (DEBUG) fprintf (stderr, "adding a new client (%d)\n", client);
   clients[Nclients] = client;
   ALLOCATE (buffers[Nclients], IOBuffer, 1);
 …
     REALLOCATE (buffers, IOBuffer *, NCLIENTS);
+  }
+  ClientUnlock();
+}
 …
   int i, j;
+  if (DEBUG) gprint (GP_LOG, "deleting a client (%d)\n", client);
+  ClientLock();
+  if (DEBUG) fprintf (stderr, "deleting a client (%d)\n", client);
   for (i = 0; i < Nclients; i++) {
     if (clients[i] == client) {
 …
         REALLOCATE (buffers, IOBuffer *, NCLIENTS);
+      }
+      ClientUnlock();
       return TRUE;
+    }
+  }
   // did not find the client
+  ClientUnlock();
   return FALSE;
+}
 …
   gprintInit ();  // each thread needs to init the printing system
+  // define server output log files
+  if (VarConfig ("PANTASKS_SERVER_STDOUT", "%s", log_stdout) != NULL) {
+    gprintSetFileThisThread (GP_LOG, log_stdout);
+  } else {
+    strcpy (log_stdout, "stdout");
+  }
+  if (VarConfig ("PANTASKS_SERVER_STDERR", "%s", log_stderr) != NULL) {
+    gprintSetFileThisThread (GP_ERR, log_stderr);
+  } else {
+    strcpy (log_stderr, "stderr");
+  }
+  /* set buffers for the output for this client */
+  gprintSetBuffer (GP_LOG);
+  gprintSetBuffer (GP_ERR);
   while (1) {
 …
     /* place all of the clients in the fdSet */
-    Ncurrent = Nclients;
     Nmax = 0;
     FD_ZERO (&fdSet);
+    ClientLock();
+    Ncurrent = Nclients;
+    ClientUnlock();
     for (i = 0; i < Ncurrent; i++) {
       Nmax = MAX (Nmax, clients[i]);
 …
     /* block until we have some data on the pipes (or timeout) */
     if (DEBUG) gprint (GP_ERR, "listening to %d clients\n", Ncurrent);
+    if (DEBUG) fprintf (stderr, "listening to %d clients\n", Ncurrent);
     status = select (Nmax, &fdSet, NULL, NULL, &timeout);
     if (status == -1) {
 …
       if ((Nread == 0) || (Nread == -2)) {
         /* error: do something */
         if (DEBUG && (Nread == 0)) gprint (GP_ERR, "socket is closed\n");
         if (DEBUG && (Nread == -2)) gprint (GP_ERR, "error reading from socket\n");
+        if (DEBUG && (Nread == 0)) fprintf (stderr, "socket is closed\n");
+        if (DEBUG && (Nread == -2)) fprintf (stderr, "error reading from socket\n");
         DeleteClient (clients[i]);
         continue;
+        break;  // the other thread could also have modified the list; restart with new Ncurrent
+      }
       if (DEBUG) gprint (GP_ERR, "read %d total bytes\n", buffers[i][0].Nbuffer);
+      if (DEBUG) fprintf (stderr, "read %d total bytes\n", buffers[i][0].Nbuffer);
       /* see if we have a complete message waiting; if not, keep waiting for messages */
 …
       if (*line) {
-        /* set buffers for the output for this client */
-        gprintSetBuffer (GP_LOG);
-        gprintSetBuffer (GP_ERR);
         /* run the command, return the exit status */
+        CommandLock();
         status = multicommand (line);
+        CommandUnlock();
         SendMessage (clients[i], "STATUS %d", status);
 …
           SendMessageFixed (clients[i], 0, "");
+        }
+        FlushIOBuffer (outbuffer);
         // return the stdout messages first
 …
           SendMessageFixed (clients[i], 0, "");
+        }
+        /* clear and reset the output buffers to their last output file names */
+        gprintSetFileAllThreads (GP_LOG, NULL);
+        gprintSetFileAllThreads (GP_ERR, NULL);
+        FlushIOBuffer (outbuffer);
+      }
       free (line);

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/LocalJob.c

-              r18161
+              r23594
   pid = fork ();
+  if (pid == -1) {
+    gprint_syserror (GP_ERR, errno, "error starting local job: ");
+    goto pipe_error;
+  }
   if (!pid) { /* must be child process */
     if (VerboseMode()) gprint (GP_ERR, "starting local job\n");

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/Makefile

-              r18085
+              r23594
 $(SRC)/pantasks.$(ARCH).o \
 $(SRC)/thread_locks.$(ARCH).o \
+$(SRC)/job_threads.$(ARCH).o \
+$(SRC)/task_threads.$(ARCH).o \
+$(SRC)/jobs_and_tasks_thread.$(ARCH).o \
 $(SRC)/controller_threads.$(ARCH).o
 …
 $(SRC)/pantasks_server.$(ARCH).o \
 $(SRC)/server_run.$(ARCH).o \
-$(SRC)/server_load.$(ARCH).o \
-$(SRC)/InputQueue.$(ARCH).o \
 $(SRC)/ListenClients.$(ARCH).o \
 $(SRC)/server.$(ARCH).o \
 …
 $(SRC)/CheckPassword.$(ARCH).o \
 $(SRC)/thread_locks.$(ARCH).o \
+$(SRC)/job_threads.$(ARCH).o \
+$(SRC)/task_threads.$(ARCH).o \
+$(SRC)/controller_threads.$(ARCH).o \
+$(SRC)/input_threads.$(ARCH).o
+$(SRC)/jobs_and_tasks_thread.$(ARCH).o \
+$(SRC)/controller_threads.$(ARCH).o
 funcs = \
 …
 cmds = \
-$(SRC)/pulse.$(ARCH).o \
 $(SRC)/status.$(ARCH).o \
 $(SRC)/flush.$(ARCH).o \

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/controller.c

r18085	r23594
50	50	}
51	51
	52	ControlLock(__func__);
52	53	status = (*func)(argc - 1, argv + 1);
	54	ControlUnlock(__func__);
53	55	return (status);
54	56	}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/controller_host.c

-              r19125
+              r23594
   if (argc != 3) goto usage;
+  if (max_threads && strcasecmp (argv[1], "ADD")) goto usage;
   /* start controller connection (if needed) */
 …
   if (!strcasecmp (argv[1], "ADD")) {
     AddHost (argv[2], max_threads);
+  } else {
+    if (max_threads) goto usage;
+  }
+  }
   if (!strcasecmp (argv[1], "DELETE")) {
 …
   InitIOBuffer (&buffer, 0x100);
-  SerialThreadLock ();
   status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
   if (status) gwrite (buffer.buffer, 1, buffer.Nbuffer, GP_LOG);
 …
   return (FALSE);
+}
-/* should I keep an internal host table so I can reload the
-   hosts if the controller exits?
-   alternatively, that could be a user-level choice
-*/

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/controller_jobstack.c

-              r18085
+              r23594
   InitIOBuffer (&buffer, 0x100);
-  SerialThreadLock ();
   status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
   if (status) {

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/controller_status.c

-              r8548
+              r23594
   InitIOBuffer (&buffer, 0x100);
-  SerialThreadLock ();
   status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
   if (status) {

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/controller_threads.c

-              r15791
+              r23594
     // one run of the task checker
     SerialThreadLock ();
+    ControlLock(__func__);
     CheckController ();
+    ControlUnlock(__func__);
+    ControlLock(__func__);
     CheckControllerOutput ();
+    SerialThreadUnlock ();
+    ControlUnlock(__func__);
     if (VerboseMode() == 2) fprintf (stderr, "C");
     // fprintf (stderr, "**** C ****");

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/controller_verbose.c

-              r18085
+              r23594
   InitIOBuffer (&buffer, 0x100);
-  SerialThreadLock ();
   status = ControllerCommand (command, CONTROLLER_PROMPT, &buffer);
-  SerialThreadUnlock ();
   if (status) {

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/delete.c

-              r13542
+              r23594
     JobID = atoi (argv[2]);
+    JobTaskLock();
     job = FindJob (JobID);
     if (job == NULL) {
       gprint (GP_LOG, "job not found\n");
+      JobTaskUnlock();
       return (TRUE);
+    }
 …
         job[0].state = JOB_HUNG;
         if (VerboseMode()) gprint (GP_LOG, "child process %d is hung, cannot kill\n", job[0].pid);
+        JobTaskUnlock();
         return (FALSE);
+      }
 …
         job[0].state = JOB_HUNG;
         if (VerboseMode()) gprint (GP_LOG, "child process %d is hung, cannot kill\n", job[0].pid);
+        JobTaskUnlock();
         return (FALSE);
+      }
 …
     DeleteJob (job);
     gprint (GP_LOG, "job removed\n");
+    JobTaskUnlock();
     return (TRUE);
+  }
 …
 usage:
     gprint (GP_ERR, "USAGE: delete job (JobID)\n");
     gprint (GP_ERR, "USAGE: delete task (TaskName)\n");
     return (FALSE);
+  gprint (GP_ERR, "USAGE: delete job (JobID)\n");
+  gprint (GP_ERR, "USAGE: delete task (TaskName)\n");
+  return (FALSE);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/flush.c

-              r14590
+              r23594
   if (!strcasecmp (argv[1], "jobs")) {
+    JobTaskLock();
     FlushJobs ();
+    JobTaskUnlock();
     return (TRUE);
+  }

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/init.c

-              r16453
+              r23594
 int halt            PROTO((int, char **));
 int flush_jobs      PROTO((int, char **));
-int pulse           PROTO((int, char **));
 int showtask        PROTO((int, char **));
 int status_sys      PROTO((int, char **));
 …
   {1, "options",    task_options,  "define optional variables associated with the job task"},
   {1, "periods",    task_periods,  "define time scales for a task"},
-  {1, "pulse",      pulse,         "set the scheduler update period"},
   {1, "run",        run,           "run the scheduler"},
   {1, "flush",      flush_jobs,    "flush all jobs from the queue"},

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/init_server.c

-              r16903
+              r23594
 int task_stdout     PROTO((int, char **));
 int task_stderr     PROTO((int, char **));
-int pulse           PROTO((int, char **));
 int flush_jobs      PROTO((int, char **));
 int status_server   PROTO((int, char **));
 …
   {1, "npending",   task_npending, "define maximum number of outstanding jobs for a task"},
   {1, "periods",    task_periods,  "define time scales for a task"},
-  {1, "pulse",      pulse,         "set the scheduler update period"},
   {1, "flush",      flush_jobs,    "flush all jobs from the queue"},
   {1, "server",     server,        "server-specific commands"},
 …
   InitJobs ();
   InitJobIDs ();
-  InitInputs ();
   for (i = 0; i < sizeof (cmds) / sizeof (Command); i++) {
 …
   FreeJobs ();
   FreeJobIDs ();
-  FreeInputs ();
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/kill.c

-              r13542
+              r23594
   JobID = atoi (argv[1]);
+  JobTaskLock();
   job = FindJob (JobID);
   if (job == NULL) {
     gprint (GP_LOG, "job not found\n");
+    JobTaskUnlock();
     return (TRUE);
+  }
 …
       job[0].state = JOB_HUNG;
       if (VerboseMode()) gprint (GP_LOG, "child process %d is hung, cannot kill\n", job[0].pid);
+      JobTaskUnlock();
       return (FALSE);
+    }
 …
       job[0].state = JOB_HUNG;
       if (VerboseMode()) gprint (GP_LOG, "child process %d is hung, cannot kill\n", job[0].pid);
+      JobTaskUnlock();
       return (FALSE);
+    }
 …
   DeleteJob (job);
   gprint (GP_LOG, "job removed\n");
+  JobTaskUnlock();
   return (TRUE);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/notes.txt

-              r7892
+              r23594
+stdout_cntl:
+  GetJobOutout
+    CheckControllerJob
+      CheckController
+        controller_threads (controlThread)
+  StartController
+  ControllerCommand
+    CheckController
+      controller_threads (controlThread)
+    controller_check (clientThread)
+    controller_host (clientThread)
+    controller_jobstack (clientThread)
+    controller_run (clientThread)
+    controller_status (clientThread)
+    controller_verbose (clientThread)
+    DeleteControllerJob
+      CheckControllerJob
+        CheckController
+          controller_threads (controlThread)
+    CheckControllerJobStatus
+      CheckControllerJob
+        CheckController
+          controller_threads (controlThread)
+    SubmitControllerJob
+      SubmitJob
+        CheckTasks (JobTaskThread)
+    PrintControllerBusyJobs
+    KillControllerJob
+    QuitController
+    RestartController
+  CheckControllerOutput
+    controller_output (clientThread)
+    controller_threads (controlThread)
+20090322
+  We've been surviving with a slightly broken threading / locking
+  model.  I would like to clean it up.  Previously, the threads where
+  blocking at a very coarse level, and there were certain interactions
+  that were not thread-safe.  Here are my notes on re-working this:
+  * we have 6 threads:
+    * main (top-level parent) : after it spawns the 5 child threads,
+      this thread waits for new clients and adds them to the list of
+      active clients with 'AddNewClient'
+    * clientsThread (ListenClients): this thread is listening for
+      commands from the clients; when it receives a complete command,
+      it executes the command using 'multicommand'.
+    * tasksThread
+      CheckTasks
+        NextTask
+    * jobsThread
+    * controllerThread
+      CheckControllerStatus : race condition is irrelevant
+    * inputsThread
+      AddNewInput called by clientThread
+      (not sure this is really being used:
+           server input (filename) calls 'input'
+           server module (filename) calls 'module'
+  * functions which need to be thread-safe:
+    * AddNewClient
+    * gprint and supporting functions must be thread safe (extensively
+      used...)
+* race condition is irrelevant for this variable
+static int ControllerStatus = FALSE;
+static int stdin_cntl, stdout_cntl, stderr_cntl;
+GetJobOutput -> CheckControllerJob -> CheckController -> controllerThread
+ControllerCommand -> controllerThread / clientThread
+CheckControllerOutput -> controllerThread / clientThread
+StartController -> controllerThread / clientThread
+StopController
+static IOBuffer stdout_buffer;
+static IOBuffer stderr_buffer;
+static int ControllerPID = 0;
+/* local static variables to track the controller host properties */
+static Host *hosts = NULL;
+static int Nhosts = 0;
+static int NHOSTS = 0;
 PanTasks Client / Server design

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/pantasks.c.in

-              r16453
+              r23594
 void program_init (int *argc, char **argv) {
+  pthread_t jobsThread;
+  pthread_t tasksThread;
+  pthread_t JobsAndTasksThread;
   pthread_t controllerThread;
 …
   /* start up the background threads here */
+  pthread_create (&tasksThread,      NULL, &CheckTasksThread,      NULL);
+  pthread_create (&jobsThread,       NULL, &CheckJobsThread,       NULL);
+  pthread_create (&controllerThread, NULL, &CheckControllerThread, NULL);
+  // pthread_create (&tasksThread,      NULL, &CheckTasksThread,           NULL);
+  // pthread_create (&jobsThread,       NULL, &CheckJobsThread,            NULL);
+  pthread_create (&JobsAndTasksThread, NULL, &CheckJobsAndTasksThread, NULL);
+  pthread_create (&controllerThread,   NULL, &CheckControllerThread,   NULL);
   return;
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/pantasks_server.c.in

-              r16453
+              r23594
   char log_stdout[128], log_stderr[128];
+  pthread_t jobsThread;
+  pthread_t tasksThread;
+  pthread_t inputsThread;
+  pthread_t JobsAndTasksThread;
   pthread_t clientsThread;
   pthread_t controllerThread;
 …
   /* start up the background threads here */
+  pthread_create (&clientsThread,    NULL, &ListenClients,         NULL);
+  pthread_create (&tasksThread,      NULL, &CheckTasksThread,      NULL);
+  pthread_create (&jobsThread,       NULL, &CheckJobsThread,       NULL);
+  pthread_create (&controllerThread, NULL, &CheckControllerThread, NULL);
+  pthread_create (&inputsThread,     NULL, &CheckInputsThread,     NULL);
+  pthread_create (&clientsThread,       NULL, &ListenClients,           NULL);
+  pthread_create (&JobsAndTasksThread,  NULL, &CheckJobsAndTasksThread, NULL);
+  pthread_create (&controllerThread,    NULL, &CheckControllerThread,   NULL);
   /* in this loop, we listen for incoming connections, validate, and

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/server.c

-              r16463
+              r23594
 int input           PROTO((int, char **));
 int module          PROTO((int, char **));
-int server_load     PROTO((int, char **));
 int server_run      PROTO((int, char **));
 int server_stop     PROTO((int, char **));
 …
   {1, "input",  input,  "load input file on server"},
   {1, "module", module, "load module file on server"},
-  {1, "load",   server_load, "load input file on server"},
   {1, "run",    server_run,  "run scheduler"},
   {1, "stop",   server_stop, "stop scheduler"},

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/server_run.c

-              r11084
+              r23594
   CheckJobsSetState (TRUE);
   CheckControllerSetState (TRUE);
-  CheckInputsSetState (TRUE);
   return (TRUE);
+}
 …
   CheckTasksSetState (FALSE);
-  // CheckJobsSetState (FALSE);
-  // CheckControllerSetState (FALSE);
-  CheckInputsSetState (FALSE);
   return (TRUE);
+}
 …
   CheckJobsSetState (FALSE);
   CheckControllerSetState (FALSE);
-  CheckInputsSetState (FALSE);
   return (TRUE);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/showtask.c

r12468	r23594
8	8	}
9	9
	10	JobTaskLock();
10	11	ShowTask (argv[1]);
	12	JobTaskUnlock();
	13
11	14	return (TRUE);
12	15	}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/status_server.c

-              r16012
+              r23594
   if ((N = get_argument (argc, argv, "-tasks"))) {
     remove_argument (N, &argc, argv);
+    JobTaskLock();
     ListTasks (FALSE);
+    JobTaskUnlock();
     return (TRUE);
+  }
 …
   if ((N = get_argument (argc, argv, "-taskinfo"))) {
     remove_argument (N, &argc, argv);
+    JobTaskLock();
     ListTasks (TRUE);
+    JobTaskUnlock();
     return (TRUE);
+  }
 …
   if ((N = get_argument (argc, argv, "-taskstats"))) {
     remove_argument (N, &argc, argv);
+    JobTaskLock();
     if (argc == 2) {
       ListTaskStats (argv[N]);
 …
       ListTaskStats (NULL);
+    }
+    JobTaskUnlock();
     return (TRUE);
+  }
 …
   if ((N = get_argument (argc, argv, "-taskstatsreset"))) {
     remove_argument (N, &argc, argv);
+    JobTaskLock();
     if (argc == 2) {
       ResetTaskStats (argv[N]);
 …
       ResetTaskStats (NULL);
+    }
+    JobTaskUnlock();
     return (TRUE);
+  }
 …
     gprint (GP_LOG, " Controller is stopped\n");
+  }
+  JobTaskLock();
   ListTasks (FALSE);
   ListJobs ();
+  JobTaskUnlock();
+  ControlLock(__func__);
+  PrintControllerBusyJobs();
+  ControlUnlock(__func__);
   return (TRUE);

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task.c

-              r14590
+              r23594
   if (argc != 2) goto usage;
+  JobTaskLock();
   task = FindTask (argv[1]);
   if (task == NULL) { /**** new task ****/
 …
     SetNewTask (task);
+  }
+  JobTaskUnlock();
   /* While a task is being defined, it is removed from the task list.  The new task is added to the task list
      when the definition process is complete.
 …
       case TASK_END:
-        /* I need to add in a test here to see if all task elements
-           have been defined.  delete the task if not */
         free (input);
         /* validate the new task: all mandatory elements defined? */
 …
           return (FALSE);
+        }
+        JobTaskLock();
         RegisterNewTask ();
+        JobTaskUnlock();
         return (TRUE);
         break;

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_active.c

-              r10734
+              r23594
   if (argc != 2) goto usage;
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
     gprint (GP_ERR, "ERROR: not defining or running a task\n");
+    JobTaskUnlock();
     return (FALSE);
+  }
 …
   if (!strcasecmp (argv[1], "true")) {
     task[0].active = TRUE;
+    JobTaskUnlock();
     return (TRUE);
+  }
   if (!strcasecmp (argv[1], "false")) {
     task[0].active = FALSE;
+    JobTaskUnlock();
     return (TRUE);
+  }
   gprint (GP_ERR, "ERROR: invalid option: %s\n", argv[1]);
+  JobTaskUnlock();
   return (FALSE);

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_command.c

r7917	r23594
12	12	}
13	13
	14	JobTaskLock();
14	15	task = GetNewTask ();
15	16	if (task == NULL) {
…	…
17	18	if (task == NULL) {
18	19	gprint (GP_ERR, "ERROR: not defining or running a task\n");
	20	JobTaskUnlock();
19	21	return (FALSE);
20	22	}
…	…
35	37	task[0].argv[i] = strcreate (argv[i+1]);
36	38	}
	39	JobTaskUnlock();
37	40	return (TRUE);
38	41	}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_host.c

-              r7917
+              r23594
   task = GetNewTask ();
+  JobTaskLock();
   if (task == NULL) {
     task = GetActiveTask ();
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
+    }
 …
   task[0].host = NULL;
+  if (!strcasecmp (argv[1], "LOCAL")) return (TRUE);
+  if (!strcasecmp (argv[1], "LOCAL")) {
+    JobTaskUnlock();
+    return (TRUE);
+  }
   task[0].host = strcreate (argv[1]);
+  JobTaskUnlock();
   return (TRUE);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_macros.c

r10647	r23594
22	22	}
23	23
	24	JobTaskLock();
24	25	task = GetNewTask ();
25	26	if (task == NULL) {
26	27	gprint (GP_ERR, "ERROR: not defining or running a task\n");
	28	JobTaskUnlock();
27	29	return (FALSE);
28	30	}
…	…
135	137	free (input);
136	138	REALLOCATE (macro[0].line, char *, MAX (1, macro[0].Nlines));
	139	JobTaskUnlock();
137	140	return (TRUE);
138	141	}
…	…
148	151	}
149	152	}
	153	JobTaskUnlock();
150	154	return (TRUE);
151	155	}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_nmax.c

-              r11055
+              r23594
   if (argc != 2) goto usage;
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
     gprint (GP_ERR, "ERROR: not defining or running a task\n");
+    JobTaskUnlock();
     return (FALSE);
+  }
   task[0].Nmax = atoi (argv[1]);
+  JobTaskUnlock();
   return (TRUE);
 …
   if (argc != 2) goto usage;
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
     gprint (GP_ERR, "ERROR: not defining or running a task\n");
+    JobTaskUnlock();
     return (FALSE);
+  }
   task[0].NpendingMax = atoi (argv[1]);
+  JobTaskUnlock();
   return (TRUE);

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_options.c

r8129	r23594
12	12	}
13	13
	14	JobTaskLock();
14	15	task = GetNewTask ();
15	16	if (task == NULL) {
…	…
17	18	if (task == NULL) {
18	19	gprint (GP_ERR, "ERROR: not defining or running a task\n");
	20	JobTaskUnlock();
19	21	return (FALSE);
20	22	}
…	…
35	37	task[0].optv[i] = strcreate (argv[i+1]);
36	38	}
	39	JobTaskUnlock();
37	40	return (TRUE);
38	41	}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_periods.c

-              r7917
+              r23594
+  }
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
 …
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
+    }
 …
   if (Timeout) task[0].timeout_period = TimeoutValue;
+  JobTaskUnlock();
   return (TRUE);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_stdout.c

-              r12332
+              r23594
+  }
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
 …
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
+    }
 …
   if (task[0].stdout_dump != NULL) free (task[0].stdout_dump);
   task[0].stdout_dump = strcreate (argv[1]);
+  JobTaskUnlock();
   return (TRUE);
+}
 …
+  }
+  JobTaskLock();
   task = GetNewTask ();
   if (task == NULL) {
 …
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
+    }
 …
   if (task[0].stderr_dump != NULL) free (task[0].stderr_dump);
   task[0].stderr_dump = strcreate (argv[1]);
+  JobTaskUnlock();
   return (TRUE);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_trange.c

-              r15487
+              r23594
     if (argc != 1) goto usage;
+    JobTaskLock();
     task = GetNewTask ();
     if (task == NULL) {
       gprint (GP_ERR, "ERROR: not defining or running a task\n");
+      JobTaskUnlock();
       return (FALSE);
+    }
 …
     task[0].Nranges = 0;
     REALLOCATE (task[0].ranges, TimeRange, 1);
+    JobTaskUnlock();
     return (TRUE);
+  }
 …
   if (argc != 3) goto usage;
-  task = GetNewTask ();
-  if (task == NULL) {
-    gprint (GP_ERR, "ERROR: not defining or running a task\n");
-    return (FALSE);
+  }
   /* test for Mon[@HH:MM:SS] - both must match */
 …
 valid:
+  JobTaskLock();
+  task = GetNewTask ();
+  if (task == NULL) {
+    gprint (GP_ERR, "ERROR: not defining or running a task\n");
+    JobTaskUnlock();
+    return (FALSE);
+  }
   N = task[0].Nranges;
   task[0].Nranges ++;
 …
   task[0].ranges[N] = range;
+  JobTaskUnlock();
   return (TRUE);

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/thread_locks.c

-              r11084
+              r23594
 # include "pantasks.h"
+/* this mutex is used by the serialized threads */
+/* mutex to lock Client table operations */
+static pthread_mutex_t ClientMutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+void SerialThreadLock () {
+    pthread_mutex_lock (&mutex);
+void ClientLock () {
+  pthread_mutex_lock (&ClientMutex);
+}
 void SerialThreadUnlock () {
     pthread_mutex_unlock (&mutex);
+void ClientUnlock () {
+  pthread_mutex_unlock (&ClientMutex);
+}
+/* mutex to lock Command / Multicommand operations */
+static pthread_mutex_t CommandMutex = PTHREAD_MUTEX_INITIALIZER;
+void CommandLock () {
+  //  fprintf (stderr, "command lock\n");
+  pthread_mutex_lock (&CommandMutex);
+}
+void CommandUnlock () {
+  //  fprintf (stderr, "command unlock\n");
+  pthread_mutex_unlock (&CommandMutex);
+}
+/* mutex to lock Control operations */
+static pthread_mutex_t ControlMutex = PTHREAD_MUTEX_INITIALIZER;
+void ControlLock (char *func) {
+  // fprintf (stderr, "control lock %s\n", func);
+  pthread_mutex_lock (&ControlMutex);
+}
+void ControlUnlock (char *func) {
+  // fprintf (stderr, "control unlock %s\n", func);
+  pthread_mutex_unlock (&ControlMutex);
+}
+/* mutex to lock Job / Task operations */
+static pthread_mutex_t JobTaskMutex = PTHREAD_MUTEX_INITIALIZER;
+void JobTaskLock () {
+  //  fprintf (stderr, "jobtask lock\n");
+  pthread_mutex_lock (&JobTaskMutex);
+}
+void JobTaskUnlock () {
+  //  fprintf (stderr, "jobtask unlock\n");
+  pthread_mutex_unlock (&JobTaskMutex);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pcontrol/JobOps.c

-              r21379
+              r23594
   ALLOCATE (job, Job, 1);
+  job[0].JobID    = NextJobID();
+  if (job[0].JobID < 0) {
+    free (job);
+    return -1;
+  }
   job[0].argc     = argc;
   job[0].argv     = argv;
 …
   job[0].dtime = 0.0;
   job[0].pid = 0;
-  job[0].JobID    = NextJobID();
   job[0].host     = NULL;

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pcontrol/job.c

-              r19124
+              r23594
       return (FALSE);
+  }
+  if (get_argument (argc, argv, "-h")) goto usage;
+  if (get_argument (argc, argv, "-help")) goto usage;
+  if (get_argument (argc, argv, "--help")) goto usage;
   Host = NULL;
 …
   if (argc < 2) {
-    gprint (GP_ERR, "USAGE: job [options] (arg0) (arg1) ... (argN)\n");
-    gprint (GP_ERR, "  arguments of the form @MAX_THREADS@ will be replaced when the job is launched\n");
     FREE (Host);
     return (FALSE);
+    goto usage;
+  }
 …
+  }
+  // a JobID < 0 mean the job was not accepted
   JobID = AddJob (Host, Mode, Timeout, targc, targv);
   gprint (GP_LOG, "JobID: %d\n", (int) JobID);
   return (TRUE);
+ usage:
+    gprint (GP_ERR, "USAGE: job [options] (arg0) (arg1) ... (argN)\n");
+    gprint (GP_ERR, "  options: -host, +host, -timeout\n");
+    gprint (GP_ERR, "  arguments of the form @MAX_THREADS@ will be replaced when the job is launched\n");
+    return (FALSE);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/relastro/src/ImageOps.c

r23344	r23594
297	297	mask = TRUE;
298	298	}
	299	if (~finite(catalog[c].measure[m].dR) \|\| ~finite(catalog[c].measure[m].dD)) mask = TRUE;
	300
299	301	raw[i].mask = mask;
300	302

branches/cnb_branches/cnb_branch_20090301/Ohana/src/relastro/src/UpdateObjects.c

-              r21508
+              r23594
       for (k = 0; k < catalog[i].average[j].Nmeasure; k++, m++) {
+        //exclude measurements which have non-finite astrometry
+        if (~finite(catalog[i].measure[m].dR) || ~finite(catalog[i].measure[m].dD)) continue;
         // exclude measurements by previous outlier detection

branches/cnb_branches/cnb_branch_20090301/Ohana/src/relastro/src/bcatalog.c

-              r23344
+              r23594
   int mask;
   PhotCode *code;
+  int skipFew = skipPhotCodeKeep = skipPhotCodeSkip = skipTime = skipPhotFlag = skipSigmaLim = skipImagSelect = 0;
   // XXX in the future, use catalog[0].Nsecfilt only?  allow catalogs to have variable Nsecfilt?
 …
   /* exclude stars not in range or with too few measurements */
   for (i = 0; i < catalog[0].Naverage; i++) {
+    if (catalog[0].average[i].Nmeasure <= SRC_MEAS_TOOFEW) continue;
+    if (catalog[0].average[i].Nmeasure <= SRC_MEAS_TOOFEW) {
+      skipFew++;
+      continue;
+    }
     /* start with all stars good */
     subcatalog[0].average[Naverage] = catalog[0].average[i];
 …
           if (photcodesKeep[k][0].code == GetPhotcodeEquivCodebyCode(catalog[0].measure[offset].photcode)) found = TRUE;
+        }
+        if (!found) continue;
+        if (!found) {
+          skipPhotCodeKeep++;
+          continue;
+        }
+      }
       if (NphotcodesSkip > 0) {
 …
           if (photcodesSkip[k][0].code == GetPhotcodeEquivCodebyCode(catalog[0].measure[offset].photcode)) found = TRUE;
+        }
+        if (found) continue;
+        if (found) {
+          skipPhotCodeSkip++;
+          continue;
+        }
+      }
       /* select measurements by time */
       if (TimeSelect) {
+        if (catalog[0].measure[offset].t < TSTART) continue;
+        if (catalog[0].measure[offset].t > TSTOP) continue;
+        if (catalog[0].measure[offset].t < TSTART) {
+          skipTime++;
+          continue;
+        }
+        if (catalog[0].measure[offset].t > TSTOP) {
+          skipTime++;
+          continue;
+        }
+      }
       /* select measurements by quality */
       // XXX FIX THIS!!
       // if (DophotSelect && (catalog[0].measure[offset].dophot != DophotValue)) continue;
       /* select measurements by quality */
       if (PhotFlagSelect) {
 …
           mask = code[0].astromBadMask;
+        }
+        if (mask & catalog[0].measure[offset].photFlags) continue;
+        if (mask & catalog[0].measure[offset].photFlags) {
+          skipPhotFlag++;
+          continue;
+        }
+      }
       /* select measurements by measurement error */
+      if ((SIGMA_LIM > 0) && (catalog[0].measure[offset].dM > SIGMA_LIM)) continue;
+      if ((SIGMA_LIM > 0) && (catalog[0].measure[offset].dM > SIGMA_LIM)) {
+        skipSigmaLim++;
+        continue;
+      }
       /* select measurements by mag limit */
       if (ImagSelect) {
         mag = PhotInst (&catalog[0].measure[offset]);
+        if (mag < ImagMin) continue;
+        if (mag > ImagMax) continue;
+        if (mag < ImagMin || mag > ImagMax) {
+          skipImagSelect++;
+          continue;
+        }
+      }
       // re-assess on each run of relastro if a measurement should be used
 …
   if (VERBOSE) {
+    fprintf(stderr, "Reasons for exclusion in bcatalog:\n");
+    fprintf(stderr, "\ntoo few: %d \nphotCodeSkip: %d \nphotCodeKeep: %d \ntime: %d\n",
+            skipfew, skipPhotCodeKeep, skipPhotCodeSkip, skipTime);
+    fprintf(stderr, "photFlag: %d \nmagnitude error: %d \nImag: %d\n",
+            skipPhotFlag, skipSigmaLim, skipImagSelect);
+  }
+  if (VERBOSE) {
     fprintf (stderr, "%d: using %d stars (%d measures) for catalog\n", i,
              subcatalog[0].Naverage, subcatalog[0].Nmeasure);

branches/cnb_branches/cnb_branch_20090301/PS-IPP-Config/lib/PS/IPP/Config.pm

-              r23352
+              r23594
+    }
     return 1 if defined $self->{nebulous}; # Already started
+    return $self->{nebulous} if defined $self->{nebulous}; # Already started
     my $server = metadataLookupStr( $self->{_siteConfig}, 'NEB_SERVER' ); # Nebulous server
 …
     my $neb = eval { Nebulous::Client->new( proxy => $server ); };
     if ($@ or not defined $neb) {
         carp "Unable to find NEB_SERVER in camera configuration file.";
+        carp "Unable to create a Nebulous::Client object with proxy => $server";
         return undef;
+    }
 …
     $self->{nebulous} = $neb;
+    return 1;
+    return $neb;
+}
+sub nebulous
+{
+    my $self = shift;
+    carp "no parameters are allowed" if @_;
+    # check to see if we already have a live nebulous object
+    return $self->{nebulous} if defined $self->{nebulous};
+    # if not, call _neb_start() and return the new object
+    return $self->_neb_start() or ( carp "Can't start Nebulous" and return undef );
+}
 …
         if ($scheme eq 'neb') {
+            $self->_neb_start() or ( carp "Can't start Nebulous" and return undef );
+            my $neb = $self->{nebulous}; # Nebulous handle
+            my $neb = $self->nebulous; # Nebulous handle
             if ($create_if_doesnt_exist) {
                 my $status = eval { $neb->stat( $name ); };
 …
         $scheme = lc($scheme);
         if ($scheme eq 'neb') {
+            $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+            my $fh = eval { $self->{nebulous}->open_create( $name ); };
+            my $fh = eval { $self->nebulous->open_create( $name ); };
             if ($@ or not defined $fh) {
                 carp "Unable to open/create Nebulous handle $name";
 …
         $scheme = lc($scheme);
         if ($scheme eq 'neb') {
+            $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+            my $fh = eval { $self->{nebulous}->open_create( $name ) };
+            my $fh = eval { $self->nebulous->open_create( $name ) };
             if ($@ or not defined $fh) {
                 carp "Unable to open/create Nebulous handle $name";
 …
     my $scheme = file_scheme($name); # The scheme, e.g., file://, path://
     if (defined $scheme and lc($scheme) eq 'neb') {
+        $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+        $name = eval { $self->{nebulous}->create( $name ) };
+        $name = eval { $self->nebulous->create( $name ) };
         if ($@ or not defined $name) {
             carp "Unable to create Nebulous handle $name";
 …
     my $scheme = file_scheme($name); # The scheme, e.g., file://, path://
     if (defined $scheme and lc($scheme) eq 'neb') {
+        $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+        my $found = eval { $self->{nebulous}->find_instances( $name ); };
+        my $found = eval { $self->nebulous->find_instances( $name ); };
         ( carp "Unable to find instances of Nebulous handle $name" and return undef ) if $@;
         return (defined $found ? 1 : 0);
 …
     my $scheme = file_scheme($target); # The scheme, e.g., file://, path://
     if (defined $scheme and lc($scheme) eq 'neb') {
+        $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+        $target = eval { $self->{nebulous}->create( $target ); };
+        $target = eval { $self->nebulous->create( $target ); };
         if ($@ or not defined $target) {
             carp "Unable to create Nebulous handle";
 …
     my $scheme = file_scheme($name); # The scheme, e.g., file://, path://
     if (defined $scheme and lc($scheme) eq 'neb') {
+        $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+        $status = eval { $self->{nebulous}->delete( $name ); };
+        $status = eval { $self->nebulous->delete( $name ); };
         ( carp "Unable to delete Nebulous handle $name" and return undef ) if $@;
     } else {

branches/cnb_branches/cnb_branch_20090301/dbconfig/changes.txt

-              r23352
+              r23594
 ALTER TABLE diffSkyfile ADD COLUMN magicked TINYINT;
 -- drop exiting foreign key constraints
+-- drop existing foreign key constraints
 ALTER TABLE diffInputSkyfile drop FOREIGN KEY diffInputSkyfile_ibfk_1;
 ALTER TABLE diffInputSkyfile drop FOREIGN KEY diffInputSkyfile_ibfk_2;
 …
 show create table detResidImfile;
 alter table detResidImfile drop foreign key detResidImfile_ibfk_3;
+-- Version: 1.1.50  magic and distribution changes
+ALTER TABLE magicDSRun ADD COLUMN label VARCHAR(64) after cam_id;
+ALTER TABLE magicDSRun ADD KEY(label);
+-- replace null values for magicked flags with zero
+UPDATE rawImfile SET magicked = 0 WHERE magicked IS NULL;
+UPDATE chipProcessedImfile SET magicked = 0 WHERE magicked IS NULL;
+UPDATE warpSkyfile SET magicked = 0 WHERE magicked IS NULL;
+UPDATE diffSkyfile SET magicked = 0 WHERE magicked IS NULL;
+CREATE TABLE distRun (
+    dist_id     BIGINT AUTO_INCREMENT,
+    target_id   BIGINT,
+    stage       VARCHAR(64),
+    stage_id    BIGINT,
+    label       VARCHAR(64),
+    outroot     VARCHAR(255),
+    clean       TINYINT,
+    state       VARCHAR(64),
+    time_stamp  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    fault       SMALLINT,
+    PRIMARY KEY(dist_id),
+    KEY(state),
+    KEY(label),
+    FOREIGN KEY(target_id) REFERENCES distTarget(target_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE distComponent (
+    dist_id     BIGINT,
+    component   VARCHAR(64),
+    bytes       INT,
+    md5sum      VARCHAR(32),
+    state       VARCHAR(64),
+    name        VARCHAR(255),
+    fault       SMALLINT,
+    PRIMARY KEY(dist_id, component),
+    KEY(state),
+    FOREIGN KEY(dist_id) REFERENCES distRun(dist_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+-- change state fields to match the rest of the system
+UPDATE magicRun SET state = 'new' WHERE state = 'run';
+UPDATE magicRun SET state = 'full' WHERE state = 'stop';
+UPDATE magicDSRun SET state = 'new' WHERE state = 'run';
+UPDATE magicDSRun SET state = 'full' WHERE state = 'stop';
+ALTER TABLE rawExp ADD COLUMN magicked TINYINT;
+ALTER TABLE chipRun ADD COLUMN magicked TINYINT;
+ALTER TABLE warpRun ADD COLUMN magicked TINYINT;
+ALTER TABLE diffRun ADD COLUMN magicked TINYINT;
+update rawExp  SET magicked = 0;
+update chipRun SET magicked = 0;
+update warpRun SET magicked = 0;
+update diffRun SET magicked = 0;
+-- The following only applies to the gpc1 database (and recent copies)
+update chipRun SET magicked = 1 where chip_id = 11955;
+CREATE TABLE distTarget (
+    target_id   BIGINT AUTO_INCREMENT,
+    obs_mode    VARCHAR(64),
+    stage       VARCHAR(64),
+    clean       TINYINT,
+    state       VARCHAR(64),
+    comment     VARCHAR(255),
+    PRIMARY KEY(target_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcDSProduct (
+    prod_id     BIGINT AUTO_INCREMENT,
+    name        VARCHAR(64),
+    dbname      VARCHAR(64),
+    dbhost      VARCHAR(64),
+    prod_root   VARCHAR(255),
+    PRIMARY KEY(prod_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcDestination (
+    dest_id     BIGINT AUTO_INCREMENT,
+    prod_id     BIGINT,
+    name        VARCHAR(64),
+    status_uri  VARCHAR(255),
+    comment     VARCHAR(255),
+    last_fileset VARCHAR(255),
+    state       VARCHAR(64),
+    PRIMARY KEY(dest_id),
+    FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcInterest (
+    int_id      BIGINT AUTO_INCREMENT,
+    dest_id     BIGINT,
+    target_id   BIGINT,
+    prod_id     BIGINT,
+    state       VARCHAR(64),
+    PRIMARY KEY(int_id),
+    FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id),
+    FOREIGN KEY(target_id) REFERENCES distTarget(target_id),
+    FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcDSFileset (
+    fs_id       BIGINT AUTO_INCREMENT,
+    dist_id     BIGINT,
+    prod_id     BIGINT,
+    name        VARCHAR(64),
+    state       VARCHAR(64),
+    registered  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(fs_id),
+    FOREIGN KEY(dist_id) REFERENCES distRun(dist_id),
+    FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcRun (
+    rc_id       BIGINT AUTO_INCREMENT,
+    fs_id       BIGINT,
+    dest_id     BIGINT,
+    state       VARCHAR(64),
+    registered  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(rc_id),
+    FOREIGN KEY(fs_id) REFERENCES rcDSFileset(fs_id),
+    FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;

branches/cnb_branches/cnb_branch_20090301/dbconfig/chip.md

r21310	r23594
11	11	tess_id STR 64
12	12	end_stage STR 64 # Key
	13	magicked BOOL f
13	14	END
14	15

branches/cnb_branches/cnb_branch_20090301/dbconfig/config.md

r23352	r23594
2	2	pkg_name STR ippdb
3	3	pkg_namespace STR ippdb
4		pkg_version STR 1.1.49
	4	pkg_version STR 1.1.50
5	5	END

branches/cnb_branches/cnb_branch_20090301/dbconfig/diff.md

r21310	r23594
11	11	tess_id STR 64 # Key
12	12	exp_id S64 0 # fkey(exp_id) ref rawExp(exp_id)
	13	magicked BOOL f
13	14	END
14	15

branches/cnb_branches/cnb_branch_20090301/dbconfig/dist.md

-              r23352
+              r23594
     stage       STR         64
     stage_id    S64         0
-    chip_id     S64         0
     label       STR         64      # Key
     outroot     STR         255
     clean       BOOL        f
     state       STR         64      # Key
+    time_stamp  UTC         0001-01-01T00:00:00Z
     fault       S16         0
 END
 …
 END
+distTarget METADATA
+    target_id   S64        0       # Primary Key
+    obs_mode    STR        64
+    clean       BOOL       f
+    state       STR        64
+    comment     STR        255
+END

branches/cnb_branches/cnb_branch_20090301/dbconfig/ipp.m4

r17886	r23594
22	22	include(flatcorr.md)
23	23	include(pstamp.md)
	24	include(dist.md)

branches/cnb_branches/cnb_branch_20090301/dbconfig/magic.md

-              r20973
+              r23594
 magicDSRun METADATA
     magic_ds_id S64         0       # Primary Key fkey(magic_ds_id)
+    magic_ds_id S64         0       # Primary Key
     magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
     state       STR         0       # Key
 …
     stage_id    S64         0
     cam_id      S64         0
+    label       STR         64      # key
     outroot     STR         255
     recoveryroot    STR     255
 …
 magicDSFile METADATA
     magic_ds_id S64         0       # Primary Key
+    magic_ds_id S64         0       # Primary Key fkey(magic_ds_id) ref magicDSRun(magic_ds_id)
     component   STR         64
     backup_path_base  STR         255

branches/cnb_branches/cnb_branch_20090301/dbconfig/tasks.md

-              r20973
+              r23594
     fault       S16         0       # Key NOT NULL
     epoch       UTC         0001-01-01T00:00:00Z
+    magicked    BOOL        f
 END
 …
     moon_alt    F32         0.0
     moon_phase  F32         0.0
     ignored     BOOL        FALSE
+    ignored     BOOL        f
     hostname    STR         64
     fault       S16         0       # Key NOT NULL

branches/cnb_branches/cnb_branch_20090301/dbconfig/warp.md

r21310	r23594
21	21	end_stage STR 64 # Key
22	22	registered TAI NULL
23		# if magic is T then look for the exp_id in the magic output tables
	23	magicked BOOL f
24	24	END
25	25

branches/cnb_branches/cnb_branch_20090301/glueforge/templates/psdb/configure_ac.tt

-              r14443
+              r23594
 CFLAGS=${TMP_CFLAGS}
 AC_PATH_PROG([PERL], [perl], [missing])
 if test "$PERL" = "missing" ; then
 …
 fi
+AC_PATH_PROG([DOXYGEN], [doxygen], [])
+AC_ARG_ENABLE(doxygen,
+              [AS_HELP_STRING(--enable-doxygen,doxygen-erated documentation)],
+              [AC_MSG_RESULT(doxygen enabled)
+               AC_PATH_PROG([DOXYGEN], [doxygen], [])],
+              [AC_MSG_RESULT(doxygen disabled)])
 AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"])

branches/cnb_branches/cnb_branch_20090301/ippMonitor/raw/ipp.php

r21137	r23594
40	40
41	41	// make this a DB lookup
42		$sql = "SELECT projname FROM projects";
	42	$sql = "SELECT projname FROM projects order by projname";
43	43
44	44	$qry = $db->query($sql);

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/chip_imfile.pl

-              r23352
+              r23594
+}
+my @ARGS = @ARGV;
 # Parse the command-line arguments
 my ( $exp_id, $chip_id, $class_id, $chip_imfile_id, $uri, $camera, $outroot, $dbname, $run_state, $reduction, $threads, $verbose,
      $no_update, $no_op, $redirect );
+     $no_update, $no_op, $redirect, $magicked, $deburned );
 GetOptions(
     'exp_id=s'          => \$exp_id,    # Exposure identifier
 …
     'reduction=s'       => \$reduction, # Reduction class
     'run-state=s'       => \$run_state, # current state of the run (new, update)
+    'magicked'          => \$magicked,  # magicked state of input file
+    'deburned=s'        => \$deburned,  # does deburned image exist?
     'threads=s'         => \$threads,   # Number of threads to use for ppImage
     'verbose'           => \$verbose,   # Print to stdout
 …
     print STDOUT "\n\n";
     print STDOUT "Starting script $0 on $host\n\n";
     print STDOUT "COMMAND IS: @ARGV\n\n";
+    print STDOUT "FULL COMMAND: $0 @ARGS\n\n";
+}
 …
     my $command;
     my $do_stats;
+    ## get the ppImage recipe for this camera and CHIP reduction
+    $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe_ppImage -";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
+    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    ## XXX make the feature more general?
+    my $useDeburnedImage = metadataLookupBool($recipeData, 'USE.DEBURNED.IMAGE');
+    if ($useDeburnedImage && $deburned) {
+        $uri =~ s/fits$/burn.fits/;
+        &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri);
+    }
     if ($run_state eq "new") {
 …
+    }
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
     unless ($success) {
 …
         &my_die("Unable to perform ppImage: $error_code", $exp_id, $chip_id, $class_id, $error_code);
+    }
-    ## get the ppImage recipe for this camera and CHIP reduction
-    $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe_ppImage -";
-    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+    }
-    my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
     ## allow the output images to be optional, depending on the recipe / reduction class
 …
     $command .= " -uri $outputImage";
     $command .= " -path_base $outroot";
+    $command .= " -magicked" if $magicked;
     $command .= " -hostname $host" if defined $host;
     $command .= " -dbname $dbname" if defined $dbname;

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_apply.pl

r23352	r23594
89	89	'darkmask' => 'mask',
90	90	'flatmask' => 'mask',
	91	'ctemask' => 'mask',
91	92	};
92	93

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_calc.pl

r23352	r23594
81	81	'darkmask' => 0,
82	82	'flatmask' => 0,
	83	'ctemask' => 0,
83	84	};
84	85

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_process_imfile.pl

r23352	r23594
79	79	'DARK' => undef,
80	80	'DARK_PREMASK' => undef,
	81	'CTEMASK' => undef,
81	82	'SHUTTER' => 'PPIMAGE.OUTPUT.DETREND',
82	83	'FLAT_PREMASK' => 'PPIMAGE.OUTPUT.DETREND',

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_exp.pl

-              r23352
+              r23594
     # write the normalizations to a file as a metadata config file in the form: class_id F32 value
     # XXX a possible optimization: if there is only one imfile, skip normalization
+    # Correcting the resid imfiles and stats requires us to *divide* by the normalization
+    # we do this by inverting the normalization here:
     my $normsFile;
     ($normsFile, $normsName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.norms.XXXX", UNLINK => !$save_temps );
 …
     foreach my $norm (@$normsMD) {
         my $class_id = $norm->{class_id};
         my $normalization = $norm->{norm};
+        my $normalization = 1.0 / $norm->{norm};
         $norms{$class_id} = $normalization;

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_imfile.pl

-              r23352
+              r23594
     'darkmask'         => '-mask',      # Specify the mask frame
     'flatmask'         => '-mask',      # Specify the mask frame
+    'ctemask'          => '-mask',      # Specify the mask frame
 };
 …
 my $FILERULES = { 'FLATMASK'         => 'PPIMAGE.OUTPUT.RESID',
                   'DARKMASK'         => 'PPIMAGE.OUTPUT.RESID',
+                  'CTEMASK'          => 'PPIMAGE.OUTPUT.RESID',
                   'MASK'             => 'PPIMAGE.OUTPUT.RESID',
                   'BIAS'             => 'PPIMAGE.OUTPUT.RESID',

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_stack.pl

-              r23352
+              r23594
 my $FILERULES = { 'FLATMASK'         => 'PPMERGE.OUTPUT.MASK',
                   'DARKMASK'         => 'PPMERGE.OUTPUT.MASK',
+                  'CTEMASK'          => 'PPMERGE.OUTPUT.MASK',
                   'MASK'             => 'PPMERGE.OUTPUT.MASK',
                   'BIAS'             => 'PPMERGE.OUTPUT.BIAS',
 …
 my $STATRECIPES = {'FLATMASK'         => 'DETSTATS',
                    'DARKMASK'         => 'DETSTATS',
+                   'CTEMASK'          => 'DETSTATS',
                    'MASK'             => 'DETSTATS',
                    'BIAS'             => 'DETSTATS',

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/diff_skycell.pl

-              r23352
+              r23594
 # Identify the input and the template
 my ($input, $inputMask, $inputVariance, $inputPath); # Input files and path
+my ($input, $inputMask, $inputVariance, $inputPath, $inputSources); # Input files and path
 my ($template, $templateMask, $templateVariance, $templatePath, $templateSources); # Template files and path
 my $tess_id;                    # Tesselation identifier
 my $camera;                     # Camera
+my $magicked_0;
+my $magicked_1;
 foreach my $file (@$files) {
     if (defined $file->{template} and $file->{template}) {
 …
             $templateVariance = "PPSTACK.OUTPUT.VARIANCE";
             $templateSources = "PSPHOT.OUT.CMF.MEF";  ## this must be consistent with the value in stack_skycell.pl:161
+            # template is a stack so it doesn't need to be magicked
+            $magicked_1 = 1;
             ## use an explicit stack name for psphot output objects
         } else {
 …
             $templateVariance = "PSWARP.OUTPUT.VARIANCE";
             $templateSources = "PSWARP.OUTPUT.SOURCES";
+            $magicked_1 = $file->{magicked};
+        }
     } else {
         $input = $file->{uri};
         $inputPath = $file->{path_base};
+        $magicked_0 = $file->{magicked};    # if input is a stack the output can't be "magicked"
         if ($file->{warp_id} == 0) {
             $inputMask = "PPSTACK.OUTPUT.MASK";
             $inputVariance = "PPSTACK.OUTPUT.VARIANCE";
+            $inputSources = "PSPHOT.OUT.CMF.MEF";  ## this must be consistent with the value in stack_skycell.pl:161
         } else {
             $inputMask = "PSWARP.OUTPUT.MASK";
             $inputVariance = "PSWARP.OUTPUT.VARIANCE";
+            $inputSources = "PSWARP.OUTPUT.SOURCES";
+        }
+    }
 …
 &my_die("Unable to identify camera", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless defined $camera;
 $ipprc->define_camera($camera);
+# Compute the magicked status of the output.
+# The output file will be considered magicked if the input has been magicked and the
+# template is either a stack or a warp that has been magicked.
+# note that difftool -inputskyfile outputs the magicked boolean as an int not T or F
+# because the output is constructed from a union of two selects
+my $magicked = $magicked_0 && $magicked_1;
 # Recipes to use based on reduction class
 …
 # print "inputVariance: $inputVariance\n";
 # print "templateSources: $templateSources\n";
+# print "inputSources: $inputSources\n";
+$inputMask = $ipprc->filename($inputMask, $inputPath);
+$inputVariance = $ipprc->filename($inputVariance, $inputPath);
+$inputSources = $ipprc->filename($inputSources, $inputPath);
 $templateMask = $ipprc->filename($templateMask, $templatePath);
-$inputMask = $ipprc->filename($inputMask, $inputPath);
 $templateVariance = $ipprc->filename($templateVariance, $templatePath);
-$inputVariance = $ipprc->filename($inputVariance, $inputPath);
 $templateSources = $ipprc->filename($templateSources, $templatePath);
 …
 print "inputVariance: $inputVariance\n";
 print "templateSources: $templateSources\n";
+print "inputSources: $inputSources\n";
 &my_die("Couldn't find input: $template", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($template);
 &my_die("Couldn't find input: $templateMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateMask);
 &my_die("Couldn't find input: $templateVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateVariance);
+&my_die("Couldn't find input: $templateSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateSources);
 &my_die("Couldn't find input: $input", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input);
 &my_die("Couldn't find input: $inputMask", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputMask);
 &my_die("Couldn't find input: $inputVariance", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputVariance);
 &my_die("Couldn't find input: $templateSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($templateSources);
+&my_die("Couldn't find input: $inputSources", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($inputSources);
 # Get the output filenames
 …
 # Perform subtraction
 unless ($no_op) {
+    my $command = "$ppSub $input $template $outroot";
+    my $command = "$ppSub $outroot";
+    $command .= " -inimage $input";
+    $command .= " -refimage $template";
     $command .= " -inmask $inputMask";
     $command .= " -refmask $templateMask";
     $command .= " -invariance $inputVariance";
     $command .= " -refvariance $templateVariance";
+    $command .= " -insources $inputSources";
+    $command .= " -refsources $templateSources";
     $command .= " -stats $outputStats";
     $command .= " -threads $threads" if defined $threads;
 …
     $command .= " -F PSPHOT.OUTPUT PSPHOT.OUT.CMF.MEF";
     $command .= " -F PSPHOT.BACKMDL PSPHOT.BACKMDL.MEF";
-    $command .= " -sources $templateSources";
     $command .= " -photometry";
     $command .= " -tracedest $traceDest -log $logDest";
 …
         my $command = "$difftool -adddiffskyfile -diff_id $diff_id -skycell_id $skycell_id -uri $outputName -path_base $outroot";
         $command .= " $cmdflags";
+        $command .= " -magicked" if $magicked;
         $command .= (" -dtime_script " . ((DateTime->now->mjd - $mjd_start) * 86400));
         $command .= " -hostname $host" if defined $host;

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/dist_component.pl

-              r23352
+              r23594
 my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean);
 my ($outroot, $run_state, $data_state, $magicked);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 GetOptions(
            'dist_id=s'  => \$dist_id,# Magic destreak run identifier
+           'dist_id=s'      => \$dist_id,    # distribution run identifier
            'camera=s'       => \$camera,     # camera for evaluating file rules
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
 …
            'state=s'        => \$run_state,  # state of the run
            'data_state=s'   => \$data_state, # data_state for this component
            'magicked=s'     => \$magicked,   # data_state for this component
+           'magicked'       => \$magicked,   # magicked state for this component
            'outroot=s'      => \$outroot,    # "directory" for outputs
            'clean=s'        => \$clean,      # create clean distribution
+           'clean'          => \$clean,      # create clean distribution
            'save-temps'     => \$save_temps, # Save temporary files?
            'dbname=s'       => \$dbname,     # Database name
            'verbose'        => \$verbose,    # Print stuff?
            'no-update'      => \$no_update,  # Don't update the database?
-           'no-op'          => \$no_op,      # Don't do any operations?
            'logfile=s'      => \$logfile,
            ) or pod2usage( 2 );
 …
 my $basename = basename($path_base);
+#
-# are we creating a clean distribution?
-if (defined($clean) and ($clean eq "T")) {
-    $clean = 1;
-} else {
-    $clean = 0;
+}
 # making a clean bundle of raw images doesn't make sense
 if (($stage eq "raw") and $clean) {
 …
     &my_die("cannot create clean run at raw stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
 # create the output directories if it is not a nebulous path and it doesn't exist
 …
 foreach my $file (@$file_list) {
     my $base = basename($file);
+    my $path = $ipprc->file_resolve($file);
+    # if file is not found, just skip it.
+    # This is not always the right thing to do.
+    # For example if warpSkyfile is ignored, there won't be many of the data products
+    # the new config dump will only list the files actually produced.
+    next if !$path;
     push @base_list, $base;
-    my $path = $ipprc->file_resolve($file);
     symlink $path, "$tmpdir/$base";
+}
 …
     # Here is where we determine whether or not a GPC1 image can be released.
-    if ($magicked and ($magicked eq "F")) {
-        $magicked = 0;
+    }
     if ($camera eq "GPC1" and !$magicked) {
 …
     $command .= " -chip_mask $mask" if ($stage eq 'chip' and $mask);
     $command .= " -weight $variance" if $variance;
+    unless (defined $no_op) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
+        }
+    } else {
+        print "skipping command $command\n";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
+    }
+}
 …
     my $command = "tar -C $tmpdir -czhf $tarfile .";
+    unless (defined $no_op) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
+        }
+    } else {
+        print "skipping command $command\n";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
+    }
 …
     $bytes = $finfo[7];
+    if (!$save_temps) {
+        # delete the temp directory and it's contents
+        system "rm -r $tmpdir";
+    }
+    delete_tmpdir($tmpdir);
+}
+{
 …
         # TODO: these data will eventually come from the CONFIG dump
+        my $fh;
+        open $fh, "/data/ipp004.0/home/bills/ipp/ippScripts/scripts/clean.mdc" or
+            &my_die("cannot find clean.mdc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+        my @data = <$fh>;
+        my $clean_mdc = join "", @data;
+        my $clean_mdc = get_legacy_file_mdc();
         my $mdlist = $mdcParser->parse($clean_mdc) or
 …
+    }
+    delete_tmpdir();
     carp($msg);
     exit $exit_code;
+}
+sub delete_tmpdir
+{
+    if (!$save_temps) {
+        system "rm -r $tmpdir";
+    }
+}
+# list of output data products for runs that were made before the configuration re-work
+sub get_legacy_file_mdc
+{
+    my $list =
+"
+#
+# interesting things we might want to save in this file
+#   1. whether file is to be distributed in 'clean' mode
+#   2. whether file is to be cleaned (this should be the same thing)
+#   3. type of file 'no, not our business, precious'
+# this data should probably be computed by the relevant program and saved as part
+# of the configuration dump that is output.
+PROD_LIST MULTI
+PROD_LIST   METADATA
+        STAGE                    STR     RAW
+    # there is isn't really a file rule for raw images
+        DUMMY                   BOOL    F
+END
+# list of data products for a gpc1 chipProcessedImfile  (made by ppImage)
+PROD_LIST   METADATA
+        STAGE                    STR     CHIP
+        PPIMAGE.CONFIG          BOOL    T
+#       PPIMAGE.CHIP            BOOL    F
+#       PPIMAGE.CHIP.MASK       BOOL    F
+#       PPIMAGE.CHIP.VARIANCE   BOOL    F
+        PPIMAGE.BIN1            BOOL    T
+        PPIMAGE.BIN2            BOOL    T
+        PSPHOT.OUT.CMF.SPL      BOOL    T
+        PSPHOT.BACKMDL          BOOL    T
+        PPIMAGE.STATS           BOOL    T
+        LOG.IMFILE              BOOL    T
+        TRACE.IMFILE            BOOL    T
+    # where do we put exposure level data such as LOG.EXP ?
+END
+# exposure level output products from camera processing made by psastro and ppImage (jpegs)
+PROD_LIST   METADATA
+        STAGE                    STR     CAM
+        PSASTRO.CONFIG          BOOL    T
+        PSASTRO.OUTPUT          BOOL    T
+        PSASTRO.STATS           BOOL    T
+        PPIMAGE.JPEG1           BOOL    T
+        PPIMAGE.JPEG2           BOOL    T
+        LOG.EXP                 BOOL    T
+        TRACE.EXP               BOOL    T
+END
+# chip lelel output products from camera processing made by psastro
+PROD_LIST   METADATA
+        STAGE                    STR CAM_CHIP
+#       PSASTRO.OUTPUT.MASK     BOOL    F
+END
+PROD_LIST   METADATA
+        STAGE                    STR     FAKE
+#       PPSIM.OUTPUT            BOOL    F
+END
+# list of data products for a gpc1 warpSkyfile (pswarp)
+PROD_LIST   METADATA
+        STAGE                    STR     WARP
+        PSWARP.CONFIG           BOOL    T
+#       PSWARP.OUTPUT           BOOL    F
+#       PSWARP.OUTPUT.MASK      BOOL    F
+#       PSWARP.OUTPUT.VARIANCE  BOOL    F
+        PSWARP.OUTPUT.SOURCES   BOOL    T
+        PSPHOT.BACKMDL.MEF      BOOL    T
+        PSPHOT.PSF.SKY.SAVE     BOOL    T
+        SKYCELL.STATS           BOOL    T
+        SKYCELL.TEMPLATE        BOOL    T
+        LOG.EXP                 BOOL    T
+        TRACE.EXP               BOOL    T
+END
+# outputs from diffRun (ppSub)
+PROD_LIST   METADATA
+        STAGE                    STR     DIFF
+        PPSUB.CONFIG            BOOL    T
+#       PPSUB.OUTPUT            BOOL    F
+#       PPSUB.OUTPUT.MASK       BOOL    F
+#       PPSUB.OUTPUT.VARIANCE   BOOL    F
+        PPSUB.OUTPUT.KERNELS    BOOL    T
+        PPSUB.OUTPUT.JPEG1      BOOL    T
+        PPSUB.OUTPUT.JPEG2      BOOL    T
+        PSPHOT.OUT.CMF.MEF      BOOL    T
+        PSPHOT.BACKMDL.MEF      BOOL    T
+        SKYCELL.STATS           BOOL    T
+        LOG.EXP                 BOOL    T
+        TRACE.EXP               BOOL    T
+END
+PROD_LIST   METADATA
+        STAGE                    STR     STACK
+        PPSTACK.CONFIG          BOOL    T
+#       PPSTACK.OUTPUT          BOOL    F
+#       PPSTACK.OUTPUT.MASK     BOOL    F
+#       PPSTACK.OUTPUT.VARIANCE BOOL    F
+        PPSTACK.TARGET.PSF      BOOL    T
+        PSPHOT.OUT.CMF.MEF      BOOL    T
+        PSPHOT.BACKMDL.MEF      BOOL    T
+        SKYCELL.STATS           BOOL    T
+        PPSTACK.OUTPUT.JPEG1    BOOL    T
+        PPSTACK.OUTPUT.JPEG2    BOOL    T
+        LOG.EXP                 BOOL    T
+        TRACE.EXP               BOOL    T
+END
+";
+    return $list;
+}
 __END__

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/register_imfile.pl

-              r23352
+              r23594
 # we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type
 if (uc(&value_for_flag ($cmdflags, "-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
 if (uc(&value_for_flag ($cmdflags, "-inst"))      eq "NULL") { &my_die ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
 if (uc(&value_for_flag ($cmdflags, "-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
 if (uc(&value_for_flag ($cmdflags, "-class_id"))  eq "NULL") { &my_die ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
 if (uc(&value_for_flag ($cmdflags, "-exp_type"))  eq "NULL") { &my_die ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-telescope")) eq "NULL") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-inst"))      eq "NULL") { &my_die ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-filelevel")) eq "NULL") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-class_id"))  eq "NULL") { &my_die ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&value_for_flag ($cmdflags, "NULL", "-exp_type"))  eq "NULL") { &my_die ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR); }
 my $command = "$regtool -addprocessedimfile";
 …
 # determine solar-system parameters
 my $longitude = &value_for_flag($cmdflags, "-longitude");
 my $latitude  = &value_for_flag($cmdflags, "-latitude");
 my $elevation = &value_for_flag($cmdflags, "-elevation");
 my $ra        = &value_for_flag($cmdflags, "-ra");
 my $dec       = &value_for_flag($cmdflags, "-decl");
 my $dateobs   = &value_for_flag($cmdflags, "-dateobs");
+my $longitude = &value_for_flag($cmdflags, 0.0, "-longitude");
+my $latitude  = &value_for_flag($cmdflags, 0.0, "-latitude");
+my $elevation = &value_for_flag($cmdflags, 0.0, "-elevation");
+my $ra        = &value_for_flag($cmdflags, 0.0, "-ra");
+my $dec       = &value_for_flag($cmdflags, 0.0, "-decl");
+my $dateobs   = &value_for_flag($cmdflags, 0.0, "-dateobs");
 # if the needed data is available, pass it to sunmoon:
 …
+{
     my $cmdflags = shift;
+    my $default = shift;
     my $flag = shift;
     my $value = 0.0;
+    my $value = $default;
     if ($cmdflags =~ m|$flag|) {
         ($value) = $cmdflags =~ m|$flag\s+(\S+)|;

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/stack_skycell.pl

r23352	r23594
203	203	# Perform stacking
204	204	unless ($no_op) {
205		my $command = "$ppStack $listName $outroot";
	205	my $command = "$ppStack -input $listName $outroot";
206	206	$command .= " -stats $outputStats" if $do_stats;;
207	207	$command .= " -recipe PPSTACK $recipe_ppStack";

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/summit_copy.pl

-              r23352
+              r23594
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage( -msg => "Required options: --uri --filename --exp_name --inst --telescope --class --class_id --workdir",
+pod2usage( -msg => "Required options: --uri --filename --exp_name --inst --telescope --class --class_id",
        -exitval => 3)
     unless defined $uri
 …
     and defined $telescope
     and defined $class
+    and defined $class_id
+    and defined $workdir;
+    and defined $class_id;
 # Look for programs we need
 …
 $command .= " -class_id $class_id";
 $command .= " -uri $filename";
 $command .= " -workdir $workdir";
+# $command .= " -workdir $workdir";
 $command .= " -hostname $host";
 $command .= " -end_stage $end_stage" if defined $end_stage;
+# $command .= " -end_stage $end_stage" if defined $end_stage;
 $command .= " -dbname $dbname" if defined $dbname;

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/warp_skycell.pl

-              r23352
+              r23594
+}
 my ($warp_id, $skycell_id, $warp_skyfile_id, $tess_dir, $camera, $dbname, $outroot, $threads, $run_state, $verbose, $no_update, $no_op, $redirect, $save_temps);
+my ($warp_id, $skycell_id, $warp_skyfile_id, $tess_dir, $camera, $dbname, $outroot, $threads, $run_state, $magicked, $verbose, $no_update, $no_op, $redirect, $save_temps);
 GetOptions(
     'warp_id|i=s'         => \$warp_id, # Warp identifier
 …
     'threads=s'           => \$threads,   # Number of threads to use for pswarp
     'run-state=s'         => \$run_state,  # 'new' or 'update'
+    'magicked'            => \$magicked,  # input run has been magicked already?
     'verbose'             => \$verbose,   # Print to stdout
     'no-update'           => \$no_update, # Don't update the database?
 …
             $command .= " -tess_id $tess_dir";
             $command .= " -path_base $outroot"; # needed for logfile lookups
+            $command .= " -magicked" if $magicked;
             $command .= " -uri $outputImage" if $accept;

branches/cnb_branches/cnb_branch_20090301/ippTasks/Makefile.am

-              r23352
+              r23594
 task_files = \
-        detrend.mkruns.pro \
         detrend.norm.pro \
         detrend.process.pro \
 …
         detrend.stack.pro \
         detrend.correct.pro \
+        detrend.cleanup.pro \
         flatcorr.pro \
         automate.pro \
 …
         simtest.flatcorr.config \
         simtest.flatcorr.auto \
+        simtest.ctemask.config \
+        simtest.ctemask.auto \
         simtest.stack.config \
         simtest.stack.auto

branches/cnb_branches/cnb_branch_20090301/ippTasks/chip.pro

-              r23352
+              r23594
     book getword chipPendingImfile $pageName exp_id -var EXP_ID
     book getword chipPendingImfile $pageName exp_tag -var EXP_TAG
+    book getword chipPendingImfile $pageName raw_magicked -var RAW_MAGICKED
+    book getword chipPendingImfile $pageName deburned -var DEBURNED
     book getword chipPendingImfile $pageName chip_id -var CHIP_ID
     book getword chipPendingImfile $pageName chip_imfile_id -var CHIP_IMFILE_ID
 …
     book getword chipPendingImfile $pageName state -var RUN_STATE
+    if ("$RAW_MAGICKED" == "T")
+        $MAGICKED_ARG = "--magicked"
+    else
+        $MAGICKED_ARG = ""
+    end
     # specify choice of local or remote host based on camera and chip (class_id)
     set.host.for.camera $CAMERA $CLASS_ID
 …
     stderr $LOGDIR/chip.imfile.log
     $run = chip_imfile.pl --threads @MAX_THREADS@ --exp_id $EXP_ID --chip_id $CHIP_ID --chip_imfile_id $CHIP_IMFILE_ID --class_id $CLASS_ID --uri $URI --camera $CAMERA --run-state $RUN_STATE --outroot $outroot --redirect-output
+    $run = chip_imfile.pl --threads @MAX_THREADS@ --exp_id $EXP_ID --chip_id $CHIP_ID --chip_imfile_id $CHIP_IMFILE_ID --class_id $CLASS_ID --uri $URI --camera $CAMERA --run-state $RUN_STATE $MAGICKED_ARG --deburned $DEBURNED --outroot $outroot --redirect-output
     if ("$REDUCTION" != "NULL")
       $run = $run --reduction $REDUCTION

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.norm.pro

-              r23352
+              r23594
 book init detPendingNormImfile
 book init detPendingNormExp
-book init detCleanupNormStatImfile
-book init detCleanupNormImfile
-book init detCleanupNormExp
 macro detnorm.reset
 …
   book init detPendingNormImfile
   book init detPendingNormExp
-  book init detCleanupNormStatImfile
-  book init detCleanupNormImfile
-  book init detCleanupNormExp
 end
 …
   book listbook detPendingNormImfile
   book listbook detPendingNormExp
-  book listbook detCleanupNormStatImfile
-  book listbook detCleanupNormImfile
-  book listbook detCleanupNormExp
 end
 …
     active true
   end
-  task detrend.cleanup.norm.load
-    active true
-  end
-  task detrend.cleanup.norm.run
-    active true
-  end
-  task detrend.cleanup.normexp.load
-    active true
-  end
-  task detrend.cleanup.normexp.run
-    active true
-  end
-  task detrend.cleanup.normstat.load
-    active true
-  end
-  task detrend.cleanup.normstat.run
-    active true
-  end
 end
 …
   end
   task detrend.normstat.run
-    active false
-  end
-  task detrend.cleanup.norm.load
-    active false
-  end
-  task detrend.cleanup.norm.run
-    active false
-  end
-  task detrend.cleanup.normexp.load
-    active false
-  end
-  task detrend.cleanup.normexp.run
-    active false
-  end
-  task detrend.cleanup.normstat.load
-    active false
-  end
-  task detrend.cleanup.normstat.run
     active false
   end
 …
 $detPendingNormImfile_DB = 0
 $detPendingNormExp_DB = 0
-$detCleanupNormStatImfile_DB = 0
-$detCleanupNormImfile_DB = 0
-$detCleanupNormExp_DB = 0
 # select images ready for copy
 …
   end
 end
-########## cleanup normstat ###########
-task           detrend.cleanup.normstat.load
-  host         local
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-  stdout NULL
-  stderr $LOGDIR/detrend.cleanup.normstat.log
-  task.exec
-    $run = dettool -pendingcleanup_normalizedstat
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupNormStatImfile_DB
-      $run = $run -dbname $DB:$detCleanupNormStatImfile_DB
-      $detCleanupNormStatImfile_DB ++
-      if ($detCleanupNormStatImfile_DB >= $DB:n) set detCleanupNormStatImfile_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupNormStatImfile -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupNormStatImfile
-    end
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupNormStatImfile
-  end
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-# run the ipp_cleanup.pl script on pending images
-task           detrend.cleanup.normstat.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-  task.exec
-    book npages detCleanupNormStatImfile -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    # look for new images in detCleanupNormStatImfile (pantaskState == INIT)
-    book getpage detCleanupNormStatImfile 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-    book setword detCleanupNormStatImfile $pageName pantaskState RUN
-    book getword detCleanupNormStatImfile $pageName det_id   -var DET_ID
-    book getword detCleanupNormStatImfile $pageName iteration -var ITERATION
-    book getword detCleanupNormStatImfile $pageName camera -var CAMERA
-    book getword detCleanupNormStatImfile $pageName state -var CLEANUP_MODE
-    book getword detCleanupNormStatImfile $pageName dbname -var DBNAME
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-    stdout $LOGDIR/detrend.cleanup.normstat.log
-    stderr $LOGDIR/detrend.cleanup.normstat.log
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.normstat.imfile --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-    # save the pageName for future reference below
-    options $pageName
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-  # default exit status
-  task.exit    default
-    process_exit detCleanupNormStatImfile $options:0 $JOB_STATUS
-  end
-  task.exit    crash
-    showcommand crash
-    book setword detCleanupNormStatImfile $options:0 pantaskState CRASH
-  end
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupNormStatImfile $options:0 pantaskState TIMEOUT
-  end
-end
-########## cleanup norm (normalized.imfile) ###########
-task           detrend.cleanup.norm.load
-  host         local
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-  stdout NULL
-  stderr $LOGDIR/detrend.cleanup.norm.log
-  task.exec
-    $run = dettool -pendingcleanup_normalizedimfile
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupNormImfile_DB
-      $run = $run -dbname $DB:$detCleanupNormImfile_DB
-      $detCleanupNormImfile_DB ++
-      if ($detCleanupNormImfile_DB >= $DB:n) set detCleanupNormImfile_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupNormImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupNormImfile
-    end
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupNormImfile
-  end
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-# run the ipp_cleanup.pl script on pending images
-task           detrend.cleanup.norm.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-  task.exec
-    book npages detCleanupNormImfile -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    # look for new images in detCleanupNormImfile (pantaskState == INIT)
-    book getpage detCleanupNormImfile 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-    book setword detCleanupNormImfile $pageName pantaskState RUN
-    book getword detCleanupNormImfile $pageName det_id   -var DET_ID
-    book getword detCleanupNormImfile $pageName iteration -var ITERATION
-    book getword detCleanupNormImfile $pageName class_id -var CLASS_ID
-    book getword detCleanupNormImfile $pageName camera -var CAMERA
-    book getword detCleanupNormImfile $pageName state -var CLEANUP_MODE
-    book getword detCleanupNormImfile $pageName dbname -var DBNAME
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-    stdout $LOGDIR/detrend.cleanup.norm.log
-    stderr $LOGDIR/detrend.cleanup.norm.log
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.norm.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-    # save the pageName for future reference below
-    options $pageName
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-  # default exit status
-  task.exit    default
-    process_exit detCleanupNormImfile $options:0 $JOB_STATUS
-  end
-  task.exit    crash
-    showcommand crash
-    book setword detCleanupNormImfile $options:0 pantaskState CRASH
-  end
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupNormImfile $options:0 pantaskState TIMEOUT
-  end
-end
-########## cleanup normexp ###########
-task           detrend.cleanup.normexp.load
-  host         local
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-  stdout NULL
-  stderr $LOGDIR/detrend.normexp.cleanup.log
-  task.exec
-    $run = dettool -pendingcleanup_normalizedexp
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupNormExp_DB
-      $run = $run -dbname $DB:$detCleanupNormExp_DB
-      $detCleanupNormExp_DB ++
-      if ($detCleanupNormExp_DB >= $DB:n) set detCleanupNormExp_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupNormExp -key det_id:iteration -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupNormExp
-    end
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupNormExp
-  end
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-# run the ipp_cleanup.pl script on pending images
-task           detrend.cleanup.normexp.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-  task.exec
-    book npages detCleanupNormExp -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    # look for new images in detCleanupNormExp (pantaskState == INIT)
-    book getpage detCleanupNormExp 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-    book setword detCleanupNormExp $pageName pantaskState RUN
-    book getword detCleanupNormExp $pageName det_id   -var DET_ID
-    book getword detCleanupNormExp $pageName iteration -var ITERATION
-    book getword detCleanupNormExp $pageName camera -var CAMERA
-    book getword detCleanupNormExp $pageName state -var CLEANUP_MODE
-    book getword detCleanupNormExp $pageName dbname -var DBNAME
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-    stdout $LOGDIR/detrend.cleanup.normexp.log
-    stderr $LOGDIR/detrend.cleanup.normexp.log
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.norm.exp --stage_id $DET_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-    # save the pageName for future reference below
-    options $pageName
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-  # default exit status
-  task.exit    default
-    process_exit detCleanupNormExp $options:0 $JOB_STATUS
-  end
-  task.exit    crash
-    showcommand crash
-    book setword detCleanupNormExp $options:0 pantaskState CRASH
-  end
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupNormExp $options:0 pantaskState TIMEOUT
-  end
-end

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.process.pro

-              r23352
+              r23594
 book init detPendingProcessedImfile
 book init detPendingProcessedExp
-book init detCleanupProcessedImfile
-book init detCleanupProcessedExp
 macro detproc.reset
   book init detPendingProcessedImfile
   book init detPendingProcessedExp
-  book init detCleanupProcessedImfile
-  book init detCleanupProcessedExp
 end
 …
   echo detPendingProcessedExp
   book listbook detPendingProcessedExp
-  echo detCleanupProcessedImfile
-  book listbook detCleanupProcessedImfile
-  echo detCleanupProcessedExp
-  book listbook detCleanupProcessedExp
 end
 …
     active true
   end
-  task detrend.cleanup.process.load
-    active true
-  end
-  task detrend.cleanup.process.run
-    active true
-  end
-  task detrend.cleanup.processexp.load
-    active true
-  end
-  task detrend.cleanup.processexp.run
-    active true
-  end
 end
 …
     active false
   end
-  task detrend.cleanup.process.load
-    active false
-  end
-  task detrend.cleanup.process.run
-    active false
-  end
-  task detrend.cleanup.processexp.load
-    active false
-  end
-  task detrend.cleanup.processexp.run
-    active false
-  end
 end
 …
 $detPendingProcessedImfile_DB = 0
 $detPendingProcessedExp_DB = 0
-$detCleanupProcessedImfile_DB = 0
-$detCleanupProcessedExp_DB = 0
 # select images ready for copy
 …
   end
 end
-######## cleanup process imfile ########
-task           detrend.cleanup.process.load
-  host         local
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-  stdout NULL
-  stderr $LOGDIR/detrend.cleanup.process.imfile.log
-  task.exec
-    $run = dettool -pendingcleanup_processedimfile
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupProcessedImfile_DB
-      $run = $run -dbname $DB:$detCleanupProcessedImfile_DB
-      $detCleanupProcessedImfile_DB ++
-      if ($detCleanupProcessedImfile_DB >= $DB:n) set detCleanupProcessedImfile_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupProcessedImfile -key det_id:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupProcessedImfile
-    end
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupProcessedImfile
-  end
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-# run the ipp_cleanup.pl script on pending images
-task           detrend.cleanup.process.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-  task.exec
-    book npages detCleanupProcessedImfile -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    # look for new images in detCleanupProcessedImfile (pantaskState == INIT)
-    book getpage detCleanupProcessedImfile 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-    book setword detCleanupProcessedImfile $pageName pantaskState RUN
-    book getword detCleanupProcessedImfile $pageName det_id   -var DET_ID
-    book getword detCleanupProcessedImfile $pageName exp_id   -var EXP_ID
-    book getword detCleanupProcessedImfile $pageName class_id -var CLASS_ID
-    book getword detCleanupProcessedImfile $pageName camera   -var CAMERA
-    book getword detCleanupProcessedImfile $pageName state    -var CLEANUP_MODE
-    book getword detCleanupProcessedImfile $pageName dbname   -var DBNAME
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-    stdout $LOGDIR/detrend.cleanup.process.imfile.log
-    stderr $LOGDIR/detrend.cleanup.process.imfile.log
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.process.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-    # save the pageName for future reference below
-    options $pageName
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-  # default exit status
-  task.exit    default
-    process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS
-  end
-  # locked list
-  task.exit    crash
-    showcommand crash
-    echo "hostname: $JOB_HOSTNAME"
-    book setword detCleanupProcessedImfile $options:0 pantaskState CRASH
-  end
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupProcessedImfile $options:0 pantaskState TIMEOUT
-  end
-end
-######## cleanup process exp ########
-task           detrend.cleanup.processexp.load
-  host         local
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-  stdout NULL
-  stderr $LOGDIR/detrend.cleanup.process.exp.log
-  task.exec
-    $run = dettool -pendingcleanup_processedexp
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupProcessedExp_DB
-      $run = $run -dbname $DB:$detCleanupProcessedExp_DB
-      $detCleanupProcessedExp_DB ++
-      if ($detCleanupProcessedExp_DB >= $DB:n) set detCleanupProcessedExp_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupProcessedExp -key det_id:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupProcessedExp
-    end
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupProcessedExp
-  end
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-# run the ipp_cleanup.pl script on pending images
-task           detrend.cleanup.processexp.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-  task.exec
-    book npages detCleanupProcessedExp -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    # look for new images in detCleanupProcessedExp (pantaskState == INIT)
-    book getpage detCleanupProcessedExp 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-    book setword detCleanupProcessedExp $pageName pantaskState RUN
-    book getword detCleanupProcessedExp $pageName det_id   -var DET_ID
-    book getword detCleanupProcessedExp $pageName exp_id   -var EXP_ID
-    book getword detCleanupProcessedExp $pageName camera -var CAMERA
-    book getword detCleanupProcessedExp $pageName state -var CLEANUP_MODE
-    book getword detCleanupProcessedExp $pageName dbname -var DBNAME
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-    stdout $LOGDIR/detrend.cleanup.process.exp.log
-    stderr $LOGDIR/detrend.cleanup.process.exp.log
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.process.exp --stage_id $DET_ID --exp_id $EXP_ID --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-    # save the pageName for future reference below
-    options $pageName
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-  # default exit status
-  task.exit    default
-    process_exit detCleanupProcessedExp $options:0 $JOB_STATUS
-  end
-  # locked list
-  task.exit    crash
-    showcommand crash
-    echo "hostname: $JOB_HOSTNAME"
-    book setword detCleanupProcessedExp $options:0 pantaskState CRASH
-  end
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupProcessedExp $options:0 pantaskState TIMEOUT
-  end
-end

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.resid.pro

-              r23352
+              r23594
 book init detPendingResidImfile
 book init detPendingResidExp
-book init detCleanupResidImfile
-book init detCleanupResidExp
 macro detresid.reset
   book init detPendingResidImfile
   book init detPendingResidExp
-  book init detCleanupResidImfile
-  book init detCleanupResidExp
 end
 …
   book listbook detPendingResidImfile
   book listbook detPendingResidExp
-  book listbook detCleanupResidImfile
-  book listbook detCleanupResidExp
 end
 …
     active true
   end
-  task detrend.cleanup.resid.load
-    active true
-  end
-  task detrend.cleanup.resid.run
-    active true
-  end
-  task detrend.cleanup.residexp.load
-    active true
-  end
-  task detrend.cleanup.residexp.run
-    active true
-  end
 end
 …
     active false
   end
-  task detrend.cleanup.resid.load
-    active false
-  end
-  task detrend.cleanup.resid.run
-    active false
-  end
-  task detrend.cleanup.residexp.load
-    active false
-  end
-  task detrend.cleanup.residexp.run
-    active false
-  end
 end
 …
 $detPendingResidImfile_DB = 0
 $detPendingResidExp_DB = 0
-$detCleanupResidImfile_DB = 0
-$detCleanupResidExp_DB = 0
 # select images ready for copy
 …
   end
 end
-######## cleanup resid imfile ########
-task           detrend.cleanup.resid.load
-  host         local
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-  stdout NULL
-  stderr $LOGDIR/detrend.cleanup.resid.imfile.log
-  task.exec
-    $run = dettool -pendingcleanup_residimfile
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupResidImfile_DB
-      $run = $run -dbname $DB:$detCleanupResidImfile_DB
-      $detCleanupResidImfile_DB ++
-      if ($detCleanupResidImfile_DB >= $DB:n) set detCleanupResidImfile_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupResidImfile -key det_id:iteration:exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupResidImfile
-    end
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupResidImfile
-  end
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-# run the ipp_cleanup.pl script on pending images
-task           detrend.cleanup.resid.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-  task.exec
-    book npages detCleanupResidImfile -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    # look for new images in detCleanupResidImfile (pantaskState == INIT)
-    book getpage detCleanupResidImfile 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-    book setword detCleanupResidImfile $pageName pantaskState RUN
-    book getword detCleanupResidImfile $pageName det_id   -var DET_ID
-    book getword detCleanupResidImfile $pageName exp_id   -var EXP_ID
-    book getword detCleanupResidImfile $pageName class_id -var CLASS_ID
-    book getword detCleanupResidImfile $pageName iteration -var ITERATION
-    book getword detCleanupResidImfile $pageName camera -var CAMERA
-    book getword detCleanupResidImfile $pageName state -var CLEANUP_MODE
-    book getword detCleanupResidImfile $pageName dbname -var DBNAME
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-    stdout $LOGDIR/detrend.cleanup.resid.imfile.log
-    stderr $LOGDIR/detrend.cleanup.resid.imfile.log
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.resid.imfile --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --iteration $ITERATION --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-    # save the pageName for future reference below
-    options $pageName
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-  # default exit status
-  task.exit    default
-    process_exit detCleanupResidImfile $options:0 $JOB_STATUS
-  end
-  # locked list
-  task.exit    crash
-    showcommand crash
-    echo "hostname: $JOB_HOSTNAME"
-    book setword detCleanupResidImfile $options:0 pantaskState CRASH
-  end
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupResidImfile $options:0 pantaskState TIMEOUT
-  end
-end
-######## cleanup resid exp ########
-task           detrend.cleanup.residexp.load
-  host         local
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-  stdout NULL
-  stderr $LOGDIR/detrend.cleanup.resid.exp.log
-  task.exec
-    $run = dettool -pendingcleanup_residexp
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupResidExp_DB
-      $run = $run -dbname $DB:$detCleanupResidExp_DB
-      $detCleanupResidExp_DB ++
-      if ($detCleanupResidExp_DB >= $DB:n) set detCleanupResidExp_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupResidExp -key det_id:iteration:exp_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupResidExp
-    end
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupResidExp
-  end
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-# run the ipp_cleanup.pl script on pending images
-task           detrend.cleanup.residexp.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-  task.exec
-    book npages detCleanupResidExp -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    # look for new images in detCleanupResidExp (pantaskState == INIT)
-    book getpage detCleanupResidExp 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-    book setword detCleanupResidExp $pageName pantaskState RUN
-    book getword detCleanupResidExp $pageName det_id    -var DET_ID
-    book getword detCleanupResidExp $pageName exp_id    -var EXP_ID
-    book getword detCleanupResidExp $pageName iteration -var ITERATION
-    book getword detCleanupResidExp $pageName camera    -var CAMERA
-    book getword detCleanupResidExp $pageName state     -var CLEANUP_MODE
-    book getword detCleanupResidExp $pageName dbname    -var DBNAME
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-    stdout $LOGDIR/detrend.cleanup.resid.exp.log
-    stderr $LOGDIR/detrend.cleanup.resid.exp.log
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.resid.exp --stage_id $DET_ID --exp_id $EXP_ID --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-    # save the pageName for future reference below
-    options $pageName
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-  # default exit status
-  task.exit    default
-    process_exit detCleanupResidExp $options:0 $JOB_STATUS
-  end
-  # locked list
-  task.exit    crash
-    showcommand crash
-    echo "hostname: $JOB_HOSTNAME"
-    book setword detCleanupResidExp $options:0 pantaskState CRASH
-  end
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupResidExp $options:0 pantaskState TIMEOUT
-  end
-end

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.stack.pro

-              r23352
+              r23594
 book init detPendingStackedImfile
-book init detCleanupStackedImfile
 macro detstack.reset
   book init detPendingStackedImfile
-  book init detCleanupStackedImfile
 end
 …
   echo detPendingStackedImfile
   book listbook detPendingStackedImfile
-  echo detCleanupStackedImfile
-  book listbook detCleanupStackedImfile
 end
 …
   end
   task detrend.stack.run
-    active true
-  end
-  task detrend.cleanup.stack.load
-    active true
-  end
-  task detrend.cleanup.stack.run
     active true
   end
 …
     active false
   end
-  task detrend.cleanup.stack.load
-    active false
-  end
-  task detrend.cleanup.stack.run
-    active false
-  end
 end
 # this variable will cycle through the known database names
 $detPendingStackedImfile_DB = 0
-$detCleanupStackedImfile_DB = 0
 # select images ready for detrend_stack.pl
 …
   end
 end
-########## cleanup stack ###########
-task           detrend.cleanup.stack.load
-  host         local
-  periods      -poll $LOADPOLL
-  periods      -exec $LOADEXEC
-  periods      -timeout 30
-  npending     1
-  active       true
-  stdout NULL
-  stderr $LOGDIR/detrend.cleanup.stack.log
-  task.exec
-    $run = dettool -pendingcleanup_stacked
-    if ($DB:n == 0)
-      option DEFAULT
-    else
-      # save the DB name for the exit tasks
-      option $DB:$detCleanupStackedImfile_DB
-      $run = $run -dbname $DB:$detCleanupStackedImfile_DB
-      $detCleanupStackedImfile_DB ++
-      if ($detCleanupStackedImfile_DB >= $DB:n) set detCleanupStackedImfile_DB = 0
-    end
-    add_poll_args run
-    command $run
-  end
-  # success
-  task.exit    0
-    # convert 'stdout' to book format
-    ipptool2book stdout detCleanupStackedImfile -key det_id:iteration:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
-    if ($VERBOSE > 2)
-      book listbook detCleanupStackedImfile
-    end
-    # delete existing entries in the appropriate pantaskStates
-    process_cleanup detCleanupStackedImfile
-  end
-  # locked list
-  task.exit    default
-    showcommand failure
-  end
-  task.exit    crash
-    showcommand crash
-  end
-  # operation times out?
-  task.exit    timeout
-    showcommand timeout
-  end
-end
-# run the ipp_cleanup.pl script on pending images
-task           detrend.cleanup.stack.run
-  periods      -poll $RUNPOLL
-  periods      -exec $RUNEXEC
-  periods      -timeout 60
-  active       true
-  task.exec
-    book npages detCleanupStackedImfile -var N
-    if ($N == 0) break
-    if ($NETWORK == 0) break
-    # look for new images in detCleanupStackedImfile (pantaskState == INIT)
-    book getpage detCleanupStackedImfile 0 -var pageName -key pantaskState INIT
-    if ("$pageName" == "NULL") break
-    book setword detCleanupStackedImfile $pageName pantaskState RUN
-    book getword detCleanupStackedImfile $pageName det_id   -var DET_ID
-    book getword detCleanupStackedImfile $pageName iteration -var ITERATION
-    book getword detCleanupStackedImfile $pageName class_id -var CLASS_ID
-    book getword detCleanupStackedImfile $pageName camera -var CAMERA
-    book getword detCleanupStackedImfile $pageName state -var CLEANUP_MODE
-    book getword detCleanupStackedImfile $pageName dbname -var DBNAME
-    # specify choice of local or remote host based on camera and chip (class_id)
-    set.host.for.camera $CAMERA FPA
-    stdout $LOGDIR/detrend.cleanup.stack.log
-    stderr $LOGDIR/detrend.cleanup.stack.log
-    # XXX is everything listed here needed?
-    $run = ipp_cleanup.pl --stage detrend.stack.imfile --stage_id $DET_ID --iteration $ITERATION --class_id $CLASS_ID --camera $CAMERA --mode $CLEANUP_MODE
-    add_standard_args run
-    # save the pageName for future reference below
-    options $pageName
-    # create the command line
-    if ($VERBOSE > 1)
-      echo command $run
-    end
-    command $run
-  end
-  # default exit status
-  task.exit    default
-    process_exit detCleanupStackedImfile $options:0 $JOB_STATUS
-  end
-  # locked list
-  task.exit    crash
-    showcommand crash
-    echo "hostname: $JOB_HOSTNAME"
-    book setword detCleanupStackedImfile $options:0 pantaskState CRASH
-  end
-  # operation timed out?
-  task.exit    timeout
-    showcommand timeout
-    book setword detCleanupStackedImfile $options:0 pantaskState TIMEOUT
-  end
-end

branches/cnb_branches/cnb_branch_20090301/ippTasks/dist.pro

-              r23352
+              r23594
     book getword distToProcess $pageName outroot -var OUTROOT
     book getword distToProcess $pageName dbname -var DBNAME
+    if ("$CLEAN" = "T")
+        $CLEAN_ARG = "--clean"
+    else
+        $CLEAN_ARG = ""
+    end
+    # magicked is output as integer due to the union in the sql
+    if ($MAGICKED)
+        $MAGICKED_ARG = "--magicked"
+    else
+        $MAGICKED_ARG = ""
+    end
 #    set.host.for.camera $CAMERA $MAGIC_ID
 …
     sprintf logfile "%s/dist.%s.%s.log" $OUTROOT $DIST_ID $COMPONENT
     $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE --magicked $MAGICKED --clean $CLEAN --outroot $OUTROOT --logfile $logfile
+    $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $MAGICKED_ARG $CLEAN_ARG --outroot $OUTROOT --logfile $logfile
     add_standard_args run

branches/cnb_branches/cnb_branch_20090301/ippTasks/nebulous.site.pro.in

-              r19748
+              r23594
 # customize these variables for your site
+# and copy the file to share/pantasks/modules directory in your installation
 $NEB_DB    = nebulous
 $NEB_HOST  = alala
+$NEB_DB    = your_nebulous_database_name
+$NEB_HOST  = host_for_nebulous_mysql_database
 $NEB_USER  = XXX
 $NEB_PASS  = XXX

branches/cnb_branches/cnb_branch_20090301/ippTasks/pantasks.pro

r22422	r23594
13	13	$KEEP_FAILURES = 0
14	14
15		$LOADPOLL = ~~0.25~~
	15	$LOADPOLL = 1.0
16	16	$LOADEXEC = 5
17	17	$RUNPOLL = 0.5

branches/cnb_branches/cnb_branch_20090301/ippTasks/replicate.pro

-              r23352
+              r23594
   # modify these after the tasks are tested
   periods      -poll 10
   periods      -exec 10
+  periods      -poll 0.5
+  periods      -exec 5
   periods      -timeout 1500
   npending     1
 …
   task.exec
+      book npages replicatePending -var N
+      if ($N > 2000)
+        process_cleanup replicatePending
+        break
+      end
       # command does not need to be dynamic, but having it so allows us to adjust the periods
       # so that we dont have to wait 10 minutes for things to start up
       # XXX smaller limited?  7500 will be a huge book of things to do...
+      command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 7500 --so_id_start $SO_ID_START --so_id_range $SO_ID_RANGE
+      periods      -exec 1800
+  end
+  # success
+  task.exit $EXIT_SUCCESS
+      command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 500 --so_id_start $SO_ID_START --so_id_range $SO_ID_RANGE
+      # periods      -exec 1800
+  end
+  # success : 0 -- we did not hit the limit, advance so_id counter
+  task.exit 0
     # advance the so_id counter
+    $SO_ID_START += $SO_ID_RANGE
+    $SO_ID_START = $SO_ID_START + $SO_ID_RANGE
+    # convert 'stdout' to book format
+    ipptool2book stdout replicatePending -key key -uniq -setword pantaskState INIT
+    if ($VERBOSE > 2)
+      book listbook replicatePending
+    end
+    # delete existing entries in the appropriate pantaskStates
+    process_cleanup replicatePending
+  end
+  # success : 1 -- we DID hit the limit, do NOT advance so_id counter
+  task.exit 1
     # convert 'stdout' to book format
     ipptool2book stdout replicatePending -key key -uniq -setword pantaskState INIT
 …
 # create the desired replicas
 task           replicate.run
   periods      -poll $RUNPOLL
   periods      -exec 30
+  periods      -poll 0.5
+  periods      -exec 5
   periods      -timeout 30
 …
     if ($NETWORK == 0) break
     if ($N == 0)
         periods -exec 30
+        periods -exec 5
         break
     end

branches/cnb_branches/cnb_branch_20090301/ippTasks/simtest.pro

-              r21049
+              r23594
     echo "  simtest.setup.detverify : run detrend creation and detrend verification"
     echo "  simtest.setup.flatcorr : run a flat-field correction demonstration"
+    echo "  simtest.setup.ctemask : run a ctemask demonstration"
     break
   end
 …
 end
+macro simtest.setup.ctemask
+  $PPSIM_RECIPE = BADCTE.TEST
+  $SIMTEST_SEQUENCE = simtest.ctemask.config
+  $SIMTEST_AUTO = simtest.ctemask.auto
+end
 macro simtest.setup.flatcorr
   $PPSIM_RECIPE = FLATCORR

branches/cnb_branches/cnb_branch_20090301/ippTasks/summit.copy.pro

-              r23352
+              r23594
 # list of summit imfiles that need to be downloaded
 book init pzPendingImfile
+# list of pzDownloadExps that have completed downloading
+book init pzPendingAdvance
 macro copy.on
 …
       active true
   end
+  task summit.toadvance
+      active true
+  end
   task summit.advance
       active true
 …
   end
   task pztool.clearfault
+      active false
+  end
+  task summit.toadvance
       active false
   end
 …
 $pztoolPendingExp_DB = 0
 $pztoolPendingImfile_DB = 0
+$pztoolPendingAdvance_DB = 0
 $pztoolClearFault_DB = 0
 $pztoolAdvance_DB = 0;
 …
         stderr $LOGDIR/summit.copy.log
         $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --end_stage reg --workdir $workdir --dbname $DBNAME --timeout 120 --verbose --copies 2
+        $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --dbname $DBNAME --timeout 120 --verbose --copies 2
         if ($COMPRESS)
             $run = $run --compress
 …
 end
+# promote exposures for which all imfiles have been copied
+task           summit.advance
+  host         local
+  periods      -poll $LOADPOLL
+  periods      -exec $LOADEXEC
+  periods      -timeout 30
+  npending     1
+  stdout NULL
+  stderr $LOGDIR/summit.advance.log
+  task.exec
+    $run = pztool -advance
+    if ($DB:n == 0)
+      option DEFAULT
+    else
+      # save the DB name for the exit tasks
+      option $DB:$pztoolAdvance_DB
+      $run = $run -dbname $DB:$pztoolAdvance_DB
+      $pztoolAdvance_DB ++
+      if ($pztoolAdvance_DB >= $DB:n) set pztoolAdvance_DB = 0
+    end
+    add_poll_args run
+    command $run
+  end
+  # success
+  task.exit    0
+  end
+  # locked list
+  task.exit    default
+    showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
+  end
+  # operation times out?
+  task.exit    timeout
+    showcommand timeout
+  end
+end
+# build a book of exposures that have completed downloading
+task summit.toadvance
+    host         local
+    periods      -exec     30
+    periods      -poll      1
+    periods      -timeout  120
+    # trage       16:00 23:59
+    # trage       00:00 04:00
+    npending     1
+    # select entries from the current DB; cycle to the next DB, if it exists
+    # iff the DB list is not set, use the value defined in .ipprc
+    task.exec
+      if ($DB:n == 0)
+        option DEFAULT
+        command pztool -toadvance -limit 40
+      else
+        # save the DB name for the exit tasks
+        option $DB:$pztoolPendingAdvance_DB
+        command pztool -toadvance -limit 60 -dbname $DB:$pztoolPendingAdvance_DB
+        $pztoolPendingAdvance_DB ++
+        if ($pztoolPendingAdvance_DB >= $DB:n) set pztoolPendingAdvance_DB = 0
+      end
+    end
+    # success
+    task.exit    0
+        # convert 'stdout' to book format
+        ipptool2book stdout pzPendingAdvance -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT
+        book shuffle pzPendingAdvance
+        # delete existing entries in the appropriate pantaskStates
+        process_cleanup pzPendingAdvance
+    end
+    task.exit     default
+        showcommand failure
+    end
+    task.exit     crash
+        showcommand crash
+    end
+    task.exit     timeout
+        showcommand timeout
+    end
+end
+task summit.advance
+    periods      -exec     5
+    periods      -poll     0.05
+    periods      -timeout  650
+    # trage       16:00 23:59
+    # trage       00:00 04:00
+    task.exec
+        if ($NETWORK == 0) break
+        # if we are waiting on data, make the interval long
+        book npages pzPendingAdvance -var N
+        if ($N == 0)
+            periods -exec 20
+            break
+        end
+        periods -exec 0.05
+        # find an exp that needs imfiles fetched
+        book getpage pzPendingAdvance 0 -var pageName -key pantaskState INIT
+        if ("$pageName" == "NULL") break
+        # set that exp to run
+        book setword pzPendingAdvance $pageName pantaskState RUN
+        book getword pzPendingAdvance $pageName exp_name  -var EXP_NAME
+        book getword pzPendingAdvance $pageName camera    -var CAMERA
+        book getword pzPendingAdvance $pageName telescope -var TELESCOPE
+        book getword pzPendingAdvance $pageName dbname    -var DBNAME
+        # 2007-08-30T05:09:59Z
+        substr $DATEOBS 0 4 YEAR
+        substr $DATEOBS 5 2 MONTH
+        substr $DATEOBS 8 2 DAY
+        # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship
+        # this function uses workdir_template, default_host, volume_template, volume_default,
+        # it sets workdir and volume
+        set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base
+        $workdir = $workdir_template/$CAMERA/$YEAR\$MONTH\$DAY
+        # workdir examples:
+        # file://data/@HOST@.0/gpc1/20080130
+        # neb://@HOST@.0/gpc1/20080130
+        stdout $LOGDIR/summit.advance.log
+        stderr $LOGDIR/summit.advance.log
+        $run = pztool -advance -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -end_stage reg -workdir $workdir -dbname $DBNAME
+        # store the pageName for future reference below
+        options $pageName
+        # create the command line
+        if ($VERBOSE > 1)
+          echo command $run
+        end
+        command $run
+    end
+    # default exit status
+    task.exit default
+        process_exit pzPendingAdvance $options:0 $JOB_STATUS
+    end
+    task.exit crash
+        showcommand crash
+        book setword pzPendingAdvance $options:0 pantaskState CRASH
+    end
+    # operation timed out?
+    task.exit timeout
+        showcommand timeout
+        book setword pzPendingAdvance $options:0 pantaskState TIMEOUT
+    end
+end

branches/cnb_branches/cnb_branch_20090301/ippTasks/warp.pro

-              r23352
+              r23594
     book getword warpPendingSkyCell $pageName exp_tag -var EXP_TAG
     book getword warpPendingSkyCell $pageName state -var RUN_STATE
+    book getword warpPendingSkyCell $pageName magicked -var CHIP_MAGICKED
+    if ("$CHIP_MAGICKED" == "T")
+        $MAGICKED_ARG = "--magicked"
+    else
+        $MAGICKED_ARG = ""
+    end
     # set the host and workdir based on the skycell hash
 …
     stderr $LOGDIR/warp.skycell.log
     $run = warp_skycell.pl --threads @MAX_THREADS@ --warp_id $WARP_ID --warp_skyfile_id $WARP_SKYFILE_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output --run-state $RUN_STATE
+    $run = warp_skycell.pl --threads @MAX_THREADS@ --warp_id $WARP_ID --warp_skyfile_id $WARP_SKYFILE_ID --skycell_id $SKYCELL_ID --tess_dir $TESS_DIR --camera $CAMERA --outroot $outroot --redirect-output --run-state $RUN_STATE $MAGICKED_ARG
     add_standard_args run

branches/cnb_branches/cnb_branch_20090301/ippTools/configure.ac

r21308	r23594
18	18	PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
19	19	PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
20		PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.48])
	20	PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.50])
21	21
22	22	PXTOOLS_CFLAGS="${PSLIB_CFLAGS=} ${PSMODULES_CFLAGS=} ${IPPDB_CFLAGS=}"

branches/cnb_branches/cnb_branch_20090301/ippTools/share/Makefile.am

-              r23352
+              r23594
      difftool_definebyquery_part2.sql \
      difftool_definebyquery_temp_create.sql \
+     difftool_definewarpwarp_select.sql \
+     difftool_definewarpwarp_insert.sql \
      difftool_donecleanup.sql \
      difftool_inputskyfile.sql \
 …
      regtool_revertprocessedexp.sql \
      regtool_revertprocessedimfile.sql \
+     regtool_updateprocessedimfile.sql \
      stacktool_definebyquery_insert.sql \
      stacktool_definebyquery_insert_random_part1.sql \
 …
      warptool_donecleanup.sql \
      warptool_exp.sql \
+     warptool_finished_run_select.sql \
+     warptool_finish_run.sql \
      warptool_imfile.sql \
      warptool_pendingcleanuprun.sql \

branches/cnb_branches/cnb_branch_20090301/ippTools/share/camtool_find_processedexp.sql

r21443	r23594
1	1	SELECT DISTINCT
2	2	camProcessedExp.*,
	3	chipRun.chip_id,
3	4	rawExp.exp_tag,
4	5	rawExp.exp_name,

branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_change_imfile_data_state.sql

-              r23352
+              r23594
 -- handle changes in data_state. Used for the modes tocleanedimfile and topurgedimfile
+-- args are new data_state, chip_id, class_id, and current expected state for chipRun
+-- args are new data_state, a possibly empty string for updating the magicked state,
+-- chip_id and class_id
 UPDATE chipProcessedImfile
+JOIN rawImfile USING(exp_id, class_id)
     SET
+    data_state = '%s'
+    chipProcessedImfile.data_state = '%s'
+    -- set magicked hook %s
 WHERE
     chip_id = %lld

branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_completely_processed_exp.sql

-              r20608
+              r23594
     dvodb,
     tess_id,
+    end_stage
+    end_stage,
+    all_files_magicked as magicked
 FROM
     (SELECT
         chipRun.*,
         rawImfile.class_id as rawimfile_class_id,
+        chipProcessedImfile.class_id
+        chipProcessedImfile.class_id,
+        SUM(!chipProcessedImfile.magicked) = 0 as all_files_magicked
     FROM chipRun
     JOIN rawImfile

branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_pendingimfile.sql

r21334	r23594
4	4	rawImfile.class_id,
5	5	rawImfile.uri,
	6	rawImfile.magicked as raw_magicked,
	7	(rawImfile.user_1 is not NULL and rawImfile.user_1 > 0.5) as deburned,
6	8	rawExp.exp_tag,
7	9	rawExp.exp_name,

branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_completed_runs.sql

-              r20973
+              r23594
 SELECT DISTINCT
+    diff_id
+    diff_id,
+    all_magicked as magicked
 FROM (
     SELECT
         COUNT(diffInputSkyfile.skycell_id), COUNT(diffSkyfile.skycell_id),
+        diffSkyfile.*
+        diffSkyfile.*,
+        SUM(!diffSkyfile.magicked) = 0 as all_magicked
     FROM diffRun
     JOIN diffInputSkyfile USING(diff_id)

branches/cnb_branches/cnb_branch_20090301/ippTools/share/difftool_inputskyfile.sql

-              r20976
+              r23594
         warpSkyfile.uri,
         warpSkyfile.path_base,
+        warpSkyfile.magicked as magicked,
 as template,
         rawExp.camera
 …
         warpSkyfile.uri,
         warpSkyfile.path_base,
+        warpSkyfile.magicked as magicked,
 as template,
         rawExp.camera
 …
         stackSumSkyfile.uri,
         stackSumSkyfile.path_base,
+as magicked,
 as template,
         rawExp.camera
 …
         stackSumSkyfile.uri,
         stackSumSkyfile.path_base,
+as magicked,
 as template,
         rawExp.camera

branches/cnb_branches/cnb_branch_20090301/ippTools/share/disttool_pendingcomponent.sql

-              r23352
+              r23594
 SELECT
     distRun.dist_id,
+    distRun.label,
     stage,
     stage_id,
 …
     rawExp.camera,
     outroot,
     rawImfile.uri as path_base,
+    rawImfile.uri as path_base,         -- change this once rawImfile has path_base
     chipProcessedImfile.path_base as chip_path_base,
     NULL as state,
 …
 FROM distRun
 JOIN rawExp ON exp_id = stage_id
+JOIN rawImfile using(exp_id)
+JOIN (                      -- find the last magicked chip run
+    SELECT
+        exp_id,
+        MAX(chip_id) AS chip_id
+    FROM chipRun
+    WHERE
+        chipRun.state = 'full'
+        AND chipRun.exp_id = exp_id
+        --   AND chipRun.magicked
+        -- magicked hook 1 %s
+        GROUP BY exp_id
+    ) AS bestChipRun
+    USING(exp_id)
+JOIN rawImfile USING(exp_id)
 JOIN chipProcessedImfile
+    ON distRun.chip_id = chipProcessedImfile.chip_id
+    AND rawImfile.class_id = chipProcessedImfile.class_id
+    USING(exp_id, chip_id, class_id)
 LEFT JOIN distComponent
     ON distRun.dist_id = distComponent.dist_id
 …
     AND distRun.stage = 'raw'
     AND distComponent.dist_id IS NULL
+    -- if magicked add AND rawImfile.magicked here
+    -- where hook 1 %s
 -- chip stage
 …
 SELECT
     distRun.dist_id,
+    distRun.label,
     stage,
     stage_id,
 …
     AND distRun.stage = 'chip'
     AND distComponent.dist_id IS NULL
+UNION
+SELECT
+    distRun.dist_id,
+    -- where hook 2 %s
+UNION
+SELECT
+    distRun.dist_id,
+    distRun.label,
+    stage,
+    stage_id,
+    chipProcessedImfile.class_id AS component,
+    clean,
+    rawExp.camera,
+    outroot,
+    camProcessedExp.path_base,
+    chipProcessedImfile.path_base as chip_path_base,
+    camRun.state,
+    NULL,
+FROM distRun
+JOIN camRun ON camRun.cam_id = distRun.stage_id
+JOIN camProcessedExp USING(cam_id)
+JOIN chipRun USING(chip_id)
+JOIN chipProcessedImfile USING(exp_id, chip_id)
+JOIN rawExp using(exp_id)
+LEFT JOIN distComponent
+    ON distRun.dist_id = distComponent.dist_id
+    AND chipProcessedImfile.class_id = distComponent.component
+WHERE
+    distRun.state = 'new'
+    AND distRun.stage = 'camera'
+    AND distComponent.dist_id IS NULL
+    -- where hook 3 %s
+UNION
+SELECT
+    distRun.dist_id,
+    distRun.label,
+    stage,
+    stage_id,
+    fakeProcessedImfile.class_id AS component,
+    clean,
+    rawExp.camera,
+    outroot,
+    fakeProcessedImfile.path_base,
+    NULL,
+    fakeRun.state,
+    NULL,
+FROM distRun
+JOIN fakeRun ON fakeRun.fake_id = distRun.stage_id
+JOIN fakeProcessedImfile USING(fake_id)
+JOIN camRun USING(cam_id)
+JOIN chipRun USING(chip_id, exp_id)
+JOIN rawExp using(exp_id)
+LEFT JOIN distComponent
+    ON distRun.dist_id = distComponent.dist_id
+    AND fakeProcessedImfile.class_id = distComponent.component
+WHERE
+    distRun.state = 'new'
+    AND distRun.stage = 'fake'
+    AND distComponent.dist_id IS NULL
+    -- where hook 4 %s
+UNION
+SELECT
+    distRun.dist_id,
+    distRun.label,
     stage,
     stage_id,
 …
     AND distRun.stage = 'warp'
     AND distComponent.dist_id IS NULL
+UNION
+SELECT
+    distRun.dist_id,
+    -- where hook 5 %s
+UNION
+SELECT
+    distRun.dist_id,
+    distRun.label,
     stage,
     stage_id,
 …
     AND distRun.stage = 'diff'
     AND distComponent.dist_id IS NULL
+    -- where hook 6 %s
 UNION
 SELECT DISTINCT
     distRun.dist_id,
+    distRun.label,
     stage,
     stage_id,
 …
     AND distRun.stage = 'stack'
     AND distComponent.dist_id IS NULL
+    -- where hook 7 %s
 ) as Foo

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_completed_runs.sql

-              r20973
+              r23594
         AND magicDSFile.component = rawImfile.class_id
     WHERE
         magicDSRun.state = 'run'
+        magicDSRun.state = 'new'
         AND magicDSRun.stage = 'raw'
     GROUP BY
 …
         AND magicDSFile.component = chipProcessedImfile.class_id
     WHERE
         magicDSRun.state = 'run'
+        magicDSRun.state = 'new'
         AND magicDSRun.stage = 'chip'
     GROUP BY
 …
         AND magicDSFile.component = warpSkyfile.skycell_id
     WHERE
         magicDSRun.state = 'run'
+        magicDSRun.state = 'new'
         AND magicDSRun.stage = 'warp'
         AND warpSkyfile.fault = 0
 …
         AND magicDSFile.component = diffSkyfile.skycell_id
     WHERE
         magicDSRun.state = 'run'
+        magicDSRun.state = 'new'
         AND magicDSRun.stage = 'diff'
         AND diffSkyfile.fault = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magicdstool_todestreak.sql

-              r20973
+              r23594
     AND magicDSFile.component = rawImfile.class_id
 WHERE
     magicDSRun.state = 'run'
+    magicDSRun.state = 'new'
     AND magicDSRun.stage = 'raw'
     AND magicDSFile.component IS NULL
 …
     AND magicDSFile.component = chipProcessedImfile.class_id
 WHERE
     magicDSRun.state = 'run'
+    magicDSRun.state = 'new'
     AND magicDSRun.stage = 'chip'
     AND chipRun.state = 'full'
 …
     AND magicDSFile.component = warpSkyfile.skycell_id
 WHERE
     magicDSRun.state = 'run'
+    magicDSRun.state = 'new'
     AND magicDSRun.stage = 'warp'
     AND warpRun.state = 'full'
 …
     AND magicDSFile.component = diffSkyfile.skycell_id
 WHERE
     magicDSRun.state = 'run'
+    magicDSRun.state = 'new'
     AND magicDSRun.stage = 'diff'
     AND diffSkyfile.fault = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_addmask.sql

-              r18526
+              r23594
     magicRun
 SET
     state = 'stop'
+    state = 'full'
 WHERE
     state != 'stop'
+    state != 'full'

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_chipprocessedimfile.sql

r20706	r23594
26	26	chipRun.state = 'full'
27	27	AND chipProcessedImfile.fault = 0
28		-- AND magicRun.state = '~~stop~~'
	28	-- AND magicRun.state = 'full'
29	29	-- AND magicMask.fault = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_diffskyfile.sql

r20701	r23594
24	24	diffRun.state = 'full'
25	25	AND diffSkyfile.fault = 0
26		-- AND magicRun.state = '~~stop~~'
	26	-- AND magicRun.state = 'full'
27	27	-- AND magicMask.fault = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_inputskyfile.sql

r20973	r23594
9	9	AND magicInputSkyfile.node = diffSkyfile.skycell_id
10	10	WHERE
11		magicRun.state = '~~run~~'
	11	magicRun.state = 'new'

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_mask.sql

r20735	r23594
6	6	USING(magic_id)
7	7	WHERE
8		magicRun.state = '~~stop~~'
	8	magicRun.state = 'full'
9	9	AND magicMask.fault = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_rawimfile.sql

r20706	r23594
24	24	WHERE
25	25	rawImfile.fault = 0
26		-- AND magicRun.state = '~~stop~~'
	26	-- AND magicRun.state = 'full'
27	27	-- AND magicMask.fault = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_tomask.sql

r18781	r23594
10	10	LEFT JOIN magicNodeResult USING(magic_id, node)
11	11	WHERE
12		magicRun.state = '~~run~~'
	12	magicRun.state = 'new'
13	13	AND magicNodeResult.node = 'root'
14	14	AND magicNodeResult.fault = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_toprocess_inputs.sql

r21004	r23594
19	19	AND magicTree.node = magicNodeResult.node
20	20	WHERE
21		magicRun.state = '~~run~~'
	21	magicRun.state = 'new'
22	22	AND magicNodeResult.magic_id IS NULL
23	23	AND magicNodeResult.node IS NULL

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_toprocess_tree.sql

r21099	r23594
15	15	AND magicTree.node = magicNodeResult.node
16	16	WHERE
17		magicRun.state = '~~run~~'
	17	magicRun.state = 'new'
18	18	-- where hook %s
19	19	ORDER BY

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_toskyfilemask.sql

r15348	r23594
7	7	USING(magic_id)
8	8	WHERE
9		magicRun.state = '~~run~~'
	9	magicRun.state = 'new'
10	10	AND magicSkyfileMask.magic_id IS NULL

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_totree.sql

r20474	r23594
12	12	USING(magic_id)
13	13	WHERE
14		magicRun.state = '~~run~~'
	14	magicRun.state = 'new'
15	15	AND magicTree.node IS NULL
16	16	AND magicRun.fault = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/magictool_warpskyfile.sql

r20701	r23594
21	21	warpRun.state = 'full'
22	22	AND warpSkyfile.fault = 0
23		-- AND magicRun.state = '~~stop~~'
	23	-- AND magicRun.state = 'full'
24	24	-- AND magicMask.fault = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/pxadmin_create_tables.sql

-              r23352
+              r23594
     fault SMALLINT NOT NULL,
     epoch TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    magicked TINYINT,
     PRIMARY KEY(exp_id),
     KEY(exp_name),
 …
     tess_id VARCHAR(64),
     end_stage VARCHAR(64),
+    magicked TINYINT,
     PRIMARY KEY(chip_id),
     KEY(chip_id), KEY(exp_id),
 …
     end_stage VARCHAR(64),
     registered DATETIME,
+    magicked TINYINT,
     PRIMARY KEY(warp_id),
     KEY(warp_id),
 …
         tess_id VARCHAR(64),
         exp_id  BIGINT,
+        magicked TINYINT,
         PRIMARY KEY(diff_id),
         KEY(diff_id),
 …
         stage_id BIGINT,
         cam_id BIGINT,
+        label VARCHAR(64),
         outroot VARCHAR(255),
         recoveryroot VARCHAR(255),
 …
         KEY(magic_ds_id),
         KEY(state),
+        KEY(magic_id),
+        KEY(label),
         FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 …
         KEY(job_id),
         FOREIGN KEY (req_id)  REFERENCES  pstampRequest(req_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE distTarget (
+    target_id   BIGINT AUTO_INCREMENT,
+    obs_mode    VARCHAR(64),
+    stage       VARCHAR(64),
+    clean       TINYINT,
+    state       VARCHAR(64),
+    comment     VARCHAR(255),
+    PRIMARY KEY(target_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE distRun (
+    dist_id     BIGINT AUTO_INCREMENT,
+    target_id   BIGINT,
+    stage       VARCHAR(64),
+    stage_id    BIGINT,
+    label       VARCHAR(64),
+    outroot     VARCHAR(255),
+    clean       TINYINT,
+    state       VARCHAR(64),
+    time_stamp  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    fault       SMALLINT,
+    PRIMARY KEY(dist_id),
+    KEY(state),
+    KEY(label),
+    FOREIGN KEY(target_id) REFERENCES distTarget(target_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE distComponent (
+    dist_id     BIGINT,
+    component   VARCHAR(64),
+    bytes       INT,
+    md5sum      VARCHAR(32),
+    state       VARCHAR(64),
+    name        VARCHAR(255),
+    fault       SMALLINT,
+    PRIMARY KEY(dist_id, component),
+    KEY(state),
+    FOREIGN KEY(dist_id) REFERENCES distRun(dist_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcDSProduct (
+    prod_id     BIGINT AUTO_INCREMENT,
+    name        VARCHAR(64),
+    dbname      VARCHAR(64),
+    dbhost      VARCHAR(64),
+    prod_root   VARCHAR(255),
+    PRIMARY KEY(prod_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcDestination (
+    dest_id     BIGINT AUTO_INCREMENT,
+    prod_id     BIGINT,
+    name        VARCHAR(64),
+    status_uri  VARCHAR(255),
+    comment     VARCHAR(255),
+    last_fileset VARCHAR(255),
+    state       VARCHAR(64),
+    PRIMARY KEY(dest_id),
+    FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcInterest (
+    int_id      BIGINT AUTO_INCREMENT,
+    dest_id     BIGINT,
+    target_id   BIGINT,
+    prod_id     BIGINT,
+    state       VARCHAR(64),
+    PRIMARY KEY(int_id),
+    FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id),
+    FOREIGN KEY(target_id) REFERENCES distTarget(target_id),
+    FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcDSFileset (
+    fs_id       BIGINT AUTO_INCREMENT,
+    dist_id     BIGINT,
+    prod_id     BIGINT,
+    name        VARCHAR(64),
+    state       VARCHAR(64),
+    registered  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(fs_id),
+    FOREIGN KEY(dist_id) REFERENCES distRun(dist_id),
+    FOREIGN KEY(prod_id) REFERENCES rcDSProduct(prod_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+CREATE TABLE rcRun (
+    rc_id       BIGINT AUTO_INCREMENT,
+    fs_id       BIGINT,
+    dest_id     BIGINT,
+    state       VARCHAR(64),
+    registered  TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    PRIMARY KEY(rc_id),
+    FOREIGN KEY(fs_id) REFERENCES rcDSFileset(fs_id),
+    FOREIGN KEY(dest_id) REFERENCES rcDestination(dest_id)
+)  ENGINE=innodb DEFAULT CHARSET=latin1;
+-- This comment line is here to avoid empty query error.
+-- Another way to avoid that problem is to omit the semicolon above but I think that is untidy.

branches/cnb_branches/cnb_branch_20090301/ippTools/share/pxadmin_drop_tables.sql

-              r21308
+              r23594
 DROP TABLE IF EXISTS calRun;
 DROP TABLE IF EXISTS flatcorrRun;
+DROP TABLE IF EXISTS flatcorrExp;
 DROP TABLE IF EXISTS flatcorrChipLink;
 DROP TABLE IF EXISTS flatcorrCamLink;
 …
 DROP TABLE IF EXISTS pstampRequest;
 DROP TABLE IF EXISTS pstampJob;
+DROP TABLE IF EXISTS distRun;
+DROP TABLE IF EXISTS distTarget;
+DROP TABLE IF EXISTS distComponent;
+DROP TABLE IF EXISTS rcDSProduct;
+DROP TABLE IF EXISTS rcDestination;
+DROP TABLE IF EXISTS rcInterest;
+DROP TABLE IF EXISTS rcDSFileset;
+DROP TABLE IF EXISTS rcRun;
 SET FOREIGN_KEY_CHECKS=1

branches/cnb_branches/cnb_branch_20090301/ippTools/share/pztool_find_completed_exp.sql

-              r17925
+              r23594
 SELECT DISTINCT
     exp_name, -- return should match pzDownloadExp
+    exp_name,
     camera,
     telescope,
+    state,
+    NULL as epoch    -- epoch
+    state
 FROM (
     SELECT

branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_change_skyfile_data_state.sql

r23352	r23594
8	8	warp_id = %lld
9	9	AND skycell_id = '%s'
10		~~-- only update if chipRun.state has the expected value~~
11		~~-- AND (~~
12		~~-- SELECT state from warpRun where warpRun.warp_id = warpSkyfile.warp_id~~
13		~~-- ) = '%s'~~

branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_definebyquery.sql

r20041	r23594
9	9	rawExp.dateobs,
10	10	rawExp.exp_tag,
	11	rawExp.exp_name,
11	12	rawExp.exp_type,
12	13	rawExp.filelevel,

branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_pendingcleanuprun.sql

r19521	r23594
13	13	USING (exp_id)
14	14	WHERE
15		(warpRun.state = 'goto_cleaned' OR warpRun.state = 'goto_purged')
	15	(warpRun.state = 'goto_cleaned' OR warpRun.state = 'goto_scrubbed' OR warpRun.state = 'goto_purged')

branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_pendingcleanupskyfile.sql

-              r19521
+              r23594
     USING(warp_id)
 WHERE
     (warpRun.state = 'goto_cleaned' AND warpSkyfile.data_state = 'full')
+   ((warpRun.state = 'goto_cleaned'  AND warpSkyfile.data_state = 'full')
     OR
+    (warpRun.state = 'goto_purged' AND warpSkyfile.data_state != 'purged')
+    (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state = 'full')
+    OR
+    (warpRun.state = 'goto_purged'   AND warpSkyfile.data_state != 'purged'))

branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_towarped.sql

-              r21308
+              r23594
     rawExp.camera,
     rawExp.exp_tag,
+    warpRun.workdir
+    warpRun.workdir,
+    chipRun.magicked
 FROM warpRun
 JOIN warpSkyCellMap

branches/cnb_branches/cnb_branch_20090301/ippTools/src

Property svn:ignore

old	new
32	32	flatcorr
33	33	pstamptool
	34	disttool

branches/cnb_branches/cnb_branch_20090301/ippTools/src/Makefile.am

-              r20789
+              r23594
         dettool \
         difftool \
+        disttool \
         flatcorr \
         magictool \
 …
         dettool.h \
         difftool.h \
+        disttool.h \
         flatcorr.h \
         faketool.h \
 …
     difftoolConfig.c
+disttool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+disttool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+disttool_SOURCES = \
+    disttool.c \
+    disttoolConfig.c
 stacktool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
 stacktool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la

branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtool.c

-              r23352
+              r23594
     psMetadata *where = psMetadataAlloc();
     pxcamGetSearchArgs (config, where);
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "chipRun.label",     "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
     if (!psListLength(where->list)
 …
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "==");
     pxcamGetSearchArgs (config, where);
+    PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "camRun.state", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",    "camRun.cam_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "camRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-state",     "camRun.state", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "==");
     if (!psListLength(where->list)
 …
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "==");
     pxcamGetSearchArgs (config, where);
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",    "camRun.cam_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "camRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "==");
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 …
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-cam_id",   "cam_id",   "==");
-    PXOPT_COPY_S64(config->args, where, "-chip_id",  "chip_id",  "==");
-    PXOPT_COPY_STR(config->args, where, "-class",    "class",    "==");
-    PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
     pxcamGetSearchArgs (config, where);
+    PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",    "camRun.cam_id",                "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "camRun.label",                 "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction",             "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id",   "chipRun.chip_id",              "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id",  "chipProcessedImfile.class_id", "==");
+    // XXX is this used? PXOPT_COPY_STR(config->args, where, "-class",    "class",         "==");
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 …
     // generate restrictions
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-cam_id",   "cam_id",   "==");
     pxcamGetSearchArgs (config, where);
+    PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",    "camRun.cam_id",    "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "camRun.label",     "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "==");
     psString query = pxDataGet("camtool_find_processedexp.sql");
 …
+    }
+    // order by cam_id so that the postage stamp parser can easliy find the 'latest' astrometry
+    psStringAppend(&query, " ORDER BY cam_id");
     // treat limit == 0 as "no limit"
     if (limit) {
 …
+    }
-    // order by cam_id so that the postage stamp parser can easliy find the 'latest' astrometry
-    psStringAppend(&query, "\nORDER BY cam_id");
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
 …
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "==");
     pxcamGetSearchArgs (config, where);
+    PXOPT_COPY_STR(config->args, where, "-label", "camRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-code", "camProcessedExp.fault", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",    "camRun.cam_id",         "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "camRun.label",          "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction",      "==");
+    PXOPT_COPY_S16(config->args, where, "-code",      "camProcessedExp.fault", "==");
     if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtoolConfig.c

-              r23352
+              r23594
     psMetadata *definebyqueryArgs = psMetadataAlloc();
     pxcamSetSearchArgs(definebyqueryArgs);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",              0, "search by chipRun label", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",          0, "search by chipRun reduction class", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir",        0, "define workdir", NULL);
 …
     // XXX need to allow multiple exp_ids
     psMetadata *updaterunArgs = psMetadataAlloc();
+    pxcamSetSearchArgs(updaterunArgs);
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id",             0, "search by cam_id", 0);
+    pxcamSetSearchArgs(updaterunArgs);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search for state", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",              0, "search by camRun label", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state",              0, "search by camRun state", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction",          0, "search by camRun reduction class", NULL);
     psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all",               0, "allow everything to be queued without search terms", false);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state",              0, "set state", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label",              0, "set label", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state",          0, "set state", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label",          0, "set label", NULL);
     // -pendingexp
     psMetadata *pendingexpArgs = psMetadataAlloc();
-    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id",            0, "search by camtool ID", 0);
     pxcamSetSearchArgs(pendingexpArgs);
+    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL);
+    psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id",            0, "search by cam_id", 0);
+    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label",             0, "search by camRun label", NULL);
+    psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction",         0, "search by camRun reduction class", NULL);
     psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit",             0, "limit result set to N items", 0);
     psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple",           0, "use the simple output format", false);
 …
     // -pendingimfile
     psMetadata *pendingimfileArgs = psMetadataAlloc();
-    psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-cam_id", 0,            "search by camtool ID", 0);
     pxcamSetSearchArgs(pendingimfileArgs);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0,            "search by class", NULL);
+    psMetadataAddS64(pendingimfileArgs, PS_LIST_TAIL, "-cam_id",   0,            "search by camtool ID", 0);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-label",    0,            "search by camRun label", NULL);
+    psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-reduction",0,            "search by camRun reduction class", NULL);
     psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class_id", 0,            "search by class ID", NULL);
+    psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple", 0,            "use the simple output format", false);
+    psMetadataAddBool(pendingimfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
+    // XXX is this used? psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class",    0,            "search by class", NULL);
     // -addprocessedexp
 …
     // -processedexp
     psMetadata *processedexpArgs = psMetadataAlloc();
-    psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0,            "search by camtool ID", 0);
     pxcamSetSearchArgs(processedexpArgs);
+    psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL);
+    psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id",   0,            "search by cam_id", 0);
+    psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label",    0,            "search by camRun label", NULL);
+    psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0,            "search by camRun reduction class", NULL);
     psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 …
     // XXX need to allow multiple exp_ids
     psMetadata *revertprocessedexpArgs = psMetadataAlloc();
-    psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id",  0,            "search by cam_id", 0);
     pxcamSetSearchArgs(revertprocessedexpArgs);
+    psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", 0, "search for label", NULL);
+    psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id",   0,            "search by cam_id", 0);
+    psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label",    0,            "search by camRun label", NULL);
+    psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0,            "search by camRun reduction class", NULL);
+    psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code",     0,            "search by fault code", 0);
     psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all",  0,            "allow everything to be queued without search terms", false);
-    psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
     // -updateprocessedexp

branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptool.c

-              r23352
+              r23594
             return false;
+        }
+        //
         // queue the exp
         if (!pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) {
 …
     PXOPT_LOOKUP_F32(n_cr, config->args,           "-n_cr", false, false);
     PXOPT_LOOKUP_STR(path_base, config->args,      "-path_base", false, false);
+    PXOPT_LOOKUP_BOOL(magicked, config->args,      "-magicked", false);
     // default values
 …
                                    path_base,
                                    code,
     // magic_ds_id
+                                   magicked
             )) {
         // rollback
 …
     PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
     PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "LIKE");
+    PXOPT_COPY_S32(config->args, where, "-magicked", "chipRun.magicked", "==");
     psString query = pxDataGet("chiptool_processedimfile.sql");
 …
+        }
         // set chipRun.state to 'stop'
         if (!pxchipRunSetState(config, chipRun->chip_id, "full")) {
+        // set chipRun.state to 'stop' and update the magicked state
+        if (!pxchipRunSetState(config, chipRun->chip_id, "full", chipRun->magicked)) {
             psError(PS_ERR_UNKNOWN, false, "failed to change chipRun.state for chip_id: %" PRId64, chipRun->chip_id);
             psFree(chipRun);
 …
+    }
+    psString set_magic = "";
+    if (!strcmp(data_state, "full")) {
+        // copy the magicked state from the input to the output when transitioning to full state
+        set_magic = "\n , chipProcessedImfile.magicked = rawImfile.magicked";
+    }
     // note only updates if chipRun.state = run_state
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, class_id, run_state)) {
+    // XXX note that we have removed this constraint for now
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magic, chip_id, class_id)) {
+        psFree(query);
         psError(PS_ERR_UNKNOWN, false, "database error");
         // rollback
 …
     query = pxDataGet("chiptool_change_exp_state.sql");
     if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, data_state)) {
+        psFree(query);
         // rollback
         if (!psDBRollback(config->dbh)) {
 …
         return false;
+    }
+    psFree(query);
     if (!psDBCommit(config->dbh)) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptoolConfig.c

-              r23352
+              r23594
     pxchipSetSearchArgs (updaterunArgs);
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id",              0,            "search by chip ID", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0,            "set state", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0,            "search by state", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",  0,          "search by label", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0,        "set state", NULL);
 …
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-path_base",  0,            "define base output location", NULL);
     psMetadataAddS16(addprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
+    psMetadataAddBool(addprocessedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "define magicked status", false);
     // -processedimfile
 …
     psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-reduction",          0, "search by reduction class", NULL);
     psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-label",              0, "search by chipRun label (LIKE comparison)", NULL);
+    psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "search by magicked status", false);
     pxchipSetSearchArgs(processedimfileArgs);
     psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit",  0,           "limit result set to N items", 0);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.c

-              r23352
+              r23594
 static bool definepoprunMode(pxConfig *config);
 static bool definebyqueryMode(pxConfig *config);
+static bool definewarpwarpMode(pxConfig *config);
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupskyfileMode(pxConfig *config);
 …
 static bool importrunMode(pxConfig *config);
 static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state);
+static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked);
 static bool diffRunComplete(pxConfig *config);
 …
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
         MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY,         definebyqueryMode);
+        MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
 …
             registered,
             tess_id,
+            exp_id
+            exp_id,
+            false
     );
     if (!run) {
 …
     if (state) {
         // set detRun.state to state
         return setdiffRunState(config, diff_id, state);
+        return setdiffRunState(config, diff_id, state, false);
+    }
 …
     if (count == 2) {
         if (!setdiffRunState(config, diff_id, "new")) {
+        if (!setdiffRunState(config, diff_id, "new", false)) {
             if (!psDBRollback(config->dbh)) {
                 psError(PS_ERR_UNKNOWN, false, "database error");
 …
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
+    PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false);
     if (!psDBTransaction(config->dbh)) {
 …
                            good_frac,
                            code,
         // magic_ds_id
+                           magicked
           )) {
         if (!psDBRollback(config->dbh)) {
 …
 static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state)
+static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, bool magicked)
+{
     PS_ASSERT_PTR_NON_NULL(state, false);
 …
+    }
+    char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = %"PRId64;
+    if (!p_psDBRunQueryF(config->dbh, query, state, diff_id)) {
+    char *query = "UPDATE diffRun SET state = '%s', magicked = %d WHERE diff_id = %"PRId64;
+    if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) {
         psError(PS_ERR_UNKNOWN, false,
                 "failed to change state for diff_id %"PRId64, diff_id);
 …
             registered,
             tess_id,
+            exp_id
+            exp_id,
+            false       // magicked
     );
 …
     psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery);
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
     if (!p_psDBRunQueryF(config->dbh, query, warpQuery, expQuery, diffQuery)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
+    }
 …
         psErrorCode err = psErrorCodeLast();
         switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+          case PS_ERR_DB_CLIENT:
+            psError(PXTOOLS_ERR_SYS, false, "database error");
+            break;
+          case PS_ERR_DB_SERVER:
+            psError(PXTOOLS_ERR_PROG, false, "database error");
+            break;
+          default:
+            psError(PXTOOLS_ERR_PROG, false, "unknown error");
+            break;
+        }
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
+    }
 …
         psTrace("difftool", PS_LOG_INFO, "no rows found");
         psFree(output);
+        if (!psDBCommit(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+        }
         return true;
+    }
 …
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
         return false;
+    }
 …
         if (!mdok) {
             psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
             psFree(stackQuery);
             psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
             psFree(stackQuery);
             psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
             psFree(stackQuery);
             psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
             psFree(stackQuery);
             psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
             psFree(stackQuery);
             psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
             psFree(stackQuery);
             psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
                 registered,
                 tess_id,
+                exp_id
+                exp_id,
+                false       // magicked
         );
 …
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(run);
+            return true;
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
         run->diff_id = psDBLastInsertID(config->dbh);
 …
             psFree(skycell_query);
             psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
             return false;
+        }
 …
             psFree(skycell_query);
             psFree(query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
         query = NULL;
         if (!setdiffRunState(config, run->diff_id, "new")) {
+        if (!setdiffRunState(config, run->diff_id, "new", false)) {
             psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
                 run->diff_id);
 …
             psFree(skycell_query);
             psFree(query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
     psFree(skycell_query);
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(list);
+        return false;
+    }
     if (!diffRunPrintObjects(stdout, list, !simple)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print object");
 …
     return true;
+}
+static bool definewarpwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    psMetadata *selectWhere = psMetadataAlloc();
+    psMetadata *insertWhere = psMetadataAlloc();
+    // Restrictions for selecting warps
+    PXOPT_COPY_S64(config->args, selectWhere, "-warp_id", "inputWarpRun.warp_id", "==");
+    PXOPT_COPY_S64(config->args, selectWhere, "-exp_id", "inputRawExp.exp_id", "==");
+    PXOPT_COPY_STR(config->args, selectWhere, "-filter", "inputRawExp.filter", "==");
+    PXOPT_COPY_STR(config->args, selectWhere, "-input_label", "inputWarpRun.label", "==");
+    PXOPT_COPY_STR(config->args, selectWhere, "-template_label", "templateWarpRun.label", "==");
+    PXOPT_COPY_F32(config->args, selectWhere, "-timediff",
+                   "ABS(TIME_TO_SEC(TIMEDIFF(inputRawExp.dateobs, templateRawExp.dateobs)))", "<=");
+    PXOPT_COPY_F32(config->args, selectWhere, "-rotdiff", "ABS(inputRawExp.posang - templateRawExp.posang)", "<=");
+    // Restrictions for inserting skycells
+    PXOPT_COPY_F32(config->args, insertWhere, "-good_frac", "inputWarpSkyfile.good_frac", ">=");
+    PXOPT_COPY_F32(config->args, insertWhere, "-good_frac", "templateWarpSkyfile.good_frac", ">=");
+    // Additional controls
+    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+    // Settings to apply to defined run
+    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options
+    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option
+    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option
+    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    psString select = pxDataGet("difftool_definewarpwarp_select.sql");
+    if (!select) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+    psString whereClause = NULL; // The WHERE part of the query
+    if (psListLength(selectWhere->list)) {
+        psString new = psDBGenerateWhereConditionSQL(selectWhere, NULL);
+        psStringAppend(&whereClause, "\n%s %s", whereClause ? "AND" : "WHERE", new);
+        psFree(new);
+    }
+    psFree(selectWhere);
+    if (!rerun) {
+        psStringAppend(&whereClause, "\n%s diffRun.diff_id IS NULL", whereClause ? "AND" : "WHERE");
+    }
+    if (!available) {
+        psStringAppend(&whereClause, "\n%s inputWarpRun.state = 'full'", whereClause ? "AND" : "WHERE");
+    }
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(select);
+        psFree(whereClause);
+        psFree(insertWhere);
+        return false;
+    }
+    if (!p_psDBRunQueryF(config->dbh, select,
+                         !rerun ? "\n" : "", // Activate LEFT JOIN against diffRun?
+                         whereClause)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to run query: %s [WITH] %s", select, whereClause);
+        psFree(select);
+        psFree(whereClause);
+        psFree(insertWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(select);
+    psFree(whereClause);
+    psArray *results = p_psDBFetchResult(config->dbh); // Results of query
+    if (!results) {
+        psErrorCode err = psErrorCodeLast(); // Code for error
+        switch (err) {
+          case PS_ERR_DB_CLIENT:
+            psError(PXTOOLS_ERR_SYS, false, "database error");
+            break;
+          case PS_ERR_DB_SERVER:
+            psError(PXTOOLS_ERR_PROG, false, "database error");
+            break;
+          default:
+            psError(PXTOOLS_ERR_PROG, false, "unknown error");
+            break;
+        }
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psFree(insertWhere);
+        return false;
+    }
+    if (!psArrayLength(results)) {
+        psTrace("difftool", 1, "no rows found");
+        psFree(results);
+        psFree(insertWhere);
+        if (!psDBCommit(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+        }
+        return true;
+    }
+    psString insert = pxDataGet("difftool_definewarpwarp_insert.sql"); // Insertion for each new run
+    if (psListLength(insertWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(insertWhere, NULL);
+        psStringAppend(&insert, "\n%s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(insertWhere);
+    psArray *list = psArrayAllocEmpty(16); // List of runs defined, to print
+    long numGood = 0;                   // Number of good rows added
+    for (long i = 0; i < results->n; i++) {
+        psMetadata *row = results->data[i]; // Result row from query
+        psS64 exp_id = psMetadataLookupS64(NULL, row, "input_exp_id");
+        psS64 input_id = psMetadataLookupS64(NULL, row, "input_warp_id");
+        psS64 template_id = psMetadataLookupS64(NULL, row, "template_warp_id");
+        const char *tess_id = psMetadataLookupStr(NULL, row, "tess_id");
+        if (!exp_id || !input_id || !template_id || !tess_id) {
+            psError(PXTOOLS_ERR_PROG, false, "Identifiers not found");
+            psFree(list);
+            psFree(insert);
+            psFree(results);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered,
+                                          tess_id, exp_id, false); // Run to insert
+        if (!diffRunInsertObject(config->dbh, run)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(run);
+            psFree(list);
+            psFree(insert);
+            psFree(results);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        run->diff_id = psDBLastInsertID(config->dbh); // Difference run identifier
+        // Convert identifiers to string, for insertion into query
+        psString diff = NULL, input = NULL, template = NULL; // String versions of identifiers
+        psStringAppend(&diff, "%" PRId64, run->diff_id);
+        psStringAppend(&input, "%" PRId64, input_id);
+        psStringAppend(&template, "%" PRId64, template_id);
+        if (!p_psDBRunQueryF(config->dbh, insert, diff, input, template, input, template)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(input);
+            psFree(template);
+            psFree(diff);
+            psFree(run);
+            psFree(list);
+            psFree(insert);
+            psFree(results);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(input);
+        psFree(template);
+        psFree(diff);
+        if (!setdiffRunState(config, run->diff_id, "new", false)) {
+            psError(PS_ERR_UNKNOWN, false, "Failed to change diffRun.state for diff_id: %" PRId64,
+                    run->diff_id);
+            psFree(run);
+            psFree(list);
+            psFree(insert);
+            psFree(results);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psArrayAdd(list, list->n, run);
+        psFree(run);                    // Drop reference
+        numGood++;
+    }
+    psFree(insert);
+    psFree(results);
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(list);
+        return false;
+    }
+    if (!diffRunPrintObjects(stdout, list, !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(list);
+        return false;
+    }
+    psFree(list);
+    return true;
+}
 static bool pendingcleanuprunMode(pxConfig *config)
 …
         psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");
+        bool magicked = psMetadataLookupBool(NULL, row, "magicked");
         // set diffRun.state to 'stop'
         if (!setdiffRunState(config, diff_id, "full")) {
+        if (!setdiffRunState(config, diff_id, "full", magicked)) {
             psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
                 diff_id);
 …
     char sqlFilename[80];
   } ExportTable;
   int numExportFiles = 3;
 …
+{
   unsigned int nFail;
   int numImportTables = 2;
   char tables[2] [80] = {"diffInputSkyfile", "diffSkyfile"};
   PS_ASSERT_PTR_NON_NULL(config, NULL);
   PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
 …
     psAssert (item, "entry not in input?");
     psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
     switch (i) {
       case 0:
 …
+        }
         break;
       case 1:
         for (int i = 0; i < item->data.list->n; i++) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.h

r23352	r23594
35	35	DIFFTOOL_MODE_DEFINEPOPRUN,
36	36	DIFFTOOL_MODE_DEFINEBYQUERY,
	37	DIFFTOOL_MODE_DEFINEWARPWARP,
37	38	DIFFTOOL_MODE_PENDINGCLEANUPRUN,
38	39	DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE,

branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftoolConfig.c

-              r23352
+              r23594
     psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-kernel_xy",  0, "define kernel xy moment", NAN);
     psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-kernel_yy",  0, "define kernel yy moment", NAN);
+    psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-sources",  0,            "define number of sources", 0);
+    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0,            "define hostname", 0);
+    psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
+    psMetadataAddS32(adddiffskyfileArgs, PS_LIST_TAIL, "-sources",  0,   "define number of sources", 0);
+    psMetadataAddStr(adddiffskyfileArgs, PS_LIST_TAIL, "-hostname", 0,   "define hostname", 0);
+    psMetadataAddF32(adddiffskyfileArgs, PS_LIST_TAIL, "-good_frac",  0, "define %% of good pixels", NAN);
+    psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", false);
     // -diffskyfile
 …
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    // -definewarpwarp
+    psMetadata *definewarpwarpArgs = psMetadataAlloc();
+    psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0);
+    psMetadataAddS64(definewarpwarpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
+    psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL);
+    psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-timediff", 0, "limit time difference between input and template", NAN);
+    psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-rotdiff", 0, "limit rotator difference between input and template", NAN);
+    psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-input_label", 0, "search by warp label for input", NULL);
+    psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-template_label", 0, "search by warp label for template", NULL);
+    psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN);
+    psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL);
+    psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-label",  0, "define label", NULL);
+    psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-reduction",  0, "define reduction class", NULL);
+    psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
+    psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false);
+    psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false);
+    psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
     // -pendingcleanuprun
     psMetadata *pendingcleanuprunArgs = psMetadataAlloc();
 …
     PXOPT_ADD_MODE("-definepoprun",     "", DIFFTOOL_MODE_DEFINEPOPRUN,      definepoprunArgs);
     PXOPT_ADD_MODE("-definebyquery",    "", DIFFTOOL_MODE_DEFINEBYQUERY,     definebyqueryArgs);
+    PXOPT_ADD_MODE("-definewarpwarp",   "", DIFFTOOL_MODE_DEFINEWARPWARP,    definewarpwarpArgs);
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
     PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttool.c

-              r23352
+              r23594
     PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
-    bool require_chip_id = false;
-    if (!strcmp(stage, "raw")) {
-        // need to know the chip id for raw stage so that we can find the masks
-        require_chip_id = true;
+    }
-    PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id",  require_chip_id, false);
     // TODO: check that stage has an expected value
+    // XXX: all of the following concerns will be managed properly by definebyquery
     // TODO: check that stage_id actually exists for stage
     // in magicdstool we queue off of a magic_id so the stage_id, exp_id, and cam_id get looked up
 …
     // change between the time that it is queued.
     if (!distRunInsert(config->dbh, 0, stage, stage_id, chip_id, set_label, outroot, clean, "new", 0)) {
+    if (!distRunInsert(config->dbh, 0, stage, stage_id, set_label, outroot, clean, "new", NULL, 0)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
 …
+    }
     psString query = psStringCopy("UPDATE distRun");
+    psString query = psStringCopy("UPDATE distRun SET time_stamp = UTC_TIMESTAMP()");
     if (state) {
         psStringAppend(&query, " SET state = '%s'", state);
+        psStringAppend(&query, " , state = '%s'", state);
+    }
     if (label) {
         psStringAppend(&query, " SET label = '%s'", label);
+        psStringAppend(&query, " , label = '%s'", label);
+    }
     if (code) {
         psStringAppend(&query, " SET fault = %d", code);
+        psStringAppend(&query, " , fault = %d", code);
+    }
 …
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(need_magic, config->args, "-need_magic", false);
     // look for "inputs" that need to processed
 …
+    }
+    if (!p_psDBRunQuery(config->dbh, query)) {
+    psString    chip_magic = "";
+    psString    raw_where = "";
+    psString    chip_where = "";
+    psString    camera_where = "";
+    psString    fake_where = "";
+    psString    warp_where = "";
+    psString    diff_where = "";
+    psString    stack_where = "";
+    if (need_magic) {
+        chip_magic = psStringCopy("\nAND chipRun.magicked");
+        raw_where  = psStringCopy("\nAND rawExp.magicked");
+        chip_where = psStringCopy("\nAND (distRun.clean OR chipRun.magicked)");
+        // chipRun must be magicked to allow release camera level masks
+        camera_where = psStringCopy("\nAND (distRun.clean OR chipRun.magicked)");
+        warp_where = psStringCopy("\nAND (distRun.clean OR warpRun.magicked)");
+        diff_where = psStringCopy("\nAND (distRun.clean OR diffRun.magicked)");
+    }
+    if (!p_psDBRunQueryF(config->dbh,
+            query,
+            chip_magic,
+            raw_where,
+            chip_where,
+            camera_where,
+            fake_where,
+            warp_where,
+            diff_where,
+            stack_where)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/disttoolConfig.c

-              r23352
+              r23594
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage",         0, "define stage for bundle (required)", NULL);
     psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id (required)", 0);
-    psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip_id (required for raw stage", 0);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot",  0, "define output destination (required)", NULL);
     psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-clean", 0,   "build clean distribution bundle", false);
 …
     psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
     psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage",    0, "limit results to runs for stage", NULL);
+    psMetadataAddBool(pendingcomponentArgs, PS_LIST_TAIL, "-need_magic", 0, "magic is needed", false);
     psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label",    0, "limit results to label", NULL);
     psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/magicdstool.c

-              r21433
+              r23594
 static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state);
 static bool magicDSRunComplete(pxConfig *config);
+static bool magicDSRunComplete(pxConfig *config, bool setmagicked);
 static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id);
 …
         // create a new magicRun for this group
         magicRunRow *run = magicRunRowAlloc(0, exp_id, "run", workdir, "dirty", label, dvodb, registered, 0);
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, "new", workdir, "dirty", label, dvodb, registered, 0);
         if (!run) {
             psAbort("failed to alloc magicRun object");
 …
     PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
     PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false);
+    // optional
     PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false);
     PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false);
     PXOPT_LOOKUP_BOOL(remove, config->args, "-remove", false);
-    // optional
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     psS64 stage_id = 0, cam_id = 0;
 …
 ,          // ID
             magic_id,
             "run",      // state
+            "new",      // state
             stage,
             stage_id,
             cam_id,
+            label,
             outroot,
             recoveryroot,
 …
     PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
 //    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 …
+}
+static bool
+setRunMagicked(pxConfig *config, psS64 magic_ds_id)
+{
+    // first query the magicDSRun to find the stage and the stage_id
+    psString query = "SELECT stage, stage_id from magicDSRun where magic_ds_id = %" PRId64;
+    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psError(PS_ERR_UNKNOWN, true, "magicDSRun not found for magic_ds_id %" PRId64, magic_ds_id);
+        psFree(output);
+        return false;
+    }
+    if (psArrayLength(output) > 1) {
+        psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found %ld for magic_ds_id %" PRId64,
+            psArrayLength(output), magic_ds_id);
+        psFree(output);
+        return false;
+    }
+    psMetadata *row = output->data[0];
+    psString stage = psMetadataLookupStr(NULL, row, "stage");
+    psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
+    // chose the appropriate query based on the stage
+    if (!strcmp(stage, "raw")) {
+        query = "UPDATE rawExp SET magicked = 1 where exp_id = %" PRId64;
+    } else if (!strcmp(stage, "chip")) {
+        query = "UPDATE chipRun SET magicked = 1 where chip_id = %" PRId64;
+    } else if (!strcmp(stage, "warp")) {
+        query = "UPDATE warpRun SET magicked = 1 where warp_id = %" PRId64;
+    } else if (!strcmp(stage, "diff")) {
+        query = "UPDATE diffRun SET magicked = 1 where diff_id = %" PRId64;
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage);
+        psFree(output);
+        return false;
+    }
+    if (!p_psDBRunQueryF(config->dbh, query, stage_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(output);
+    psU64 affected = psDBAffectedRows(config->dbh);
+    if (affected != 1) {
+        psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
+        return false;
+    }
+    return true;
+}
 static bool adddestreakedfileMode(pxConfig *config)
+{
 …
     if (setmagicked) {
+        // set the image file's magicked flag
         if (!setMagicked(config, magic_ds_id, component)) {
             psError(PS_ERR_UNKNOWN, false, "setMagicked failed");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
+        }
 …
+    }
     if (!magicDSRunComplete(config)) {
+    if (!magicDSRunComplete(config, setmagicked)) {
             // rollback
         if (!psDBRollback(config->dbh)) {
 …
     if (!strcmp(stage, "diff")) {
         // don't need these ids for diff stage
+        // don't need these ids for diff stage because diff_id is in the magicRun
         *stage_id = 0;
         *cam_id = 0;
 …
+}
 static bool magicDSRunComplete(pxConfig *config)
+static bool magicDSRunComplete(pxConfig *config, bool setmagicked)
+{
     PS_ASSERT_PTR_NON_NULL(config, false);
 …
         psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id");
+        // set magicDSRun.state to 'stop'
+        if (!setmagicDSRunState(config, magic_ds_id, "stop")) {
+        // if requested, set stageRun.magicked
+        if (setmagicked && !setRunMagicked(config, magic_ds_id)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64,
+                magic_ds_id);
+            return false;
+        }
+        // set magicDSRun.state to 'full'
+        if (!setmagicDSRunState(config, magic_ds_id, "full")) {
             psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64,
                 magic_ds_id);
 …
     // check that state is a valid string value
     if (!(
             (strncmp(state, "run", 4) == 0)
             || (strncmp(state, "stop", 5) == 0)
+            (strncmp(state, "new", 4) == 0)
+            || (strncmp(state, "full", 5) == 0)
+        )
     ) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/magicdstoolConfig.c

-              r21433
+              r23594
     psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-replace", 0, "use the simple output format", false);
     psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false);
 //    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL);
+    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL);
     psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 …
     psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
     psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
 //    psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0);
+    psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL);
     psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/magictool.c

-              r21402
+              r23594
         // create a new magicRun for this group
         magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "run", workdir, "dirty", label, dvodb, registered, 0);
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, "new", workdir, "dirty", label, dvodb, registered, 0);
         if (!run) {
             psAbort("failed to alloc magicRun object");
 …
     if (code > 0) {
         char *query = "UPDATE magicRun SET fault = %d, state = 'stop' WHERE magic_id = %" PRId64;
+        char *query = "UPDATE magicRun SET fault = %d, state = 'full' WHERE magic_id = %" PRId64;
         if (!p_psDBRunQueryF(config->dbh, query, code, magic_id)) {
             psError(PS_ERR_UNKNOWN, false,
 …
+    }
     // Set to "run"
+    // Set to "new"
+    {
         psString query = psStringCopy("UPDATE magicRun JOIN magicMask USING(magic_id) "
                                       "SET magicRun.state = 'run' WHERE magicMask.fault != 0");
+                                      "SET magicRun.state = 'new' WHERE magicMask.fault != 0");
         if (psListLength(where->list)) {
 …
     // check that state is a valid string value
     if (!(
             (strncmp(state, "run", 4) == 0)
             || (strncmp(state, "stop", 5) == 0)
+            (strncmp(state, "new", 4) == 0)
+            || (strncmp(state, "full", 5) == 0)
             || (strncmp(state, "reg", 4) == 0)
+        )

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxcam.c

-              r21402
+              r23594
     psMetadataAddStr(md,  PS_LIST_TAIL, "-comment",            0, "search by comment", NULL);
     psMetadataAddStr(md,  PS_LIST_TAIL, "-filelevel",          0, "search by filelevel", NULL);
-    psMetadataAddStr(md,  PS_LIST_TAIL, "-reduction",          0, "search by reduction class", NULL);
     psMetadataAddStr(md,  PS_LIST_TAIL, "-filter",             0, "search for filter", NULL);
     psMetadataAddF64(md,  PS_LIST_TAIL, "-airmass_min",        0, "define min airmass", NAN);
 …
 bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where) {
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "==");
+    PXOPT_COPY_STR(config->args, where, "-inst", "camera", "==");
+    PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<=");
+    PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "==");
+    PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter", "filter", "==");
+    PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">=");
+    PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<");
+    PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">=");
+    PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<");
+    PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">=");
+    PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">=");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<");
+    PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">=");
+    PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<");
+    PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">=");
+    PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<");
+    PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">=");
+    PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<");
+    PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">=");
+    PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<");
+    PXOPT_COPY_STR(config->args, where, "-object", "object", "==");
+    PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">=");
+    PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<");
+    PXOPT_COPY_S64(config->args,  where, "-chip_id",            "chipRun.chip_id",       "==");
+    PXOPT_COPY_S64(config->args,  where, "-exp_id",             "rawExp.exp_id",         "==");
+    PXOPT_COPY_STR(config->args,  where, "-exp_name",           "rawExp.exp_name",       "==");
+    PXOPT_COPY_STR(config->args,  where, "-inst",               "rawExp.camera",         "==");
+    PXOPT_COPY_STR(config->args,  where, "-telescope",          "rawExp.telescope",      "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin",      "rawExp.dateobs",        ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",        "rawExp.dateobs",        "<=");
+    PXOPT_COPY_STR(config->args,  where, "-exp_tag",            "rawExp.exp_tag",        "==");
+    PXOPT_COPY_STR(config->args,  where, "-exp_type",           "rawExp.exp_type",       "==");
+    PXOPT_COPY_STR(config->args,  where, "-comment",            "rawExp.comment",        "LIKE");
+    PXOPT_COPY_STR(config->args,  where, "-filelevel",          "rawExp.filelevel",      "==");
+    PXOPT_COPY_STR(config->args,  where, "-filter",             "rawExp.filter",         "==");
+    PXOPT_COPY_F64(config->args,  where, "-airmass_min",        "rawExp.airmass",        ">=");
+    PXOPT_COPY_F64(config->args,  where, "-airmass_max",        "rawExp.airmass",        "<");
+    PXOPT_COPY_F64(config->args,  where, "-ra_min",             "rawExp.ra",             ">=");
+    PXOPT_COPY_F64(config->args,  where, "-ra_max",             "rawExp.ra",             "<");
+    PXOPT_COPY_F64(config->args,  where, "-decl_min",           "rawExp.decl",           ">=");
+    PXOPT_COPY_F64(config->args,  where, "-decl_max",           "rawExp.decl",           "<");
+    PXOPT_COPY_F32(config->args,  where, "-exp_time_min",       "rawExp.exp_time",       ">=");
+    PXOPT_COPY_F32(config->args,  where, "-exp_time_max",       "rawExp.exp_time",       "<");
+    PXOPT_COPY_F32(config->args,  where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args,  where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args,  where, "-bg_min",             "rawExp.bg",             ">=");
+    PXOPT_COPY_F64(config->args,  where, "-bg_max",             "rawExp.bg",             "<");
+    PXOPT_COPY_F64(config->args,  where, "-bg_stdev_min",       "rawExp.bg_stdev",       ">=");
+    PXOPT_COPY_F64(config->args,  where, "-bg_stdev_max",       "rawExp.bg_stdev",       "<");
+    PXOPT_COPY_F64(config->args,  where, "-bg_mean_stdev_min",  "rawExp.bg_mean_stdev",  ">=");
+    PXOPT_COPY_F64(config->args,  where, "-bg_mean_stdev_max",  "rawExp.bg_mean_stdev",  "<");
+    PXOPT_COPY_F64(config->args,  where, "-alt_min",            "rawExp.alt",            ">=");
+    PXOPT_COPY_F64(config->args,  where, "-alt_max",            "rawExp.alt",            "<");
+    PXOPT_COPY_F64(config->args,  where, "-az_min",             "rawExp.az",             ">=");
+    PXOPT_COPY_F64(config->args,  where, "-az_max",             "rawExp.az",             "<");
+    PXOPT_COPY_F32(config->args,  where, "-ccd_temp_min",       "rawExp.ccd_temp",       ">=");
+    PXOPT_COPY_F32(config->args,  where, "-ccd_temp_max",       "rawExp.ccd_temp",       "<");
+    PXOPT_COPY_F64(config->args,  where, "-posang_min",         "rawExp.posang",         ">=");
+    PXOPT_COPY_F64(config->args,  where, "-posang_max",         "rawExp.posang",         "<");
+    PXOPT_COPY_STR(config->args,  where, "-object",             "rawExp.object",         "==");
+    PXOPT_COPY_F32(config->args,  where, "-solang_min",         "rawExp.solang",         ">=");
+    PXOPT_COPY_F32(config->args,  where, "-solang_max",         "rawExp.solang",         "<");
     return true;

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxchip.c

-              r21402
+              r23594
 bool pxchipSetSearchArgs (psMetadata *md) {
+    // psMetadataAddStr(md,  PS_LIST_TAIL, "-class_id",           0, "search by class ID", NULL);
+    // psMetadataAddStr(md,  PS_LIST_TAIL, "-reduction",          0, "search by reduction class", NULL);
     // XXX need to allow multiple exp_ids
     psMetadataAddS64(md,  PS_LIST_TAIL, "-exp_id",             0, "search by exp_id", 0);
     psMetadataAddStr(md,  PS_LIST_TAIL, "-exp_name",           0, "search by exp_name", NULL);
-    // psMetadataAddStr(md,  PS_LIST_TAIL, "-class_id",           0, "search by class ID", NULL);
     psMetadataAddStr(md,  PS_LIST_TAIL, "-inst",               0, "search for camera", NULL);
     psMetadataAddStr(md,  PS_LIST_TAIL, "-telescope",          0, "search for telescope", NULL);
 …
     psMetadataAddStr(md,  PS_LIST_TAIL, "-exp_type",           0, "search by exp_type", NULL);
     psMetadataAddStr(md,  PS_LIST_TAIL, "-filelevel",          0, "search by filelevel", NULL);
-    // psMetadataAddStr(md,  PS_LIST_TAIL, "-reduction",          0, "search by reduction class", NULL);
     psMetadataAddStr(md,  PS_LIST_TAIL, "-filter",             0, "search for filter", NULL);
     psMetadataAddF32(md,  PS_LIST_TAIL, "-airmass_min",        0, "search by min airmass", NAN);
 …
+}
 bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state)
+bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked)
+{
     PS_ASSERT_PTR_NON_NULL(config, false);
 …
+    }
     char *query = "UPDATE chipRun SET state = '%s' WHERE chip_id = %" PRId64;
     if (!p_psDBRunQueryF(config->dbh, query, state, chip_id)) {
+    char *query = "UPDATE chipRun SET state = '%s', magicked = %d WHERE chip_id = %" PRId64;
+    if (!p_psDBRunQueryF(config->dbh, query, state, magicked, chip_id)) {
         psError(PS_ERR_UNKNOWN, false,
                 "failed to change state for chip_id %" PRId64, chip_id);
 …
     if (!pxIsValidState(state)) {
         psError(PS_ERR_UNKNOWN, false, "invalid chipRun state: %s", state);
+        return false;
+    }
+    if (!strcmp(state, "full")) {
+        // There are states that need to be met for a run to be set to full that we don't
+        // check here.
+        // for example all of the run's Imfiles must have chipProcessedImfile.data_state == "full"
+        // chipRun.magicked = (SUM(!chipProcessedImfile.magicked) = 0)
+        // so don't do allow setting the state to full
+        psError(PS_ERR_UNKNOWN, true, "cannot use -updaterun so set chipRun state to full");
         return false;
+    }
 …
             dvodb,
             tess_id,
+            end_stage)
+            end_stage,
+           // magicked
+            )
     ) {
         if (!psDBRollback(config->dbh)) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxchip.h

r19324	r23594
25	25	#include "pxtools.h"
26	26
27		bool pxchipRunSetState(pxConfig config, psS64 chip_id, const char state);
	27	bool pxchipRunSetState(pxConfig config, psS64 chip_id, const char state, const bool magicked);
28	28	bool pxchipRunSetStateByQuery(pxConfig config, psMetadata where, const char *state);
29	29	bool pxchipRunSetLabel(pxConfig config, psS64 chip_id, const char label);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxerrors.c

-              r17611
+              r23594
 #include "pxtools.h"
 psExit pxerrorGetExitStatus () {
+psExit pxerrorGetExitStatus(void) {
     psErrorCode err = psErrorCodeLast ();
 …
       default:
     return PS_EXIT_UNKNOWN_ERROR;
+    }
+    }
     return PS_EXIT_UNKNOWN_ERROR;
+}

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtools.h

-              r18561
+              r23594
 bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code);
 psExit pxerrorGetExitStatus ();
+psExit pxerrorGetExitStatus(void);
 void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet);
 …
 #define PXOPT_ADD_WHERE_BOOL(name) \
 { \
     bool value = false;  \
+    bool value = false;  \
     bool status = false; \
     if ((value = psMetadataLookupBool(&status, config->args, "-" #name))) { \
 …
+}
 #define PXOPT_ADD_WHERE_BOOL_ALIAS(flag,name)   \
+#define PXOPT_ADD_WHERE_BOOL_ALIAS(flag,name)   \
 { \
     bool value = 0; \
     bool status = false; \
     if ((value = psMetadataLookupBool(&status, config->args, flag))) {  \
+    if ((value = psMetadataLookupBool(&status, config->args, flag))) {  \
         if (!psMetadataAddBOOL(config->where, PS_LIST_TAIL, name, 0, "==", value)) { \
             psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \
 …
+}
 #define PXOPT_ADD_WHERE_S16_ALIAS(flag,name)    \
+#define PXOPT_ADD_WHERE_S16_ALIAS(flag,name)    \
 { \
     psS16 s16 = 0; \
     bool status = false; \
     if ((s16= psMetadataLookupS16(&status, config->args, flag))) {      \
+    if ((s16= psMetadataLookupS16(&status, config->args, flag))) {      \
         if (!psMetadataAddS16(config->where, PS_LIST_TAIL, name, 0, "==", s16)) { \
             psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \
 …
+}
 #define PXOPT_ADD_WHERE_S32_ALIAS(flag,name)    \
+#define PXOPT_ADD_WHERE_S32_ALIAS(flag,name)    \
 { \
     psS32 s32 = 0; \
     bool status = false; \
     if ((s32= psMetadataLookupS32(&status, config->args, flag))) {      \
+    if ((s32= psMetadataLookupS32(&status, config->args, flag))) {      \
         if (!psMetadataAddS32(config->where, PS_LIST_TAIL, name, 0, "==", s32)) { \
             psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \
 …
+}
 #define PXOPT_ADD_WHERE_S64_ALIAS(flag,name)    \
+#define PXOPT_ADD_WHERE_S64_ALIAS(flag,name)    \
 { \
     psS64 s64 = 0; \
     bool status = false; \
     if ((s64= psMetadataLookupS64(&status, config->args, flag))) {      \
+    if ((s64= psMetadataLookupS64(&status, config->args, flag))) {      \
         if (!psMetadataAddS64(config->where, PS_LIST_TAIL, name, 0, "==", s64)) { \
             psError(PS_ERR_UNKNOWN, false, "failed to add item %s", flag); \
 …
+}
 #define PXOPT_ADD_WHERE_F32_ALIAS(flag,name)    \
+#define PXOPT_ADD_WHERE_F32_ALIAS(flag,name)    \
 { \
     psF32 var = 0; \
     bool status = false; \
     if ((var = psMetadataLookupF32(&status, config->args, flag))) {     \
+    if ((var = psMetadataLookupF32(&status, config->args, flag))) {     \
         if (!isnan(var))  { \
             if (!psMetadataAddF32(config->where, PS_LIST_TAIL, name, 0, "==", var)) { \
 …
+}
 #define PXOPT_ADD_WHERE_F64_ALIAS(flag,name)    \
+#define PXOPT_ADD_WHERE_F64_ALIAS(flag,name)    \
 { \
     psF64 var = 0; \
     bool status = false; \
     if ((var = psMetadataLookupF64(&status, config->args, flag))) {     \
+    if ((var = psMetadataLookupF64(&status, config->args, flag))) {     \
         if (!isnan(var))  { \
             if (!psMetadataAddF64(config->where, PS_LIST_TAIL, name, 0, "==", var)) { \

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxwarp.c

-              r23352
+              r23594
 #include "pxtools.h"
 #include "pxwarp.h"
+bool pxwarpSetSearchArgs (psMetadata *md) {
+    psMetadataAddS64(md, PS_LIST_TAIL, "-fake_id",            0, "search by fake_id", 0);
+    psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id",             0, "search by cam_id", 0);
+    psMetadataAddS64(md, PS_LIST_TAIL, "-chip_id",            0, "search by chip_id", 0);
+    psMetadataAddS64(md, PS_LIST_TAIL, "-exp_id",             0, "search by exp_id", 0);
+    psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name",           0, "search by exp_name", NULL);
+    psMetadataAddStr(md, PS_LIST_TAIL, "-inst",               0, "search for camera", NULL);
+    psMetadataAddStr(md, PS_LIST_TAIL, "-telescope",          0, "search for telescope", NULL);
+    psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin",     0, "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end",       0, "search for exposures by time (<)", NULL);
+    psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag",            0, "search by exp_tag", NULL);
+    psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type",           0, "search by exp_type", NULL);
+    psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel",          0, "search by filelevel", NULL);
+    psMetadataAddStr(md, PS_LIST_TAIL, "-filter",             0, "search for filter", NULL);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min",        0, "search by min airmass", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max",        0, "search by max airmass", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min",             0, "search by min", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max",             0, "search by max", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min",           0, "search by min", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max",           0, "search by max", NAN);
+    psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min",       0, "search by min", NAN);
+    psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max",       0, "search by max", NAN);
+    psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "search by max fraction of saturated pixels", NAN);
+    psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "search by min fraction of saturated pixels", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min",             0, "search by max", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max",             0, "search by max", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min",       0, "search by max", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max",       0, "search by max", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min",  0, "search by max", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max",  0, "search by max", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min",            0, "search by min", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max",            0, "search by max", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-az_min",             0, "search by min", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-az_max",             0, "search by max", NAN);
+    psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min",       0, "search by min ccd tempature", NAN);
+    psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max",       0, "search by max ccd tempature", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min",         0, "search by min rotator position angle", NAN);
+    psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max",         0, "search by max rotator position angle", NAN);
+    psMetadataAddStr(md, PS_LIST_TAIL, "-object",             0, "search by exposure object", NULL);
+    psMetadataAddF32(md, PS_LIST_TAIL, "-solang_min",         0, "search by min solar angle", NAN);
+    psMetadataAddF32(md, PS_LIST_TAIL, "-solang_max",         0, "search by max solar angle", NAN);
+    return true;
+}
+bool pxwarpGetSearchArgs (pxConfig *config, psMetadata *where) {
+    PXOPT_COPY_S64(config->args, where, "-fake_id",            "fakeRun.fake_id",       "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",             "camRun.cam_id",         "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id",            "chipRun.chip_id",       "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id",             "newExp.exp_id",         "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name",           "rawExp.exp_name",       "==");
+    PXOPT_COPY_STR(config->args, where, "-inst",               "rawExp.camera",         "==");
+    PXOPT_COPY_STR(config->args, where, "-telescope",          "rawExp.telescope",      "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin",     "rawExp.dateobs",        ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",       "rawExp.dateobs",        "<=");
+    PXOPT_COPY_STR(config->args, where, "-exp_tag",            "rawExp.exp_tag",        "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_type",           "rawExp.exp_type",       "==");
+    PXOPT_COPY_STR(config->args, where, "-filelevel",          "rawExp.filelevel",      "==");
+    PXOPT_COPY_STR(config->args, where, "-filter",             "rawExp.filter",         "==");
+    PXOPT_COPY_F64(config->args, where, "-airmass_min",        "rawExp.airmass",        ">=");
+    PXOPT_COPY_F64(config->args, where, "-airmass_max",        "rawExp.airmass",        "<");
+    PXOPT_COPY_F64(config->args, where, "-ra_min",             "rawExp.ra",             ">=");
+    PXOPT_COPY_F64(config->args, where, "-ra_max",             "rawExp.ra",             "<");
+    PXOPT_COPY_F64(config->args, where, "-decl_min",           "rawExp.decl",           ">=");
+    PXOPT_COPY_F64(config->args, where, "-decl_max",           "rawExp.decl",           "<");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_min",       "rawExp.exp_time",       ">=");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_max",       "rawExp.exp_time",       "<");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_min",             "rawExp.bg",             ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_max",             "rawExp.bg",             "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min",       "rawExp.bg_stdev",       ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max",       "rawExp.bg_stdev",       "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min",  "rawExp.bg_mean_stdev",  ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max",  "rawExp.bg_mean_stdev",  "<");
+    PXOPT_COPY_F64(config->args, where, "-alt_min",            "rawExp.alt",            ">=");
+    PXOPT_COPY_F64(config->args, where, "-alt_max",            "rawExp.alt",            "<");
+    PXOPT_COPY_F64(config->args, where, "-az_min",             "rawExp.az",             ">=");
+    PXOPT_COPY_F64(config->args, where, "-az_max",             "rawExp.az",             "<");
+    PXOPT_COPY_F32(config->args, where, "-ccd_temp_min",       "rawExp.ccd_temp",       ">=");
+    PXOPT_COPY_F32(config->args, where, "-ccd_temp_max",       "rawExp.ccd_temp",       "<");
+    PXOPT_COPY_F64(config->args, where, "-posang_min",         "rawExp.posang",         ">=");
+    PXOPT_COPY_F64(config->args, where, "-posang_max",         "rawExp.posang",         "<");
+    PXOPT_COPY_STR(config->args, where, "-object",             "rawExp.object",         "==");
+    PXOPT_COPY_F32(config->args, where, "-solang_min",         "rawExp.solang",         ">=");
+    PXOPT_COPY_F32(config->args, where, "-solang_max",         "rawExp.solang",         "<");
+    return true;
+}
 bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state)
 …
         tess_id,
         end_stage,
+        NULL       // registered
+        NULL,      // registered
+          // magicked zero when created will get updated when warpRun goes to 'full'
     )) {
         psError(PS_ERR_UNKNOWN, false, "database error");

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxwarp.h

-              r18029
+              r23594
 #include "pxtools.h"
+bool pxwarpGetSearchArgs (pxConfig *config, psMetadata *where);
+bool pxwarpSetSearchArgs (psMetadata *md);
 bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state);
 bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pztool.c

-              r21479
+              r23594
 static bool clearcommonfaultsMode(pxConfig *config);
+static bool toadvanceMode(pxConfig *config);
 static bool advanceMode(pxConfig *config);
 static bool copydoneCompleteExp(pxConfig *config);
+// static bool copydoneCompleteExp(pxConfig *config);
 static psArray *pzGetPendingCameras(pxConfig *config);
 static psArray *pzArrayZip(psArray *arraySet, psS64 limit);
 …
         MODECASE(PZTOOL_MODE_REVERTCOPIED, revertcopiedMode);
         MODECASE(PZTOOL_MODE_CLEARCOMMONFAULTS, clearcommonfaultsMode);
+        MODECASE(PZTOOL_MODE_TOADVANCE, toadvanceMode);
         MODECASE(PZTOOL_MODE_ADVANCE, advanceMode);
         default:
 …
     PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
-    // NOTE : the rest of the command-line args are parsed in copydoneCompleteExp
     // start a transaction early so it will contain any row level locks
     if (!psDBTransaction(config->dbh)) {
 …
+    }
-#ifdef notdef
-    // we've changed to use -advance instead
-    if (!copydoneCompleteExp(config)) {
-        // rollback
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
-        psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed");
-        return false;
+    }
-#endif
     // point of no return
     if (!psDBCommit(config->dbh)) {
 …
+}
+static bool copydoneCompleteExp(pxConfig *config)
+{
+    // THIS FUNCTION MUST BE INVOKED FROM INSIDE A TRANSACTION!!!
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    // XXX this is an ugly hack!
+    // we are passing exp level info to a imfile level mode (-copydone)
+    // these options are thrown away unless we just -copydone'd the last imfile
+    // in an exp.
+    // optional
+    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
+    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
+    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false);
+    PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
+static bool toadvanceMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    // optional args
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 …
     PXOPT_COPY_STR(config->args, where,  "-exp_name", "exp_name", "==");
     PXOPT_COPY_STR(config->args, where,  "-inst", "camera", "==");
+    PXOPT_COPY_STR(config->args, where,  "-telescope", "telescope", "==");
+    PXOPT_COPY_STR(config->args, where,  "-inst", "camera", "==");
+    PXOPT_COPY_STR(config->args, where,  "-label", "label", "==");
     if (psListLength(where->list)) {
 …
+    }
     psFree(where);
     // treat limit == 0 as "no limit"
 …
+    }
+   for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+        pzDownloadExpRow *doneExp = pzDownloadExpObjectFromMetadata(row);
+        if (!doneExp) {
+            psError(PS_ERR_UNKNOWN, false, "pzDownloadExpObjectFromMetadata() failed");
+            psFree(doneExp);
+            psFree(output);
+            return false;
+        }
+        if (!newExpInsert(config->dbh,
+x0,                // exp_id
+                    doneExp->exp_name,  // tmp_exp_name
+                    doneExp->camera,    // tmp_camera
+                    doneExp->telescope, // tmp_telescope
+                    "run",              // state
+                    workdir,            // workdir
+                    "dirty",            // workdir state
+                    NULL,               // reduction class
+                    dvodb,              // dvodb
+                    tess_id,            // tess_id
+                    end_stage,          // end_stage
+                    label,
+                    NULL                // epoch
+                )
+    if (!ippdbPrintMetadatas(stdout, output, "toadvance", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+    return true;
+}
+static bool advanceMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    // required
+    PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false);
+    PXOPT_LOOKUP_STR(inst, config->args, "-inst", true, false);
+    PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false);
+    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
+    // optional
+    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
+    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false);
+    PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
+    // start a transaction so it's all rows or nothing
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!newExpInsert(config->dbh,
+x0,        // exp_id
+            exp_name,   // tmp_exp_name
+            inst,       // tmp_camera
+            telescope,  // tmp_telescope
+            "run",      // state
+            workdir,    // workdir
+            "dirty",    // workdir state
+            NULL,       // reduction class
+            dvodb,      // dvodb
+            tess_id,    // tess_id
+            end_stage,  // end_stage
+            label,      // label
+            NULL        // epoch
+            )
         ) {
             psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(doneExp);
-            psFree(output);
             return false;
+        }
 …
                 "INSERT INTO newImfile"
                 "   SElECT"
                 "       %" PRId64 "," // exp_id
+                "       %" PRId64 ","               // exp_id
                 "       pzDownloadImfile.class_id," // tmp_class_id
                 "       pzDownloadImfile.uri," // uri
                 "       NULL" // epoch
+                "       pzDownloadImfile.uri,"      // uri
+                "       NULL"                       // epoch
                 "   FROM pzDownloadImfile"
                 "   WHERE"
 …
                 "       AND pzDownloadImfile.telescope = '%s'";
+            if (!p_psDBRunQueryF(config->dbh, query, exp_id, doneExp->exp_name, doneExp->camera, doneExp->telescope)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(doneExp);
+                psFree(output);
+                return false;
+            }
+            // sanity check: we should have inserted at least one row
+            psU64 affected = psDBAffectedRows(config->dbh);
+            if (psDBAffectedRows(config->dbh) < 1) {
+                psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected);
+                psFree(doneExp);
+                psFree(output);
+                return false;
+            }
+        }
+        // set pzDownloadExp.state to 'stop'
+        if (!pzDownloadExpSetState(config, doneExp->exp_name, doneExp->camera, doneExp->telescope, "stop")) {
+            psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %s:%s:%s", doneExp->exp_name, doneExp->camera, doneExp->telescope);
+            psFree(doneExp);
+            psFree(output);
+        if (!p_psDBRunQueryF(config->dbh, query, exp_id, exp_name, inst, telescope)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
             return false;
+        }
+        psFree(doneExp);
+    }
+    psFree(output);
+        }
+        // sanity check: we should have inserted at least one row
+        psU64 affected = psDBAffectedRows(config->dbh);
+        if (psDBAffectedRows(config->dbh) < 1) {
+            psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected);
+            return false;
+        }
+    }
+    // set pzDownloadExp.state to 'stop'
+    if (!pzDownloadExpSetState(config, exp_name, inst, telescope, "stop")) {
+        psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %s:%s:%s", exp_name, inst, telescope);
+        return false;
+    }
+    // point of no return
+    if (!psDBCommit(config->dbh)) {
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
     return true;
 …
+}
-static bool advanceMode(pxConfig *config)
+{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-    // NOTE : the command-line args are parsed in copydoneCompleteExp
-    // start a transaction so it's all rows or nothing
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
+    }
-    if (!copydoneCompleteExp(config)) {
-        // rollback
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
-        psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed");
-        return false;
+    }
-    // point of no return
-    if (!psDBCommit(config->dbh)) {
-        // rollback
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
+    }
-    return true;
+}
 static bool pzDownloadExpSetState(pxConfig *config, const char *exp_name, const char *camera, const char *telescope, const char *state)

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pztool.h

r18314	r23594
35	35	PZTOOL_MODE_REVERTCOPIED,
36	36	PZTOOL_MODE_CLEARCOMMONFAULTS,
	37	PZTOOL_MODE_TOADVANCE,
37	38	PZTOOL_MODE_ADVANCE
38	39	} pztoolMode;

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pztoolConfig.c

-              r21479
+              r23594
     psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
     psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0,     "lock pzDownImfile rows while advancing an exposure", false);
+    // XXX: remove this once advance is fixed
+    psMetadataAddU64(copydoneArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     // -copied
 …
     // -clearcommonfaults
     psMetadata *clearcommonfaultsArgs = psMetadataAlloc();
+    //
+    // -toadvance
+    psMetadata *toadvanceArgs = psMetadataAlloc();
+    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-exp_name", 0,      "define exposure ID", NULL);
+    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-inst", 0,          "define camera ID", NULL);
+    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-telescope", 0,     "define telescope ID", NULL);
+    psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label",  0,        "define the label for the chip stage", NULL);
+    psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple",  0,      "use the simple output format", false);
+    psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit",  0,        "limit result set to N items", 0);
     // -advance
     psMetadata *advanceArgs = psMetadataAlloc();
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-exp_name", 0,            "define exposure ID", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-inst", 0,            "define camera ID", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-telescope", 0,            "define telescope ID", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-workdir",  0,        "define the \"default\" workdir for this exposure", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-dvodb",  0,        "define the dvodb for the next processing step", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-tess_id",  0,        "define the tess_id for the next processing step", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-end_stage",  0,        "define the end goal processing step", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label",  0,        "define the label for the chip stage", NULL);
+    psMetadataAddU64(advanceArgs, PS_LIST_TAIL, "-limit",  0,         "limit result set to N items", 0);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-exp_name", 0,   "define exposure ID (required)", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-inst", 0,       "define camera ID (required)", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-telescope", 0,  "define telescope ID (required)", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-workdir",  0,   "define the \"default\" workdir for this exposure (required)", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-dvodb",  0,     "define the dvodb for the next processing step", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-tess_id",  0,   "define the tess_id for the next processing step", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-end_stage",  0, "define the end goal processing step", NULL);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label",  0,     "define the label for the chip stage", NULL);
     psMetadata *argSets = psMetadataAlloc();
 …
     PXOPT_ADD_MODE("-revertcopied",    "", PZTOOL_MODE_REVERTCOPIED,revertcopiedArgs);
     PXOPT_ADD_MODE("-clearcommonfaults","", PZTOOL_MODE_CLEARCOMMONFAULTS,clearcommonfaultsArgs);
+    PXOPT_ADD_MODE("-toadvance",          "", PZTOOL_MODE_TOADVANCE,    toadvanceArgs);
     PXOPT_ADD_MODE("-advance",          "", PZTOOL_MODE_ADVANCE,    advanceArgs);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtool.c

-              r23352
+              r23594
     psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where,  "-exp_id", "exp_id", "==");
+    PXOPT_COPY_STR(config->args, where,  "-exp_name", "exp_name", "==");
+    PXOPT_COPY_STR(config->args, where,  "-class_id", "class_id", "==");
+    PXOPT_COPY_S64(config->args, where,  "-exp_id",        "exp_id",   "==");
+    PXOPT_COPY_STR(config->args, where,  "-exp_name",      "exp_name", "==");
+    PXOPT_COPY_STR(config->args, where,  "-class_id",      "class_id", "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs",  ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "dateobs",  "<=");
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 …
     PXOPT_COPY_STR(config->args, where,  "-class_id",     "class_id", "==");
     PXOPT_COPY_S16(config->args, where,  "-code",         "fault", "==");
+    PXOPT_COPY_S64(config->args, where,  "-exp_id_begin", "exp_id", ">=");
+    PXOPT_COPY_S64(config->args, where,  "-exp_id_end", "exp_id", "<=");
     psString query = pxDataGet("regtool_revertprocessedimfile.sql");
 …
         psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
+    } else {
+        psFree(where);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        return false;
+    }
     psFree(where);
 …
     psFree(query);
+    if (psDBAffectedRows(config->dbh) < 1) {
+        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
+        return false;
+    }
+    int numUpdated = psDBAffectedRows(config->dbh);
+    psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawImfile", numUpdated);
     return true;
 …
+{
     PS_ASSERT_PTR_NON_NULL(config, false);
+    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false);
+    PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
+    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+    PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false);
+    if ((code == INT16_MAX) && !isfinite(user_1)) {
+        psError(PS_ERR_UNKNOWN, false, "one of -code or -user_1 must be selected");
+        return false;
+    }
+    if ((code != INT16_MAX) && isfinite(user_1)) {
+        psError(PS_ERR_UNKNOWN, false, "only one of -code or -user_1 must be selected");
+        return false;
+    }
     psMetadata *where = psMetadataAlloc();
 …
     PXOPT_COPY_STR(config->args, where,  "-class_id",     "class_id", "==");
+    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);
+    if (!pxSetFaultCode(config->dbh, "rawImfile", where, code)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+        psFree (where);
+        return false;
+    if (code != INT16_MAX) {
+        // this is fairly dangerous : can set all if the where is not set...
+        if (!pxSetFaultCode(config->dbh, "rawImfile", where, code)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
+            psFree (where);
+            return false;
+        }
+    }
+    if (isfinite(user_1)) {
+        psString query = pxDataGet("regtool_updateprocessedimfile.sql");
+        if (!query) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            return false;
+        }
+        if (!p_psDBRunQueryF(config->dbh, query, user_1, exp_id, class_id)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(query);
+            return false;
+        }
+        psFree(query);
+    }
     psFree (where);
 …
         hostname,
         code,
+        NULL
+        NULL,
     )) {
         // rollback
 …
                 exp_id,
                 workdir,
             label,
+                label,
                 reduction,
                 NULL, // expgroup
 …
     PXOPT_COPY_S64(config->args, where,  "-exp_id",       "exp_id", "==");
     PXOPT_COPY_S16(config->args, where,  "-code",         "fault", "==");
+    PXOPT_COPY_S64(config->args, where,  "-exp_id_begin", "exp_id", ">=");
+    PXOPT_COPY_S64(config->args, where,  "-exp_id_end", "exp_id", "<=");
     psString query = pxDataGet("regtool_revertprocessedexp.sql");
 …
         psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp");
         psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    } else {
         psFree(where);
+        psFree(whereClause);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        return false;
+    }
     psFree(where);
 …
     psFree(query);
+    if (psDBAffectedRows(config->dbh) < 1) {
+        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
+        return false;
+    }
+    int numUpdated = psDBAffectedRows(config->dbh);
+    psLogMsg("regtool", PS_LOG_INFO, "Updated %d rawExp", numUpdated);
     return true;

branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtoolConfig.c

-              r23352
+              r23594
     ADD_OPT(Str,  processedimfileArgs, "-exp_name",  "search by exposure name",               NULL);
     ADD_OPT(Str,  processedimfileArgs, "-class_id",  "search by class ID",                    NULL);
+    ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL);
+    ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL);
     ADD_OPT(U64,  processedimfileArgs, "-limit",     "limit result set to N items",           0);
     ADD_OPT(Bool, processedimfileArgs, "-faulted",   "only return imfiles with a fault status set", false);
 …
     // -revertprocessedimfile
     psMetadata *revertprocessedimfileArgs = psMetadataAlloc();
     ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id",        "search by exposure ID (required)", 0);
+    ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id",        "search by exposure ID", 0);
     ADD_OPT(Str, revertprocessedimfileArgs, "-tmp_class_id",  "searcy by temp. class ID", NULL);
     ADD_OPT(Str, revertprocessedimfileArgs, "-class_id",      "search by class ID", NULL);
     ADD_OPT(S16, revertprocessedimfileArgs, "-code",          "search by fault code", 0);
+    ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id_begin",  "search by exposure ID", 0);
+    ADD_OPT(S64, revertprocessedimfileArgs, "-exp_id_end",    "search by exposure ID", 0);
     // -updateprocessedimfile
 …
     ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id",        "search by exposure ID", 0);
     ADD_OPT(Str, updateprocessedimfileArgs, "-class_id",      "search by class ID", NULL);
+    ADD_OPT(S16, updateprocessedimfileArgs, "-code",          "set fault code (required)", INT16_MAX);
+    ADD_OPT(F64, updateprocessedimfileArgs, "-user_1",        "set user stat (1)", NAN);
+    ADD_OPT(S16, updateprocessedimfileArgs, "-code",          "set fault code", INT16_MAX);
     // -pendingexp
 …
     // -revertprocessedexp
     psMetadata *revertprocessedexpArgs = psMetadataAlloc();
     psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id",   0,            "search by exposure ID (required)", 0);
+    psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id",   0,            "search by exposure ID", 0);
     psMetadataAddS16(revertprocessedexpArgs, PS_LIST_TAIL, "-code",     0,            "search by fault code", 0);
+    psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id_begin",   0,      "search by exposure ID", 0);
+    psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-exp_id_end",   0,      "search by exposure ID", 0);
     // -updatedprocessedexp

branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptool.c

-              r23352
+              r23594
             tess_id,
             end_stage,
+            registered
+            registered,
+            false       // magicked
     );
     if (!warpRun) {
 …
     psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "newExp.exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "==");
+    PXOPT_COPY_STR(config->args, where, "-inst", "camera", "==");
+    PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<=");
+    PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "==");
+    PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter", "filter", "==");
+    PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">=");
+    PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<");
+    PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">=");
+    PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<");
+    PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">=");
+    PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">=");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<");
+    PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">=");
+    PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<");
+    PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">=");
+    PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<");
+    PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">=");
+    PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<");
+    PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">=");
+    PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<");
+    PXOPT_COPY_STR(config->args, where, "-object", "object", "==");
+    PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">=");
+    PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+    pxwarpGetSearchArgs (config, where);
+    PXOPT_COPY_STR(config->args, where, "-reduction", "fakeRun.reduction", "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "fakeRun.label",     "==");
     if (!psListLength(where->list) &&
 …
     psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "==");
+    PXOPT_COPY_STR(config->args, where, "-inst", "camera", "==");
+    PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<=");
+    PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "==");
+    PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter", "filter", "==");
+    PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">=");
+    PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<");
+    PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">=");
+    PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<");
+    PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">=");
+    PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">=");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<");
+    PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">=");
+    PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<");
+    PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">=");
+    PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<");
+    PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "ccd_temp", ">=");
+    PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "ccd_temp", "<");
+    PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">=");
+    PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<");
+    PXOPT_COPY_STR(config->args, where, "-object", "object", "==");
+    PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">=");
+    PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<");
+    pxwarpGetSearchArgs (config, where);
+    PXOPT_COPY_S64(config->args, where, "-warp_id",   "warpRun.warp_id",   "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "warpRun.reduction", "==");
+    PXOPT_COPY_STR(config->args, where, "-label",     "warpRun.label",     "==");
+    PXOPT_COPY_STR(config->args, where, "-state",     "warpRun.state",     "==");
     if (!psListLength(where->list)
 …
+    }
     PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
     if ((!state) && (!label)) {
 …
     PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
     PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false);
+    PXOPT_LOOKUP_BOOL(magicked, config->args, "-magicked", false);
     // default values
 …
                            !accept,
                            code,
             // magic_ds_id
+                           magicked
         )) {
         if (!psDBRollback(config->dbh)) {
 …
     PS_ASSERT_PTR_NON_NULL(config, false);
+    // XXX this SQL has not been broken out to into seperate files as the MYSQL
+    // < 5 & MYSQL 5 versions need to be kept in sync
+#undef MYSQL5
+#if MYSQL5
+    // XXX at MySQL 4.1.21 (probably all of 4.1.x) chokes and dies on this
+    // statement as it thinks it is trying to select from the table being
+    // updated. The 4.1 manual says that nested sub-queries are explicited
+    // allowed to do this with update statements as a temporary table is
+    // created so that you are not actually selecting from the table you are
+    // modifying.
+    char *query =
+        "UPDATE warpRun\n"
+        "   SET warpRun.state = 'stop'\n"
+        " WHERE\n"
+        "   warpRun.warp_id =\n"
+        "   (SELECT DISTINCT\n"
+        "       warp_id\n"
+        "   FROM\n"
+        "       (SELECT DISTINCT\n"
+        "           warpRun.warp_id,\n"
+        "           warpSkyCellMap.warp_id as foo,\n"
+        "           warpSkyfile.warp_id as bar\n"
+        "       FROM warpRun\n"
+        "       JOIN warpSkyCellMap\n"
+        "           USING(warp_id)\n"
+        "       LEFT JOIN warpSkyfile\n"
+        "           USING(warp_id, skycell_id, tess_id)\n"
+        "       WHERE\n"
+        "           warpRun.state = 'new'\n"
+        "       GROUP BY\n"
+        "           warpRun.warp_id\n"
+        "       HAVING\n"
+        "       COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n"
+        "       ) as Foo\n"
+        "   )\n";
+    psString query = pxDataGet("warptool_finished_run_select.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retrieve SQL statement");
+        return false;
+    }
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+#else // if MYSQL5
+{
+    char *query =
+        "CREATE TEMPORARY TABLE finished\n"
+        " (warp_id INT, PRIMARY KEY(warp_id)) ENGINE=MEMORY\n";
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+}
+{
+    char *query =
+        "INSERT INTO finished\n"
+        " SELECT\n"
+        "   warp_id\n"
+        " FROM\n"
+        "   (SELECT DISTINCT\n"
+        "       warpRun.warp_id,\n"
+        "       warpSkyCellMap.warp_id as foo,\n"
+        "       warpSkyfile.warp_id as bar\n"
+        "   FROM warpRun\n"
+        "   JOIN warpSkyCellMap\n"
+        "       USING(warp_id)\n"
+        "   LEFT JOIN warpSkyfile\n"
+        "       USING(warp_id, skycell_id)\n"
+        "   WHERE\n"
+        "       warpRun.state = 'new'\n"
+        "   GROUP BY\n"
+        "       warpRun.warp_id\n"
+        "   HAVING\n"
+        "       COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n"
+        " ) as Foo \n";
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+}
+{
+    char *query =
+        "UPDATE warpRun\n"
+        "   SET warpRun.state = 'full'\n"
+        " WHERE\n"
+        "   warpRun.warp_id =\n"
+        "   (SELECT DISTINCT\n"
+        "       warp_id\n"
+        "   FROM finished\n"
+        "   )\n";
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+}
+#endif // if MYSQL5
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("warptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    query = pxDataGet("warptool_finish_run.sql");
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+        bool status;
+        psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id");
+            psFree(output);
+            psFree(query);
+            return false;
+        }
+        psS32 magicked = psMetadataLookupS64(&status, row, "magicked");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to look up value for magicked");
+            psFree(output);
+            psFree(query);
+            return false;
+        }
+        if (!p_psDBRunQueryF(config->dbh, query, magicked, warp_id)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            psFree(query);
+            return false;
+        }
+        psS64 numUpdated = psDBAffectedRows(config->dbh);
+        if (numUpdated != 1) {
+            psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
+            psFree(query);
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(output);
+    psFree(query);
     return true;
 …
     psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "==");
+    pxwarpGetSearchArgs (config, where);
+    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "==");
+    PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
+    PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "==");
+    PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "==");
+    PXOPT_COPY_F32(config->args, where, "-airmass_min", "rawExp.airmass", ">=");
+    PXOPT_COPY_F32(config->args, where, "-airmass_max", "rawExp.airmass", "<");
+    PXOPT_COPY_F64(config->args, where, "-ra_min", "rawExp.ra", ">=");
+    PXOPT_COPY_F64(config->args, where, "-ra_max", "rawExp.ra", "<");
+    PXOPT_COPY_F64(config->args, where, "-decl_min", "rawExp.decl", ">=");
+    PXOPT_COPY_F64(config->args, where, "-decl_max", "rawExp.decl", "<");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">=");
+    PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">=");
+    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<");
+    PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">=");
+    PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<");
+    PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">=");
+    PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<");
+    PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">=");
+    PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<");
+    PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">=");
+    PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<");
+    PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "==");
+    PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">=");
+    PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<");
+    PXOPT_COPY_S16(config->args, where, "-code", "warpSkyfile.fault", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",    "warpSkyfile.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction",  "rawExp.reduction", "==");
+    PXOPT_COPY_STR(config->args, where, "-label",      "warpRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-code",       "warpSkyfile.fault", "==");
     if (!psListLength(where->list)
 …
     int numUpdated;                     // Number updated
+    {
+        // This query is no longer necessary because we do not set warpRun's to full statte
+        // if they have faulted skyfiles.
+        // We do have runs in the DB that follow the old convention so we leave this in for
+        // now
         psString query = pxDataGet("warptool_revertwarped_update.sql");
         if (!query) {
 …
         numUpdated = psDBAffectedRows(config->dbh);
+#ifdef notdef
+        // new warpRuns won't get changed (they're already new) so don't require an update
         if (numUpdated < 1) {
             psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
+            psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row");
             if (!psDBRollback(config->dbh)) {
                 psError(PS_ERR_UNKNOWN, false, "database error");
 …
             return false;
+        }
+#endif
+    }
 …
     PS_ASSERT_PTR_NON_NULL(config, NULL);
     PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false);
+    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     psMetadata *where = psMetadataAlloc();
+    if (warp_id) {
+        PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
+    }
+    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
     PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
 …
     // note only updates if warpRun.state = run_state
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id, run_state)) {
+    // XXX note that we have removed this constraint for now
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, warp_id, skycell_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         // rollback

branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptoolConfig.c

-              r23352
+              r23594
     // XXX need to allow multiple exp_ids
     psMetadata *definebyqueryArgs = psMetadataAlloc();
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-fake_id",            0, "search by fake_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id",             0, "search by cam_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id",            0, "search by chip_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id",             0, "search by exp_id", 0);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_name",           0, "search by exp_name", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-inst",               0, "search for camera", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-telescope",          0, "search for telescope", NULL);
+    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-dateobs_begin",     0, "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-dateobs_end",       0, "search for exposures by time (<)", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_tag",            0, "search by exp_tag", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-exp_type",           0, "search by exp_type", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filelevel",          0, "search by filelevel", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",          0, "search by reduction class", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-filter",             0, "search for filter", NULL);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_min",        0, "define min airmass", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-airmass_max",        0, "define max airmass", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_min",             0, "define min", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-ra_max",             0, "define max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_min",           0, "define min", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-decl_max",           0, "define max", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_min",       0, "define min", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-exp_time_max",       0, "define max", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define min fraction of saturated pixels", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_min",             0, "define max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_max",             0, "define max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_min",       0, "define max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_stdev_max",       0, "define max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_min",  0, "define max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-bg_mean_stdev_max",  0, "define max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_min",            0, "define min", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-alt_max",            0, "define max", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_min",             0, "define min", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-az_max",             0, "define max", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_min",       0, "define min ccd tempature", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-ccd_temp_max",       0, "define max ccd tempature", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_min",         0, "define min rotator position angle", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-posang_max",         0, "define max rotator position angle", NAN);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-object",             0, "search by exposure object", NULL);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_min",         0, "define min solar angle", NAN);
+    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-solang_max",         0, "define max solar angle", NAN);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "search on fakeRun label", NULL);
+    pxwarpSetSearchArgs (definebyqueryArgs);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction",          0, "search by fakeRun reduction class", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",              0, "search on fakeRun label", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_mode",           0, "define mode (warp, diff, stack, magic)", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir",        0, "define workdir", NULL);
 …
     // XXX need to allow multiple exp_ids
     psMetadata *updaterunArgs = psMetadataAlloc();
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warptool ID", 0);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-fake_id",  0,            "search by fake_id", 0);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id",  0,            "search by chip_id", 0);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exp_id", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_name",  0,            "search by exp_name", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-inst",  0,            "search for camera", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-telescope",  0,            "search for telescope", NULL);
+    psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_begin", 0,            "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(updaterunArgs, PS_LIST_TAIL, "-dateobs_end", 0,            "search for exposures by time (<)", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_tag",  0,            "search by exp_tag", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-exp_type",  0,            "search by exp_type", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filelevel",  0,            "search by filelevel", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction",  0,            "search by reduction class", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-filter",  0,            "search for filter", NULL);
+    psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_min",  0,            "define min airmass", NAN);
+    psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-airmass_max",  0,            "define max airmass", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_min",  0,            "define min", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ra_max",  0,            "define max", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_min",  0,            "define min", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-decl_max",  0,            "define max", NAN);
+    psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_min",  0,            "define min", NAN);
+    psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-exp_time_max",  0,            "define max", NAN);
+    psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_min",  0,            "define max fraction of saturated pixels", NAN);
+    psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sat_pixel_frac_max",  0,            "define min fraction of saturated pixels", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_min",  0,            "define max", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_max",  0,            "define max", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_min",  0,            "define max", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_stdev_max",  0,            "define max", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_min",  0,            "define max", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-bg_mean_stdev_max",  0,            "define max", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_min",  0,            "define min", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-alt_max",  0,            "define max", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_min",  0,            "define min", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-az_max",  0,            "define max", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_min",  0,            "define min ccd tempature", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-ccd_temp_max",  0,            "define max ccd tempature", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_min",  0,            "define min rotator position angle", NAN);
+    psMetadataAddF64(updaterunArgs, PS_LIST_TAIL, "-posang_max",  0,            "define max rotator position angle", NAN);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-object",  0,            "search by exposure object", NULL);
+    psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_min",  0,            "define min solar angle", NAN);
+    psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-solang_max",  0,            "define max solar angle", NAN);
+    psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all",  0,            "allow everything to be queued without search terms", false);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0,            "set state", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0,            "set label", NULL);
+#if 0
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-workdir", 0,            "define workdir (required)", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-registered",  0,            "time detrend run was registered", now);
+#endif
+    pxwarpSetSearchArgs (updaterunArgs);
+    psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-warp_id", 0,    "search by warptool ID", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0,  "search by warpRun reduction class", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0,      "search by warpRun state", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0,      "search by warpRun label", NULL);
+    psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all",  0,      "allow everything to be queued without search terms", false);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0,  "set state", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0,  "set label", NULL);
     // -exp
 …
     psMetadataAddF32(addwarpedArgs, PS_LIST_TAIL, "-good_frac",  0,            "define %% of good pixels", NAN);
     psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-accept",  0, "define if this skycell should be accepted", false);
+    psMetadataAddBool(addwarpedArgs, PS_LIST_TAIL, "-magicked",  0, "define if this skycell has been magicked", false);
     psMetadataAddS16(addwarpedArgs, PS_LIST_TAIL, "-code",  0,            "set fault code", 0);
 …
     // XXX need to allow multiple exp_ids
     psMetadata *revertwarpedArgs = psMetadataAlloc();
+    pxwarpSetSearchArgs(revertwarpedArgs); // XXX does this work here?
     psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warptool ID", 0);
     psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-skycell_id",  0,            "search by skycell ID", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-tess_id",  0,            "searcy by tessellation ID", NULL);
+    psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-fake_id",  0,            "search by fake_id", 0);
+    psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-chip_id",  0,            "search by chip_id", 0);
+    psMetadataAddS64(revertwarpedArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exp_id", 0);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_name",  0,            "search by exp_name", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-label",  0,            "search by label", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-inst",  0,            "search for camera", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-telescope",  0,            "search for telescope", NULL);
+    psMetadataAddTime(revertwarpedArgs, PS_LIST_TAIL, "-dateobs_begin", 0,            "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(revertwarpedArgs, PS_LIST_TAIL, "-dateobs_end", 0,            "search for exposures by time (<)", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_tag",  0,            "search by exp_tag", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-exp_type",  0,            "search by exp_type", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-filelevel",  0,            "search by filelevel", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-reduction",  0,            "search by reduction class", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-filter",  0,            "search for filter", NULL);
+    psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-airmass_min",  0,            "define min airmass", NAN);
+    psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-airmass_max",  0,            "define max airmass", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ra_min",  0,            "define min", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ra_max",  0,            "define max", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-decl_min",  0,            "define min", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-decl_max",  0,            "define max", NAN);
+    psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-exp_time_min",  0,            "define min", NAN);
+    psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-exp_time_max",  0,            "define max", NAN);
+    psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-sat_pixel_frac_min",  0,            "define max fraction of saturated pixels", NAN);
+    psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-sat_pixel_frac_max",  0,            "define min fraction of saturated pixels", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_min",  0,            "define max", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_max",  0,            "define max", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_stdev_min",  0,            "define max", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_stdev_max",  0,            "define max", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_mean_stdev_min",  0,            "define max", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-bg_mean_stdev_max",  0,            "define max", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-alt_min",  0,            "define min", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-alt_max",  0,            "define max", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-az_min",  0,            "define min", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-az_max",  0,            "define max", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ccd_temp_min",  0,            "define min ccd tempature", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-ccd_temp_max",  0,            "define max ccd tempature", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-posang_min",  0,            "define min rotator position angle", NAN);
+    psMetadataAddF64(revertwarpedArgs, PS_LIST_TAIL, "-posang_max",  0,            "define max rotator position angle", NAN);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-object",  0,            "search by exposure object", NULL);
+    psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-solang_min",  0,            "define min solar angle", NAN);
+    psMetadataAddF32(revertwarpedArgs, PS_LIST_TAIL, "-solang_max",  0,            "define max solar angle", NAN);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-tess_id",  0,            "search by tessellation ID", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-reduction",  0,            "search by warpRun reduction class", NULL);
+    psMetadataAddStr(revertwarpedArgs, PS_LIST_TAIL, "-label",  0,            "search by warpRun label", NULL);
     psMetadataAddS16(revertwarpedArgs, PS_LIST_TAIL, "-code",  0,            "search by fault code", 0);
     psMetadataAddBool(revertwarpedArgs, PS_LIST_TAIL, "-all",  0,            "allow everything to be queued without search terms", false);
 …
     psMetadata *pendingcleanupskyfileArgs = psMetadataAlloc();
     psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label",  0,            "list blocks for specified label", NULL);
     psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,          "search by warp ID", 0);
+    psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,          "search by warp ID (required)", 0);
     psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
     psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppImage.config

-              r23352
+              r23594
 OLDDARK                 BOOL    FALSE
+# use the deburned image instead of the raw, if it exists
+USE.DEBURNED.IMAGE      BOOL    TRUE           # use burntool-repaired image?
 ## XXX use these local variations until we are building real detrend images

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/Makefile.am

r20196	r23594
9	9	addstar.config \
10	10	masks.config \
	11	masks.16bit.config \
11	12	reductionClasses.mdc \
12	13	rejections.config \

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-mef.mdc

-              r23352
+              r23594
 ### File rules for PHU=FPA, EXT=CHIP (eg, Megacam)
 ### this file specifies compression types for various output files
+### The image types are defined in fitstypes.mdc
+### The default type of NONE means no compression
 ### Redirections (MEF)
 …
 PSASTRO.INPUT.CMP       INPUT    @FILES        CHIP       CMP
 PSASTRO.INPUT.CMF       INPUT    @FILES        CHIP       CMF
+PSASTRO.INPUT.MASK      INPUT    @FILES        CHIP       MASK
+PSASTRO.REFMASK         INPUT    @DETDB        CHIP       MASK
 PSASTRO.WCS             INPUT    @FILES        CHIP       CMF
 PSASTRO.MODEL           INPUT    @DETDB        FPA        ASTROM
+PSASTRO.EXTRACT.ASTROM  INPUT    @FILES        CHIP       CMF
+PSASTRO.EXTRACT.INPUT   INPUT    @FILES        CHIP       IMAGE
 ## files used by pswarp
 …
 PPSTACK.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits                  VARIANCE  COMP_WT    FPA        TRUE      NONE
 PPSTACK.TARGET.PSF      OUTPUT {OUTPUT}.target.psf               PSF       NONE       CHIP       TRUE      NONE
+PPSTACK.CONV.KERNEL     OUTPUT {OUTPUT}.{FILE.INDEX}.kernel      SUBKERNEL NONE       FPA        TRUE      NONE
 PPSTACK.OUTPUT.JPEG1    OUTPUT {OUTPUT}.b1.jpg                   JPEG      NONE       FPA        TRUE      NONE
 PPSTACK.OUTPUT.JPEG2    OUTPUT {OUTPUT}.b2.jpg                   JPEG      NONE       FPA        TRUE      NONE

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-simple.mdc

-              r23352
+              r23594
 ### File rules for PHU=FPA, EXT=NONE
+### The default type of NONE means no compression
 PSASTRO.INPUT      STR PSASTRO.INPUT.CMF
 …
 PPIMAGE.INPUT.PSF       INPUT    @FILES        READOUT    PSF
 PPIMAGE.INPUT.SRC       INPUT    @FILES        READOUT    CMF
 PPIMAGE.MASK            INPUT    @DETDB        CHIP       IMAGE
+PPIMAGE.MASK            INPUT    @DETDB        CHIP       MASK
 PPIMAGE.BIAS            INPUT    @DETDB        CHIP       IMAGE
 PPIMAGE.DARK            INPUT    @DETDB        CHIP       DARK
 …
 PSASTRO.INPUT.CMP       INPUT    @FILES        CHIP       CMP
 PSASTRO.INPUT.CMF       INPUT    @FILES        CHIP       CMF
+PSASTRO.INPUT.MASK      INPUT    @FILES        CHIP       MASK
+PSASTRO.REFMASK         INPUT    @DETDB        CHIP       MASK
 PSASTRO.WCS             INPUT    @FILES        CHIP       CMF
 PSASTRO.MODEL           INPUT    @DETDB        FPA        ASTROM
+PSASTRO.EXTRACT.ASTROM  INPUT    @FILES        CHIP       CMF
+PSASTRO.EXTRACT.INPUT   INPUT    @FILES        CHIP       IMAGE
 ## files used by pswarp
 …
 PPSTACK.OUTPUT.VARIANCE OUTPUT {OUTPUT}.weight.fits  VARIANCE  NONE       FPA        TRUE      NONE
 PPSTACK.TARGET.PSF      OUTPUT {OUTPUT}.target.psf   PSF       NONE       CHIP       TRUE      NONE
+PPSTACK.CONV.KERNEL     OUTPUT {OUTPUT}.{FILE.INDEX}.kernel SUBKERNEL NONE FPA       TRUE      NONE
 PPSTACK.OUTPUT.JPEG1    OUTPUT {OUTPUT}.b1.jpg       JPEG      NONE       FPA        TRUE      NONE
 PPSTACK.OUTPUT.JPEG2    OUTPUT {OUTPUT}.b2.jpg       JPEG      NONE       FPA        TRUE      NONE

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-split.mdc

-              r23352
+              r23594
 ### File rules for PHU=CHIP, EXT=CELL (eg, GPC1)
 ### this file specifies compression types for various output files
+### The default type of NONE means no compression
 # for basic chip analysis, psphot should produce SPLIT output
 …
 PSASTRO.WCS             INPUT    @FILES        CHIP       WCS
 PSASTRO.MODEL           INPUT    @DETDB        FPA        ASTROM
+PSASTRO.EXTRACT.ASTROM  INPUT    @FILES        CHIP       CMF
+PSASTRO.EXTRACT.INPUT   INPUT    @FILES        CHIP       IMAGE
 ## files used by pswarp
 …
 PPIMAGE.JPEG1           OUTPUT {OUTPUT}.b1.jpg                   JPEG      NONE       FPA        TRUE      NONE
 PPIMAGE.JPEG2           OUTPUT {OUTPUT}.b2.jpg                   JPEG      NONE       FPA        TRUE      NONE
 PPIMAGE.STATS           OUTPUT {OUTPUT}.{CHIP.NAME}.stats        STATS     NONE       CHIP       TRUE      NONE
 …
 PPSTACK.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits                  VARIANCE  COMP_WT    FPA        TRUE      NONE
 PPSTACK.TARGET.PSF      OUTPUT {OUTPUT}.target.psf               PSF       NONE       CHIP       TRUE      NONE
+PPSTACK.CONV.KERNEL     OUTPUT {OUTPUT}.{FILE.INDEX}.kernel      SUBKERNEL NONE       FPA        TRUE      NONE
 PPSTACK.OUTPUT.JPEG1    OUTPUT {OUTPUT}.b1.jpg                   JPEG      NONE       FPA        TRUE      NONE
 PPSTACK.OUTPUT.JPEG2    OUTPUT {OUTPUT}.b2.jpg                   JPEG      NONE       FPA        TRUE      NONE

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/jpeg.mdc

-              r23352
+              r23594
         END
 END
+CTEMAP  METADATA
+        PPIMAGE.JPEG1            METADATA
+                COLORMAP        STR     rainbow
+                SCALE.MODE      STR     VALUE
+                SCALE.MIN       F32     0.0
+                SCALE.MAX       F32     5.0
+        END
+        PPIMAGE.JPEG2            METADATA
+                COLORMAP        STR     rainbow
+                SCALE.MODE      STR     VALUE
+                SCALE.MIN       F32     0.0
+                SCALE.MAX       F32     5.0
+        END
+END

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/masks.16bit.config

-              r21185
+              r23594
 # Mask values which represent features of the detector
+DETECTOR        U8      0x0001          # Something is wrong with the detector
+FLAT            U8      0x0002          # Pixel doesn't flat-field properly
+DARK            U8      0x0004          # Pixel doesn't dark-subtract properly
+BLANK           U8      0x0008          # Pixel doesn't contain valid data
+DETECTOR        U16     0x0001          # Something is wrong with the detector
+FLAT            U16     0x0002          # Pixel doesn't flat-field properly
+DARK            U16     0x0004          # Pixel doesn't dark-subtract properly
+BLANK           U16     0x0008          # Pixel doesn't contain valid data
+CTE             U16     0x0010          # Pixel has poor Charge Transfer Efficiency
 # Mask values which represent invalid signal ranges
+RANGE           U8      0x0010          # Pixel is out-of-range of linearity
+SAT             U8      0x0020          # Pixel is saturated
+BAD             U8      0x0040          # Pixel is low (can we rename this 'LOW'?)
+SAT             U16     0x0020          # Pixel is saturated or non-linear
+LOW             U16     0x0040          # Pixel is low
+SUSPECT         U16     0x0080          # Pixel is suspected of being bad
 # Mask values which represent non-astronomical structures
+GHOST           U8      0x0100          # Pixel contains an optical ghost
+CR              U8      0x0200          # Pixel contains a cosmic ray
+STREAK          U8      0x0400          # Pixel contains a streak
+CR              U16     0x0100          # Pixel contains a cosmic ray
+SPIKE           U16     0x0200          # Pixel contains a diffraction spike
+GHOST           U16     0x0400          # Pixel contains an optical ghost
+STREAK          U16     0x0800          # Pixel contains a streak
+STARCORE        U16     0x1000          # Pixel contains a bright star core
 # Mask values which identify pixels badly affected by convolutions and interpolations
 BAD.WARP        U8      0x1000          # Pixel is bad after convolution with a bad pixel
 POOR.WARP       U8      0x2000          # Pixel is poor after convolution with a bad pixel
+CONV.BAD        U16     0x2000          # Pixel is bad after convolution with a bad pixel
+CONV.POOR       U16     0x4000          # Pixel is poor after convolution with a bad pixel
+# Spare value for temporary marking
+# MARK          U16     0x8000          # Pixel is temporarily marked

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/masks.config

-              r20631
+              r23594
 FLAT            U8      0x04            # Pixel doesn't flat-field properly =4
 DARK            U8      0x40            # Pixel doesn't dark-subtract properly =64
+CTE             U8      0x40            # Pixel has poor cte
 # Filling in the rest with whatever makes some sense
 BLANK           U8      0x01            # Pixel doesn't contain valid data
 …
 GHOST           U8      0x20            # Pixel contains an optical ghost
 STREAK          U8      0x20            # Pixel contains a streak
+SPIKE           U8      0x20            # Pixel contains a diffraction spike
+STARCORE        U8      0x20            # Pixel contains a streak
 ###### The following values are what I'm aiming to have in the long term (PAP, 2008-09-09)

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppImage.config

-              r23352
+              r23594
 ASTROM.MOSAIC      BOOL    FALSE           # Astrometry for mosaic?
 BACKGROUND         BOOL    FALSE           # Subtract model background?
+CHECK.CTE          BOOL    FALSE           # measure CTE errors?
+USE.DEBURNED.IMAGE BOOL    FALSE           # use burntool-repaired image?
 # output data formats to save
 …
 BIN2.XBIN               S32     16
 BIN2.YBIN               S32     16
+CTE.XBIN                S32     20
+CTE.YBIN                S32     20
 PPIMAGE.JPEG1  METADATA
 …
   BASE.FITS        BOOL    TRUE            # Save base detrended image?
   BASE.MASK.FITS   BOOL    TRUE            # Save base detrended image?
   BASE.VARIANCE.FITS BOOL    TRUE            # Save base detrended image?
   CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image?
   CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image?
   CHIP.VARIANCE.FITS BOOL    FALSE           # Save chip-mosaic-ed image?
+  BASE.VARIANCE.FITS BOOL  TRUE            # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.VARIANCE.FITS BOOL  FALSE           # Save chip-mosaic-ed image?
   OVERSCAN         BOOL    TRUE            # Overscan subtraction
   BIAS             BOOL    TRUE            # Bias subtraction
 …
 END
+# generate CTE map image
+PPIMAGE_CTEMAP     METADATA
+  BASE.FITS        BOOL    TRUE            # Save base image?
+  BASE.MASK.FITS   BOOL    TRUE            # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL  TRUE            # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.VARIANCE.FITS BOOL  FALSE           # Save chip-mosaic-ed image?
+  OVERSCAN         BOOL    FALSE           # Overscan subtraction
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    TRUE            # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    TRUE            # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+  CHECK.CTE        BOOL    TRUE            # measure CTE errors?
+END
 # For SDSS cameras
 PPIMAGE_OA_SOFT   METADATA

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppMerge.config

-              r21370
+              r23594
 FRINGE.XSMOOTH  S32     5               # Number of smoothing regions in x
 FRINGE.YSMOOTH  S32     11              # Number of smoothing regions in y
+CTE.MIN         F32     0.2             # regions lower than this in the CTE image are masked
 SHUTTER.SIZE    S32     128             # Size for shutter measurement regions
 MASK.SUSPECT    F32     5.0             # Threshold for suspect pixels (sigma)
 …
 END
+# CTEMASK generation combines the images like a flat or bias (mean after outlier rejection),
+# but it then sets a mask based on the resulting data values (using CTE.MIN)
+PPMERGE_CTEMASK METADATA
+        REJ             F32     3.0             # Rejection threshold (sigma)
+        ITER            S32     2               # Number of rejection iterations
+        FRACHIGH        F32     0.0             # Fraction of high pixels to reject immediately
+        FRACLOW         F32     0.0             # Fraction of low pixels to reject immediately
+        VARIANCES       BOOL    FALSE           # Use image variances?
+        COMBINE         STR     CLIPPED         # Statistic to use for combination:
+        CTE.MIN         F32     0.2             # regions lower than this in the CTE image are masked
+        MASK.SET.VALUE  STR     CTE             # set this bit in the output mask
+END
 # Shutter generation --- already included in default, above
 PPMERGE_SHUTTER METADATA

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSim.config

-              r20911
+              r23594
 GALAXY.MODEL     STR    PS_MODEL_GAUSS
+BADCTE          BOOL    FALSE           # create a region of 'bad CTE'?
+BADCTE.REGION   STR     [0:200,0:200]   # region of 'bad CTE'
+BADCTE.SIGMA    F32     2.0             # 'bad CTE' smoothing scale
+BADCTE.NSIGMA   S32     2               # 'bad CTE' smoothing scale
 BADPIX.SEED     U64     123456789       # Seed for RNG in creating deterministic bad pixels
 …
     BADPIX.MODE STR     NAN             # Mode for bad pixels: RANDOM|NAN
 END
+# recipe for a CTE test
+BADCTE.TEST     METADATA
+  BADCTE BOOL TRUE
+END

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStack.config

-              r23352
+              r23594
 # Recipe configuration for ppStack (image combination)
+CONVOLVE        BOOL    TRUE            # Convolve images when stacking?
 ITER            S32     1               # Number of rejection iterations
 COMBINE.REJ     F32     3.0             # Rejection threshold in combination (sigma)
 COMBINE.SYS     F32     0.08            # Relative systematic error in combination
 COMBINE.DISCARD F32     0.2             # Discard fraction for Olympic weighted mean
 MASK.VAL        STR     MASK.VALUE,BAD.WARP     # Mask value of input bad pixels
+MASK.VAL        STR     MASK.VALUE,CONV.BAD     # Mask value of input bad pixels
 MASK.BAD        STR     BLANK           # Mask value to give bad pixels
 MASK.POOR       STR     POOR.WARP       # Mask value to give poor pixels
+MASK.POOR       STR     CONV.POOR       # Mask value to give poor pixels
 POOR.FRACTION   F32     0.01            # Maximum fraction of bad variance for poor pixels
 THRESHOLD.MASK  F32     0.5             # Threshold for mask deconvolution (0..1)

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSub.config

-              r23352
+              r23594
 # EAM : we now ignore these: BAD.WARP is always bad, POOR.WARP is always defined
 MASK.IN         STR     MASK.VALUE,BAD.WARP     # Mask value for input
 MASK.BAD        STR     BLANK           # Mask value to give bad pixels
 MASK.POOR       STR     POOR.WARP       # Mask value to give poor pixels
+MASK.IN         STR     MASK.VALUE,CONV.BAD     # Mask value for input
+MASK.BAD        STR     CONV.BAD        # Mask value to give bad pixels
+MASK.POOR       STR     CONV.POOR       # Mask value to give poor pixels
 POOR.FRACTION   F32     0.10            # Maximum fraction of bad weight for poor pixels

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/psastro.config

r20797	r23594
124	124	REFSTAR_MASK_BLEED_MAG_SLOPE F32 5.0
125	125
	126	EXTRACT_MAX_MAG F32 -15.0
	127
126	128	# 2MASS default configuration (use 2MASS_J as ref magnitude)
127	129	PSASTRO.CATDIR STR 2MASS

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/reductionClasses.mdc

-              r23352
+              r23594
         FLATMASK_JPEG_RESID     STR     FLAT_RESID
+        CTEMASK_PROCESS         STR     PPIMAGE_CTEMAP
+        CTEMASK_RESID           STR     PPIMAGE_M
+        CTEMASK_VERIFY          STR     PPIMAGE_M
+        CTEMASK_STACK           STR     PPMERGE_CTEMASK
+        CTEMASK_JPEG_IMAGE      STR     CTEMAP
+        CTEMASK_JPEG_RESID      STR     CTEMAP
         FRINGE_PROCESS          STR     PPIMAGE_OBDSF
         FRINGE_RESID            STR     PPIMAGE_R

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/rejections.config

-              r19379
+              r23594
 FLATMASK METADATA
+  FILTER             STR  *
+  EXPECTED           F32  0.0
+  IMFILE.MEAN        F32  0.0
+  IMFILE.STDEV       F32  0.0
+  IMFILE.MEANSTDEV   F32  0.0
+  IMFILE.SKEWNESS    F32  0.0
+  IMFILE.KURTOSIS    F32  0.0
+  IMFILE.SNR         F32  0.0
+  IMFILE.BIN.STDEV   F32  0.0
+  IMFILE.BIN.SNR     F32  0.0
+  IMFILE.FLUX        F32  0.0
+  EXP.MEAN           F32  0.0
+  EXP.STDEV          F32  0.0
+  EXP.MEANSTDEV      F32  0.0
+  EXP.SNR            F32  0.0
+  EXP.BIN.STDEV      F32  0.0
+  EXP.BIN.SNR        F32  0.0
+  EXP.FLUX           F32  0.0
+  ENSEMBLE.MEAN      F32  0.0
+  ENSEMBLE.STDEV     F32  0.0
+  ENSEMBLE.MEANSTDEV F32  0.0
+END
+CTEMASK METADATA
   FILTER             STR  *
   EXPECTED           F32  0.0

branches/cnb_branches/cnb_branch_20090301/ippconfig/system.config

r20387	r23594
36	36
37	37	RECIPES METADATA # Site-level recipes
38		MASKS STR recipes/masks.config # Mask values
	38	MASKS STR recipes/masks.16bit.config # Mask values
39	39	REJECTIONS STR recipes/rejections.config # Rejection for detrend creation
40	40	PPIMAGE STR recipes/ppImage.config # Image reduction

branches/cnb_branches/cnb_branch_20090301/operations/registration/input

-              r22426
+              r23594
     controller host add ipp007 -threads $1
 # ipp008 is flaky
+    controller host add ipp009 -threads $1
+# ipp009 is registration server
     controller host add ipp010 -threads $1
     controller host add ipp011 -threads $1
 …
     controller host add ipp007
 # ipp008 is flaky
+    controller host add ipp009
+# ipp009 is registration server
     controller host add ipp010
     controller host add ipp011

branches/cnb_branches/cnb_branch_20090301/operations/registration/ptolemy.rc

r22425	r23594
2	2	PANTASKS_SERVER_STDOUT pantasks.stdout.log
3	3	PANTASKS_SERVER_STDERR pantasks.stderr.log
4		PANTASKS_SERVER ipp008
	4	PANTASKS_SERVER ipp009
5	5	PASSWORD foobar
6	6

branches/cnb_branches/cnb_branch_20090301/ppImage

Property svn:mergeinfo changed
/branches/eam_branches/eam_branch_20090312/ppImage (added) merged: 23312,23316,23320,23406,23408
/trunk/ppImage merged: 23366,23411,23426,23534-23535

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/Makefile.am

r23352	r23594
21	21	ppMergeFileGroup.c \
22	22	ppMergeReadChunk.c \
23		ppMergeLoop~~_Threaded.c~~ \
	23	ppMergeLoop.c \
24	24	ppMergeSetThreads.c \
25	25	ppMergeMask.c \

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMerge.c

r23352	r23594
55	55	case PPMERGE_TYPE_FLAT:
56	56	case PPMERGE_TYPE_FRINGE:
	57	case PPMERGE_TYPE_CTEMASK:
57	58	if (!ppMergeScaleZero(config)) {
58	59	psErrorStackPrint(stderr, "Error getting scale and zero-points.");

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMerge.h

r23352	r23594
40	40	PPMERGE_TYPE_DARK, ///< (Multi-)Dark frame
41	41	PPMERGE_TYPE_MASK, ///< Mask frame
	42	PPMERGE_TYPE_CTEMASK, ///< CTE Mask based on flat variance
42	43	PPMERGE_TYPE_SHUTTER, ///< Shutter frame
43	44	PPMERGE_TYPE_FLAT, ///< Flat-field frame (dome or sky)

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeArguments.c

-              r21365
+              r23594
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-fringe-ysmooth", 0, "Number of smoothing regions in y", 0);
+    /** CTEMASK construction parameters */
+    psMetadataAddF32(arguments, PS_LIST_TAIL, "-cte-min", 0, "min allowed value for good CTE", NAN);
     /** Shutter construction parameters */
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-shutter-size", 0, "Size for shutter measurement regions", 0);
 …
       goto VALID;
+    }
+    if (strcasecmp(typeStr, "CTEMASK") == 0) {
+      type = PPMERGE_TYPE_CTEMASK;
+      goto VALID;
+    }
     if (strcasecmp(typeStr, "MASK") == 0 ||
         strcasecmp(typeStr, "DARKMASK") == 0 ||
 …
     VALUE_ARG_RECIPE_INT("-fringe-xsmooth", "FRINGE.XSMOOTH", S32, 0);
     VALUE_ARG_RECIPE_INT("-fringe-ysmooth", "FRINGE.YSMOOTH", S32, 0);
+    /** CTEMASK construction parameters */
+    VALUE_ARG_RECIPE_FLOAT("-cte-min",      "CTE.MIN",        F32);
     /** Shutter construction parameters */

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeCamera.c

-              r21365
+              r23594
         fileType = PM_FPA_FILE_MASK;
         break;
+      case PPMERGE_TYPE_CTEMASK:
+        fileType = PM_FPA_FILE_MASK;
+        break;
       case PPMERGE_TYPE_SHUTTER:
         fileType = PM_FPA_FILE_IMAGE;

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeFiles.c

-              r21365
+              r23594
         outSuffix = "MASK";
         break;
+      case PPMERGE_TYPE_CTEMASK:
+        outSuffix = "MASK";
+        break;
       case PPMERGE_TYPE_SHUTTER:
         outSuffix = "SHUTTER";

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeScaleZero.c

r23352	r23594
35	35	case PPMERGE_TYPE_BIAS:
36	36	case PPMERGE_TYPE_DARK:
	37	case PPMERGE_TYPE_CTEMASK:
37	38	// Nothing to measure
38	39	return true;

branches/cnb_branches/cnb_branch_20090301/ppSim/src/Makefile.am

r23352	r23594
45	45	ppSimRandomGaussian.c \
46	46	ppSimBadPixels.c \
	47	ppSimBadCTE.c \
47	48	ppSimVersion.c
48	49

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSim.h

-              r23352
+              r23594
 } ppSimGalaxy;
 ppSimStar *ppSimStarAlloc ();
 ppSimGalaxy *ppSimGalaxyAlloc ();
+ppSimStar *ppSimStarAlloc(void);
+ppSimGalaxy *ppSimGalaxyAlloc(void);
 /// Parse command-line arguments
 …
+// add a bad CTE region
+bool ppSimBadCTE(psImage *image,        // Signal image, modified and returned
+                 const pmConfig *config // configuration
+  );
 /// Add bad pixels to an image
 bool ppSimBadPixels(pmReadout *readout, ///< Readout for which to generate bad pixels
 …
 double ppSimRandomGaussian (const psRandom *rnd, double mean, double sigma);
 double ppSimRandomGaussianNorm (const psRandom *rnd);
 void ppSimRandomGaussianFree();
+void ppSimRandomGaussianFree(void);
 bool ppSimPhotomFiles (pmConfig *config, pmFPAfile *fakeFile, pmFPAfile *forceFile);

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimLoadSpots.c

r18047	r23594
57	57
58	58	// load refstars from the catalog
59		psArray *spots = psastroLoadRefstars(config);
	59	psArray *spots = psastroLoadRefstars(config, "PSASTRO.INPUT");
60	60	if (!spots \|\| spots->n == 0) {
61	61	psError(PS_ERR_UNKNOWN, false, "Unable to find reference stars.");

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimLoadStars.c

r20040	r23594
41	41	psMetadataAdd(astroRecipe, PS_LIST_TAIL, "DEC_MIN", PS_DATA_F32 \| PS_META_REPLACE, "", dec0 - radius);
42	42	psMetadataAdd(astroRecipe, PS_LIST_TAIL, "DEC_MAX", PS_DATA_F32 \| PS_META_REPLACE, "", dec0 + radius);
43		psArray *refStars = psastroLoadRefstars(config);
	43	psArray *refStars = psastroLoadRefstars(config, "PSASTRO.INPUT");
44	44	if (!refStars) {
45	45	psError(PS_ERR_UNKNOWN, false, "Unable to find reference stars.");

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimLoop.c

-              r23352
+              r23594
             done:
                 if (!ppSimAddNoise(readout->image, readout->variance, cell, config, rng)) ESCAPE (PS_ERR_UNKNOWN, "problem adding noise");
+                if (!ppSimBadCTE(readout->image, config)) ESCAPE (PS_ERR_UNKNOWN, "problem inducing bad cte");
                 if (!ppSimSaturate(readout, config)) ESCAPE (PS_ERR_UNKNOWN, "problem setting saturation levels");

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimRandomGaussian.c

-              r18011
+              r23594
 /* integrate a gaussian from -5 sigma to +5 sigma */
 void p_ppSimRandomGaussianInit () {
+void p_ppSimRandomGaussianInit (void) {
   int i;
   long A, B;
   double val, x, dx, dx1, dx2, dx3, df;
   double mean, sigma;
   /* no need to generate this if it already exists */
   if (gaussint) return;
 …
   for (B = 0; A == time(NULL); B++);
   srand48(B);
   Ngaussint = 0x1000;
   ppSimRandomGaussianAlloc (Ngaussint + 1);
 …
   mean = 0.0;
   sigma = 1.0;
   for (i = 0, x = -7.0; (i < Ngaussint) && (x < 7.0); x += dx)  {
     df = (3.0*p_ppSimGaussian(x    , mean, sigma) +
+    df = (3.0*p_ppSimGaussian(x    , mean, sigma) +
 .0*p_ppSimGaussian(x+dx1, mean, sigma) +
 .0*p_ppSimGaussian(x+dx2, mean, sigma) +
+.0*p_ppSimGaussian(x+dx2, mean, sigma) +
 .0*p_ppSimGaussian(x+dx3, mean, sigma)) * (dx1/8.0);
     val += df;
 …
+}
 // XXX we are using drand48() rather than the random var supplied by rnd
+// XXX we are using drand48() rather than the random var supplied by rnd
 double ppSimRandomGaussian (const psRandom *rnd, double mean, double sigma) {
   int i;
   double y;
   if (gaussint == NULL) {
       p_ppSimRandomGaussianInit ();
 …
   i = Ngaussint*y;
   y = gaussint[i]*sigma + mean;
   return (y);
+}
 // XXX we are using drand48() rather than the random var supplied by rnd
+// XXX we are using drand48() rather than the random var supplied by rnd
 double ppSimRandomGaussianNorm (const psRandom *rnd) {
   int i;
   double y;
   if (gaussint == NULL) {
       p_ppSimRandomGaussianInit ();
 …
   i = Ngaussint*y;
   y = gaussint[i];
   return (y);
+}

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimStars.c

-              r17557
+              r23594
+}
 ppSimStar *ppSimStarAlloc () {
+ppSimStar *ppSimStarAlloc(void) {
     ppSimStar *star = (ppSimStar *) psAlloc(sizeof(ppSimStar));
 …
+}
 ppSimGalaxy *ppSimGalaxyAlloc () {
+ppSimGalaxy *ppSimGalaxyAlloc(void) {
     ppSimGalaxy *galaxy = (ppSimGalaxy *) psAlloc(sizeof(ppSimGalaxy));

branches/cnb_branches/cnb_branch_20090301/ppStack

Property svn:mergeinfo changed
/trunk/ppStack merged: 23357,23360,23362,23364,23367-23368,23371-23373,23379,23462,23573,23575-23577

branches/cnb_branches/cnb_branch_20090301/ppStack/src/Makefile.am

r23352	r23594
29	29	ppStackPrepare.c \
30	30	ppStackConvolve.c \
	31	ppStackCombinePrepare.c \
31	32	ppStackCombineInitial.c \
32	33	ppStackReject.c \

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStack.h

-              r23352
+              r23594
 #include <pslib.h>
 #include <psmodules.h>
+#include "ppStackOptions.h"
 // Mask values for inputs
 …
 /// Convolve image to match specified seeing
 bool ppStackMatch(pmReadout *readout,   // Readout to be convolved; replaced with output
+                  psArray **regions,    // Array of regions used in each PSF matching, returned
+                  psArray **kernels,    // Array of kernels used in each PSF matching, returned
+                  float *chi2,          // Chi^2 from the stamps
+                  float *weighting,     // Stack weighting (1/noise^2)
+                  psArray *sources,     // Array of sources
+                  const pmPSF *psf,     // Target PSF
+                  psRandom *rng,        // Random number generator
+                  ppStackOptions *options, // Options for stacking
+                  int index,            // Index of image to match
                   const pmConfig *config // Configuration
     );
 …
 ///
 /// Corrects the source PSF photometry to a common system.  Return the sum of the exposure times.
+float ppStackSourcesTransparency(const psArray *sourceLists, // Sources for each input
+                                 psVector *inputMask, // Indicates bad input
+                                 const pmFPAview *view, // View to readout
+                                 const pmConfig *config // Configuration
+bool ppStackSourcesTransparency(ppStackOptions *options, // Stacking options
+                                const pmFPAview *view, // View to readout
+                                const pmConfig *config // Configuration
     );

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackArguments.c

-              r23352
+              r23594
     fprintf(stderr, "\nPan-STARRS Image combination\n\n");
     fprintf(stderr,
             "Usage: %s INPUTS.mdc OUTPUT_ROOT [-sources STAMPS.cmf | -stamps STAMPS.dat]\n"
+            "Usage: %s -input INPUTS.mdc OUTPUT_ROOT [-sources STAMPS.cmf | -stamps STAMPS.dat]\n"
             "where INPUTS.mdc contains various METADATAs, each with:\n"
             "\tIMAGE(STR):     Image filename\n"
 …
             "\tVARIANCE(STR):  Variance map filename\n"
             "\tPSF(STR):       PSF filename\n"
+            "\tSOURCES(STR):   Sources filename\n"
+            "\tWEIGHTING(F32): Relative weighting to be applied\n",
+            "\tSOURCES(STR):   Sources filename\n",
             program);
     fprintf(stderr, "\n");
 …
         psArgumentRemove(argNum, &argc, argv);
+    }
     psMetadata *arguments = config->arguments; // Command-line arguments
 …
     psMetadataAddBool(arguments, PS_LIST_TAIL, "-visual", 0, "visualisation", false);
     if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 3) {
+    if (argc == 1) {
         usage(argv[0], arguments, config);
+    }
+    if ((argNum = psArgumentGet(argc, argv, "-input"))) {
+        psArgumentRemove(argNum, &argc, argv);
+        if (argNum >= argc) {
+            usage(argv[0], arguments, config);
+        }
+        unsigned int numBad = 0;                     // Number of bad lines
+        psMetadata *inputs = psMetadataConfigRead(NULL, &numBad, argv[argNum], false); // Input file info
+        if (!inputs || numBad > 0) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to cleanly read MDC file with inputs.");
+            return false;
+        }
+        psMetadataAddMetadata(arguments, PS_LIST_TAIL, "INPUTS", 0, "Metadata with input details", inputs);
+        psFree(inputs);
+        psArgumentRemove(argNum, &argc, argv);
+    }
+    if (!psArgumentParse(arguments, &argc, argv) || argc != 2) {
+        usage(argv[0], arguments, config);
+    }
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "OUTPUT", 0, "Root name of the output image list", argv[1]);
     const char *stampsName = psMetadataLookupStr(NULL, arguments, "-stamps"); // Name of stamps file
     psMetadataAddStr(arguments, PS_LIST_TAIL, "STAMPS", 0, "Stamps file", stampsName);
-    unsigned int numBad = 0;                     // Number of bad lines
-    psMetadata *inputs = psMetadataConfigRead(NULL, &numBad, argv[1], false); // Information about inputs
-    if (!inputs || numBad > 0) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to cleanly read MDC file with inputs.");
-        return false;
+    }
-    psMetadataAddMetadata(arguments, PS_LIST_TAIL, "INPUTS", 0, "Metadata with input details", inputs);
-    psFree(inputs);
-    psMetadataAddStr(arguments, PS_LIST_TAIL, "OUTPUT", 0, "Root name of the output image list", argv[2]);
     valueArgStr(arguments, "-stats", "STATS", arguments);
 …
     psTrace("ppStack", 1, "Done parsing arguments\n");
+    // Dump configuration, now that's it's settled
+    bool status;
+    psString dump_file =  psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
+    if (dump_file) {
+        pmConfigCamerasCull(config, NULL);
+        pmConfigRecipesCull(config, "PPSTACK,PPSUB,PPSTATS,PSPHOT,MASKS,JPEG");
+        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSTACK.INPUT"); // Input file
+        pmConfigDump(config, input->fpa, dump_file);
+    }
+    pmConfigCamerasCull(config, NULL);
+    pmConfigRecipesCull(config, "PPSTACK,PPSUB,PPSTATS,PSPHOT,MASKS,JPEG");
     return true;

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackCamera.c

-              r21366
+              r23594
 #include "ppStack.h"
+#if 0
+// Define an output convolved image file
+static pmFPAfile *defineOutputConvolved(const char *name, // FPA file name
+                                        pmFPA *fpa, // FPA to bind
+                                        const pmConfig *config, // Configuration
+                                        pmFPAfileType type // Expected type
+    )
+// Define a file
+static pmFPAfile *defineFile(pmConfig *config, // Configuration
+                             pmFPAfile *bind, // File to which to bind
+                             const char *name, // Name of file rule
+                             const char *filename, // Name of file
+                             pmFPAfileType type // Type of file
+                             )
+{
+    pmFPAfile *file = pmFPAfileDefineOutput(config, fpa, name);
+    if (!file) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to define output convolved file %s", name);
+    psArray *dummy = psArrayAlloc(1);   // Dummy array of filenames for this FPA
+    dummy->data[0] = psStringCopy(filename);
+    psMetadataAddArray(config->arguments, PS_LIST_TAIL, "FILENAMES", PS_META_REPLACE,
+                       "Filenames for file rule definition", dummy);
+    psFree(dummy);
+    bool found = false;             // Found the file?
+    pmFPAfile *file = bind ? pmFPAfileBindFromArgs(&found, bind, config, name, "FILENAMES") :
+        pmFPAfileDefineFromArgs(&found, config, name, "FILENAMES");
+    if (!file || !found) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to define file %s from %s", name, filename);
         return NULL;
+    }
     if (file->type != PM_FPA_FILE_IMAGE) {
         psError(PS_ERR_IO, true, "PPSTACK.OUTCONV is not of type %s", pmFPAfileStringFromType(type));
+    if (file->type != type) {
+        psError(PS_ERR_IO, true, "%s is not of type %s", name, pmFPAfileStringFromType(type));
         return NULL;
+    }
 …
+}
-// Define an input convolved image file, using the output as a basis
-static pmFPAfile *defineInputConvolved(const char *inputName, // Input FPA file name
-                                       pmFPAfile *outFile, // Corresponding output FPA file
-                                       pmConfig *config, // Configuration
-                                       pmFPAfileType type // Expected type
+    )
+{
-    pmFPAview *view = pmFPAviewAlloc(0); // View into sky cells
-    view->chip = view->cell = view->readout = 0;
-    psString imageName = pmFPANameFromRule(outFile->filerule, outFile->fpa, view);
-    psArray *imageNames = psArrayAlloc(1);
-    imageNames->data[0] = imageName;
-    psMetadataAddArray(config->arguments, PS_LIST_TAIL, "INCONV.FILENAMES", PS_META_REPLACE,
-                       "Filenames of input convolved image files", imageNames);
-    psFree(imageNames);
-    bool found = false;                 // Found the file?
-    pmFPAfile *imageFile = pmFPAfileDefineFromArgs(&found, config, "PPSTACK.INCONV",
-                                                   "INCONV.FILENAMES");
-    psMetadataRemoveKey(config->arguments, "INCONV.FILENAMES");
-    if (!imageFile || !found) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to define %s file", inputName);
-        return NULL;
+    }
-    if (imageFile->type != type) {
-        psError(PS_ERR_IO, true, "PPSTACK.INCONV is not of type %s",
-                pmFPAfileStringFromType(type));
-        return NULL;
+    }
-    return imageFile;
+}
-#endif
 bool ppStackCamera(pmConfig *config)
+{
+    bool haveVariances = false;           // Do we have variance maps?
+    int num = 0;                        // Number of inputs
+    bool haveVariances = false;         // Do we have variance maps?
     bool havePSFs = false;              // Do we have PSFs?
+    psMetadata *inputs = psMetadataLookupMetadata(NULL, config->arguments, "INPUTS"); // The inputs info
+    psMetadataIterator *iter = psMetadataIteratorAlloc(inputs, PS_LIST_HEAD, NULL); // Iterator
+    psMetadataItem *item;               // Item from iteration
+    int i = 0;                          // Counter
+    while ((item = psMetadataGetAndIncrement(iter))) {
+        if (item->type != PS_DATA_METADATA) {
+            psError(PS_ERR_BAD_PARAMETER_TYPE, true,
+                    "Component %s of the input metadata is not of type METADATA", item->name);
+            psFree(iter);
+            return false;
+        }
+        psMetadata *input = item->data.md; // The input metadata of interest
+        psString image = psMetadataLookupStr(NULL, input, "IMAGE"); // Name of image
+        if (!image || strlen(image) == 0) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Component %s lacks IMAGE of type STR", item->name);
+            psFree(iter);
+            return false;
+        }
+        bool mdok;
+        psString mask = psMetadataLookupStr(&mdok, input, "MASK"); // Name of mask
+        psString variance = psMetadataLookupStr(&mdok, input, "VARIANCE"); // Name of variance map
+        psString psf = psMetadataLookupStr(&mdok, input, "PSF"); // Name of PSF
+        psString sources = psMetadataLookupStr(&mdok, input, "SOURCES"); // Name of sources
+        // Add the image file
+        psArray *imageFiles = psArrayAlloc(1); // Array of filenames for this FPA
+        imageFiles->data[0] = psMemIncrRefCounter(image);
+        psMetadataAddArray(config->arguments, PS_LIST_TAIL, "IMAGE.FILENAMES", PS_META_REPLACE,
+                           "Filenames of image files", imageFiles);
+        psFree(imageFiles);
+        bool found = false;             // Found the file?
+        pmFPAfile *imageFile = pmFPAfileDefineFromArgs(&found, config, "PPSTACK.INPUT", "IMAGE.FILENAMES");
+        if (!imageFile || !found) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define file from image %d (%s)", i, image);
+            return false;
+        }
+        if (imageFile->type != PM_FPA_FILE_IMAGE) {
+            psError(PS_ERR_IO, true, "PPSTACK.INPUT is not of type IMAGE");
+            return false;
+        }
+        // Optionally add the mask file
+        if (mask && strlen(mask) > 0) {
+            psArray *maskFiles = psArrayAlloc(1); // Array of filenames for this FPA
+            maskFiles->data[0] = psMemIncrRefCounter(mask);
+            psMetadataAddArray(config->arguments, PS_LIST_TAIL, "MASK.FILENAMES", PS_META_REPLACE,
+                               "Filenames of mask files", maskFiles);
+            psFree(maskFiles);
+            bool status;
+            pmFPAfile *maskFile = pmFPAfileBindFromArgs(&status, imageFile, config, "PPSTACK.INPUT.MASK",
+                                                        "MASK.FILENAMES");
+    bool status = false;                // Status of file definition
+    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // Recipe for ppSim
+    if (!recipe) {
+        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSTACK_RECIPE);
+        return false;
+    }
+    bool convolve = psMetadataLookupBool(NULL, recipe, "CONVOLVE"); // Convolve images before stack?
+    psArray *runImages = pmFPAfileDefineMultipleFromRun(&status, NULL, config,
+                                                        "PPSTACK.INPUT"); // Input images from previous run
+    if (runImages) {
+        // Defining files from the RUN metadata
+        num = runImages->n;
+        psArray *runMasks = pmFPAfileDefineMultipleFromRun(&status, runImages, config,
+                                                           "PPSTACK.INPUT.MASK"); // Input masks
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define input masks from RUN metadata.");
+            psFree(runImages);
+            return false;
+        }
+        psFree(runMasks);
+        psArray *runVars = pmFPAfileDefineMultipleFromRun(&status, runImages, config,
+                                                          "PPSTACK.INPUT.VARIANCE"); // Input variances
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define input variances from RUN metadata.");
+            psFree(runImages);
+            return false;
+        }
+        if (runVars) {
+            haveVariances = true;
+        }
+        psFree(runVars);
+        psArray *runPSF = pmFPAfileDefineMultipleFromRun(&status, runImages, config,
+                                                         "PPSTACK.INPUT.PSF"); // Input PSFs
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define input PSFs from RUN metadata.");
+            psFree(runImages);
+            return false;
+        }
+        if (runPSF) {
+            havePSFs = true;
+        }
+        psFree(runPSF);
+        psArray *runSrc = pmFPAfileDefineMultipleFromRun(&status, runImages, config,
+                                                         "PPSTACK.INPUT.SOURCES"); // Input sources
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define input sources from RUN metadata.");
+            psFree(runImages);
+            return false;
+        }
+        if (!runSrc) {
+            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to define input sources from RUN metadata.");
+            psFree(runImages);
+            return false;
+        }
+        psFree(runSrc);
+        if (convolve) {
+            psArray *runKernel = pmFPAfileDefineMultipleFromRun(&status, runImages, config,
+                                                                "PPSTACK.CONV.KERNEL"); // Convolution kernels
             if (!status) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to define convolution kernels from RUN metadata.");
+                psFree(runImages);
+                return false;
+            }
+            if (!runKernel) {
+                psError(PS_ERR_UNEXPECTED_NULL, true,
+                        "Unable to define convolution kernels from RUN metadata.");
+                psFree(runImages);
+                return false;
+            }
+            psFree(runKernel);
+        }
+        psFree(runImages);
+    } else {
+        // Defining files from the input metadata
+        psMetadata *inputs = psMetadataLookupMetadata(NULL, config->arguments, "INPUTS"); // The inputs info
+        if (!inputs) {
+            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find inputs.");
+            return false;
+        }
+        psMetadataIterator *iter = psMetadataIteratorAlloc(inputs, PS_LIST_HEAD, NULL); // Iterator
+        psMetadataItem *item;               // Item from iteration
+        int i = 0;                          // Counter
+        while ((item = psMetadataGetAndIncrement(iter))) {
+            if (item->type != PS_DATA_METADATA) {
+                psError(PS_ERR_BAD_PARAMETER_TYPE, true,
+                        "Component %s of the input metadata is not of type METADATA", item->name);
+                psFree(iter);
+                return false;
+            }
+            psMetadata *input = item->data.md; // The input metadata of interest
+            psString image = psMetadataLookupStr(NULL, input, "IMAGE"); // Name of image
+            if (!image || strlen(image) == 0) {
+                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Component %s lacks IMAGE of type STR", item->name);
+                psFree(iter);
+                return false;
+            }
+            bool mdok;
+            psString mask = psMetadataLookupStr(&mdok, input, "MASK"); // Name of mask
+            psString variance = psMetadataLookupStr(&mdok, input, "VARIANCE"); // Name of variance map
+            psString psf = psMetadataLookupStr(&mdok, input, "PSF"); // Name of PSF
+            psString sources = psMetadataLookupStr(&mdok, input, "SOURCES"); // Name of sources
+            pmFPAfile *imageFile = defineFile(config, NULL, "PPSTACK.INPUT",
+                                              image, PM_FPA_FILE_IMAGE); // File for image
+            if (!imageFile) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to define file from image %d (%s)", i, image);
+                return false;
+            }
+            if (mask && strlen(mask) > 0 &&
+                !defineFile(config, imageFile, "PPSTACK.INPUT.MASK", mask, PM_FPA_FILE_MASK)) {
                 psError(PS_ERR_UNKNOWN, false, "Unable to define file from mask %d (%s)", i, mask);
                 return false;
+            }
+            if (maskFile->type != PM_FPA_FILE_MASK) {
+                psError(PS_ERR_IO, true, "PPSTACK.INPUT.MASK is not of type MASK");
+                return false;
+            }
+        }
+        // Optionally add the variance file
+        if (variance && strlen(variance) > 0) {
+            haveVariances = true;
+            psArray *varianceFiles = psArrayAlloc(1); // Array of filenames for this FPA
+            varianceFiles->data[0] = psMemIncrRefCounter(variance);
+            psMetadataAddArray(config->arguments, PS_LIST_TAIL, "VARIANCE.FILENAMES", PS_META_REPLACE,
+                               "Filenames of variance files", varianceFiles);
+            psFree(varianceFiles);
+            bool status;
+            pmFPAfile *varianceFile = pmFPAfileBindFromArgs(&status, imageFile, config,
+                                                            "PPSTACK.INPUT.VARIANCE", "VARIANCE.FILENAMES");
+            if (!status) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to define file from variance %d (%s)", i, variance);
+                return false;
+            }
+            if (varianceFile->type != PM_FPA_FILE_VARIANCE) {
+                psError(PS_ERR_IO, true, "PPSTACK.INPUT.VARIANCE is not of type VARIANCE");
+                return false;
+            }
+        }
+        // Add the psf file
+        if (!psf || strlen(psf) == 0) {
+            if (havePSFs) {
+                psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find PSF %d", i);
+                return false;
+            }
+        } else {
+            if (!havePSFs && i != 0) {
+                psWarning("PSF not provided for all inputs --- ignoring.");
+            } else {
+                psArray *psfFiles = psArrayAlloc(1); // Array of filenames for this FPA
+                psfFiles->data[0] = psMemIncrRefCounter(psf);
+                psMetadataAddArray(config->arguments, PS_LIST_TAIL, "PSF.FILENAMES", PS_META_REPLACE,
+                                   "Filenames of PSF files", psfFiles);
+                psFree(psfFiles);
+                bool status;
+                pmFPAfile *psfFile = pmFPAfileBindFromArgs(&status, imageFile, config, "PPSTACK.INPUT.PSF",
+                                                           "PSF.FILENAMES");
+                if (!status) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to define file from psf %d (%s)", i, psf);
+            if (variance && strlen(variance) > 0) {
+                haveVariances = true;
+                if (!defineFile(config, imageFile, "PPSTACK.INPUT.VARIANCE", variance,
+                                PM_FPA_FILE_VARIANCE)) {
+                    psError(PS_ERR_UNKNOWN, false,
+                            "Unable to define file from variance %d (%s)", i, variance);
                     return false;
+                }
+                if (psfFile->type != PM_FPA_FILE_PSF) {
+                    psError(PS_ERR_IO, true, "PPSTACK.INPUT.PSF is not of type PSF");
+            }
+            if (psf && strlen(psf) > 0) {
+                if (i != 0 && !havePSFs) {
+                    psWarning("PSF not provided for all inputs --- ignoring.");
+                } else {
+                    havePSFs = true;
+                    if (!defineFile(config, imageFile, "PPSTACK.INPUT.PSF", psf, PM_FPA_FILE_PSF)) {
+                        psError(PS_ERR_UNKNOWN, false, "Unable to define file from psf %d (%s)", i, psf);
+                        return false;
+                    }
+                }
+            } else if (havePSFs) {
+                psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find PSF %d", i);
+                return false;
+            }
+            if (!sources || strlen(sources) == 0) {
+                psError(PS_ERR_UNEXPECTED_NULL, true, "SOURCES not provided for file %d", i);
+                return false;
+            }
+            if (!defineFile(config, imageFile, "PPSTACK.INPUT.SOURCES", sources, PM_FPA_FILE_CMF)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to define file from sources %d (%s)",
+                        i, sources);
+                return false;
+            }
+            if (convolve) {
+                pmFPAfile *kernel = pmFPAfileDefineOutput(config, imageFile->fpa, "PPSTACK.CONV.KERNEL");
+                if (!kernel) {
+                    psError(PS_ERR_IO, false, _("Unable to generate output file from PPSTACK.CONV.KERNEL"));
                     return false;
+                }
+                havePSFs = true;
+            }
+        }
+        // Add the sources file
+        {
+            psArray *sourcesFiles = psArrayAlloc(1); // Array of filenames for this FPA
+            sourcesFiles->data[0] = psMemIncrRefCounter(sources);
+            psMetadataAddArray(config->arguments, PS_LIST_TAIL, "SOURCES.FILENAMES", PS_META_REPLACE,
+                               "Filenames of SOURCES files", sourcesFiles);
+            psFree(sourcesFiles);
+            bool status;
+            pmFPAfile *sourcesFile = pmFPAfileBindFromArgs(&status, imageFile, config,
+                                                           "PPSTACK.INPUT.SOURCES", "SOURCES.FILENAMES");
+            if (!status) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to define file from sources %d (%s)",
+                        i, sources);
+                return false;
+            }
+            if (sourcesFile->type != PM_FPA_FILE_CMF) {
+                psError(PS_ERR_IO, true, "PPSTACK.INPUT.SOURCES is not of type CMF");
+                return false;
+            }
+        }
+#if 0
+        // Output convolved files
+        pmFPAfile *outconvImage    = defineOutputConvolved("PPSTACK.OUTCONV", imageFile->fpa, config,
+                                                           PM_FPA_FILE_IMAGE);
+        pmFPAfile *outconvMask     = defineOutputConvolved("PPSTACK.OUTCONV.MASK", imageFile->fpa, config,
+                                                           PM_FPA_FILE_MASK);
+        pmFPAfile *outconvVariance = defineOutputConvolved("PPSTACK.OUTCONV.VARIANCE", imageFile->fpa, config,
+                                                           PM_FPA_FILE_VARIANCE);
+        if (!outconvImage || !outconvMask || !outconvVariance) {
+            return false;
+        }
+        // Input convolved files
+        pmFPAfile *inconvImage    = defineInputConvolved("PPSTACK.INCONV", outconvImage, config,
+                                                         PM_FPA_FILE_IMAGE);
+        pmFPAfile *inconvMask     = defineInputConvolved("PPSTACK.INCONV.MASK", outconvMask, config,
+                                                         PM_FPA_FILE_MASK);
+        pmFPAfile *inconvVariance = defineInputConvolved("PPSTACK.INCONV.VARIANCE", outconvVariance, config,
+                                                         PM_FPA_FILE_VARIANCE);
+        if (!inconvImage || !inconvMask || !inconvVariance) {
+            return false;
+        }
+#endif
+        i++;
+    }
+    psFree(iter);
+    psMetadataRemoveKey(config->arguments, "IMAGE.FILENAMES");
+    if (psMetadataLookup(config->arguments, "MASK.FILENAMES")) {
+        psMetadataRemoveKey(config->arguments, "MASK.FILENAMES");
+    }
+    if (psMetadataLookup(config->arguments, "VARIANCE.FILENAMES")) {
+        psMetadataRemoveKey(config->arguments, "VARIANCE.FILENAMES");
+    }
+    if (psMetadataLookup(config->arguments, "PSF.FILENAMES")) {
+        psMetadataRemoveKey(config->arguments, "PSF.FILENAMES");
+    }
+    if (psMetadataLookup(config->arguments, "SOURCES.FILENAMES")) {
+        psMetadataRemoveKey(config->arguments, "SOURCES.FILENAMES");
+    }
+    psMetadataAddS32(config->arguments, PS_LIST_TAIL, "INPUTS.NUM", 0, "Number of input files", i);
+                kernel->save = true;
+            }
+            i++;
+        }
+        psFree(iter);
+        psMetadataRemoveKey(config->arguments, "FILENAMES");
+        num = i;
+    }
+    psMetadataAddS32(config->arguments, PS_LIST_TAIL, "INPUTS.NUM", 0, "Number of input files", num);
     psMetadataAddBool(config->arguments, PS_LIST_TAIL, "HAVE.PSF", 0, "Have PSFs available?", havePSFs);
 …
+    }
     jpeg2->save = true;
-    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // Recipe for ppSim
-    if (!recipe) {
-        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSTACK_RECIPE);
-        return false;
+    }
     // For photometry, we operate on the chip-mosaicked image

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackCombineInitial.c

-              r23352
+              r23594
     psAssert(config, "Require configuration");
+    psTimerStart("PPSTACK_FINAL");
+    if (!options->convolve) {
+        // No need to do initial combination when we haven't convolved
+        return true;
+    }
+    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe
+    psAssert(recipe, "We've thrown an error on this before.");
+    psTimerStart("PPSTACK_INITIAL");
     psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe
 …
                                           "KERNEL.SIZE"); // Overlap by kernel size between consecutive scans
-    pmFPAview *view = NULL;             // View to readout
-    int row0, col0;                 // Offset for readout
-    int numCols, numRows;           // Size of readout
-    ppStackThread *thread = stack->threads->data[0]; // Representative thread
-    if (!pmReadoutStackSetOutputSize(&col0, &row0, &numCols, &numRows, thread->readouts)) {
-        psError(PS_ERR_UNKNOWN, false, "problem setting output readout size.");
-        return false;
+    }
-    pmFPAfileActivate(config->files, false, NULL);
-    ppStackFileActivation(config, PPSTACK_FILES_COMBINE, true);
-    view = ppStackFilesIterateDown(config);
-    if (!view) {
-        return false;
+    }
-    pmCell *outCell = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT"); // Output cell
-    options->outRO = pmReadoutAlloc(outCell); // Output readout
-    psFree(view);
-    psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits to mask for bad
-    psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels
-    if (!pmReadoutStackDefineOutput(options->outRO, col0, row0, numCols, numRows, true, true, maskBad)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to prepare output.");
-        return false;
+    }
-    psFree(options->cells); options->cells = NULL;
     bool status;                    // Status of read
 …
+        }
-        // call: ppStackReadoutInitial(config, outRO, readouts, subRegions, subKernels)
         psThreadJob *job = psThreadJobAlloc("PPSTACK_INITIAL_COMBINE"); // Job to start
         psArrayAdd(job->args, 1, thread);
 …
     if (options->stats) {
         psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_FINAL", 0,
                          "Time to make final stack", psTimerMark("PPSTACK_FINAL"));
+        psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_INITIAL", 0,
+                         "Time to make final stack", psTimerMark("PPSTACK_INITIAL"));
+    }

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackConvolve.c

-              r23352
+              r23594
     int num = options->num;             // Number of inputs
     options->cells = psArrayAlloc(num); // Cells for convolved images --- a handle for reading again
     options->subKernels = psArrayAlloc(num); // Subtraction kernels --- required in the stacking
     options->subRegions = psArrayAlloc(num); // Subtraction regions --- required in the stacking
+    options->kernels = psArrayAlloc(num); // PSF-matching kernels --- required in the stacking
+    options->regions = psArrayAlloc(num); // PSF-matching regions --- required in the stacking
     int numGood = 0;                    // Number of good frames
     options->numCols = 0;
 …
     psVectorInit(options->matchChi2, NAN);
     options->weightings = psVectorAlloc(num, PS_TYPE_F32); // Combination weightings for images (1/noise^2)
     psVectorInit(options->weightings, NAN);
+    psVectorInit(options->weightings, 0.0);
     options->covariances = psArrayAlloc(num); // Covariance matrices
 …
         // Background subtraction, scaling and normalisation is performed automatically by the image matching
-        psArray *regions = NULL, *kernels = NULL; // Regions and kernels used in subtraction
         psTimerStart("PPSTACK_MATCH");
+        if (!ppStackMatch(readout, &regions, &kernels, &options->matchChi2->data.F32[i],
+                          &options->weightings->data.F32[i], options->sourceLists->data[i],
+                          options->psf, rng, config)) {
+        if (!ppStackMatch(readout, options, i, config)) {
             psErrorStackPrint(stderr, "Unable to match image %d --- ignoring.", i);
             options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_MATCH;
+            options->inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PPSTACK_MASK_MATCH;
             psErrorClear();
             continue;
 …
         if (options->stats) {
-            pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, readout->analysis,
-                                                                PM_SUBTRACTION_ANALYSIS_KERNEL);// Conv kernel
             psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_MATCH", PS_META_DUPLICATE_OK,
                              "Time to match PSF", psTimerMark("PPSTACK_MATCH"));
-            psMetadataAddF32(options->stats, PS_LIST_TAIL, "STAMP.MEAN", PS_META_DUPLICATE_OK,
-                             "Mean deviation for stamps", kernels->mean);
-            psMetadataAddF32(options->stats, PS_LIST_TAIL, "STAMP.RMS", PS_META_DUPLICATE_OK,
-                             "RMS deviation for stamps", kernels->rms);
-            psMetadataAddF32(options->stats, PS_LIST_TAIL, "STAMP.NUM", PS_META_DUPLICATE_OK,
-                             "Number of stamps", kernels->numStamps);
-            float deconv = psMetadataLookupF32(NULL, readout->analysis,
-                                               PM_SUBTRACTION_ANALYSIS_DECONV_MAX); // Deconvolution fraction
-            psMetadataAddF32(options->stats, PS_LIST_TAIL, "KERNEL.DECONV", PS_META_DUPLICATE_OK,
-                             "Deconvolution fraction for kernel", deconv);
             psMetadataAddF32(options->stats, PS_LIST_TAIL, "PPSTACK.WEIGHTING", PS_META_DUPLICATE_OK,
                              "Weighting for image", options->weightings->data.F32[i]);
+            if (options->convolve) {
+                // Pull parameters out of convolution kernel
+                pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, readout->analysis,
+                                                                    PM_SUBTRACTION_ANALYSIS_KERNEL);
+                psMetadataAddF32(options->stats, PS_LIST_TAIL, "STAMP.MEAN", PS_META_DUPLICATE_OK,
+                                 "Mean deviation for stamps", kernels->mean);
+                psMetadataAddF32(options->stats, PS_LIST_TAIL, "STAMP.RMS", PS_META_DUPLICATE_OK,
+                                 "RMS deviation for stamps", kernels->rms);
+                psMetadataAddF32(options->stats, PS_LIST_TAIL, "STAMP.NUM", PS_META_DUPLICATE_OK,
+                                 "Number of stamps", kernels->numStamps);
+                float deconv = psMetadataLookupF32(NULL, readout->analysis,
+                                                   PM_SUBTRACTION_ANALYSIS_DECONV_MAX);
+                psMetadataAddF32(options->stats, PS_LIST_TAIL, "KERNEL.DECONV", PS_META_DUPLICATE_OK,
+                                 "Deconvolution fraction for kernel", deconv);
+            }
+        }
         psLogMsg("ppStack", PS_LOG_INFO, "Time to match image %d: %f sec", i, psTimerClear("PPSTACK_MATCH"));
-        options->subRegions->data[i] = regions;
-        options->subKernels->data[i] = kernels;
         // Write the temporary convolved files
+        pmHDU *hdu = readout->parent->parent->parent->hdu; // HDU for convolved image
+        assert(hdu);
+        ppStackWriteImage(options->imageNames->data[i], hdu->header, readout->image, config);
+        psMetadata *maskHeader = psMetadataCopy(NULL, hdu->header); // Copy of header, for mask
+        pmConfigMaskWriteHeader(config, maskHeader);
+        ppStackWriteImage(options->maskNames->data[i], maskHeader, readout->mask, config);
+        psFree(maskHeader);
+        psImageCovarianceTransfer(readout->variance, readout->covariance);
+        ppStackWriteImage(options->varianceNames->data[i], hdu->header, readout->variance, config);
+        if (options->convolve) {
+            pmHDU *hdu = readout->parent->parent->parent->hdu; // HDU for convolved image
+            assert(hdu);
+            ppStackWriteImage(options->imageNames->data[i], hdu->header, readout->image, config);
+            psMetadata *maskHeader = psMetadataCopy(NULL, hdu->header); // Copy of header, for mask
+            pmConfigMaskWriteHeader(config, maskHeader);
+            ppStackWriteImage(options->maskNames->data[i], maskHeader, readout->mask, config);
+            psFree(maskHeader);
+            psImageCovarianceTransfer(readout->variance, readout->covariance);
+            ppStackWriteImage(options->varianceNames->data[i], hdu->header, readout->variance, config);
 #ifdef TESTING
+        {
             psString name = NULL;
             psStringAppend(&name, "covariance_%d.fits", i);
             ppStackWriteImage(name, hdu->header, readout->covariance->image, config);
             pmStackVisualPlotTestImage(readout->covariance->image, name);
             psFree(name);
+        }
+            {
+                psString name = NULL;
+                psStringAppend(&name, "covariance_%d.fits", i);
+                ppStackWriteImage(name, hdu->header, readout->covariance->image, config);
+                pmStackVisualPlotTestImage(readout->covariance->image, name);
+                psFree(name);
+            }
 #endif
+        }
         pmCell *inCell = readout->parent; // Input cell
 …
     psFree(rng);
+    psFree(options->norm); options->norm = NULL;
     psFree(options->sourceLists); options->sourceLists = NULL;
     psFree(options->psf); options->psf = NULL;
 …
     // Reject images out-of-hand on the basis of their match chi^2
+    {
+    if (options->convolve) {
         psVector *values = psVectorAllocEmpty(num, PS_TYPE_F32); // Values to sort
         for (int i = 0; i < num; i++) {

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackFiles.c

-              r23352
+              r23594
 /// Files required for the convolution
+static char *filesConvolve[] = { "PPSTACK.INPUT", "PPSTACK.INPUT.MASK", "PPSTACK.INPUT.VARIANCE", NULL };
+static char *filesConvolve[] = { "PPSTACK.INPUT", "PPSTACK.INPUT.MASK", "PPSTACK.INPUT.VARIANCE",
+                                 "PPSTACK.CONV.KERNEL", NULL };
 /// Output files for the combination

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackFinish.c

-              r23352
+              r23594
+    }
+    // Dump configuration
+    bool mdok;                          // Status of MD lookup
+    psString dump = psMetadataLookupStr(&mdok, config->arguments, "DUMP_CONFIG"); // File for config
+    if (dump) {
+        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSTACK.INPUT"); // Input file
+        pmConfigDump(config, input->fpa, dump);
+    }
     return true;
+}

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackLoop.c

-              r23352
+              r23594
         return false;
+    }
+    psFree(options->cells); options->cells = NULL;
+    // Prepare for combination
+    if (!ppStackCombinePrepare(stack, options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to prepare for combination.");
+        psFree(stack);
+        psFree(options);
+        return false;
+    }
     // Initial combination
 …
     // Clean up
     psTrace("ppStack", 2, "Cleaning up after combination....\n");
     if (!ppStackCombineFinal(stack, options, config)) {
+    if (!ppStackCleanup(stack, options, config)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to clean up.");
         psFree(stack);

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackLoop.h

-              r23352
+              r23594
     );
+// Prepare for combination
+bool ppStackCombinePrepare(
+    ppStackThreadData *stack,           // Stack
+    ppStackOptions *options,            // Options
+    pmConfig *config                    // Configuration
+    );
 // Initial combination
 bool ppStackCombineInitial(
 …
     );
+// Cleanup following combination
+bool ppStackCleanup(
+    ppStackThreadData *stack,           // Stack
+    ppStackOptions *options,            // Options
+    pmConfig *config                    // Configuration
+    );
 // Photometry
 bool ppStackPhotometry(

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackMatch.c

-              r23352
+              r23594
+bool ppStackMatch(pmReadout *readout, psArray **regions, psArray **kernels, float *chi2, float *weighting,
+                  psArray *sources, const pmPSF *psf, psRandom *rng, const pmConfig *config)
+bool ppStackMatch(pmReadout *readout, ppStackOptions *options, int index, const pmConfig *config)
+{
     assert(readout);
+    assert(regions && !*regions);
+    assert(kernels && !*kernels);
+    assert(options);
     assert(config);
-    *weighting = 0.0;
     psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe
 …
+    }
+    pmReadout *output = pmReadoutAlloc(NULL); // Output readout, for holding results temporarily
+    static int numInput = -1;            // Index of input file
+    numInput++;
+    // Match the PSF
+    if (options->convolve) {
+        pmReadout *conv = pmReadoutAlloc(NULL); // Conv readout, for holding results temporarily
 #ifdef TESTING
+    // Read previously produced kernel
+    if (psMetadataLookupBool(NULL, config->arguments, "PPSTACK.DEBUG.STACK")) {
+        const char *outName = psMetadataLookupStr(NULL, config->arguments, "OUTPUT"); // Output root
+        assert(outName);
+        // Read convolution kernel
+        psString filename = NULL;   // Output filename
+        psStringAppend(&filename, "%s.%d.kernel", outName, numInput);
+        psString resolved = pmConfigConvertFilename(filename, config, false, false); // Resolved filename
+        psFree(filename);
+        psFits *fits = psFitsOpen(resolved, "r"); // FITS file for subtraction kernel
+        psFree(resolved);
+        if (!fits || !pmReadoutReadSubtractionKernels(output, fits)) {
+            psError(PS_ERR_IO, false, "Unable to read previously produced kernel");
+        // This is a hack to use the temporary convolved images and kernel generated previously.
+        // This makes the 'matching' operation much faster, allowing debugging of the stack process easier.
+        // It implicitly assumes the output root name is the same between invocations.
+        // Read previously produced kernel
+        if (psMetadataLookupBool(NULL, config->arguments, "PPSTACK.DEBUG.STACK")) {
+            pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.CONV.KERNEL", index);
+            psAssert(file, "Require file");
+            pmFPAview *view = pmFPAviewAlloc(0); // View to readout of interest
+            view->chip = view->cell = view->readout = 0;
+            psString filename = pmFPAfileNameFromRule(filerule->rule, file, view); // Filename of interest
+            // Read convolution kernel
+            psString resolved = pmConfigConvertFilename(filename, config, false, false); // Resolved filename
+            psFree(filename);
+            psFits *fits = psFitsOpen(resolved, "r"); // FITS file for subtraction kernel
+            psFree(resolved);
+            if (!fits || !pmReadoutReadSubtractionKernels(conv, fits)) {
+                psError(PS_ERR_IO, false, "Unable to read previously produced kernel");
+                psFitsClose(fits);
+                return false;
+            }
             psFitsClose(fits);
+            return false;
+        }
+        psFitsClose(fits);
+        // Add in variance factor
+        pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, output->analysis,
+                                                            PM_SUBTRACTION_ANALYSIS_KERNEL); // Kernels
+        float vf = pmSubtractionVarianceFactor(kernels, 0.0, 0.0, false); // Variance factor
+        psMetadataItem *vfItem = psMetadataLookup(readout->parent->concepts, "CELL.VARFACTOR");
+        if (!isfinite(vf)) {
+            vf = 1.0;
+        }
+        if (isfinite(vfItem->data.F32)) {
+            vfItem->data.F32 *= vf;
+        } else {
+            vfItem->data.F32 = vf;
+        }
+        // Read image, mask, variance
+        const char *tempImage = psMetadataLookupStr(NULL, recipe, "TEMP.IMAGE"); // Suffix for image
+        const char *tempMask = psMetadataLookupStr(NULL, recipe, "TEMP.MASK"); // Suffix for mask
+        const char *tempVariance = psMetadataLookupStr(NULL, recipe, "TEMP.VARIANCE"); // Suffix for variance map
+        psString imageName = NULL, maskName = NULL, varianceName = NULL; // Names for convolved images
+        psStringAppend(&imageName, "%s.%d.%s", outName, numInput, tempImage);
+        psStringAppend(&maskName, "%s.%d.%s", outName, numInput, tempMask);
+        psStringAppend(&varianceName, "%s.%d.%s", outName, numInput, tempVariance);
+        if (!readImage(&readout->image, imageName, config) || !readImage(&readout->mask, maskName, config) ||
+            !readImage(&readout->variance, varianceName, config)) {
+            psError(PS_ERR_IO, false, "Unable to read previously produced image.");
+            // Add in variance factor
+            pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, conv->analysis,
+                                                                PM_SUBTRACTION_ANALYSIS_KERNEL); // Kernels
+            float vf = pmSubtractionVarianceFactor(kernels, 0.0, 0.0, false); // Variance factor
+            psMetadataItem *vfItem = psMetadataLookup(readout->parent->concepts, "CELL.VARFACTOR");
+            if (!isfinite(vf)) {
+                vf = 1.0;
+            }
+            if (isfinite(vfItem->data.F32)) {
+                vfItem->data.F32 *= vf;
+            } else {
+                vfItem->data.F32 = vf;
+            }
+            if (!readImage(&readout->image, options->imageNames->data[index], config) ||
+                !readImage(&readout->mask, options->maskNames->data[index], config) ||
+                !readImage(&readout->variance, options->varianceNames->data[index], config)) {
+                psError(PS_ERR_IO, false, "Unable to read previously produced image.");
+                psFree(imageName);
+                psFree(maskName);
+                psFree(varianceName);
+                return false;
+            }
             psFree(imageName);
             psFree(maskName);
             psFree(varianceName);
+            return false;
+        }
+        psFree(imageName);
+        psFree(maskName);
+        psFree(varianceName);
+        psRegion *region = psMetadataLookupPtr(NULL, output->analysis,
+                                               PM_SUBTRACTION_ANALYSIS_REGION); // Convolution region
+        pmSubtractionAnalysis(readout->analysis, kernels, region,
+                              readout->image->numCols, readout->image->numRows);
+        psKernel *kernel = pmSubtractionKernel(kernels, 0.0, 0.0, false); // Convolution kernel
+        psKernel *covar = psImageCovarianceCalculate(kernel, readout->covariance); // New covariance matrix
+        psFree(readout->covariance);
+        readout->covariance = covar;
+        psFree(kernel);
+    } else {
+#endif
+        // Normal operations here
+        if (psMetadataLookupBool(&mdok, config->arguments, "HAVE.PSF")) {
+            assert(psf);
+            assert(sources);
+            psRegion *region = psMetadataLookupPtr(NULL, conv->analysis,
+                                                   PM_SUBTRACTION_ANALYSIS_REGION); // Convolution region
+            pmSubtractionAnalysis(readout->analysis, kernels, region,
+                                  readout->image->numCols, readout->image->numRows);
+            psKernel *kernel = pmSubtractionKernel(kernels, 0.0, 0.0, false); // Convolution kernel
+            psKernel *covar = psImageCovarianceCalculate(kernel, readout->covariance); // Covariance matrix
+            psFree(readout->covariance);
+            readout->covariance = covar;
+            psFree(kernel);
+        } else {
+#endif
+            // Normal operations here
+            psAssert(options->psf, "Require target PSF");
+            psAssert(options->sourceLists && options->sourceLists->data[index], "Require source list");
             int order = psMetadataLookupS32(NULL, ppsub, "SPATIAL.ORDER"); // Spatial polynomial order
 …
             float rej = psMetadataLookupF32(NULL, ppsub, "REJ"); // Rejection threshold
             float sysError = psMetadataLookupF32(NULL, ppsub, "SYS"); // Relative systematic error in kernel
             pmSubtractionKernelsType type = pmSubtractionKernelsTypeFromString(
                 psMetadataLookupStr(NULL, ppsub, "KERNEL.TYPE")); // Kernel type
+            const char *typeStr = psMetadataLookupStr(NULL, ppsub, "KERNEL.TYPE"); // Kernel type
+            pmSubtractionKernelsType type = pmSubtractionKernelsTypeFromString(typeStr); // Kernel type
             psVector *widths = psMetadataLookupPtr(NULL, ppsub, "ISIS.WIDTHS"); // ISIS Gaussian widths
             psVector *orders = psMetadataLookupPtr(NULL, ppsub, "ISIS.ORDERS"); // ISIS Polynomial orders
 …
+            }
-#if 0
-            // Testing the normalisation of the fake image
+            {
-                pmReadout *test = pmReadoutAlloc(NULL); // Test readout
-                psArray *sources = psArrayAlloc(1);     // Array of sources
-                pmSource *source = pmSourceAlloc();     // Source
-                sources->data[0] = source;
-                source->peak = pmPeakAlloc(500, 500, 10000, PM_PEAK_LONE);
-                source->psfMag = -13.0;
-                pmReadoutFakeFromSources(test, 1000, 1000, sources, NULL, NULL, psf, 0.1, 0, false, true);
-                float sum = 0.0;
-                for (int y = 0; y < test->image->numRows; y++) {
-                    for (int x = 0; x < test->image->numCols; x++) {
-                        sum += test->image->data.F32[y][x];
+                    }
+                }
-                fprintf(stderr, "Photometry: %f --> %f = -13.0 ???\n", sum, -2.5*log10(sum));
-                psFits *fits = psFitsOpen("testphot.fits", "w");
-                psFitsWriteImage(fits, NULL, test->image, 0, NULL);
-                psFitsClose(fits);
-                exit(0);
+            }
-#endif
             pmReadout *fake = pmReadoutAlloc(NULL); // Fake readout with target PSF
             // For the sake of stamps, remove nearby sources
+            psArray *stampSources = stackSourcesFilter(sources, footprint); // Filtered list of sources
+            psArray *stampSources = stackSourcesFilter(options->sourceLists->data[index],
+                                                       footprint); // Filtered list of sources
             if (!pmReadoutFakeFromSources(fake, readout->image->numCols, readout->image->numRows,
                                           stampSources, NULL, NULL, psf, NAN, footprint + size,
+                                          stampSources, NULL, NULL, options->psf, NAN, footprint + size,
                                           false, true)) {
                 psError(PS_ERR_UNKNOWN, false, "Unable to generate fake image with target PSF.");
                 psFree(fake);
                 psFree(optWidths);
                 psFree(output);
+                psFree(conv);
                 return false;
+            }
 …
                 pmHDU *hdu = pmHDUFromCell(readout->parent);
                 psString name = NULL;
                 psStringAppend(&name, "fake_%03d.fits", numInput);
+                psStringAppend(&name, "fake_%03d.fits", index);
                 pmStackVisualPlotTestImage(fake->image, name);
                 psFits *fits = psFitsOpen(name, "w");
 …
                 pmHDU *hdu = pmHDUFromCell(readout->parent);
                 psString name = NULL;
                 psStringAppend(&name, "real_%03d.fits", numInput);
+                psStringAppend(&name, "real_%03d.fits", index);
                 pmStackVisualPlotTestImage(readout->image, name);
                 psFits *fits = psFitsOpen(name, "w");
 …
             // Do the image matching
+            if (!pmSubtractionMatch(output, NULL, readout, fake, footprint, stride, regionSize, spacing,
+                                    threshold, stampSources, stampsName, type, size, order, widths, orders,
+                                    inner, ringsOrder, binning, penalty, optimum, optWidths, optOrder,
+                                    optThresh, iter, rej, sysError, maskVal, maskBad, maskPoor, poorFrac,
+                                    badFrac, PM_SUBTRACTION_MODE_1)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to match images.");
+                psFree(fake);
+                psFree(optWidths);
+                psFree(stampSources);
+                psFree(output);
+                return false;
+            pmSubtractionKernels *kernel = psMetadataLookupPtr(&mdok, readout->analysis,
+                                                               PM_SUBTRACTION_ANALYSIS_KERNEL); // Conv kernel
+            if (kernel) {
+                if (!pmSubtractionMatchPrecalc(conv, NULL, readout, fake, readout->analysis,
+                                               stride, sysError, maskVal, maskBad, maskPoor,
+                                               poorFrac, badFrac)) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to convolve images.");
+                    psFree(fake);
+                    psFree(optWidths);
+                    psFree(stampSources);
+                    psFree(conv);
+                    return false;
+                }
+            } else {
+                if (!pmSubtractionMatch(conv, NULL, readout, fake, footprint, stride, regionSize, spacing,
+                                        threshold, stampSources, stampsName, type, size, order, widths,
+                                        orders, inner, ringsOrder, binning, penalty,
+                                        optimum, optWidths, optOrder, optThresh, iter, rej, sysError,
+                                        maskVal, maskBad, maskPoor, poorFrac, badFrac,
+                                        PM_SUBTRACTION_MODE_1)) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to match images.");
+                    psFree(fake);
+                    psFree(optWidths);
+                    psFree(stampSources);
+                    psFree(conv);
+                    return false;
+                }
+            }
 …
                 pmHDU *hdu = pmHDUFromCell(readout->parent);
                 psString name = NULL;
                 psStringAppend(&name, "conv_%03d.fits", numInput);
                 pmStackVisualPlotTestImage(output->image, name);
+                psStringAppend(&name, "conv_%03d.fits", index);
+                pmStackVisualPlotTestImage(conv->image, name);
                 psFits *fits = psFitsOpen(name, "w");
                 psFree(name);
                 psFitsWriteImage(fits, hdu->header, output->image, 0, NULL);
+                psFitsWriteImage(fits, hdu->header, conv->image, 0, NULL);
                 psFitsClose(fits);
+            }
 …
                 pmHDU *hdu = pmHDUFromCell(readout->parent);
                 psString name = NULL;
                 psStringAppend(&name, "diff_%03d.fits", numInput);
+                psStringAppend(&name, "diff_%03d.fits", index);
                 pmStackVisualPlotTestImage(fake->image, name);
                 psFits *fits = psFitsOpen(name, "w");
                 psFree(name);
                 psBinaryOp(fake->image, output->image, "-", fake->image);
+                psBinaryOp(fake->image, conv->image, "-", fake->image);
                 psFitsWriteImage(fits, hdu->header, fake->image, 0, NULL);
                 psFitsClose(fits);
 …
             // Set the variance factor
             psMetadataItem *vfItem = psMetadataLookup(readout->parent->concepts, "CELL.VARFACTOR");
             float vf = psMetadataLookupF32(NULL, output->analysis, PM_SUBTRACTION_ANALYSIS_VARFACTOR_1);
+            float vf = psMetadataLookupF32(NULL, conv->analysis, PM_SUBTRACTION_ANALYSIS_VARFACTOR_1);
             if (!isfinite(vf)) {
                 vf = 1.0;
 …
             psFree(readout->variance);
             psFree(readout->covariance);
+            readout->image  = psMemIncrRefCounter(output->image);
+            readout->mask   = psMemIncrRefCounter(output->mask);
+            readout->variance = psMemIncrRefCounter(output->variance);
+            readout->covariance = psImageCovarianceTruncate(output->covariance, COVAR_FRAC);
+        } else {
+            // Fake the convolution
+            psRegion *region = psRegionAlloc(0, readout->image->numCols - 1, 0, readout->image->numRows - 1);
+            psMetadataAddPtr(output->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_REGION,
+                             PS_DATA_REGION | PS_META_DUPLICATE_OK, "Fake subtraction region", region);
+            psFree(region);
+            pmSubtractionKernels *kernels = pmSubtractionKernelsPOIS(FAKE_SIZE, 0, penalty,
+                                                                     PM_SUBTRACTION_MODE_1);
+            // Set solution to delta function
+            kernels->solution1 = psVectorAlloc(kernels->num + 2, PS_TYPE_F64);
+            psVectorInit(kernels->solution1, 0.0);
+            int normIndex = PM_SUBTRACTION_INDEX_NORM(kernels); // Index for normalisation
+            kernels->solution1->data.F64[normIndex] = 1.0;
+            psMetadataAddPtr(output->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_KERNEL,
+                             PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK, "Fake subtraction kernel", kernels);
+            psFree(kernels);
+        }
+            readout->image  = psMemIncrRefCounter(conv->image);
+            readout->mask   = psMemIncrRefCounter(conv->mask);
+            readout->variance = psMemIncrRefCounter(conv->variance);
+            readout->covariance = psImageCovarianceTruncate(conv->covariance, COVAR_FRAC);
 #ifdef TESTING
+        // Write convolution kernel
+        }
+#endif
+        // Extract the regions and solutions used in the image matching
+        // This stops them from being freed when we iterate back up the FPA
+        psArray *regions = options->regions->data[index] = psArrayAllocEmpty(ARRAY_BUFFER); // Match regions
+        {
+            const char *outName = psMetadataLookupStr(NULL, config->arguments, "OUTPUT"); // Output root
+            assert(outName);
+            psString filename = NULL;   // Output filename
+            psStringAppend(&filename, "%s.%d.kernel", outName, numInput);
+            psString resolved = pmConfigConvertFilename(filename, config, true, false); // Resolved filename
+            psFree(filename);
+            psFits *fits = psFitsOpen(resolved, "w"); // FITS file for subtraction kernel
+            psFree(resolved);
+            pmReadoutWriteSubtractionKernels(output, fits);
+            psFitsClose(fits);
+        }
+    }
+#endif
+    readout->analysis = psMetadataCopy(readout->analysis, output->analysis);
+// Extract the regions and solutions used in the image matching
+// This stops them from being freed when we iterate back up the FPA
+    *regions = psArrayAllocEmpty(ARRAY_BUFFER); // Array of regions
+    {
+        psString regex = NULL;          // Regular expression
+        psStringAppend(&regex, "^%s$", PM_SUBTRACTION_ANALYSIS_REGION);
+        psMetadataIterator *iter = psMetadataIteratorAlloc(output->analysis, PS_LIST_HEAD, regex); // Iterator
+        psFree(regex);
+        psMetadataItem *item = NULL;// Item from iteration
+        while ((item = psMetadataGetAndIncrement(iter))) {
+            assert(item->type == PS_DATA_REGION);
+            *regions = psArrayAdd(*regions, ARRAY_BUFFER, item->data.V);
+        }
+        psFree(iter);
+    }
+    *kernels = psArrayAllocEmpty(ARRAY_BUFFER); // Array of kernels
+    {
+        psString regex = NULL;          // Regular expression
+        psStringAppend(&regex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL);
+        psMetadataIterator *iter = psMetadataIteratorAlloc(output->analysis, PS_LIST_HEAD, regex); // Iterator
+        psFree(regex);
+        psMetadataItem *item = NULL;// Item from iteration
+        while ((item = psMetadataGetAndIncrement(iter))) {
+            assert(item->type == PS_DATA_UNKNOWN);
+            // Set the normalisation dimensions, since these will be otherwise unavailable when reading the
+            // images by scans.
+            pmSubtractionKernels *kernel = item->data.V; // Kernel used in subtraction
+            kernel->numCols = readout->image->numCols;
+            kernel->numRows = readout->image->numRows;
+            *kernels = psArrayAdd(*kernels, ARRAY_BUFFER, kernel);
+        }
+        psFree(iter);
+    }
+    assert((*regions)->n == (*kernels)->n);
+    // Record chi^2
+    {
+        *chi2 = 0.0;
+        int num = 0;                    // Number of measurements of chi^2
+        psString regex = NULL;          // Regular expression
+        psStringAppend(&regex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL);
+        psMetadataIterator *iter = psMetadataIteratorAlloc(output->analysis, PS_LIST_HEAD, regex); // Iterator
+        psFree(regex);
+        psMetadataItem *item = NULL;// Item from iteration
+        while ((item = psMetadataGetAndIncrement(iter))) {
+            assert(item->type == PS_DATA_UNKNOWN);
+            pmSubtractionKernels *kernels = item->data.V; // Convolution kernels
+            *chi2 += kernels->mean;
+            num++;
+        }
+        psFree(iter);
+        *chi2 /= psImageCovarianceFactor(readout->covariance) * num;
+    }
+    // Reject image completely if the maximum deconvolution fraction exceeds the limit
+    float deconv = psMetadataLookupF32(NULL, output->analysis,
+                                       PM_SUBTRACTION_ANALYSIS_DECONV_MAX); // Maximum deconvolution fraction
+    if (deconv > deconvLimit) {
+        psWarning("Maximum deconvolution fraction (%f) exceeds limit (%f) --- rejecting\n",
+                  deconv, deconvLimit);
+        psFree(output);
+        return NULL;
+            psString regex = NULL;          // Regular expression
+            psStringAppend(&regex, "^%s$", PM_SUBTRACTION_ANALYSIS_REGION);
+            psMetadataIterator *iter = psMetadataIteratorAlloc(conv->analysis, PS_LIST_HEAD, regex);
+            psFree(regex);
+            psMetadataItem *item = NULL;// Item from iteration
+            while ((item = psMetadataGetAndIncrement(iter))) {
+                assert(item->type == PS_DATA_REGION);
+                regions = psArrayAdd(regions, ARRAY_BUFFER, item->data.V);
+            }
+            psFree(iter);
+        }
+        psArray *kernels = options->kernels->data[index] = psArrayAllocEmpty(ARRAY_BUFFER); // Match kernels
+        {
+            psString regex = NULL;          // Regular expression
+            psStringAppend(&regex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL);
+            psMetadataIterator *iter = psMetadataIteratorAlloc(conv->analysis, PS_LIST_HEAD, regex);
+            psFree(regex);
+            psMetadataItem *item = NULL;// Item from iteration
+            while ((item = psMetadataGetAndIncrement(iter))) {
+                assert(item->type == PS_DATA_UNKNOWN);
+                // Set the normalisation dimensions, since these will be otherwise unavailable when reading
+                // the images by scans.
+                pmSubtractionKernels *kernel = item->data.V; // Kernel used in subtraction
+                kernel->numCols = readout->image->numCols;
+                kernel->numRows = readout->image->numRows;
+                kernels = psArrayAdd(kernels, ARRAY_BUFFER, kernel);
+            }
+            psFree(iter);
+        }
+        psAssert((regions)->n == (kernels)->n, "Number of match regions and kernels should match");
+        // Record chi^2
+        {
+            double sum = 0.0;           // Sum of chi^2
+            int num = 0;                // Number of measurements of chi^2
+            psString regex = NULL;      // Regular expression
+            psStringAppend(&regex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL);
+            psMetadataIterator *iter = psMetadataIteratorAlloc(conv->analysis, PS_LIST_HEAD, regex);
+            psFree(regex);
+            psMetadataItem *item = NULL;// Item from iteration
+            while ((item = psMetadataGetAndIncrement(iter))) {
+                assert(item->type == PS_DATA_UNKNOWN);
+                pmSubtractionKernels *kernels = item->data.V; // Convolution kernels
+                sum += kernels->mean;
+                num++;
+            }
+            psFree(iter);
+            options->matchChi2->data.F32[index] = sum / (psImageCovarianceFactor(readout->covariance) * num);
+        }
+        // Reject image completely if the maximum deconvolution fraction exceeds the limit
+        float deconv = psMetadataLookupF32(NULL, conv->analysis,
+                                           PM_SUBTRACTION_ANALYSIS_DECONV_MAX); // Max deconvolution fraction
+        if (deconv > deconvLimit) {
+            psWarning("Maximum deconvolution fraction (%f) exceeds limit (%f) --- rejecting\n",
+                      deconv, deconvLimit);
+            psFree(conv);
+            return NULL;
+        }
+        readout->analysis = psMetadataCopy(readout->analysis, conv->analysis);
+        psFree(conv);
+    } else {
+        // Match the normalisation
+        float norm = powf(10.0, -0.4 * options->norm->data.F32[index]); // Normalisation
+        psBinaryOp(readout->image, readout->image, "*", psScalarAlloc(norm, PS_TYPE_F32));
+        psBinaryOp(readout->variance, readout->variance, "*", psScalarAlloc(PS_SQR(norm), PS_TYPE_F32));
+    }
     // Ensure the background value is zero
     psStats *bg = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics for background
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator
     if (!psImageBackground(bg, NULL, readout->image, readout->mask, maskVal | maskBad, rng)) {
         psWarning("Can't measure background for image.");
 …
     if (!psImageBackground(bg, NULL, readout->variance, readout->mask, maskVal | maskBad, rng)) {
         psError(PS_ERR_UNKNOWN, false, "Can't measure mean variance for image.");
+        psFree(output);
+        psFree(rng);
+        psFree(bg);
         return false;
+    }
     *weighting = 1.0 / (psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN) *
                         psImageCovarianceFactor(readout->covariance));
+    options->weightings->data.F32[index] = 1.0 / (psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN) *
+                                                  psImageCovarianceFactor(readout->covariance));
     psMetadataAddF32(readout->analysis, PS_LIST_TAIL, "PPSTACK.WEIGHTING", 0,
+                     "Weighting by 1/noise^2 for stack", *weighting);
+                     "Weighting by 1/noise^2 for stack", options->weightings->data.F32[index]);
+    psFree(rng);
     psFree(bg);
-#if 0
-#define RADIUS 10                       // Radius of photometry
-#define MIN_ERR 0.05                    // Minimum photometric error, mag
-#define MAX_MAG -13                     // Maximum magnitude for source
-    // Ensure the normalisation is correct
-    // XXX Ideally, would like to do proper PSF-fit photometry, but will settle for aperture photometry
-    int numSources = sources->n;        // Number of sources
-    psVector *ratio = psVectorAlloc(numSources, PS_TYPE_F32); // Flux ratios for sources
-    psVector *ratioMask = psVectorAlloc(numSources, PS_TYPE_MASK); // Mask for flux ratios
-    psVectorInit(ratioMask, 0xFF);
-    psImage *image = readout->image;    // Image of interest
-    psImage *mask = readout->mask;      // Mask of interest
-    int numCols = image->numCols, numRows = image->numRows; // Size of image
-    for (int i = 0; i < numSources; i++) {
-        pmSource *source = sources->data[i]; // Source of interest
-        if (!source || source->mode & SOURCE_MASK || !isfinite(source->psfMag) || !isfinite(source->errMag) ||
-            source->errMag > MIN_ERR || source->psfMag > MAX_MAG) {
-            continue;
+        }
-        float xSrc, ySrc;              // Source coordinates
-        coordsFromSource(&xSrc, &ySrc, source);
-        int xMin = PS_MAX(0, xSrc - RADIUS), xMax = PS_MIN(numCols - 1, xSrc + RADIUS); // Bounds in x
-        int yMin = PS_MAX(0, ySrc - RADIUS), yMax = PS_MIN(numRows - 1, ySrc + RADIUS); // Bounds in y
-        int numPix = 0;                 // Number of pixels
-        float sum = 0.0;                // Sum of pixels
-        for (int y = yMin; y <= yMax; y++) {
-            for (int x = xMin; x <= xMax; x++) {
-                if (mask->data.PS_TYPE_MASK_DATA[y][x] & maskBad) {
-                    continue;
+                }
-                sum += image->data.F32[y][x];
-                numPix++;
+            }
+        }
-        if (sum >= 0 && numPix > 0) {
-            float mag = -2.5 * log10(sum * M_PI * PS_SQR(RADIUS) / numPix); // Instrumental magnitude
-            ratio->data.F32[i] = mag - source->psfMag;
-            ratioMask->data.PS_TYPE_MASK_DATA[i] = 0;
+        }
+    }
-    psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics
-    if (!psVectorStats(stats, ratio, NULL, ratioMask, 0xFF)) {
-        psWarning("Unable to measure normalisation --- assuming correct.");
-    } else {
-        psLogMsg("ppStack", PS_LOG_INFO, "Renormalising image by %f (+/- %f) mag\n",
-                 stats->robustMedian, stats->robustStdev);
-        float norm = powf(10.0, -0.4 * stats->robustMedian); // Normalisation to apply
-        psBinaryOp(image, image, "*", psScalarAlloc(norm, PS_TYPE_F32));
+    }
-    psFree(stats);
-    psFree(ratio);
-    psFree(ratioMask);
-#endif
 #ifdef TESTING
 …
         pmHDU *hdu = pmHDUFromCell(readout->parent);
         psString name = NULL;
         psStringAppend(&name, "convolved_%03d.fits", numInput);
         pmStackVisualPlotTestImage(output->image, name);
+        psStringAppend(&name, "convolved_%03d.fits", index);
+        pmStackVisualPlotTestImage(readout->image, name);
         psFits *fits = psFitsOpen(name, "w");
         psFree(name);
         psFitsWriteImage(fits, hdu->header, output->image, 0, NULL);
+        psFitsWriteImage(fits, hdu->header, readout->image, 0, NULL);
         psFitsClose(fits);
+    }
 #endif
-    psFree(output);
     return true;

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackOptions.c

-              r23352
+              r23594
     psFree(options->inputMask);
     psFree(options->sourceLists);
+    psFree(options->norm);
     psFree(options->cells);
     psFree(options->subKernels);
     psFree(options->subRegions);
+    psFree(options->kernels);
+    psFree(options->regions);
     psFree(options->matchChi2);
     psFree(options->weightings);
 …
     psMemSetDeallocator(options, (psFreeFunc)stackOptionsFree);
+    options->convolve = true;
     options->stats = NULL;
     options->statsFile = NULL;
 …
     options->inputMask = NULL;
     options->sourceLists = NULL;
+    options->norm = NULL;
     options->cells = NULL;
     options->subKernels = NULL;
     options->subRegions = NULL;
+    options->kernels = NULL;
+    options->regions = NULL;
     options->numCols = 0;
     options->numRows = 0;

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackOptions.h

-              r23352
+              r23594
 typedef struct {
     // Setup
+    bool convolve;                      // Convolve images?
     psMetadata *stats;                  // Statistics for output
     FILE *statsFile;                    // File to which to write statistics
 …
     psVector *inputMask;                // Mask for inputs
     psArray *sourceLists;               // Individual lists of sources for matching
+    psVector *norm;                     // Normalisation for each image
     // Convolve
     psArray *cells;                     // Cells for convolved images --- a handle for reading again
     psArray *subKernels;                // Subtraction kernels --- required in the stacking
     psArray *subRegions;                // Subtraction regions --- required in the stacking
+    psArray *kernels;                   // PSF-matching kernels --- required in the stacking
+    psArray *regions;                   // PSF-matching regions --- required in the stacking
     int numCols, numRows;               // Size of image
     psVector *matchChi2;                // chi^2 for stamps from matching

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackPhotometry.c

-              r23352
+              r23594
         return false;
+    }
-    psFree(photView);
     if (!pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL") ||
 …
                          "Time to do photometry", psTimerMark("PPSTACK_PHOT"));
+    }
+    psFree(photView);
     psLogMsg("ppStack", PS_LOG_INFO, "Time to do photometry: %f sec", psTimerClear("PPSTACK_PHOT"));

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackPrepare.c

-              r23352
+              r23594
     psVectorInit(options->inputMask, 0);
     if (psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) {
         pmFPAfileActivate(config->files, false, NULL);
         ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true);
         pmFPAview *view = ppStackFilesIterateDown(config);
         if (!view) {
             return false;
+        }
         // Generate list of PSFs
         psMetadataIterator *fileIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD,
                                                                "^PPSTACK.INPUT$"); // Iterator
         psMetadataItem *fileItem; // Item from iteration
         psArray *psfs = psArrayAlloc(num); // PSFs for PSF envelope
         int numCols = 0, numRows = 0;   // Size of image
         int index = 0;                  // Index for file
+        while ((fileItem = psMetadataGetAndIncrement(fileIter))) {
             assert(fileItem->type == PS_DATA_UNKNOWN);
             pmFPAfile *inputFile = fileItem->data.V; // An input file
+    pmFPAfileActivate(config->files, false, NULL);
+    ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true);
+    pmFPAview *view = ppStackFilesIterateDown(config);
+    if (!view) {
+        return false;
+    }
+    psMetadataIterator *fileIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD, "^PPSTACK.INPUT$");
+    psMetadataItem *fileItem; // Item from iteration
+    psArray *psfs = psArrayAlloc(num); // PSFs for PSF envelope
+    int numCols = 0, numRows = 0;   // Size of image
+    int index = 0;                  // Index for file
+    while ((fileItem = psMetadataGetAndIncrement(fileIter))) {
+        assert(fileItem->type == PS_DATA_UNKNOWN);
+        pmFPAfile *inputFile = fileItem->data.V; // An input file
+        // Get list of PSFs, to determine target PSF
+        if (options->convolve) {
             pmChip *chip = pmFPAviewThisChip(view, inputFile->fpa); // The chip: holds the PSF
             pmPSF *psf = psMetadataLookupPtr(NULL, chip->analysis, "PSPHOT.PSF"); // PSF
 …
                 numRows = naxis2;
+            }
+            pmReadout *ro = pmFPAviewThisReadout(view, inputFile->fpa); // Readout with sources
+            psArray *sources = psMetadataLookupPtr(NULL, ro->analysis, "PSPHOT.SOURCES"); // Sources
+            if (!sources) {
+                psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find sources in readout.");
+                return NULL;
+            }
+            options->sourceLists->data[index] = psMemIncrRefCounter(sources);
+            // Re-do photometry if we don't trust the source lists
+            if (psMetadataLookupBool(NULL, recipe, "PHOT")) {
+                psTrace("ppStack", 2, "Photometering input %d of %d....\n", index, num);
+                pmFPAfileActivate(config->files, false, NULL);
+                ppStackFileActivationSingle(config, PPSTACK_FILES_CONVOLVE, true, index);
+                pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", index); // File
+                pmFPAview *photView = ppStackFilesIterateDown(config);
+                if (!photView) {
+                    psFree(view);
+                    return false;
+                }
+                pmReadout *ro = pmFPAviewThisReadout(view, file->fpa); // Readout of interest
+                if (!ppStackInputPhotometer(ro, sources, config)) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to do photometry on input sources");
+                    psFree(view);
+                    psFree(photView);
+                    return false;
+                }
+        }
+        pmReadout *ro = pmFPAviewThisReadout(view, inputFile->fpa); // Readout with sources
+        psArray *sources = psMetadataLookupPtr(NULL, ro->analysis, "PSPHOT.SOURCES"); // Sources
+        if (!sources) {
+            psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find sources in readout.");
+            return NULL;
+        }
+        options->sourceLists->data[index] = psMemIncrRefCounter(sources);
+        // Re-do photometry if we don't trust the source lists
+        if (psMetadataLookupBool(NULL, recipe, "PHOT")) {
+            psTrace("ppStack", 2, "Photometering input %d of %d....\n", index, num);
+            pmFPAfileActivate(config->files, false, NULL);
+            ppStackFileActivationSingle(config, PPSTACK_FILES_CONVOLVE, true, index);
+            pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", index); // File
+            pmFPAview *photView = ppStackFilesIterateDown(config);
+            if (!photView) {
+                psFree(view);
+                return false;
+            }
+            pmReadout *ro = pmFPAviewThisReadout(view, file->fpa); // Readout of interest
+            if (!ppStackInputPhotometer(ro, sources, config)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to do photometry on input sources");
+                psFree(view);
                 psFree(photView);
+                if (!ppStackFilesIterateUp(config)) {
+                    psFree(view);
+                    return false;
+                }
+                pmFPAfileActivate(config->files, false, NULL);
+                ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true);
+            }
+            index++;
+        }
+        psFree(fileIter);
+        // Zero point calibration
+        options->sumExposure = ppStackSourcesTransparency(options->sourceLists, options->inputMask,
+                                                          view, config);
+        if (!isfinite(options->sumExposure) || options->sumExposure <= 0) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to calculate transparency differences");
+            psFree(view);
+            return false;
+        }
+        // Generate target PSF
+                return false;
+            }
+            psFree(photView);
+            if (!ppStackFilesIterateUp(config)) {
+                psFree(view);
+                return false;
+            }
+            pmFPAfileActivate(config->files, false, NULL);
+            ppStackFileActivation(config, PPSTACK_FILES_PREPARE, true);
+        }
+        index++;
+    }
+    psFree(fileIter);
+    // Generate target PSF
+    if (options->convolve) {
         options->psf = ppStackPSF(config, numCols, numRows, psfs, options->inputMask);
         psFree(psfs);
 …
                          "Target PSF", options->psf);
         outChip->data_exists = true;
+    }
+    // Zero point calibration
+    if (!ppStackSourcesTransparency(options, view, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to calculate transparency differences");
         psFree(view);
+        if (!ppStackFilesIterateUp(config)) {
+            return false;
+        }
+        return false;
+    }
+    psFree(view);
+    if (!ppStackFilesIterateUp(config)) {
+        return false;
+    }

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackReadout.c

-              r23352
+              r23594
     if (!pmStackCombine(outRO, stack, maskVal | maskBad, maskBad, kernelSize, iter,
                         combineRej, combineSys, combineDiscard, true, useVariance, safe)) {
+                        combineRej, combineSys, combineDiscard, true, useVariance, safe, false)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts with rejection.");
         psFree(stack);
 …
     assert(outRO);
     assert(readouts);
+    assert(rejected);
+    assert(readouts->n == rejected->n);
+    assert(!rejected || readouts->n == rejected->n);
     assert(mask && mask->n == readouts->n && mask->type.type == PS_TYPE_VECTOR_MASK);
     assert(weightings && weightings->n == readouts->n && weightings->type.type == PS_TYPE_F32);
 …
     bool mdok;                          // Status of MD lookup
+    int iter = psMetadataLookupS32(NULL, recipe, "ITER"); // Rejection iterations
+    float combineRej = psMetadataLookupF32(NULL, recipe, "COMBINE.REJ"); // Combination threshold
+    float combineSys = psMetadataLookupF32(NULL, recipe, "COMBINE.SYS"); // Combination systematic error
+    float combineDiscard = psMetadataLookupF32(NULL, recipe, "COMBINE.DISCARD"); // Olympic discard fraction
     bool useVariance = psMetadataLookupBool(&mdok, recipe, "VARIANCE"); // Use variance for rejection?
+    bool safe = psMetadataLookupBool(&mdok, recipe, "SAFE"); // Be safe when combining small numbers of pixels
     psString maskValStr = psMetadataLookupStr(NULL, recipe, "MASK.VAL"); // Name of bits to mask going in
 …
     psArray *stack = psArrayAlloc(num); // Array for stacking
+    int numGood = num;                  // Number of good inputs: images that haven't been completely rejected
+    bool entire = true;                 // Combine entire image?
+    if (rejected) {
+        // We have rejection from a previous combination: combine without flagging pixels to inspect
+        entire = false;
+        safe = false;
+        iter = 0;
+        combineRej = NAN;
+        combineSys = NAN;
+    }
     for (int i = 0; i < num; i++) {
         pmReadout *ro = readouts->data[i];
+        if (!ro || !rejected->data[i] || mask->data.PS_TYPE_VECTOR_MASK_DATA[i]) {
+            // Image completely rejected
+            numGood--;
+        if (mask->data.U8[i] & (PPSTACK_MASK_REJECT | PPSTACK_MASK_BAD)) {
+            // Image completely rejected since previous combination
+            entire = true;
+            continue;
+        } else if (mask->data.U8[i]) {
+            // Image completely rejected before original combination
             continue;
+        }
 …
         pmStackData *data = pmStackDataAlloc(ro, weightings->data.F32[i],
                                              addVariance ? addVariance->data.F32[i] : NAN);
         data->reject = psMemIncrRefCounter(rejected->data[i]);
+        data->reject = rejected ? psMemIncrRefCounter(rejected->data[i]) : NULL;
         stack->data[i] = data;
+    }
+    if (!pmStackCombine(outRO, stack, maskVal | maskBad, maskBad, 0, 0, NAN, NAN, NAN,
+                        numGood != num, useVariance, false)) {
+    if (!pmStackCombine(outRO, stack, maskVal | maskBad, maskBad, 0,
+                        iter, combineRej, combineSys, combineDiscard,
+                        entire, useVariance, safe, !rejected)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts.");
         psFree(stack);

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackReject.c

-              r23352
+              r23594
     psAssert(options, "Require options");
     psAssert(config, "Require configuration");
+    if (!options->convolve) {
+        // No need to do complicated rejection when we haven't convolved
+        return true;
+    }
     int num = options->num;             // Number of inputs
 …
         psPixels *reject = pmStackReject(options->inspect->data[i], options->numCols, options->numRows,
                                          threshold, poorFrac, stride, options->subRegions->data[i],
                                          options->subKernels->data[i]); // Rejected pixels
+                                         threshold, poorFrac, stride, options->regions->data[i],
+                                         options->kernels->data[i]); // Rejected pixels
 #ifdef TESTING
 …
     psFree(options->inspect); options->inspect = NULL;
     psFree(options->subKernels); options->subKernels = NULL;
     psFree(options->subRegions); options->subRegions = NULL;
+    psFree(options->kernels); options->kernels = NULL;
+    psFree(options->regions); options->regions = NULL;
     if (numRejected >= num - 1) {

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackSetup.c

-              r23352
+              r23594
 #include "ppStackLoop.h"
+#define BUFFER 16                       // Buffer for name array
+// Generate an array of input filenames
+static psArray *stackNameArray(const ppStackOptions *options, // Stack options
+                               pmConfig *config, // Configuration
+                               const char *name // Name of file
+    )
+{
+    psAssert(config, "Require configuration");
+    psAssert(config, "Require file name");
+    psArray *array = psArrayAllocEmpty(options->num); // Array with filenames
+    pmFPAview *view = pmFPAviewAlloc(0);// View to readout
+    view->chip = view->cell = view->readout = -1;
+    for (int i = 0; i < options->num; i++) {
+        pmFPAfile *file = pmFPAfileSelectSingle(config->files, name, i); // File of interest
+        psString filename = pmFPAfileName(file, view, config); // Filename of interest
+        psAssert(filename, "Can't determine filename");
+        psArrayAdd(array, array->n, filename);
+        psFree(filename);               // Drop reference
+    }
+    return array;
+}
 bool ppStackSetup(ppStackOptions *options, pmConfig *config)
+{
 …
     psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe
     psAssert(recipe, "We've thrown an error on this before.");
+    options->convolve = psMetadataLookupBool(NULL, config->arguments, "CONVOLVE"); // Convolve images?
+    if (!psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) {
+        psWarning("No PSFs provided --- unable to convolve to common PSF.");
+        options->convolve = false;
+    }
     int num = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of inputs
 …
+    }
+    const char *tempDir = psMetadataLookupStr(NULL, recipe, "TEMP.DIR"); // Directory for temporary images
+    if (!tempDir) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find TEMP.DIR in recipe");
+        return false;
+    // Generate temporary names for convolved images
+    if (options->convolve) {
+        const char *tempDir = psMetadataLookupStr(NULL, recipe, "TEMP.DIR"); // Directory for temporary images
+        if (!tempDir) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find TEMP.DIR in recipe");
+            return false;
+        }
+        psString outputName = psStringCopy(psMetadataLookupStr(NULL, config->arguments,
+                                                               "OUTPUT")); // Name for temporary files
+        const char *tempName = basename(outputName);
+        if (!tempName) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to construct basename for temporary files.");
+            psFree(outputName);
+            return false;
+        }
+        const char *tempImage = psMetadataLookupStr(NULL, recipe, "TEMP.IMAGE"); // Suffix for images
+        const char *tempMask = psMetadataLookupStr(NULL, recipe, "TEMP.MASK"); // Suffix for masks
+        const char *tempVariance = psMetadataLookupStr(NULL, recipe, "TEMP.VARIANCE"); // Suffix for var maps
+        if (!tempImage || !tempMask || !tempVariance) {
+            psError(PS_ERR_BAD_PARAMETER_VALUE, false,
+                    "Unable to find TEMP.IMAGE, TEMP.MASK and TEMP.VARIANCE in recipe");
+            psFree(outputName);
+            return false;
+        }
+        options->imageNames = psArrayAlloc(num);
+        options->maskNames = psArrayAlloc(num);
+        options->varianceNames = psArrayAlloc(num);
+        for (int i = 0; i < num; i++) {
+            psString imageName = NULL, maskName = NULL, varianceName = NULL; // Names for convolved images
+            psStringAppend(&imageName, "%s/%s.%d.%s", tempDir, tempName, i, tempImage);
+            psStringAppend(&maskName, "%s/%s.%d.%s", tempDir, tempName, i, tempMask);
+            psStringAppend(&varianceName, "%s/%s.%d.%s", tempDir, tempName, i, tempVariance);
+            psTrace("ppStack", 5, "Temporary files: %s %s %s\n", imageName, maskName, varianceName);
+            options->imageNames->data[i] = imageName;
+            options->maskNames->data[i] = maskName;
+            options->varianceNames->data[i] = varianceName;
+        }
+        psFree(outputName);
+    } else {
+        options->imageNames = stackNameArray(options, config, "PPSTACK.INPUT");
+        options->maskNames = stackNameArray(options, config, "PPSTACK.INPUT.MASK");
+        options->varianceNames = stackNameArray(options, config, "PPSTACK.INPUT.VARIANCE");
+    }
-    psString outputName = psStringCopy(psMetadataLookupStr(NULL, config->arguments,
-                                                           "OUTPUT")); // Name for temporary files
-    const char *tempName = basename(outputName);
-    if (!tempName) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to construct basename for temporary files.");
-        psFree(outputName);
-        return false;
+    }
-    const char *tempImage = psMetadataLookupStr(NULL, recipe, "TEMP.IMAGE"); // Suffix for temporary images
-    const char *tempMask = psMetadataLookupStr(NULL, recipe, "TEMP.MASK"); // Suffix for temporary masks
-    const char *tempVariance = psMetadataLookupStr(NULL, recipe, "TEMP.VARIANCE"); // Suffix for temp var maps
-    if (!tempImage || !tempMask || !tempVariance) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, false,
-                "Unable to find TEMP.IMAGE, TEMP.MASK and TEMP.VARIANCE in recipe");
-        psFree(outputName);
-        return false;
+    }
-    options->imageNames = psArrayAlloc(num);
-    options->maskNames = psArrayAlloc(num);
-    options->varianceNames = psArrayAlloc(num);
-    for (int i = 0; i < num; i++) {
-        psString imageName = NULL, maskName = NULL, varianceName = NULL; // Names for convolved images
-        psStringAppend(&imageName, "%s/%s.%d.%s", tempDir, tempName, i, tempImage);
-        psStringAppend(&maskName, "%s/%s.%d.%s", tempDir, tempName, i, tempMask);
-        psStringAppend(&varianceName, "%s/%s.%d.%s", tempDir, tempName, i, tempVariance);
-        psTrace("ppStack", 5, "Temporary files: %s %s %s\n", imageName, maskName, varianceName);
-        options->imageNames->data[i] = imageName;
-        options->maskNames->data[i] = maskName;
-        options->varianceNames->data[i] = varianceName;
+    }
-    psFree(outputName);
     if (!pmConfigMaskSetBits(NULL, NULL, config)) {

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackSources.c

-              r23352
+              r23594
 //#define TESTING                         // Enable debugging output
+#ifdef TESTING
 // Size of fake image; set by hand because it's trouble to get it from other places
 #define FAKE_COLS 4861
 #define FAKE_ROWS 4913
+#endif
 #ifdef TESTING
 …
+float ppStackSourcesTransparency(const psArray *sourceLists, psVector *inputMask,
+                                 const pmFPAview *view, const pmConfig *config)
+bool ppStackSourcesTransparency(ppStackOptions *options, const pmFPAview *view, const pmConfig *config)
+{
+    PS_ASSERT_ARRAY_NON_NULL(sourceLists, NAN);
+    PS_ASSERT_VECTOR_NON_NULL(inputMask, NAN);
+    PS_ASSERT_VECTOR_TYPE(inputMask, PS_TYPE_U8, NAN);
+    PS_ASSERT_VECTOR_SIZE(inputMask, sourceLists->n, NAN);
+    PS_ASSERT_PTR_NON_NULL(view, NAN);
+    PS_ASSERT_PTR_NON_NULL(config, NAN);
+    PS_ASSERT_PTR_NON_NULL(options, false);
+    PS_ASSERT_PTR_NON_NULL(view, false);
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    psArray *sourceLists = options->sourceLists; // Source lists for each input
+    psVector *inputMask = options->inputMask; // Mask for inputs
+    PS_ASSERT_ARRAY_NON_NULL(sourceLists, false);
+    PS_ASSERT_VECTOR_NON_NULL(inputMask, false);
+    PS_ASSERT_VECTOR_TYPE(inputMask, PS_TYPE_U8, false);
+    PS_ASSERT_VECTOR_SIZE(inputMask, sourceLists->n, false);
 #if defined(TESTING) && 0
 …
     if (!airmassZP) {
         psError(PS_ERR_UNKNOWN, false, "Unable to find ZP.AIRMASS in recipe.");
         return NAN;
+        return false;
+    }
 …
                     exptime, airmass, expFilter);
             psFree(zp);
             return NAN;
+            return false;
+        }
 …
                         "Unable to find airmass term (ZP.AIRMASS) for filter %s", filter);
                 psFree(zp);
                 return NAN;
+                return false;
+            }
         } else if (strcmp(filter, expFilter) != 0) {
             psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Filters don't match: %s vs %s", filter, expFilter);
             psFree(zp);
             return NAN;
+            return false;
+        }
 …
         sumExpTime += exptime;
+    }
+    options->sumExposure = sumExpTime;
     psArray *matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches
 …
         psError(PS_ERR_UNKNOWN, false, "Unable to match sources");
         psFree(zp);
         return NAN;
+        return false;
+    }
 …
     if (!trans) {
         psError(PS_ERR_UNKNOWN, false, "Unable to measure transparencies");
         return NAN;
+        return false;
+    }
 …
     for (int i = 0; i < trans->n; i++) {
         if (!isfinite(trans->data.F32[i])) {
             inputMask->data.U8[i] = PPSTACK_MASK_CAL;
+            inputMask->data.U8[i] |= PPSTACK_MASK_CAL;
+        }
+    }
 …
     // m_0 = m_1 + zp_1 + trans_1 - c1 * airmass_0 - 2.5log(t_0)
     // We don't need to know the magnitude zero point for the filter, since it cancels out
+    options->norm = psVectorAlloc(num, PS_TYPE_F32);
     for (int i = 0; i < num; i++) {
         if (!isfinite(trans->data.F32[i])) {
 …
         psArray *sources = sourceLists->data[i]; // Sources of interest
         float magCorr = airmassTerm - 2.5*log10(sumExpTime) - zp->data.F32[i] - trans->data.F32[i];
+        options->norm->data.F32[i] = magCorr;
         psLogMsg("ppStack", PS_LOG_INFO, "Applying magnitude correction to image %d: %f\n", i, magCorr);
 …
             psError(PS_ERR_UNKNOWN, false, "Unable to match sources");
             psFree(zp);
             return NAN;
+            return false;
+        }
         psVector *trans = pmSourceMatchRelphot(matches, zp, iter, tol, starLimit, transIter, transRej,
 …
 #endif
     return sumExpTime;
+    return true;
+}

branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStats.h

-              r23352
+              r23594
 // used by ppStatsFromMetadata:
 psDataType psDataTypeFromString (char *typename);
 ppStatsEntry *ppStatsEntryAlloc ();
+psDataType psDataTypeFromString(char *typename);
+ppStatsEntry *ppStatsEntryAlloc(void);
 psArray *ppStatsFromMetadataEntries (psMetadata *recipe);

branches/cnb_branches/cnb_branch_20090301/ppSub

Property svn:mergeinfo changed
/trunk/ppSub merged: 23356,23369,23399,23402-23403,23448-23449,23505,23535

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubArguments.c

-              r23352
+              r23594
+    }
     psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[3]);
+    psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[1]);
 …
     const char *refImage = psMetadataLookupStr(NULL, arguments, "-refimage"); // Name of reference image
     if (refImage && strlen(refImage) > 0) {
         fileList("INPUT", refImage, "Name of the reference image", config);
+        fileList("REF", refImage, "Name of the reference image", config);
+    }
     const char *refMask = psMetadataLookupStr(NULL, arguments, "-refmask"); // Name of reference mask

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubCamera.c

-              r23352
+              r23594
     // look for the file on the RUN metadata
     pmFPAfile *file = pmFPAfileDefineFromRun(&status, config, filerule); // File to return
+    pmFPAfile *file = pmFPAfileDefineFromRun(&status, bind, config, filerule); // File to return
     if (!status) {
         psError(PS_ERR_UNKNOWN, false, "Failed to load file definition for %s", filerule);
 …
 static pmFPAfile *defineOutputFile(pmConfig *config, // Configuration
                                    pmFPAfile *template,    // File to use as basis for definition
+                                   bool source, // Is template a source (T), or for binding (F)?
                                    char *filerule,     // Name of file rule
                                    pmFPAfileType fileType // Type of file
 …
+{
+    pmFPAfile *file = pmFPAfileDefineFromFile(config, template, 1, 1, filerule);
+    pmFPAfile *file = source ? pmFPAfileDefineFromFile(config, template, 1, 1, filerule) :
+        pmFPAfileDefineOutput(config, template->fpa, filerule);
     if (!file) {
         psError(PS_ERR_IO, false, _("Unable to generate output file from %s"), filerule);
 …
     // look for the file on the RUN metadata
     pmFPAfile *file = pmFPAfileDefineFromRun(&status, config, filerule); // File to return
+    pmFPAfile *file = pmFPAfileDefineFromRun(&status, bind, config, filerule); // File to return
     if (!status) {
         psError(PS_ERR_UNKNOWN, false, "Failed to load file definition for %s", filerule);
 …
     // define new version of file
     if (!file) {
         pmFPAfileDefineOutput(config, bind ? bind->fpa : NULL, filerule);
+        file = pmFPAfileDefineOutput(config, bind ? bind->fpa : NULL, filerule);
         if (!status) {
             psError(PS_ERR_UNKNOWN, false, "Failed to load file definition for %s", filerule);
 …
     // Output image
     pmFPAfile *output = defineOutputFile(config, input, "PPSUB.OUTPUT", PM_FPA_FILE_IMAGE);
     pmFPAfile *outMask = defineOutputFile(config, output, "PPSUB.OUTPUT.MASK", PM_FPA_FILE_MASK);
+    pmFPAfile *output = defineOutputFile(config, input, true, "PPSUB.OUTPUT", PM_FPA_FILE_IMAGE);
+    pmFPAfile *outMask = defineOutputFile(config, output, false, "PPSUB.OUTPUT.MASK", PM_FPA_FILE_MASK);
     if (!output || !outMask) {
         psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
 …
     pmFPAfile *outVar = NULL;
     if (inVar && refVar) {
         outVar = defineOutputFile(config, output, "PPSUB.OUTPUT.VARIANCE", PM_FPA_FILE_VARIANCE);
+        outVar = defineOutputFile(config, output, false, "PPSUB.OUTPUT.VARIANCE", PM_FPA_FILE_VARIANCE);
         if (!outVar) {
             psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
 …
     // Convolved input image
+    pmFPAfile *inConvImage = defineOutputFile(config, input, "PPSUB.INPUT.CONV", PM_FPA_FILE_IMAGE);
+    pmFPAfile *inConvMask = defineOutputFile(config, input, "PPSUB.INPUT.CONV.MASK", PM_FPA_FILE_MASK);
+    pmFPAfile *inConvImage = defineOutputFile(config, input, true, "PPSUB.INPUT.CONV", PM_FPA_FILE_IMAGE);
+    pmFPAfile *inConvMask = defineOutputFile(config, inConvImage, false, "PPSUB.INPUT.CONV.MASK",
+                                             PM_FPA_FILE_MASK);
     if (!inConvImage || !inConvMask) {
         psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
 …
+    }
     if (inVar) {
         pmFPAfile *inConvVar = defineOutputFile(config, input, "PPSUB.INPUT.CONV.VARIANCE",
+        pmFPAfile *inConvVar = defineOutputFile(config, inConvImage, false, "PPSUB.INPUT.CONV.VARIANCE",
                                                 PM_FPA_FILE_VARIANCE);
         if (!inConvVar) {
 …
     // Convolved ref image
+    pmFPAfile *refConvImage = defineOutputFile(config, input, "PPSUB.REF.CONV", PM_FPA_FILE_IMAGE);
+    pmFPAfile *refConvMask = defineOutputFile(config, input, "PPSUB.REF.CONV.MASK", PM_FPA_FILE_MASK);
+    pmFPAfile *refConvImage = defineOutputFile(config, input, true, "PPSUB.REF.CONV", PM_FPA_FILE_IMAGE);
+    pmFPAfile *refConvMask = defineOutputFile(config, refConvImage, false, "PPSUB.REF.CONV.MASK",
+                                              PM_FPA_FILE_MASK);
     if (!refConvImage || !refConvMask) {
         psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
 …
+    }
     if (refVar) {
         pmFPAfile *refConvVar = defineOutputFile(config, input, "PPSUB.REF.CONV.VARIANCE",
+        pmFPAfile *refConvVar = defineOutputFile(config, refConvImage, false, "PPSUB.REF.CONV.VARIANCE",
                                                  PM_FPA_FILE_VARIANCE);
         if (!refConvVar) {

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubDefineOutput.c

-              r21524
+              r23594
     psAssert(view, "Require view");
-    // generate an output readout
     pmCell *outCell = pmFPAfileThisCell(config->files, view, "PPSUB.OUTPUT"); // Output cell
-    pmReadout *outRO = pmReadoutAlloc(outCell); // Output readout: subtraction
     pmFPA *outFPA = outCell->parent->parent; // Output FPA
     pmHDU *outHDU = outFPA->hdu; // Output HDU
     if (!outHDU->header) {
         outHDU->header = psMetadataAlloc();
+    }
+    // generate an output readout (first check if it's already there by virtue of kernels)
+    pmReadout *outRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT.KERNELS"); // RO with kernel
+    if (!outRO) {
+        outRO = pmReadoutAlloc(outCell); // Output readout: subtraction
+    }

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubMatchPSFs.c

-              r23352
+              r23594
     // Load pre-calculated kernel, if available
     pmReadout *kernelRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT.KERNEL"); // RO with kernel
+    pmReadout *kernelRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT.KERNELS"); // RO with kernel
     // Sources in image, used for stamps: these must be loaded from previous analysis stages
 …
     float optThresh = psMetadataLookupF32(&mdok, recipe, "OPTIMUM.TOL"); // Tolerance for search
-    // XXX Do we need/want to define different values for BAD and POOR subtraction vs BAD and POOR warp?
     psImageMaskType maskVal = pmConfigMaskGet("MASK.VALUE", config); // Bits to mask in inputs
     psImageMaskType maskPoor = pmConfigMaskGet("POOR.WARP", config); // Bits to mask for poor pixels
+    psImageMaskType maskPoor = pmConfigMaskGet("CONV.POOR", config); // Bits to mask for poor pixels
     psImageMaskType maskBad = pmConfigMaskGet("BLANK", config); // Bits to mask for bad pixels

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubSetMasks.c

-              r21524
+              r23594
     psAssert(satValue, "SAT must be non-zero");
+    psImageMaskType badValue = pmConfigMaskGet("BAD", config);
+    psAssert(badValue, "BAD must be non-zero");
+    psImageMaskType lowValue = pmConfigMaskGet("LOW", config);
+    if (!lowValue) {
+        // Look up old name for backward compatability
+        lowValue = pmConfigMaskGet("BAD", config);
+    }
+    psAssert(lowValue, "LOW or BAD must be non-zero");
     // input images
 …
     if (!inRO->mask) {
         if (psMetadataLookupBool(NULL, recipe, "MASK.GENERATE")) {
             pmReadoutSetMask(inRO, satValue, badValue);
+            pmReadoutSetMask(inRO, satValue, lowValue);
         } else {
             inRO->mask = psImageAlloc(numCols, numRows, PS_TYPE_IMAGE_MASK);
 …
     if (!refRO->mask) {
         if (psMetadataLookupBool(NULL, recipe, "MASK.GENERATE")) {
             pmReadoutSetMask(refRO, satValue, badValue);
+            pmReadoutSetMask(refRO, satValue, lowValue);
         } else {
             refRO->mask = psImageAlloc(numCols, numRows, PS_TYPE_IMAGE_MASK);
 …
     float poorFrac = psMetadataLookupF32(&mdok, recipe, "POOR.FRACTION"); // Fraction for "poor"
     psImageMaskType maskPoor = pmConfigMaskGet("POOR.WARP", config); // Bits to mask for poor pixels
+    psImageMaskType maskPoor = pmConfigMaskGet("CONV.POOR", config); // Bits to mask for poor pixels
     psImageMaskType maskBad = pmConfigMaskGet("BLANK", config); // Bits to mask for bad pixels

branches/cnb_branches/cnb_branch_20090301/psLib/src/db/psDB.c

-              r22708
+              r23594
     mysqlType       *mType;             // type tmp variable
     static bool     isNull = true;      // used in a MYSQL_BIND to indicate NULL
+    static my_bool isNull = true;       // used in a MYSQL_BIND to indicate NULL
     MYSQL_BIND *bind = mysqlRow->bind;
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.U8;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.U8)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.U8) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.U16;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.U16)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.U16) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.U32;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.U32)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.U32) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.U64;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.U64)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.U64) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.S8;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.S8)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.S8) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.S16;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.S16)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.S16) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.S32;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.S32)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.S32) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.S64;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.S64)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.S64) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.F32;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.F32)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.F32) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.F64;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.F64)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.F64) ? &isNull : NULL;
                 break;
+            }
 …
                 bind[i].length  = 0;
                 bind[i].buffer  = &item->data.B;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.B)
+                                  ? (my_bool *)&isNull
+                                  : NULL;
+                bind[i].is_null = psDBIsPTypeNaN(item->type, &item->data.B) ? &isNull : NULL;
                 break;
+            }
 …
                     bind[i].length  = &bind[i].buffer_length;
                     bind[i].buffer  = psStringCopy(item->data.V);
+                    bind[i].is_null = *item->data.str == '\0'
+                                      ? (my_bool *)&isNull
+                                      : NULL;
+                    bind[i].is_null = *item->data.str == '\0' ? &isNull : NULL;
                 } else {
                     // handles the case of NULL as a NULL database value
 …
                     bind[i].length  = &bind[i].buffer_length;
                     bind[i].buffer  = NULL;
                     bind[i].is_null = (my_bool *)&isNull;
+                    bind[i].is_null = &isNull;
+                }
                 break;
 …
                     bind[i].length  = &bind[i].buffer_length;
                     bind[i].buffer  = NULL;
                     bind[i].is_null = (my_bool *)&isNull;
+                    bind[i].is_null = &isNull;
+                }
                 break;

branches/cnb_branches/cnb_branch_20090301/psLib/src/fits/psFitsHeader.c

r23352	r23594
569	569	continue;
570	570	}
	571	} else if (keywordInList(name, noWriteCompressedKeys) \|\|
	572	(keyStarts && keywordStartsWith(name, noWriteCompressedKeyStarts))) {
	573	continue;
571	574	}
572	575

branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageBinning.c

r21107	r23594
31	31	}
32	32
33		psImageBinning *psImageBinningAlloc() {
	33	psImageBinning *psImageBinningAlloc(void) {
34	34	psImageBinning binning = (psImageBinning)psAlloc(sizeof(psImageBinning));
35	35	psMemSetDeallocator(binning, (psFreeFunc)psImageBinningFree);

branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageBinning.h

r21107	r23594
42	42
43	43
44		psImageBinning *psImageBinningAlloc() PS_ATTR_MALLOC;
	44	psImageBinning *psImageBinningAlloc(void) PS_ATTR_MALLOC;
45	45	bool psMemCheckBinning(psPtr ptr);
46	46

branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageCovariance.c

-              r23352
+              r23594
+{
     PS_ASSERT_IMAGE_NON_NULL(variance, NULL);
+    if (!covar) {
+        // Transferred all we could!
+        return true;
+    }
     PS_ASSERT_KERNEL_NON_NULL(covar, NULL);

branches/cnb_branches/cnb_branch_20090301/psLib/src/jpeg/psImageJpeg.c

r22681	r23594
35	35	}
36	36
37		psImageJpegColormap *psImageJpegColormapAlloc ()
	37	psImageJpegColormap *psImageJpegColormapAlloc(void)
38	38	{
39	39	psImageJpegColormap *map = psAlloc(sizeof(psImageJpegColormap));

branches/cnb_branches/cnb_branch_20090301/psLib/src/jpeg/psImageJpeg.h

-              r14452
+              r23594
 /* @file  psImageJpeg.h
  * @brief functions to generate JPEG images from psImage
+ *
+ *
  * @author EAM
  * @version $Revision: 1.3 $ $Name: not supported by cvs2svn $
 …
 // allocate a colormap (does not define the map values)
 psImageJpegColormap *psImageJpegColormapAlloc() PS_ATTR_MALLOC;
+psImageJpegColormap *psImageJpegColormapAlloc(void) PS_ATTR_MALLOC;
 // set the colormap values using the supplied name

branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizeLMM.c

-              r21183
+              r23594
     // check for non-finite entries in result
     for (int i = 0; i < Beta->n; i++) {
         if (!isfinite(Beta->data.F32[i])) {
             // psError(PS_ERR_BAD_PARAMETER_VALUE, 3, "Fit value diverges: vector[%d] is %.2f\n", i, Beta->data.F32[i]);
             return false;
+        }
+        if (!isfinite(Beta->data.F32[i])) {
+            // psError(PS_ERR_BAD_PARAMETER_VALUE, 3, "Fit value diverges: vector[%d] is %.2f\n", i, Beta->data.F32[i]);
+            return false;
+        }
+    }
 …
     // (we must do this before truncating Beta below)
     if (dLinear) {
         *dLinear = psMinLM_dLinear(Beta, beta, lambda);
+    }
     // full-length Beta for checkLimits functions
+        *dLinear = psMinLM_dLinear(Beta, beta, lambda);
+    }
+    // full-length Beta for checkLimits functions
     psVector *tmpBeta = psVectorAlloc(params->n, PS_TYPE_F32);
     psVectorInit (tmpBeta, 0.0);
 …
     // set tmpBeta values which are not masked
     for (int j = 0, n = 0; j < params->n; j++) {
         if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[j])) continue;
         tmpBeta->data.F32[j] = Beta->data.F32[n];
         n++;
+        if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[j])) continue;
+        tmpBeta->data.F32[j] = Beta->data.F32[n];
+        n++;
+    }
 …
         if (checkLimits) {
             checkLimits (PS_MINIMIZE_BETA_LIMIT, j, Params->data.F32, tmpBeta->data.F32);
+        }
+        }
         Params->data.F32[j] = params->data.F32[j] - tmpBeta->data.F32[j];
 …
             checkLimits (PS_MINIMIZE_PARAM_MIN,  j, Params->data.F32, tmpBeta->data.F32);
             checkLimits (PS_MINIMIZE_PARAM_MAX,  j, Params->data.F32, tmpBeta->data.F32);
+        }
+        }
+    }
     // apply tmpBeta after limits have been checked
     for (int j = 0, n = 0; j < params->n; j++) {
         if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[j])) continue;
         Beta->data.F32[n] = tmpBeta->data.F32[j];
         n++;
+        if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[j])) continue;
+        Beta->data.F32[n] = tmpBeta->data.F32[j];
+        n++;
+    }
 …
     // if none are available, return false
     if (!psMinLM_AllocAB (&Alpha, &Beta, params, paramMask)) {
         return false;
+        return false;
+    }
 …
     // beta only counts unmasked parameters
     for (int i = 0; i < beta->n; i++) {
         dh = lambda*B[i] + b[i];
         sh = 0.5*B[i]*dh;
         Sh += sh;
+        dh = lambda*B[i] + b[i];
+        sh = 0.5*B[i]*dh;
+        Sh += sh;
         dLinear += lambda*PS_SQR(B[i]) + B[i]*b[i];
+    }
 …
         assert (!isnan(chisq));
         // we track alpha,beta and params,deriv separately
+        // we track alpha,beta and params,deriv separately
         for (int j = 0, J = 0; j < params->n; j++) {
             if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[j])) continue;
 …
                 if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[k])) continue;
                 alpha->data.F32[J][K] += weight * deriv->data.F32[k];
                 K++;
+                K++;
+            }
             beta->data.F32[J] += weight * delta;
             J++;
+            J++;
+        }
+    }
 …
 between that function at the specified coords and the specified value at those
 coords.
 This requires F32 input data; all internal calls use F32.
 XXX Make an F64 version?
 …
     // Alpha & Beta only contain elements to represent the unmasked parameters
     if (!psMinLM_AllocAB (&Alpha, &Beta, params, paramMask)) {
         psAbort ("programming error: no unmasked parameters to be fit\n");
+    }
+        psAbort ("programming error: no unmasked parameters to be fit\n");
+    }
     psImage *alpha   = psImageAlloc(Alpha->numCols, Alpha->numRows, PS_TYPE_F32);
     psVector *beta   = psVectorAlloc(Beta->n, PS_TYPE_F32);
 …
             psTrace ("psLib.math", 5, "failure to calculate covariance matrix\n");
+        }
         // set covar values which are not masked
         psImageInit (covar, 0.0);
         for (int j = 0, J = 0; j < params->n; j++) {
             if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[j])) {
                 covar->data.F32[j][j] = 1.0;
                 continue;
+            }
             for (int k = 0, K = 0; k < params->n; k++) {
                 if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[k])) continue;
                 covar->data.F32[j][k] = Alpha->data.F32[J][K];
                 K++;
+            }
             J++;
+        }
+        // set covar values which are not masked
+        psImageInit (covar, 0.0);
+        for (int j = 0, J = 0; j < params->n; j++) {
+            if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[j])) {
+                covar->data.F32[j][j] = 1.0;
+                continue;
+            }
+            for (int k = 0, K = 0; k < params->n; k++) {
+                if (paramMask && (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[k])) continue;
+                covar->data.F32[j][k] = Alpha->data.F32[J][K];
+                K++;
+            }
+            J++;
+        }
+    }
 …
     // count unmasked parameters
     if (paramMask) {
         nParams = 0;
         for (int i = 0; i < paramMask->n; i++) {
             if (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) continue;
             nParams ++;
+        }
+    }
     if (nParams == 0) {
         return false;
+        nParams = 0;
+        for (int i = 0; i < paramMask->n; i++) {
+            if (paramMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) continue;
+            nParams ++;
+        }
+    }
+    if (nParams == 0) {
+        return false;
+    }
 …
+}
 psMinConstraint* psMinConstraintAlloc()
+psMinConstraint* psMinConstraintAlloc(void)
+{
     psMinConstraint *tmp = psAlloc(sizeof(psMinConstraint));

branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psMinimizeLMM.h

-              r15046
+              r23594
 psMinConstraint;
 psMinConstraint *psMinConstraintAlloc() PS_ATTR_MALLOC;
+psMinConstraint *psMinConstraintAlloc(void) PS_ATTR_MALLOC;
 /** Allocates a psMinimization structure.
 …
     psF32 lambda);
 // allocate alpha and beta for unmasked parameters only
+// allocate alpha and beta for unmasked parameters only
 bool psMinLM_AllocAB (psImage **Alpha, psVector **Beta, const psVector *params, const psVector *paramMask);

branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psSparse.c

-              r19501
+              r23594
 psSparseBorder *psSparseBorderAlloc(psSparse *sparse, int Nborder)
+{
+    psAssert(sparse->Nrows > 0, "Require positive size: nrows = %d", sparse->Nrows);
+    psAssert(Nborder > 0, "Require positive size: nborder = %d", Nborder);
     psSparseBorder *border = (psSparseBorder *)psAlloc(sizeof(psSparseBorder));
     psMemSetDeallocator(border, (psFreeFunc) psSparseBorderFree);
 …
         square = psImageCopy (square, border->Tjj, PS_TYPE_F64);
         if (!psMatrixGJSolve (square, Go)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to solve for lower square.");
             psFree (dG);
             psFree (Go);
             psFree (dF);
             psFree (Fo);
             psFree (square);
             return false;
+        }
+            psError(PS_ERR_UNKNOWN, false, "Unable to solve for lower square.");
+            psFree (dG);
+            psFree (Go);
+            psFree (dF);
+            psFree (Fo);
+            psFree (square);
+            return false;
+        }
         yVec = psVectorCopy (yVec, Go, PS_TYPE_F32);

branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psSpline.c

-              r12434
+              r23594
 tabulated function at n points, this routine calculates the second
 derivatives of the interpolating cubic splines at those n points.
 The first and second derivatives at the endpoints were previously undefined in
 the SDR.  From bugzilla #???, they are required to be 0.0, implementing natural
 splines.
 Endpoints are defined by
     PS_LEFT_SPLINE_DERIV
     PS_RIGHT_SPLINE_DERIV
 This routine assumes that vectors x and y are of the appropriate types/sizes
 (F32).
 XXX: use recycled vectors for internal data.
 XXX: do an F64 version?
 …
+}
 psSpline1D *psSpline1DAlloc()
+psSpline1D *psSpline1DAlloc(void)
+{
     psSpline1D *tmpSpline = (psSpline1D *) psAlloc(sizeof(psSpline1D));
 …
 psVectorFitSpline1D(): given a set of x/y vectors, this routine generates the
 linear or cublic splines which satisfy those data points.
 The formula for calculating the spline polynomials is derived from Numerical
 Recipes in C.  The basic idea is that the polynomial is
 …
 into a polynomial in terms of x, and then saving the coefficients of the
 powers of x in the spline polynomials.  This gets pretty complicated.
 XXX: What types must be supported?
  *****************************************************************************/
 …
 (vectorBinDisectF32()).  Then it evaluates the spline at that x location
 by a call to the 1D polynomial functions.
 XXX: The spline eval functions require input and output to be F32.  however
      the spline fit functions require F32 and F64.
 XXX: This only works if spline->knots if psF32.  Must we add support for psU32 and
 psF64?

branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psSpline.h

r14452	r23594
46	46	* @return psSpline1D* new 1-D spline struct
47	47	*/
48		psSpline1D *psSpline1DAlloc() PS_ATTR_MALLOC;
	48	psSpline1D *psSpline1DAlloc(void) PS_ATTR_MALLOC;
49	49
50	50	/** Evaluates 1-D spline polynomials at a specific coordinate.

branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psStats.c

-              r23352
+              r23594
     for (long i = 0; i < num; i++) {
         // Check if the data is with the specified range
+        if (maskData && (maskData[i] & maskVal)) {
+        if (!isfinite(vector[i]))
+            continue;
+        if (useRange && (vector[i] < stats->min))
             continue;
+        }
+        if (useRange) {
+            if (vector[i] < stats->min || vector[i] > stats->max) {
+                continue;
+            }
+        } else if (!isfinite(vector[i])) {
+            stats->max = stats->min = NAN;
+            psError(PS_ERR_IEEE, true, "Element %ld of vector is Inf/NaN", i);
+            return 0;
+        }
+        if (useRange && (vector[i] > stats->max))
+            continue;
+        if (maskData && (maskData[i] & maskVal))
+            continue;
         numValid++;

branches/cnb_branches/cnb_branch_20090301/psLib/src/mathtypes/psImage.c

-              r21347
+              r23594
         // drop my entry on my parent's array of children
         // lock parent before freeing child : psArrayRemoveDataNoFree is NOT thread safe
         psMutexLock (parent);
+        // lock parent before freeing child : psArrayRemoveDataNoFree is NOT thread safe
+        psMutexLock (parent);
         psArrayRemoveDataNoFree (parent->children, image);
         psMutexUnlock (parent);
+        psMutexUnlock (parent);
         // drop my reference to my parent
 …
     int rowSize = numCols * elementSize;        // row size in bytes.
+    if (numRows < 1) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                _("Specified number of rows (%d) or columns (%d) is invalid."),
+                numRows, numCols);
+        return NULL;
+    }
+    if (numCols < 1) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                _("Specified number of rows (%d) or columns (%d) is invalid."),
+                numRows, numCols);
+        return NULL;
+    if (numRows < 1 || numCols < 1) {
+        psAbort(_("Specified number of rows (%d) or columns (%d) is invalid."), numRows, numCols);
+    }

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psError.c

-              r19034
+              r23594
 psErrorCode p_psErrorV(const char* filename,
                        unsigned int lineno,
                        const char* func,
                        psErrorCode code,
                        bool new,
                        const char* format,
                        va_list ap)
+                       unsigned int lineno,
+                       const char* func,
+                       psErrorCode code,
+                       bool new,
+                       const char* format,
+                       va_list ap)
+{
   char errMsg[MAX_STRING_LENGTH];
 …
+}
 long psErrorGetStackSize()
+long psErrorGetStackSize(void)
+{
     psArray *errorStack = psErrorStackGet();

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psError.h

r19034	r23594
87	87	* @return int The number of items on the error stack
88	88	*/
89		long psErrorGetStackSize();
	89	long psErrorGetStackSize(void);
90	90
91	91

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psLogMsg.c

r23352	r23594
76	76	}
77	77
78		int psLogGetLevel()
	78	int psLogGetLevel(void)
79	79	{
80	80	return globalLogLevel;

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psLogMsg.h

-              r14452
+              r23594
  *  @return int:        The current file descriptor.
  */
 int psLogGetDestination();
+int psLogGetDestination(void);
 …
  *  @return int:        The current logging level.
  */
 int psLogGetLevel();
+int psLogGetLevel(void);
 /** This procedure sets the log format for future log messages.  The argument

branches/cnb_branches/cnb_branch_20090301/psModules

Property svn:mergeinfo changed
/trunk/psModules merged: 23354,23358,23370,23374,23378,23404-23405,23428,23432,23447,23450-23452,23476,23487,23498,23535,23574,23576-23577,23587

branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryDistortion.c

-              r21183
+              r23594
+}
 pmAstromGradient *pmAstromGradientAlloc ()
+pmAstromGradient *pmAstromGradientAlloc (void)
+{
 …
             grad = pmAstromGradientAlloc ();
             // XXX psTraceSetLevel("psLib.math.psVectorClipFitPolynomial2D", 7);
+            // XXX psTraceSetLevel("psLib.math.psVectorClipFitPolynomial2D", 7);
             // fit the collection of positions and offsets with a local 1st order gradient
 …
             grad->dTPdM.x = local->coeff[0][1];
             // XXX psTraceSetLevel("psLib.math.psVectorClipFitPolynomial2D", 0);
+            // XXX psTraceSetLevel("psLib.math.psVectorClipFitPolynomial2D", 0);
             // fit the collection of positions and offsets with a local 1st order gradient

branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryDistortion.h

r12486	r23594
27	27	pmAstromGradient;
28	28
29		pmAstromGradient *pmAstromGradientAlloc ();
	29	pmAstromGradient *pmAstromGradientAlloc (void);
30	30
31	31	/* The following function determines the position residual, in the tangent

branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryVisual.c

r23351	r23594
47	47	/* Initialization Routines */
48	48
49		bool pmAstromVisualClose()
	49	bool pmAstromVisualClose(void)
50	50	{
51	51	if(kapa != -1)

branches/cnb_branches/cnb_branch_20090301/psModules/src/astrom/pmAstrometryVisual.h

r23351	r23594
26	26	/** Close plotting windows at the end of a run
27	27	* @return true for success */
28		bool pmAstromVisualClose();
	28	bool pmAstromVisualClose(void);
29	29
30	30

branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPARead.c

-              r22699
+              r23594
+          }
           psFree(header);
+          bad = pmConfigMaskGet("BAD", config);
+          bad = pmConfigMaskGet("LOW", config);
+          if (!bad) {
+              // XXX look up old name for compatability
+              bad = pmConfigMaskGet("BAD", config);
+          }
           break;
+      }

branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAWrite.c

-              r23351
+              r23594
     if (writeBlank || writeImage) {
+        pmConceptSource source = PM_CONCEPT_SOURCE_HEADER | PM_CONCEPT_SOURCE_CELLS |
+                                 PM_CONCEPT_SOURCE_DEFAULTS | PM_CONCEPT_SOURCE_DATABASE;
+        if (!pmConceptsWriteCell(cell, source, true, config)) {
+        if (!pmConceptsWriteCell(cell, true, config)) {
             psError(PS_ERR_IO, false, "Unable to write concepts for cell.");
             return false;
 …
         if (writeBlank || writeImage) {
+            pmConceptSource source = PM_CONCEPT_SOURCE_HEADER | PM_CONCEPT_SOURCE_CELLS |
+                                     PM_CONCEPT_SOURCE_DEFAULTS | PM_CONCEPT_SOURCE_DATABASE;
+            if (!pmConceptsWriteChip(chip, source, true, true, config)) {
+            if (!pmConceptsWriteChip(chip, true, true, config)) {
                 psError(PS_ERR_IO, false, "Unable to write concepts for chip.\n");
                 return false;
 …
         if (writeBlank || writeImage) {
+            pmConceptSource source = PM_CONCEPT_SOURCE_HEADER | PM_CONCEPT_SOURCE_CELLS |
+                                     PM_CONCEPT_SOURCE_DEFAULTS | PM_CONCEPT_SOURCE_DATABASE;
+            if (!pmConceptsWriteFPA(fpa, source, true, config)) {
+            if (!pmConceptsWriteFPA(fpa, true, config)) {
                 psError(PS_ERR_IO, false, "Unable to write concepts for FPA.\n");
                 return false;

branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfile.c

-              r23351
+              r23594
 #include "pmConceptsCopy.h"
+static int fileNum = 0;                 // Number of file
 static void pmFPAfileFree(pmFPAfile *file)
+{
 …
+}
 pmFPAfile *pmFPAfileAlloc()
+pmFPAfile *pmFPAfileAlloc(void)
+{
     pmFPAfile *file = psAlloc(sizeof(pmFPAfile));
 …
     file->save = false;
+    file->index = fileNum++;
     file->imageId = 0;
 …
             psStringSubstitute(&newRule, name, "{OUTPUT}");
+        }
+    }
+    if (strstr(newRule, "{FILE.INDEX}")) {
+        // Number of the file in list
+        psString num = NULL;            // Number to use
+        psStringAppend(&num, "%d", file->index);
+        psStringSubstitute(&newRule, num, "{FILE.INDEX}");
+        psFree(num);
+    }

branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfile.h

-              r23351
+              r23594
     psString formatName;                // name of the camera format
+    int index;                          // Index of file
     psS64 imageId, sourceId;            // Image and source identifiers
 } pmFPAfile;
 // allocate an empty pmFPAfile structure
 pmFPAfile *pmFPAfileAlloc ();
+pmFPAfile *pmFPAfileAlloc(void);
 // select the readout from the named pmFPAfile; if the named file does not exist,

branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfileDefine.c

-              r23351
+              r23594
     // place the resulting file in the config system
+    psMetadataAddPtr(config->files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN, "Output file", file);
+    psMetadataAddPtr(config->files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK,
+                     "Output file", file);
     psFree(file);                       // we free this copy of file, but 'files' still has a copy
     return file;                        // the returned value is a view into the version on 'files'
 …
+}
 pmFPAfile *pmFPAfileDefineFromRun(bool *success, pmConfig *config, const char *filename)
+pmFPAfile *pmFPAfileDefineFromRun(bool *success, pmFPAfile *bind, pmConfig *config, const char *filename)
+{
     PS_ASSERT_PTR_NON_NULL(config, NULL);
 …
+    }
     pmFPAfile *file = fpaFileDefineFromArray(config, NULL, filename, filenames); // File of interest
+    pmFPAfile *file = fpaFileDefineFromArray(config, bind, filename, filenames); // File of interest
     psFree(filenames);
 …
+}
+psArray *pmFPAfileDefineMultipleFromRun(bool *success, psArray *bind, pmConfig *config, const char *filename)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
+    if (success) {
+        *success = false;
+    }
+    psArray *files = pmConfigRunFileGet(config, filename); // Filenames used, to return
+    if (!files || files->n == 0) {
+        if (success) {
+            *success = true;
+        }
+        return NULL;
+    }
+    if (bind && files->n != bind->n) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                "Length of filenames (%ld) and bind files (%ld) does not match.",
+                files->n, bind->n);
+        psFree(files);
+        return NULL;
+    }
+    psArray *dummy = psArrayAlloc(1);   // Dummy array of single filename
+    for (int i = 0; i < files->n; i++) {
+        psFree(dummy->data[0]);
+        dummy->data[0] = files->data[i];
+        pmFPAfile *bindFile = bind ? bind->data[i] : NULL; // File to which to bind
+        files->data[i] = psMemIncrRefCounter(fpaFileDefineFromArray(config, bindFile, filename, dummy));
+        if (!files->data[i]) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define file %s %d", filename, i);
+            psFree(dummy);
+            psFree(files);
+            return NULL;
+        }
+    }
+    psFree(dummy);
+    if (success) {
+        *success = true;
+    }
+    return files;
+}
 // define the named pmFPAfile from the camera->config

branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfileDefine.h

-              r23351
+              r23594
 pmFPAfile *pmFPAfileDefineFromRun(
     bool *found,                        ///< Found files?
+    pmFPAfile *bind,                    ///< File to which to bind, or NULL
     pmConfig *config,                   ///< Configuration
     const char *filename                ///< Name of file
     );
+/// Define multiple files based on the filenames in the RUN metadata in the configuration
+///
+/// An array of the files defined is returned
+psArray *pmFPAfileDefineMultipleFromRun(
+    bool *found,                        ///< Found files?
+    psArray *bind,                      ///< Files to which to bind, or NULL
+    pmConfig *config,                   ///< Configuration
+    const char *filename                ///< Name of file
+    );
 // look for the given argname on the argument list.  find the give filename from the file rules

branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfileFitsIO.c

-              r22699
+              r23594
+          }
-          pmConceptSource sources = PM_CONCEPT_SOURCE_HEADER | PM_CONCEPT_SOURCE_CELLS |
-              PM_CONCEPT_SOURCE_DEFAULTS | PM_CONCEPT_SOURCE_DATABASE; // Concept sources to write
           if (cell) {
               if (!pmConceptsWriteCell(cell, sources, true, config)) {
+              if (!pmConceptsWriteCell(cell, true, config)) {
                   psError(PS_ERR_IO, false, "Unable to write concepts for cell.\n");
                   return false;
+              }
           } else if (chip) {
               if (!pmConceptsWriteChip(chip, sources, true, true, config)) {
+              if (!pmConceptsWriteChip(chip, true, true, config)) {
                   psError(PS_ERR_IO, false, "Unable to write concepts for chip.\n");
                   return false;
+              }
           } else if (!pmConceptsWriteFPA(fpa, sources, true, config)) {
+          } else if (!pmConceptsWriteFPA(fpa, true, config)) {
               psError(PS_ERR_IO, false, "Unable to write concepts for FPA.\n");
               return false;

branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfileIO.c

-              r23351
+              r23594
+    }
     if (!pmConfigRunFileAdd(config, file)) {
+    if (!pmConfigRunFileAddRead(config, file)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to add file to run-time information");
         return false;
 …
+    }
     if (!pmConfigRunFileAdd(config, file)) {
+    if (!pmConfigRunFileAddWrite(config, file)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to add file to run-time information");
         return false;
 …
+}
+psString pmFPAfileName(const pmFPAfile *file, const pmFPAview *view, pmConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(file, NULL);
+    PS_ASSERT_PTR_NON_NULL(view, NULL);
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+    psString filename = pmFPAfileNameFromRule(file->filerule, file, view); // Filename, based on rule
+    if (!filename) {
+        psError(PS_ERR_IO, true, "Cannot determine file name from rule");
+        return false;
+    }
+    // indirect filenames: these come from a list on the command line or elsewhere
+    if (!strcasecmp(filename, "@FILES")) {
+        psString filesrc = pmFPAfileNameFromRule(file->filesrc, file, view); // Source of file name
+        if (!filesrc) {
+            psError(PS_ERR_IO, false, "error converting filesrc to name %s", file->filesrc);
+            return false;
+        }
+        psFree(filename);
+        filename = psMemIncrRefCounter(psMetadataLookupStr(NULL, file->names, filesrc));
+        if (!filename) {
+            psError(PS_ERR_IO, false, "filename lookup error (@FILES) for %s : %s", file->filesrc, filesrc);
+            psFree(filesrc);
+            return false;
+        }
+        psFree(filesrc);
+    }
+    // get name from detrend database
+    // file->detrend->detID contains the desired -det_id detID -iteration iter string
+    if (!strcasecmp(filename, "@DETDB")) {
+        if (!file->detrend) {
+            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find information about selected detrend.");
+            return false;
+        }
+        psMetadata *menu = psMetadataLookupMetadata(NULL, file->camera, "CLASSID"); // Menu of class IDs
+        if (!menu) {
+            psError(PS_ERR_IO, false, "Unable to find CLASSID metadata in camera configuration");
+            return false;
+        }
+        const char *rule = psMetadataLookupStr(NULL, menu, file->detrend->level); // Rule for class_id
+        if (!rule || strlen(rule) == 0) {
+            psError(PS_ERR_IO, false, "Unable to find %s in CLASSID in camera configuration",
+                    file->detrend->level);
+            return false;
+        }
+        psString classId = pmFPAfileNameFromRule(rule, file, view); // The class identifier, for pmDetrendFile
+        if (!classId) {
+            psError(PS_ERR_IO, false, "error converting CLASSID rule to name: %s\n", rule);
+            return false;
+        }
+        psTrace("psModules.camera", 6, "looking for detrend (%s, %s)\n", file->detrend->detID, classId);
+        psFree(filename);
+        filename = pmDetrendFile(file->detrend->detID, classId, config);
+        if (!filename) {
+            psError(PS_ERR_IO, false, "failed to find a valid detrend image for detID %s : classID %s",
+                    file->detrend->detID, classId);
+            psFree(classId);
+            return false;
+        }
+        psTrace("psModules.camera", 6, "got detrend file %s", filename);
+        psFree(classId);
+    }
+    return filename;
+}
 // open file (if not already opened).
 // this function is only called only within pmFPAfileRead or pmFPAfileWrite.
 …
     // determine the file name, free a name allocated earlier
+    psFree (file->filename);
+    file->filename = pmFPAfileNameFromRule (file->filerule, file, view);
+    if (file->filename == NULL) {
+        psError(PS_ERR_IO, true, "Filename is NULL");
+        return false;
+    }
+    // indirect filenames: these come from a list on the command line or elsewhere
+    if (!strcasecmp (file->filename, "@FILES")) {
+        char *filesrc = pmFPAfileNameFromRule (file->filesrc, file, view);
+        if (filesrc == NULL) {
+            psError(PS_ERR_IO, false, "error converting filesrc to name %s\n", file->filesrc);
+            return false;
+        }
+        psFree (file->filename);
+        file->filename = psMetadataLookupStr (&status, file->names, filesrc);
+        if (file->filename == NULL) {
+            psError(PS_ERR_IO, true, "filename lookup error (@FILES) for %s : %s\n", file->filesrc, filesrc);
+            psFree (filesrc);
+            return false;
+        }
+        // psMetadataLookupStr just returns a view, file->filename must be protected
+        psMemIncrRefCounter (file->filename);
+        psFree (filesrc);
+    }
+    // get name from detrend database
+    // file->detrend->detID contains the desired -det_id detID -iteration iter string
+    if (!strcasecmp (file->filename, "@DETDB")) {
+        if (!file->detrend) {
+            psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find information about selected detrend.");
+            return false;
+        }
+        psString classId = NULL;        // The class identifier, to pass to pmDetrendFile
+        psMetadata *menu = psMetadataLookupMetadata(&status, file->camera, "CLASSID"); // Menu of class IDs
+        if (!status || !menu) {
+            psError(PS_ERR_IO, false, "Unable to find CLASSID metadata in camera configuration");
+            return false;
+        }
+        const char *rule = psMetadataLookupStr(&status, menu, file->detrend->level); // Rule for class_id
+        if (!status || !rule || strlen(rule) == 0) {
+            psError(PS_ERR_IO, false, "Unable to find %s in CLASSID in camera configuration", file->detrend->level);
+            return false;
+        }
+        classId = pmFPAfileNameFromRule(rule, file, view);
+        if (!classId) {
+            psError(PS_ERR_IO, false, "error converting CLASSID rule to name: %s\n", rule);
+            return false;
+        }
+        psTrace ("psModules.camera", 6, "looking for detrend (%s, %s)\n", file->detrend->detID, classId);
+        psFree (file->filename);
+        file->filename = pmDetrendFile(file->detrend->detID, classId, config);
+        if (file->filename == NULL) {
+            psError(PS_ERR_IO, false, "failed to find a valid detrend image for detID %s : classID %s\n", file->detrend->detID, classId);
+            psFree (classId);
+            return false;
+        }
+        psTrace ("psModules.camera", 6, "got detrend file %s\n", file->filename);
+        psFree (classId);
+    psFree(file->filename);
+    file->filename = pmFPAfileName(file, view, config);
+    if (!file->filename) {
+        psError(PS_ERR_IO, true, "Unable to determine filename");
+        return false;
+    }

branches/cnb_branches/cnb_branch_20090301/psModules/src/camera/pmFPAfileIO.h

-              r18364
+              r23594
 /// @addtogroup Camera Camera Layout
 /// @{
+// Determine appropriate file name
+psString pmFPAfileName(const pmFPAfile *file, const pmFPAview *view, pmConfig *config);
 // open the real file corresponding to the given pmFPAfile appropriate to the current view

branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConceptsStandard.c

-              r23351
+              r23594
+}
+// Get the current value of a concept
+static psMetadataItem *conceptGet(const pmFPA *fpa, // FPA of interest
+                                  const pmChip *chip, // Chip of interest, or NULL
+                                  const pmCell *cell, // Cell of interest, or NULL
+                                  const char *name // Concept name
+    )
+{
+    psMetadataItem *item = NULL;        // Item with time system
+    if (cell) {
+        item = psMetadataLookup(cell->concepts, name);
+    }
+    if (!item && chip) {
+        item = psMetadataLookup(chip->concepts, name);
+    }
+    if (!item && fpa) {
+        item = psMetadataLookup(fpa->concepts, name);
+    }
+    if (!item) {
+        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find %s in concepts", name);
+        return NULL;
+    }
+    return item;
+}
+// Determine the corresponding TIMESYS for one of the TIME concepts
+static psTimeType conceptGetTimesysForTime(const char *name, // Concept name ("CELL.TIME" or "FPA.TIME")
+                                           const pmFPA *fpa, // FPA of interest
+                                           const pmChip *chip, // Chip of interest, or NULL
+                                           const pmCell *cell // Cell of interest, or NULL
+                                           )
+{
+    assert(name);
+    psString timesysName = psStringCopy(name); // e.g., "CELL.TIME" --> "CELL.TIMESYS"
+    psStringSubstitute(&timesysName, "TIMESYS", "TIME");
+    psMetadataItem *item = conceptGet(fpa, chip, cell, timesysName); // Time system
+    psFree(timesysName);
+    if (!item || item->type != PS_TYPE_S32) {
+        psWarning("Unable to find %s --- assuming UTC", timesysName);
+        return PS_TIME_UTC;
+    }
+    return item->data.S32;
+}
+// Set the corresponding TIMESYS for one of the TIME concepts
+static bool conceptSetTimesysForTime(const char *name, // Concept name ("CELL.TIME" or "FPA.TIME")
+                                     const pmFPA *fpa, // FPA of interest
+                                     const pmChip *chip, // Chip of interest, or NULL
+                                     const pmCell *cell, // Cell of interest, or NULL
+                                     psTimeType timeSys // The time system value
+                                     )
+{
+    assert(name);
+    psString timesysName = psStringCopy(name); // e.g., "CELL.TIME" --> "CELL.TIMESYS"
+    psStringSubstitute(&timesysName, "TIMESYS", "TIME");
+    psMetadataItem *item = conceptGet(fpa, chip, cell, timesysName); // Time system
+    psFree(timesysName);
+    if (!item) {
+        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find %s in concepts when setting %s\n",
+                timesysName, name);
+        return false;
+    }
+    if (item->data.S32 != -1 && item->data.S32 != timeSys) {
+        psWarning("Time system is set to %x; but should be %x", item->data.S32, timeSys);
+    }
+    item->data.S32 = timeSys;
+    return true;
+}
 psMetadataItem *p_pmConceptParse_TIMESYS(const psMetadataItem *concept,
 …
     if (concept->type != PS_DATA_STRING || strlen(sys) <= 0) {
         // XXX is this too low verbosity?
         psLogMsg ("psModules.concepts", PS_LOG_DETAIL, "Can't interpret %s --- assuming UTC.\n", pattern->name);
+        psWarning("Can't interpret %s --- assuming UTC.", pattern->name);
     } else if (strcasecmp(sys, "TAI") == 0) {
         timeSys = PS_TIME_TAI;
 …
     } else {
         // XXX is this too low verbosity?
+        psLogMsg ("psModules.concepts", PS_LOG_DETAIL, "Can't interpret %s --- assuming UTC.\n", pattern->name);
+        psWarning("Can't interpret %s --- assuming UTC.", pattern->name);
+    }
+    psMetadataItem *old = conceptGet(fpa, chip, cell, pattern->name); // Old value
+    if (old && old->data.S32 != -1 && old->data.S32 != timeSys) {
+        psWarning("%s is already set (%x) and not consistent with new value (%x)",
+                  pattern->name, old->data.S32, timeSys);
+    }
 …
+}
+// Determine the corresponding TIMESYS for one of the TIME concepts
+static psTimeType conceptGetTimesysForTime(const char *name, // Concept name ("CELL.TIME" or "FPA.TIME")
+                                           const pmFPA *fpa, // FPA of interest
+                                           const pmChip *chip, // Chip of interest, or NULL
+                                           const pmCell *cell // Cell of interest, or NULL
+                                           )
+{
+    assert(name);
+    psString timesysName = psStringCopy(name); // e.g., "CELL.TIME" --> "CELL.TIMESYS"
+    psStringSubstitute(&timesysName, "TIMESYS", "TIME");
+    bool mdok = false;                  // Result of MD lookup
+    psTimeType timeSys = 0xFFFFFFFF;    // The time system
+    if (cell) {
+        timeSys = psMetadataLookupS32(&mdok, cell->concepts, timesysName);
+    }
+    if (!mdok && chip) {
+        timeSys = psMetadataLookupS32(&mdok, chip->concepts, timesysName);
+    }
+    if (!mdok && fpa) {
+        timeSys = psMetadataLookupS32(&mdok, fpa->concepts, timesysName);
+    }
+    if (!mdok || (timeSys == 0xFFFFFFFF)) {
+        psWarning("Unable to find %s in concepts when parsing %s --- assuming UTC.\n",
+                  timesysName, name);
+        timeSys = PS_TIME_UTC;
+    }
+    psFree(timesysName);
+    return timeSys;
+}
 psMetadataItem *p_pmConceptParse_TIME(const psMetadataItem *concept,
 …
+    }
+    time->type = timeSys;
+    if (jdTime || mjdTime) {
+        conceptSetTimesysForTime(pattern->name, fpa, chip, cell, PS_TIME_TAI);
+    } else {
+        time->type = timeSys;
+    }
     psMetadataItem *item = psMetadataItemAllocPtr(pattern->name, PS_DATA_TIME, pattern->comment, time);

branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConceptsWrite.c

-              r22699
+              r23594
         while ((cItem = psListGetAndIncrement(cIter))) {
             if (cItem->type != PS_DATA_STRING) {
+                psWarning("psMetadataItem from list is of type %x instead of "
+                         "%x (PS_DATA_STRING) --- can't interpret.\n", cItem->type, PS_DATA_STRING);
+                psWarning("psMetadataItem from list is of type %x instead of %x (PS_DATA_STRING) --- can't interpret.\n", cItem->type, PS_DATA_STRING);
                 psFree(cIter);
                 psFree(sIter);
 …
+        }
     default:
         psWarning("Type of %s is not suitable for a FITS header --- not added.\n",
+      psWarning("Type of %s is not suitable for a FITS header --- not added.\n",
                  item->name);
         return false;
 …
+}
+// Return the camera format appropriate for a focal plane hierarchy
+static psMetadata *conceptsCameraFormat(const pmFPA *fpa, const pmChip *chip, const pmCell *cell)
+{
+    pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // The HDU at the lowest level
+    if (!hdu) {
+        return NULL;
+    }
+    return hdu->format;
+}
+// Return the DATABASE metadata from the format
+static psMetadata *conceptsDatabase(const psMetadata *format)
+{
+    bool mdok;                          // Status of MD lookup
+    return psMetadataLookupMetadata(&mdok, format, "DEFAULTS");
+}
+// Return the TRANSLATION metadata from the format
+static psMetadata *conceptsTranslation(const psMetadata *format)
+{
+    bool mdok;                          // Status of MD lookup
+    return psMetadataLookupMetadata(&mdok, format, "TRANSLATION");
+}
+// Return the DEFAULTS metadata from the format
+static psMetadata *conceptsDefaults(const psMetadata *format)
+{
+    bool mdok;                          // Status of MD lookup
+    return psMetadataLookupMetadata(&mdok, format, "DEFAULTS");
+}
 //////////////////////////////////////////////////////////////////////////////////////////////////////////////
 …
 //////////////////////////////////////////////////////////////////////////////////////////////////////////////
+bool p_pmConceptsWriteToCells(const psMetadata *specs, const pmCell *cell, const psMetadata *concepts)
+{
+    PS_ASSERT_PTR_NON_NULL(specs, false);
+    PS_ASSERT_PTR_NON_NULL(concepts, false);
+bool p_pmConceptWriteToCells(const pmCell *cell, const pmConceptSpec *spec,
+                             const psMetadataItem *conceptItem, const psMetadata *format)
+{
     if (!cell) {
         return false;
 …
+    }
+    pmHDU *hdu = pmHDUGetLowest(NULL, NULL, cell); // The HDU at the lowest level
+    if (!format) {
+        format = conceptsCameraFormat(NULL, NULL, cell);
+        if (!format) {
+            return false;
+        }
+    }
+    psMetadataItem *cameraItem = psMetadataLookup(cell->config, conceptItem->name); // Version in the config
+    if (!cameraItem) {
+        return false;
+    }
+    psString nameSource = NULL; // String with the concept name and ".SOURCE" added
+    psStringAppend(&nameSource, "%s.SOURCE", conceptItem->name);
+    bool mdok = true;       // Status of MD lookup
+    psString source = psMetadataLookupStr(&mdok, cell->config, nameSource); // The source
+    if (mdok && strlen(source) > 0) {
+        psTrace("psModules.concepts", 8, "%s is %s\n", nameSource, source);
+        if (strcasecmp(source, "HEADER") == 0) {
+            if (cameraItem->type != PS_DATA_STRING) {
+                psWarning("Concept %s is specified by header, but is not of type STR --- ignored.",
+                          conceptItem->name);
+                psFree(nameSource);
+                return false;
+            }
+            // Formatted version
+            psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_HEADER,
+                                                      format, NULL, NULL, cell);
+            if (!formatted) {
+                psFree(nameSource);
+                return true;
+            }
+            psTrace("psModules.concepts", 8, "Writing %s to header %s\n",
+                    conceptItem->name, cameraItem->data.str);
+            pmHDU *hdu = pmHDUGetLowest(NULL, NULL, cell); // Header data unit
+            if (!hdu) {
+                psError(PS_ERR_UNEXPECTED_NULL, false,
+                        "Unable to find HDU to write concept %s", conceptItem->name);
+                return false;
+            }
+            writeHeader(hdu, cameraItem->data.V, formatted);
+            psFree(formatted);
+        } else if (strcasecmp(source, "VALUE") == 0) {
+            // Formatted version
+            psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_CELLS,
+                                                      format, NULL, NULL, cell);
+            if (!formatted) {
+                psFree(nameSource);
+                return true;
+            }
+            psTrace("psModules.concepts", 8, "Checking %s against camera format.\n", conceptItem->name);
+            if (!compareConcepts(formatted, cameraItem)) {
+                psWarning("Concept %s is specified by value in the camera format, but the values don't match",
+                          conceptItem->name);
+            }
+            psFree(formatted);
+        } else {
+            psWarning("Concept source %s isn't HEADER or VALUE --- can't write", nameSource);
+        }
+    } else {
+        // Assume it's specified by value
+        psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_CELLS,
+                                                  format, NULL, NULL, cell);
+        if (!formatted) {
+            psFree(nameSource);
+            return true;
+        }
+        if (!compareConcepts(formatted, cameraItem)) {
+            psWarning("Concept %s is specified by value in the camera format, but the values don't match.",
+                      conceptItem->name);
+        }
+        psFree(formatted);
+    }
+    psFree(nameSource);
+    return true;
+}
+bool p_pmConceptWriteToDefaults(const pmFPA *fpa, const pmChip *chip, const pmCell *cell,
+                                const pmConceptSpec *spec, const psMetadataItem *conceptItem,
+                                const psMetadata *format, const psMetadata *defaults)
+{
+    if (!format) {
+        format = conceptsCameraFormat(fpa, chip, cell);
+        if (!format) {
+            return false;
+        }
+    }
+    if (!defaults) {
+        defaults = conceptsDefaults(format);
+        if (!defaults) {
+            return false;
+        }
+    }
+    psMetadataItem *defaultItem = p_pmConceptsReadSingleFromDefaults(conceptItem->name, defaults,
+                                                                     fpa, chip, cell);
+    if (!defaultItem) {
+        return false;
+    }
+    psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_DEFAULTS,
+                                              format, fpa, chip, cell);
+    if (!formatted) {
+        return true;
+    }
+    if (strcmp(defaultItem->name, conceptItem->name) != 0) {
+        // Correct the name to match the concept name
+        defaultItem = psMetadataItemCopy(defaultItem);
+        psFree(defaultItem->name);
+        defaultItem->name = psStringCopy(conceptItem->name);
+    } else {
+        psMemIncrRefCounter(defaultItem);
+    }
+    if (!compareConcepts(formatted, defaultItem)) {
+        psWarning("Concept %s is specified by the DEFAULTS in the camera format, but the values don't match.",
+                  conceptItem->name);
+    }
+    psFree(defaultItem);
+    psFree(formatted);
+    return true;
+}
+bool p_pmConceptWriteToHeader(const pmFPA *fpa, const pmChip *chip, const pmCell *cell,
+                              const pmConceptSpec *spec, const psMetadataItem *conceptItem,
+                              const psMetadata *format, const psMetadata *translation)
+{
+    if (!format) {
+        format = conceptsCameraFormat(fpa, chip, cell);
+        if (!format) {
+            return false;
+        }
+    }
+    if (!translation) {
+        translation = conceptsTranslation(format);
+        if (!translation) {
+            return false;
+        }
+    }
+    psMetadataItem *headerItem = psMetadataLookup(translation, conceptItem->name); // How to format for header
+    if (!headerItem) {
+        return false;
+    }
+    if (headerItem->type == PS_DATA_METADATA) {
+        // This is a menu
+        psTrace("psModules.concepts", 5, "%s is of type METADATA.\n", conceptItem->name);
+        headerItem = p_pmConceptsDepend(conceptItem->name, headerItem->data.md, translation, fpa, chip, cell);
+        if (!headerItem) {
+            return false;
+        }
+    }
+    if (headerItem->type != PS_DATA_STRING) {
+        psWarning("TRANSLATION keyword for concept %s isn't of type STR --- ignored.", conceptItem->name);
+        return false;
+    }
+    psTrace("psModules.concepts", 3, "Writing %s to header %s\n", conceptItem->name, headerItem->data.str);
+    psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_HEADER,
+                                              format, fpa, chip, cell);
+    if (!formatted) {
+        // Found it, but it doesn't need to be written
+        return true;
+    }
+    pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // HDU to which to write
     if (!hdu) {
+        return false;
+    }
+    psMetadata *cameraFormat = hdu->format; // The camera format
+    psMetadataIterator *specsIter = psMetadataIteratorAlloc(specs, PS_LIST_HEAD, NULL); // Iterator
+    psMetadataItem *specItem = NULL;    // Item from the specs metadata
+    while ((specItem = psMetadataGetAndIncrement(specsIter))) {
+        pmConceptSpec *spec = specItem->data.V; // The specification
+        psString name = specItem->name; // The concept name
+        psMetadataItem *cameraItem = psMetadataLookup(cell->config, name); // The concept from the camera,
+        // or NULL
+        if (cameraItem) {
+            // Grab the concept
+            psMetadataItem *conceptItem = psMetadataLookup(concepts, name); // The concept
+            psString nameSource = NULL; // String with the concept name and ".SOURCE" added
+            psStringAppend(&nameSource, "%s.SOURCE", name);
+            bool mdok = true;       // Status of MD lookup
+            psString source = psMetadataLookupStr(&mdok, cell->config, nameSource); // The source
+            if (mdok && strlen(source) > 0) {
+                psTrace("psModules.concepts", 8, "%s is %s\n", nameSource, source);
+                if (strcasecmp(source, "HEADER") == 0) {
+                    if (cameraItem->type != PS_DATA_STRING) {
+                        psWarning("Concept %s is specified by header, but is not "
+                                 "of type STR --- ignored.\n", conceptItem->name);
+                        continue;
+                    }
+                    // Formatted version
+                    psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_HEADER,
+                                                              cameraFormat, NULL, NULL, cell);
+                    if (!formatted) {
+                        continue;
+                    }
+                    psTrace("psModules.concepts", 8, "Writing %s to header %s\n", name, cameraItem->data.str);
+                    writeHeader(hdu, cameraItem->data.V, formatted);
+                    psFree(formatted);
+                } else if (strcasecmp(source, "VALUE") == 0) {
+                    // Formatted version
+                    psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_CELLS,
+                                                              cameraFormat, NULL, NULL, cell);
+                    if (!formatted) {
+                        continue;
+                    }
+                    psTrace("psModules.concepts", 8, "Checking %s against camera format.\n", name);
+                    if (! compareConcepts(formatted, cameraItem)) {
+                        psWarning("Concept %s is specified by value in the camera "
+                                 "format, but the values don't match.\n", name);
+                    }
+                    psFree(formatted);
+                } else {
+                    psWarning("Concept source %s isn't HEADER or VALUE --- can't "
+                             "write\n", nameSource);
+                }
+            } else {
+                // Assume it's specified by value
+                psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_CELLS,
+                                                          cameraFormat, NULL, NULL, cell);
+                if (!formatted) {
+                    continue;
+                }
+                if (! compareConcepts(formatted, cameraItem)) {
+                    psWarning("Concept %s is specified by value in the camera "
+                             "format, but the values don't match.\n", name);
+                }
+                psFree(formatted);
+            }
+            psFree(nameSource);
+        }
+    }
+    psFree(specsIter);
+        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find HDU to write concept %s", conceptItem->name);
+        return false;
+    }
+    writeHeader(hdu, headerItem->data.V, formatted);
+    psFree(formatted);
     return true;
+}
+bool p_pmConceptsWriteToDefaults(const psMetadata *specs, const pmFPA *fpa, const pmChip *chip,
+                                 const pmCell *cell, const psMetadata *concepts)
+{
+    PS_ASSERT_PTR_NON_NULL(specs, false);
+    PS_ASSERT_PTR_NON_NULL(concepts, false);
+    pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // The HDU at the lowest level
+    if (!hdu) {
+        return false;
+    }
+    psMetadata *cameraFormat = hdu->format; // The camera format
+    bool mdok = true;                   // Status of MD lookup
+    psMetadata *defaults = psMetadataLookupMetadata(&mdok, cameraFormat, "DEFAULTS"); // The DEFAULTS spec
+    if (!mdok || !defaults) {
+        return false;
+    }
+    psMetadataIterator *specsIter = psMetadataIteratorAlloc(specs, PS_LIST_HEAD, NULL); // Iterator
+    psMetadataItem *specItem = NULL;    // Item from the specs metadata
+    while ((specItem = psMetadataGetAndIncrement(specsIter))) {
+        pmConceptSpec *spec = specItem->data.V; // The specification
+        psString name = specItem->name; // The concept name
+        psMetadataItem *defaultItem = p_pmConceptsReadSingleFromDefaults(name, defaults, fpa, chip, cell);
+        if (!defaultItem) {
+            continue;
+        }
+        psMetadataItem *conceptItem = psMetadataLookup(concepts, name); // The item from the concepts
+        psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_DEFAULTS,
+                                                  cameraFormat, fpa, chip, cell);
+        if (!formatted) {
+            continue;
+        }
+        if (strcmp(defaultItem->name, name) != 0) {
+            // Correct the name to match the concept name
+            defaultItem = psMetadataItemCopy(defaultItem);
+            psFree(defaultItem->name);
+            defaultItem->name = psStringCopy(name);
+        } else {
+            psMemIncrRefCounter(defaultItem);
+        }
+        if (!compareConcepts(formatted, defaultItem)) {
+            psWarning("Concept %s is specified by the DEFAULTS in the camera "
+                      "format, but the values don't match.\n", name);
+        }
+        psFree(defaultItem);
+        psFree(formatted);
+    }
+    psFree(specsIter);
+bool p_pmConceptWriteToDatabase(const pmFPA *fpa, const pmChip *chip, const pmCell *cell,
+                                pmConfig *config, const pmConceptSpec *spec,
+                                const psMetadataItem *conceptItem, const psMetadata *format,
+                                const psMetadata *database)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+#ifndef HAVE_PSDB
+    return false;
+#else
+    if (!format) {
+        format = conceptsCameraFormat(fpa, chip, cell);
+        if (!format) {
+            return false;
+        }
+    }
+    if (!database) {
+        database = conceptsDatabase(format);
+        if (!database) {
+            return false;
+        }
+    }
+    psMetadataItem *dbItem = p_pmConceptsReadSingleFromDatabase(conceptItem->name, database, config,
+                                                                fpa, chip, cell); // Database version
+    if (!dbItem) {
+        return false;
+    }
+    psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_DATABASE,
+                                              format, fpa, chip, cell);
+    if (!formatted) {
+        return false;
+    }
+    if (strcmp(dbItem->name, conceptItem->name) != 0) {
+        // Correct the name to match the concept name
+        dbItem = psMetadataItemCopy(dbItem);
+        psFree(dbItem->name);
+        dbItem->name = psStringCopy(conceptItem->name);
+    } else {
+        psMemIncrRefCounter(dbItem);
+    }
+    if (!compareConcepts(formatted, dbItem)) {
+        psWarning("Concept %s is specified by the DATABASE in the camera "
+                  "format, but the values don't match.\n", conceptItem->name);
+    }
+    psFree(dbItem);
+    psFree(formatted);
     return true;
+}
+bool p_pmConceptsWriteToHeader(const psMetadata *specs, const pmFPA *fpa, const pmChip *chip,
+                               const pmCell *cell, const psMetadata *concepts)
+{
+    PS_ASSERT_PTR_NON_NULL(specs, false);
+    PS_ASSERT_PTR_NON_NULL(concepts, false);
+    pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // The HDU at the lowest level
+    if (!hdu) {
+        return false;
+    }
+    psMetadata *cameraFormat = hdu->format; // The camera format
+    bool mdok = true;                   // Status of MD lookup
+    psMetadata *translation = psMetadataLookupMetadata(&mdok, cameraFormat, "TRANSLATION"); // The TRANSLATION spec
+    if (!mdok || !translation) {
+        return false;
+    }
+    psMetadataIterator *specsIter = psMetadataIteratorAlloc(specs, PS_LIST_HEAD, NULL); // Iterator
+    psMetadataItem *specItem = NULL;    // Item from the specs metadata
+    while ((specItem = psMetadataGetAndIncrement(specsIter))) {
+        pmConceptSpec *spec = specItem->data.V; // The specification
+        psString name = specItem->name; // The concept name
+        psMetadataItem *headerItem = psMetadataLookup(translation, name); // The item from the TRANSLATION
+        if (!headerItem) {
+            continue;
+        }
+        if (headerItem->type == PS_DATA_METADATA) {
+            // This is a menu
+            psTrace("psModules.concepts", 5, "%s is of type METADATA.\n", name);
+            headerItem = p_pmConceptsDepend(name, headerItem->data.md, translation, fpa, chip, cell);
+            if (!headerItem) {
+                continue;
+            }
+        }
+        if (headerItem->type != PS_DATA_STRING) {
+            psWarning("TRANSLATION keyword for concept %s isn't of type STR --- ignored.", name);
+            continue;
+        }
+        psTrace("psModules.concepts", 3, "Writing %s to header %s\n", name, headerItem->data.str);
+        psMetadataItem *conceptItem = psMetadataLookup(concepts, name); // The item from the concepts
+        psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_HEADER,
+                                                  cameraFormat, fpa, chip, cell);
+        if (!formatted) {
+            continue;
+        }
+        writeHeader(hdu, headerItem->data.V, formatted);
+        psFree(formatted);
+    }
+    psFree(specsIter);
+#endif
+}
+bool pmConceptWriteSingle(const pmFPA *fpa, const pmChip *chip, const pmCell *cell,
+                          pmConfig *config, const psMetadataItem *conceptItem)
+{
+    pmConceptsInit();
+    psMetadata *format = conceptsCameraFormat(fpa, chip, cell); // Camera format
+    if (!format) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to retrieve camera format.");
+        return false;
+    }
+    const char *name = conceptItem->name; // Name of concept
+    psMetadata *conceptsFPA = pmConceptsSpecs(PM_FPA_LEVEL_FPA); // Concept specifications for FPA
+    bool mdok;                          // Status of MD lookup
+    pmConceptSpec *spec = psMetadataLookupPtr(&mdok, conceptsFPA, name); // Concept specification of interest
+    if (!spec) {
+        psMetadata *conceptsChip = pmConceptsSpecs(PM_FPA_LEVEL_CHIP); // Concept specifications for Chip
+        spec = psMetadataLookupPtr(&mdok, conceptsChip, name);
+        if (!spec) {
+            psMetadata *conceptsCell = pmConceptsSpecs(PM_FPA_LEVEL_CELL); // Concept specifications for Cell
+            spec = psMetadataLookupPtr(&mdok, conceptsCell, name);
+            if (!spec) {
+                psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find specification for concept %s", name);
+                return false;
+            }
+        }
+    }
+    if (!p_pmConceptWriteToCells(cell, spec, conceptItem, format) &&
+        !p_pmConceptWriteToDefaults(fpa, chip, cell, spec, conceptItem, format, NULL) &&
+        !p_pmConceptWriteToHeader(fpa, chip, cell, spec, conceptItem, format, NULL) &&
+        !p_pmConceptWriteToDatabase(fpa, chip, cell, config, spec, conceptItem, format, NULL)) {
+        return false;
+    }
     return true;
+}
-// XXX Warning: This code has not been tested at all
-bool p_pmConceptsWriteToDatabase(const psMetadata *specs, const pmFPA *fpa, const pmChip *chip,
-                                 const pmCell *cell, pmConfig *config, const psMetadata *concepts)
+{
-    PS_ASSERT_PTR_NON_NULL(specs, false);
-    PS_ASSERT_PTR_NON_NULL(concepts, false);
-    PS_ASSERT_PTR_NON_NULL(config, false);
-    #ifndef HAVE_PSDB
-    return false;
-    #else
-    pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // The HDU at the lowest level
-    if (!hdu) {
-        return false;
+    }
-    psMetadata *cameraFormat = hdu->format; // The camera format
-    bool mdok = true;                   // Status of MD lookup
-    psMetadata *database = psMetadataLookupMetadata(&mdok, cameraFormat, "DATABASE"); // The DATABASE spec
-    if (!mdok || !database) {
-        return false;
+    }
-    psMetadataIterator *specsIter = psMetadataIteratorAlloc(specs, PS_LIST_HEAD, NULL); // Iterator
-    psMetadataItem *specItem = NULL;    // Item from the specs metadata
-    while ((specItem = psMetadataGetAndIncrement(specsIter))) {
-        pmConceptSpec *spec = specItem->data.V; // The specification
-        psString name = specItem->name; // The concept name
-        psMetadataItem *dbItem = p_pmConceptsReadSingleFromDatabase(name, database, config, fpa, chip, cell);
-        if (!dbItem) {
-            continue;
+        }
-        psMetadataItem *conceptItem = psMetadataLookup(concepts, name); // The item from the concepts
-        psMetadataItem *formatted = conceptFormat(spec, conceptItem, PM_CONCEPT_SOURCE_DATABASE,
-                                                  cameraFormat, fpa, chip, cell);
-        if (!formatted) {
-            continue;
+        }
-        if (strcmp(dbItem->name, name) != 0) {
-            // Correct the name to match the concept name
-            dbItem = psMetadataItemCopy(dbItem);
-            psFree(dbItem->name);
-            dbItem->name = psStringCopy(name);
-        } else {
-            psMemIncrRefCounter(dbItem);
+        }
-        if (!compareConcepts(formatted, dbItem)) {
-            psWarning("Concept %s is specified by the DATABASE in the camera "
-                      "format, but the values don't match.\n", name);
+        }
-        psFree(dbItem);
-        psFree(formatted);
+    }
-    psFree(specsIter);
-    return true;
-    #endif
+}
 …
                           const pmChip *chip, // The chip
                           const pmCell *cell, // The cell
-                          pmConceptSource source, // The source of the concepts to write
                           pmConfig *config, // Configuration
                           psMetadata *concepts // The concepts to write out
 …
     pmConceptsInit();
+    psTrace("psModules.concepts", 3, "Writing concepts (%p %p %p): %d\n", fpa, chip, cell, source);
+    if (source & PM_CONCEPT_SOURCE_CELLS) {
+        p_pmConceptsWriteToCells(*specs, cell, concepts);
+    }
+    if (source & PM_CONCEPT_SOURCE_DEFAULTS) {
+        p_pmConceptsWriteToDefaults(*specs, fpa, chip, cell, concepts);
+    }
+    if (source & (PM_CONCEPT_SOURCE_PHU | PM_CONCEPT_SOURCE_HEADER)) {
+        p_pmConceptsWriteToHeader(*specs, fpa, chip, cell, concepts);
+    }
+    if (source & PM_CONCEPT_SOURCE_DATABASE) {
+        p_pmConceptsWriteToDatabase(*specs, fpa, chip, cell, config, concepts);
+    }
+    psTrace("psModules.concepts", 3, "Writing concepts (%p %p %p)\n", fpa, chip, cell);
+    psMetadata *format = conceptsCameraFormat(fpa, chip, cell); // Camera format
+    if (!format) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to retrieve camera format.");
+        return false;
+    }
+    psMetadata *defaults = conceptsDefaults(format); // DEFAULTS configuration
+    psMetadata *translation = conceptsTranslation(format); // TRANSLATION configuration
+    psMetadata *database = conceptsDatabase(format); // DATABASE configuration
+    psMetadataIterator *iter = psMetadataIteratorAlloc(*specs, PS_LIST_HEAD, NULL); // Iterator
+    psMetadataItem *item = NULL;    // Item from the specs metadata
+    while ((item = psMetadataGetAndIncrement(iter))) {
+        pmConceptSpec *spec = item->data.V; // The specification
+        psString name = item->name; // The concept name
+        psMetadataItem *concept = psMetadataLookup(concepts, name); // Concept to write
+        if (!p_pmConceptWriteToCells(cell, spec, concept, format) &&
+            !p_pmConceptWriteToDefaults(fpa, chip, cell, spec, concept, format, defaults) &&
+            !p_pmConceptWriteToHeader(fpa, chip, cell, spec, concept, format, translation) &&
+            !p_pmConceptWriteToDatabase(fpa, chip, cell, config, spec, concept, format, database)) {
+            psTrace("psModules.concepts", 1, "Unable to write concept %s to any output", name);
+        }
+    }
+    psFree(iter);
     return true;
 …
 bool pmConceptsWriteFPA(const pmFPA *fpa, pmConceptSource source, bool propagateDown, pmConfig *config)
+bool pmConceptsWriteFPA(const pmFPA *fpa, bool propagateDown, pmConfig *config)
+{
     PS_ASSERT_PTR_NON_NULL(fpa, false);
     psMetadata *conceptsFPA = pmConceptsSpecs(PM_FPA_LEVEL_FPA); // Concept specifications
     psTrace("psModules.concepts", 5, "Writing FPA concepts: %p %p\n", conceptsFPA, fpa->concepts);
     bool success = conceptsWrite(&conceptsFPA, fpa, NULL, NULL, source, config, fpa->concepts);
+    bool success = conceptsWrite(&conceptsFPA, fpa, NULL, NULL, config, fpa->concepts);
     if (propagateDown) {
         psArray *chips = fpa->chips;        // Array of chips
 …
             pmChip *chip = chips->data[i];  // Chip of interest
             if (chip && !chip->hdu) {
                 success &= pmConceptsWriteChip(chip, source, false, true, config);
+                success &= pmConceptsWriteChip(chip, false, true, config);
+            }
+        }
 …
+bool pmConceptsWriteChip(const pmChip *chip, pmConceptSource source, bool propagateUp,
+                         bool propagateDown, pmConfig *config)
+bool pmConceptsWriteChip(const pmChip *chip, bool propagateUp, bool propagateDown, pmConfig *config)
+{
     PS_ASSERT_PTR_NON_NULL(chip, false);
 …
     psTrace("psModules.concepts", 5, "Writing chip concepts: %p %p\n", conceptsChip, chip->concepts);
     pmFPA *fpa = chip->parent;          // FPA to which the chip belongs
     bool success = conceptsWrite(&conceptsChip, fpa, chip, NULL, source, config, chip->concepts);
+    bool success = conceptsWrite(&conceptsChip, fpa, chip, NULL, config, chip->concepts);
     if (propagateUp && !fpa->hdu) {
         psMetadata *conceptsFPA = pmConceptsSpecs(PM_FPA_LEVEL_FPA); // Concept specifications
         success &= conceptsWrite(&conceptsFPA, fpa, chip, NULL, source, config, fpa->concepts);
+        success &= conceptsWrite(&conceptsFPA, fpa, chip, NULL, config, fpa->concepts);
+    }
     if (propagateDown) {
 …
             pmCell *cell = cells->data[i];  // Cell of interest
             if (cell && !cell->hdu) {
                 success &= pmConceptsWriteCell(cell, source, false, config);
+                success &= pmConceptsWriteCell(cell, false, config);
+            }
+        }
 …
 bool pmConceptsWriteCell(const pmCell *cell, pmConceptSource source, bool propagateUp, pmConfig *config)
+bool pmConceptsWriteCell(const pmCell *cell, bool propagateUp, pmConfig *config)
+{
     PS_ASSERT_PTR_NON_NULL(cell, false);
 …
     pmFPA *fpa = chip->parent;          // FPA to which the chip belongs
     bool success = conceptsWrite(&conceptsCell, fpa, chip, cell, source, config, cell->concepts);
+    bool success = conceptsWrite(&conceptsCell, fpa, chip, cell, config, cell->concepts);
     if (propagateUp) {
         if (!chip->hdu) {
             psMetadata *conceptsChip = pmConceptsSpecs(PM_FPA_LEVEL_CHIP); // Concept specifications
             success &= conceptsWrite(&conceptsChip, fpa, chip, cell, source, config, chip->concepts);
+            success &= conceptsWrite(&conceptsChip, fpa, chip, cell, config, chip->concepts);
             if (!fpa->hdu) {
                 psMetadata *conceptsFPA = pmConceptsSpecs(PM_FPA_LEVEL_FPA); // Concept specifications
                 success &= conceptsWrite(&conceptsFPA, fpa, chip, cell, source, config, fpa->concepts);
+                success &= conceptsWrite(&conceptsFPA, fpa, chip, cell, config, fpa->concepts);
+            }
+        }

branches/cnb_branches/cnb_branch_20090301/psModules/src/concepts/pmConceptsWrite.h

-              r22699
+              r23594
 /// @{
 /// "Write" concepts to (actually, check against) the camera format file's CELLS.
+/// "Write" concept to (actually, check against) the camera format file's CELLS.
 ///
 /// Examines the CELLS metadata in the camera format file for the current type of cell, and checks that the
 /// concepts as defined there match the ones defined in the cell.  A warning is produced if the concepts do
+/// concept as defined there match the ones defined in the cell.  A warning is produced if the concept does
 /// not match.
+bool p_pmConceptsWriteToCells(const psMetadata *specs, ///< The concept specifications
+bool p_pmConceptWriteToCells(const pmCell *cell, ///< The cell
+                             const pmConceptSpec *spec, ///< Concept specification
+                             const psMetadataItem *conceptItem, ///< Concept to write
+                             const psMetadata *format ///< Camera format, or NULL
+    );
+/// "Write" concept to (actually, check against) the camera format file's DEFAULTS.
+///
+/// Examines the DEFAULTS metadata in the camera format file, and checks that the concept as defined there
+/// match the one defined in the cell.  A warning is produced if the concept does not match.
+bool p_pmConceptWriteToDefaults(const pmFPA *fpa, ///< The FPA
+                                const pmChip *chip, ///< The chip
+                                const pmCell *cell, ///< The cell
+                                const pmConceptSpec *spec, ///< Concept specification
+                                const psMetadataItem *conceptItem,      ///< Concept to write
+                                const psMetadata *format, ///< Camera format, or NULL
+                                const psMetadata *defaults ///< DEFAULTS configuration, or NULL
+    );
+/// "Write" concept to (actually, add to, pending a later write) the FITS header.
+///
+/// Examines the FITS header TRANSLATION metadata in the camera format file, and writes concept to the
+/// appropriate FITS header(s) in the HDU, in preparation for a future write of the HDU.
+bool p_pmConceptWriteToHeader(const pmFPA *fpa, ///< The FPA
+                              const pmChip *chip, ///< The chip
                               const pmCell *cell, ///< The cell
+                              const psMetadata *concepts ///< The concepts
+                             );
+                              const pmConceptSpec *spec, ///< Concept specification
+                              const psMetadataItem *conceptItem, ///< Concept to write
+                              const psMetadata *format, ///< Camera format, or NULL
+                              const psMetadata *translation ///< TRANSLATION configuration, or NULL
+                              );
 /// "Write" concepts to (actually, check against) the camera format file's DEFAULTS.
+/// Write concept to the database.
 ///
+/// Examines the DEFAULTS metadata in the camera format file, and checks that the concepts as defined there
+/// match the ones defined in the cell.  A warning is produced if the concepts do not match.
+bool p_pmConceptsWriteToDefaults(const psMetadata *specs, ///< The concept specifications
+                                 const pmFPA *fpa, ///< The FPA
+                                 const pmChip *chip, ///< The chip
+                                 const pmCell *cell, ///< The cell
+                                 const psMetadata *concepts ///< The concepts
+/// Examines the DATABASE metadata in the camera format file, and writes (actually, check against)
+/// concept to the database.
+bool p_pmConceptWriteToDatabase(const pmFPA *fpa, ///< The FPA
+                                const pmChip *chip, ///< The chip
+                                const pmCell *cell, ///< The cell
+                                pmConfig *config, ///< Configuration
+                                const pmConceptSpec *spec, ///< Concept specification
+                                const psMetadataItem *conceptItem, ///< Concept to write
+                                const psMetadata *format, ///< Camera format, or NULL
+                                const psMetadata *database ///< DATABASE configuration, or NULL
                                 );
-/// "Write" concepts to (actually, add to, pending a later write) the FITS header.
-///
-/// Examines the FITS header TRANSLATION metadata in the camera format file, and writes concepts to the
-/// appropriate FITS headers in the HDU, in preparation for a future write of the HDU.
-bool p_pmConceptsWriteToHeader(const psMetadata *specs, ///< The concept specifications
-                               const pmFPA *fpa, ///< The FPA
-                               const pmChip *chip, ///< The chip
-                               const pmCell *cell, ///< The cell
-                               const psMetadata *concepts ///< The concepts
-                              );
+/// Write concepts to the database.
+///
+/// Examines the DATABASE metadata in the camera format file, and writes concepts to the database.
+/// Warning: This function has not been tested; use at your own risk.
+bool p_pmConceptsWriteToDatabase(const psMetadata *specs, ///< The concept specifications
+                                 const pmFPA *fpa, ///< The FPA
+                                 const pmChip *chip, ///< The chip
+                                 const pmCell *cell, ///< The cell
+                                 pmConfig *config,///< Configuration
+                                 const psMetadata *concepts ///< The concepts
+                                );
+bool pmConceptWriteSingle(const pmFPA *fpa, ///< The FPA
+                          const pmChip *chip, ///< The chip
+                          const pmCell *cell, ///< The cell
+                          pmConfig *config, ///< Configuration
+                          const psMetadataItem *concept ///< Concept to write
+    );
 …
 /// written for all lower levels by iterating over the components.
 bool pmConceptsWriteFPA(const pmFPA *fpa,     ///< FPA for which to write concepts
-                        pmConceptSource source, ///< Source for concepts
                         bool propagateDown, ///< Propagate to lower levels?
                         pmConfig *config        ///< Configuration
 …
 /// written for the FPA, and the cell level by iterating over the components.
 bool pmConceptsWriteChip(const pmChip *chip,  ///< Chip for which to write concepts
-                         pmConceptSource source, ///< Source for concepts
                          bool propagateUp,///< Propagate to higher levels?
                          bool propagateDown, ///< Propagate to lower levels?
 …
 /// only the parent of this cell).
 bool pmConceptsWriteCell(const pmCell *cell,  ///< FPA for which to write concepts
-                         pmConceptSource source, ///< Source for concepts
                          bool propagateUp, ///< Propagate to higher levels?
                          pmConfig *config ///< Configuration

branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfig.c

-              r23351
+              r23594
     pmConfigLoadRecipeOptions(argc, argv, config, "-Df");
     pmConfigLoadRecipeOptions(argc, argv, config, "-Db");
+    if (!pmConfigReadRecipes(config, PM_RECIPE_SOURCE_CL)) {
+        psError(PS_ERR_IO, false, "Failed to read recipes from command-line");
+        psFree(config);
+        return NULL;
+    }
     // Look for command-line options for files to replace

branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigCamera.c

-              r23351
+              r23594
 // Don't update these skycell concepts; last one MUST be 0 (i.e., NULL).
 const static char *skycellConceptsCell[] = { "CELL.BIASSEC", "CELL.TRIMSEC", "CELL.READDIR", "CELL.XPARITY",
                                              "CELL.YPARITY", "CELL.X0", "CELL.Y0", 0 };
+                                             "CELL.YPARITY", "CELL.X0", "CELL.Y0", "CELL.TIMESYS", 0 };
 const static char *skycellConceptsChip[] = { "CHIP.XPARITY", "CHIP.YPARITY", 0 };
 const static char *skycellConceptsFPA[] = { 0 };
+const static char *skycellConceptsFPA[] = { "FPA.TIMESYS" };
 // What do we call the skycell concept in the FITS header?
 …
         psMetadataAddS32(defaults, PS_LIST_TAIL, "CHIP.X0", 0, NULL, 0);
         psMetadataAddS32(defaults, PS_LIST_TAIL, "CELL.READDIR", 0, "Read direction (rows)", 1);
+        psMetadataAddStr(defaults, PS_LIST_TAIL, "CELL.TIMESYS", 0, "Time system", "TAI");
+        psMetadataAddStr(defaults, PS_LIST_TAIL, "FPA.TIMESYS", 0, "Time system", "TAI");
         psMetadataAddMetadata(format, PS_LIST_TAIL, "DEFAULTS", 0, "Default values for concepts", defaults);

branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigMask.c

-              r21386
+              r23594
 static pmConfigMaskInfo masks[] = {
+    { "DETECTOR",  NULL,       0x00, true  },   // Something is wrong with the detector
+    { "DARK",      "DETECTOR", 0x00, true  },   // Pixel doesn't dark-subtract properly
+    { "FLAT",      "DETECTOR", 0x01, true  },   // Pixel doesn't flat-field properly
+    { "BLANK",     "DETECTOR", 0x01, true  },   // Pixel doesn't contain valid data
+    { "RANGE",     NULL,       0x00, true  },   // Pixel is out-of-range of linearity
+    { "SAT",       "RANGE",    0x01, true  },   // Pixel is saturated
+    { "BAD",       "RANGE",    0x01, true  },   // Pixel is low
+    { "BAD.WARP",  NULL,       0x01, true  },   // Pixel is bad after convolution with a bad pixel
+    { "CR",        NULL,       0x00, true  },   // Pixel contains a cosmic ray
+    { "GHOST",     NULL,       0x00, true  },   // Pixel contains an optical ghost
+    { "POOR.WARP", NULL,       0x00, false },   // Pixel is poor after convolution with a bad pixel
+    // "LOW"  Pixel is low
+    // "CONV" Pixel is bad after convolution with a bad pixel
+    // Features of the detector
+    { "DETECTOR",  NULL,       0x01, true  }, // Something is wrong with the detector
+    { "FLAT",      "DETECTOR", 0x01, true  }, // Pixel doesn't flat-field properly
+    { "DARK",      "DETECTOR", 0x01, true  }, // Pixel doesn't dark-subtract properly
+    { "BLANK",     "DETECTOR", 0x01, true  }, // Pixel doesn't contain valid data
+    { "CTE",       "DETECTOR", 0x01, true  }, // Pixel has poor CTE
+    // Invalid signal ranges
+    { "SAT",       NULL,       0x02, true  }, // Pixel is saturated or non-linear
+    { "LOW",       "SAT",      0x02, true  }, // Pixel is low
+    { "SUSPECT",   NULL,       0x04, false }, // Pixel is suspected of being bad
+    // Non-astronomical structures
+    { "CR",        NULL,       0x08, true  }, // Pixel contains a cosmic ray
+    { "SPIKE",     NULL,       0x08, true  }, // Pixel contains a diffraction spike
+    { "GHOST",     NULL,       0x08, true  }, // Pixel contains an optical ghost
+    { "STREAK",    NULL,       0x08, true  }, // Pixel contains streak data
+    { "STARCORE",  NULL,       0x08, true  }, // Pixel contains a bright star core
+    // Effects of convolution and interpolation
+    { "CONV.BAD",  NULL,       0x02, true  }, // Pixel is bad after convolution with a bad pixel
+    { "CONV.POOR", NULL,       0x04, false }, // Pixel is poor after convolution with a bad pixel
 };

branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRun.c

-              r23351
+              r23594
+}
+bool pmConfigRunFileAdd(pmConfig *config, const pmFPAfile *file)
+// Add a file to a nominated metadata in the RUN information
+static bool configRunFileAdd(pmConfig *config, // Configuration
+                             const pmFPAfile *file, // File to add
+                             const char *target // Name of metadata to which to add
+                             )
+{
     PS_ASSERT_PTR_NON_NULL(config, false);
 …
     psMetadata *run = configRun(config);// RUN information
     psAssert(run, "Require run-time information");
     psMetadata *files = configElement(run, "FILES", "Filerules used during execution");
+    psMetadata *files = configElement(run, target, "Filerules used during execution");
     psAssert(files, "Require list of files");
 …
+}
 psArray *pmConfigRunFileGet(pmConfig *config, const char *name)
+bool pmConfigRunFileAddRead(pmConfig *config, const pmFPAfile *file)
+{
     PS_ASSERT_PTR_NON_NULL(config, false);
     PS_ASSERT_STRING_NON_EMPTY(name, false);
+    PS_ASSERT_PTR_NON_NULL(file, false);
+    return configRunFileAdd(config, file, "FILES.INPUT");
+}
+bool pmConfigRunFileAddWrite(pmConfig *config, const pmFPAfile *file)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    PS_ASSERT_PTR_NON_NULL(file, false);
+    return configRunFileAdd(config, file, "FILES.OUTPUT");
+}
+// Get an array of filenames from the nominated RUN information
+static psArray *configRunFileGet(pmConfig *config, // Configuration
+                                 const char *name, // Name of file
+                                 const char *source // Source metadata for file
+                                 )
+{
     psMetadata *run = configRun(config);// RUN information
     psAssert(run, "Require run-time information");
     psMetadata *files = configElement(run, "FILES", "Filerules used during execution");
+    psMetadata *files = configElement(run, source, "Filerules used during execution");
     psAssert(files, "Require list of files");
+    if (psListLength(files->list) == 0) {
+        // Can't find anything
+        return NULL;
+    }
     psList *list = psListAlloc(NULL);   // List of file names
 …
     return array;
+}
+psArray *pmConfigRunFileGet(pmConfig *config, const char *name)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    PS_ASSERT_STRING_NON_EMPTY(name, false);
+    // Try the input and output, in turn
+    psArray *files = configRunFileGet(config, name, "FILES.INPUT"); // Files from RUN metadata
+    if (!files) {
+        configRunFileGet(config, name, "FILES.OUTPUT");
+    }
+    return files;
+}

branches/cnb_branches/cnb_branch_20090301/psModules/src/config/pmConfigRun.h

-              r23351
+              r23594
 #include <pmFPAfile.h>
+/// Add a file to the list of files used in the run-time information
+bool pmConfigRunFileAdd(
+/// Add a file to the list of files read in the run-time information
+bool pmConfigRunFileAddRead(
+    pmConfig *config,                   ///< Configuration
+    const pmFPAfile *file               ///< File to add
+    );
+/// Add a file to the list of files written in the run-time information
+bool pmConfigRunFileAddWrite(
     pmConfig *config,                   ///< Configuration
     const pmFPAfile *file               ///< File to add

branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmDetrendDB.h

r23351	r23594
63	63
64	64	pmDetrendSelectOptions pmDetrendSelectOptionsAlloc(const char camera, psTime time, pmDetrendType type);
65		pmDetrendSelectResults *pmDetrendSelectResultsAlloc();
	65	pmDetrendSelectResults *pmDetrendSelectResultsAlloc(void);
66	66	pmDetrendSelectResults pmDetrendSelect (const pmDetrendSelectOptions options, const pmConfig *config);
67	67	char pmDetrendFile (const char detID, const char classID, const pmConfig config);

branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmDetrendThreads.h

r18893	r23594
18	18
19	19	/// get the requested number of scan rows per thread
20		int pmDetrendGetScanRows ();
	20	int pmDetrendGetScanRows(void);
21	21
22	22	/// @}

branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmShutterCorrection.c

r23351	r23594
71	71	}
72	72
73		pmShutterCorrection *pmShutterCorrectionAlloc()
	73	pmShutterCorrection *pmShutterCorrectionAlloc(void)
74	74	{
75	75	pmShutterCorrection corr = (pmShutterCorrection)psAlloc(sizeof(pmShutterCorrection));

branches/cnb_branches/cnb_branch_20090301/psModules/src/detrend/pmShutterCorrection.h

r21509	r23594
69	69
70	70	/// Allocator for shutter correction parameters
71		pmShutterCorrection *pmShutterCorrectionAlloc();
	71	pmShutterCorrection *pmShutterCorrectionAlloc(void);
72	72
73	73	/// Guess a shutter correction, based on plot of counts vs exposure time

branches/cnb_branches/cnb_branch_20090301/psModules/src/extras/pmKapaPlots.c

r13896	r23594
47	47	}
48	48
49		bool pmKapaClose ()
	49	bool pmKapaClose (void)
50	50	{
51	51

branches/cnb_branches/cnb_branch_20090301/psModules/src/extras/pmKapaPlots.h

-              r12696
+              r23594
 /* @file  pmKapaPlots.h
  * @brief functions to make plots with the external program 'kapa'
+ * @brief functions to make plots with the external program 'kapa'
+ *
  * @author EAM, IfA
 …
 // move to psLib or psModules
 int pmKapaOpen (bool showWindow);
 bool pmKapaClose ();
+bool pmKapaClose(void);
 bool pmKapaPlotVectorPair (psVector *xVec, psVector *yVec);

branches/cnb_branches/cnb_branch_20090301/psModules/src/extras/psPipe.c

r12888	r23594
35	35	}
36	36
37		psPipe *psPipeAlloc ()
	37	psPipe *psPipeAlloc (void)
38	38	{
39	39	psPipe pipe = (psPipe )psAlloc(sizeof(psPipe));

branches/cnb_branches/cnb_branch_20090301/psModules/src/extras/psPipe.h

-              r11253
+              r23594
 /* @file  psPipe.h
  * @brief 3-stream pipe
+ * @brief 3-stream pipe
+ *
  * @author EAM, IfA
 …
 // psPipe functions
 psPipe *psPipeAlloc ();
+psPipe *psPipeAlloc (void);
 psPipe *psPipeOpen (char *command);
 int     psPipeClose (psPipe *pipe);

branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmStack.c

-              r23351
+              r23594
     psVector *weights;                  // Pixel weightings
     psVector *sources;                  // Pixel sources (which image did they come from?)
+    psVector *limits;                   // Rejection limits
     psVector *sort;                     // Buffer for sorting (to get a robust estimator of the standard dev)
 } combineBuffer;
 …
     psFree(buffer->weights);
     psFree(buffer->sources);
+    psFree(buffer->limits);
     psFree(buffer->sort);
     return;
 …
     buffer->weights = psVectorAlloc(numImages, PS_TYPE_F32);
     buffer->sources = psVectorAlloc(numImages, PS_TYPE_U16);
+    buffer->limits = psVectorAlloc(numImages, PS_TYPE_F32);
     buffer->sort = psVectorAlloc(numImages, PS_TYPE_F32);
     return buffer;
 …
                           bool useVariance, // Use variance for rejection when combining?
                           bool safe,    // Combine safely?
+                          bool rejectInspect, // Reject values marked for inspection from combination?
                           combineBuffer *buffer // Buffer for combination; to avoid multiple allocations
+                         )
 …
     psVector *pixelWeights = buffer->weights; // Image weights for the pixel of interest
     psVector *pixelSources = buffer->sources; // Sources for the pixel of interest
+    psVector *pixelLimits = buffer->limits; // Limits for the pixel of interest
     psVector *sort = buffer->sort;      // Sort buffer
 …
     pixelWeights->n = num;
     pixelSources->n = num;
+    pixelLimits->n = num;
 #ifdef TESTING
 …
     // Default option is that the pixel is bad
     float imageValue = NAN, varianceValue = NAN; // Value for combined image and variance map
     psImageMaskType maskValue = bad;         // Value for combined mask
+    psImageMaskType maskValue = bad;    // Value for combined mask
     switch (num) {
       case 0:
 …
               if (PS_SQR(diff) > PS_SQR(rej) * sigma2) {
                   // Not consistent: mark both for inspection
+                  combineInspect(inputs, x, y, pixelSources->data.U16[0]);
+                  combineInspect(inputs, x, y, pixelSources->data.U16[1]);
+                  if (rejectInspect) {
+                      imageValue = NAN;
+                      varianceValue = NAN;
+                      maskValue = bad;
+                  } else {
+                      combineInspect(inputs, x, y, pixelSources->data.U16[0]);
+                      combineInspect(inputs, x, y, pixelSources->data.U16[1]);
+                  }
 #ifdef TESTING
                   if (x == TEST_X && y == TEST_Y) {
 …
 #endif
                       pixelVariances->data.F32[i] = rej2 * (pixelVariances->data.F32[i] + sysVar);
+                      pixelLimits->data.F32[i] = rej2 * (pixelVariances->data.F32[i] + sysVar);
+                  }
+              }
 …
 #define MASK_PIXEL_FOR_INSPECTION() \
     pixelMasks->data.PS_TYPE_VECTOR_MASK_DATA[j] = 0xff; \
+    combineInspect(inputs, x, y, pixelSources->data.U16[j]); \
+    if (!rejectInspect) { \
+        combineInspect(inputs, x, y, pixelSources->data.U16[j]); \
+    } \
     numClipped++; \
     totalClipped++;
 …
                       // Comparing squares --- cheaper than lots of sqrts
                       // pixelVariances includes the rejection limit, from above
                       if (PS_SQR(diff) > pixelVariances->data.F32[j]) {
+                      if (PS_SQR(diff) > pixelLimits->data.F32[j]) {
                           MASK_PIXEL_FOR_INSPECTION();
 #ifdef TESTING
                           if (x == TEST_X && y == TEST_Y) {
                               fprintf(stderr, "Rejecting input %d based on variance: %f > %f\n",
                                       j, diff, sqrtf(pixelVariances->data.F32[j]));
+                                      j, diff, sqrtf(pixelLimits->data.F32[j]));
+                          }
 #endif
 …
+              }
+          }
+          if (rejectInspect && totalClipped > 0) {
+              // Get rid of the masked values
+              // The alternative to this is to make combinationMeanVariance() accept a mask
+              int good = 0;            // Index of good value
+              for (int i = 0; i < num; i++) {
+                  if (pixelMasks->data.PS_TYPE_VECTOR_MASK_DATA[i]) {
+                      continue;
+                  }
+                  if (i != good) {
+                      pixelData->data.F32[good] = pixelData->data.F32[i];
+                      pixelVariances->data.F32[good] = pixelVariances->data.F32[i];
+                      pixelWeights->data.F32[good] = pixelWeights->data.F32[i];
+                      pixelData->data.F32[good] = pixelData->data.F32[i];
+                  }
+                  good++;
+              }
+              pixelData->n = good;
+              pixelVariances->n = good;
+              pixelWeights->n = good;
+              if (combinationMeanVariance(&mean, &var, pixelData, pixelVariances, pixelWeights)) {
+                  imageValue = mean;
+                  varianceValue = var;
+                  maskValue = 0;
+              } else {
+                  imageValue = NAN;
+                  varianceValue = NAN;
+                  maskValue = bad;
+              }
+          }
           break;
+      }
 …
 bool pmStackCombine(pmReadout *combined, psArray *input, psImageMaskType maskVal, psImageMaskType bad,
                     int kernelSize, int numIter, float rej, float sys, float discard,
                     bool entire, bool useVariance, bool safe)
+                    bool entire, bool useVariance, bool safe, bool rejectInspect)
+{
     PS_ASSERT_PTR_NON_NULL(combined, false);
 …
                     combinePixels(combinedImage, combinedMask, combinedVariance, input, weights,
                                   addVariance, reject, x, y, maskVal, bad, numIter, rej, sys, discard,
                                   useVariance, safe, buffer);
+                                  useVariance, safe, rejectInspect, buffer);
+                }
+            }
 …
                 combinePixels(combinedImage, combinedMask, combinedVariance, input, weights,
                               addVariance, reject, x, y, maskVal, bad, numIter, rej, sys, discard,
                               useVariance, safe, buffer);
+                              useVariance, safe, rejectInspect, buffer);
+            }
+        }
 …
                 combinePixels(combinedImage, combinedMask, combinedVariance, input, weights,
                               addVariance, NULL, x, y, maskVal, bad, numIter, rej, sys, discard,
                               useVariance, safe, buffer);
+                              useVariance, safe, rejectInspect, buffer);
+            }
+        }

branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmStack.h

-              r21476
+              r23594
                     bool entire,        ///< Combine entire image even if rejection lists provided?
                     bool useVariance,   ///< Use variance values for rejection?
+                    bool safe           ///< Play safe with small numbers of input pixels (mask if N <= 2)?
+                    bool safe,          ///< Play safe with small numbers of input pixels (mask if N <= 2)?
+                    bool rejectInspect  ///< Reject pixels instead of marking them for inspection?
     );

branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionIO.c

-              r23351
+              r23594
 #include "pmHDU.h"
+#include "pmHDUUtils.h"
 #include "pmFPA.h"
 #include "pmFPAview.h"
 …
     psMetadata *header = psMetadataAlloc(); // Header for FITS file
+    pmCell *cell = ro->parent;          // Cell of interest
+    if (cell) {
+        pmChip *chip = cell->parent;    // Chip of interest
+        pmFPA *fpa = chip->parent;      // FPA of interest
+        pmHDU *hdu = pmHDUGetHighest(fpa, chip, cell); // HDU for readout
+        if (hdu) {
+            header = psMetadataCopy(header, hdu->header);
+        }
+    }
     // CVS tags, used to identify the version of this file (in case incompatibilities are introduced)
     psString cvsFile = psStringCopy("$RCSfile: pmSubtractionIO.c,v $");
 …
         return true;
+    }
-    if (file->fileLevel != PM_FPA_LEVEL_FPA) {
-        return true;
+    }
-    if (file->fpa->chips->n == 1) {
-        return true;
+    }
     // find the FPA phu

branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionVisual.c

-              r23351
+              r23594
 /** destroy windows at the end of a run*/
 bool pmSubtractionVisualClose()
+bool pmSubtractionVisualClose(void)
+{
     if(kapa != -1)
 …
 #else
 bool pmSubtractionVisualClose() {return true;}
+bool pmSubtractionVisualClose(void) {return true;}
 bool pmSubtractionVisualPlotConvKernels(psImage *convKernels) {return true;}
 bool pmSubtractionVisualPlotStamps(pmSubtractionStampList *stamps, pmReadout *ro) {return true;}

branches/cnb_branches/cnb_branch_20090301/psModules/src/imcombine/pmSubtractionVisual.h

r23351	r23594
2	2	#define PM_SUBTRACTION_VISUAL_H
3	3
4		bool pmSubtractionVisualClose();
	4	bool pmSubtractionVisualClose(void);
5	5	bool pmSubtractionVisualPlotConvKernels(psImage *convKernels);
6	6	bool pmSubtractionVisualPlotStamps(pmSubtractionStampList stamps, pmReadout ro);

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmDetections.c

r18828	r23594
30	30
31	31	// generate a pmDetections container with empty (allocated) footprints and peaks containers
32		pmDetections *pmDetectionsAlloc() {
	32	pmDetections *pmDetectionsAlloc(void) {
33	33
34	34	pmDetections detections = (pmDetections )psAlloc(sizeof(pmDetections));

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmDetections.h

-              r18828
+              r23594
  */
 typedef struct {
   psArray *footprints;        // collection of footprints in the image
   psArray *peaks;             // collection of all peaks contained by the footprints
   psArray *oldPeaks;          // collection of all peaks previously found
+  psArray *footprints;        // collection of footprints in the image
+  psArray *peaks;             // collection of all peaks contained by the footprints
+  psArray *oldPeaks;          // collection of all peaks previously found
   int last;
 } pmDetections;
 pmDetections *pmDetectionsAlloc ();
+pmDetections *pmDetectionsAlloc (void);
 /// @}

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmMoments.c

r23351	r23594
25	25	to zero.
26	26	*****************************************************************************/
27		pmMoments *pmMomentsAlloc()
	27	pmMoments *pmMomentsAlloc(void)
28	28	{
29	29	psTrace("psModules.objects", 10, "---- %s() begin ----\n", __func__);

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmMoments.h

r19879	r23594
53	53	*
54	54	*/
55		pmMoments *pmMomentsAlloc();
	55	pmMoments *pmMomentsAlloc(void);
56	56
57	57	/// @}

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSF.c

r20937	r23594
54	54	}
55	55
56		pmPSFOptions *pmPSFOptionsAlloc () {
	56	pmPSFOptions *pmPSFOptionsAlloc (void) {
57	57
58	58	pmPSFOptions options = (pmPSFOptions ) psAlloc(sizeof(pmPSFOptions));

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmPSF.h

r21183	r23594
96	96	pmPSF pmPSFAlloc (const pmPSFOptions options);
97	97	bool psMemCheckPSF(psPtr ptr);
98		pmPSFOptions *pmPSFOptionsAlloc();
	98	pmPSFOptions *pmPSFOptionsAlloc(void);
99	99	bool psMemCheckPSFOptions(psPtr ptr);
100	100

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSource.h

r23351	r23594
105	105	*
106	106	*/
107		pmSource *pmSourceAlloc();
	107	pmSource *pmSourceAlloc(void);
108	108
109	109	/** pmSourceCopy()

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceExtendedPars.c

-              r21363
+              r23594
+}
 pmSourceExtendedPars *pmSourceExtendedParsAlloc () {
+pmSourceExtendedPars *pmSourceExtendedParsAlloc (void) {
     pmSourceExtendedPars *pars = (pmSourceExtendedPars *) psAlloc(sizeof(pmSourceExtendedPars));
     psMemSetDeallocator(pars, (psFreeFunc) pmSourceExtendedParsFree);
 …
+}
 pmSourceRadialProfile *pmSourceRadialProfileAlloc () {
+pmSourceRadialProfile *pmSourceRadialProfileAlloc (void) {
     pmSourceRadialProfile *profile = (pmSourceRadialProfile *) psAlloc(sizeof(pmSourceRadialProfile));
 …
+}
 pmSourceIsophotalValues *pmSourceIsophotalValuesAlloc () {
+pmSourceIsophotalValues *pmSourceIsophotalValuesAlloc (void) {
     pmSourceIsophotalValues *isophot = (pmSourceIsophotalValues *) psAlloc(sizeof(pmSourceIsophotalValues));
 …
+}
 pmSourcePetrosianValues *pmSourcePetrosianValuesAlloc () {
+pmSourcePetrosianValues *pmSourcePetrosianValuesAlloc (void) {
     pmSourcePetrosianValues *petrosian = (pmSourcePetrosianValues *) psAlloc(sizeof(pmSourcePetrosianValues));
 …
+}
 pmSourceKronValues *pmSourceKronValuesAlloc () {
+pmSourceKronValues *pmSourceKronValuesAlloc (void) {
     pmSourceKronValues *kron = (pmSourceKronValues *) psAlloc(sizeof(pmSourceKronValues));
 …
+}
 pmSourceAnnuli *pmSourceAnnuliAlloc () {
+pmSourceAnnuli *pmSourceAnnuliAlloc (void) {
     pmSourceAnnuli *annuli = (pmSourceAnnuli *) psAlloc(sizeof(pmSourceAnnuli));

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceExtendedPars.h

-              r21363
+              r23594
 } pmSourceExtendedPars;
 pmSourceExtendedPars *pmSourceExtendedParsAlloc ();
+pmSourceExtendedPars *pmSourceExtendedParsAlloc(void);
 bool psMemCheckSourceExtendedPars(psPtr ptr);
 pmSourceRadialProfile *pmSourceRadialProfileAlloc ();
+pmSourceRadialProfile *pmSourceRadialProfileAlloc(void);
 bool psMemCheckSourceRadialProfile(psPtr ptr);
 pmSourceIsophotalValues *pmSourceIsophotalValuesAlloc ();
+pmSourceIsophotalValues *pmSourceIsophotalValuesAlloc(void);
 bool psMemCheckSourceIsophotalValues(psPtr ptr);
 pmSourcePetrosianValues *pmSourcePetrosianValuesAlloc ();
+pmSourcePetrosianValues *pmSourcePetrosianValuesAlloc(void);
 bool psMemCheckSourcePetrosianValues(psPtr ptr);
 pmSourceKronValues *pmSourceKronValuesAlloc ();
+pmSourceKronValues *pmSourceKronValuesAlloc(void);
 bool psMemCheckSourceKronValues(psPtr ptr);
 pmSourceAnnuli *pmSourceAnnuliAlloc ();
+pmSourceAnnuli *pmSourceAnnuliAlloc(void);
 bool psMemCheckSourceAnnuli(psPtr ptr);

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceFitSet.c

r23351	r23594
67	67	}
68	68
69		void pmSourceFitSetDone () {
	69	void pmSourceFitSetDone (void) {
70	70	psFree (fitSets);
71	71	}
…	…
150	150	}
151	151
152		pmSourceFitSetData *pmSourceFitSetDataGet () {
	152	pmSourceFitSetData *pmSourceFitSetDataGet (void) {
153	153
154	154	psAssert (fitSets, "pmSourceFitSetInit not called");
…	…
172	172	}
173	173
174		void pmSourceFitSetDataClear () {
	174	void pmSourceFitSetDataClear (void) {
175	175
176	176	int i;

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourceFitSet.h

-              r21183
+              r23594
 // use this function to init the fit sets based on the number of threads
 bool pmSourceFitSetInit (int nThreads);
 void pmSourceFitSetDone ();
+void pmSourceFitSetDone (void);
 // initialize data for a group of object models
 …
 // functions for selecting the FitSet corresponding to the current thread
 pmSourceFitSetData *pmSourceFitSetDataSet (psArray *modelSet);
 pmSourceFitSetData *pmSourceFitSetDataGet ();
 void pmSourceFitSetDataClear ();
+pmSourceFitSetData *pmSourceFitSetDataGet (void);
+void pmSourceFitSetDataClear (void);
 // function used to set limits for a group of models
 …
  */
 bool pmSourceFitSet(
     pmSource *source,                   ///< The input pmSource
     psArray *modelSet,                  ///< model to be fitted
     pmSourceFitMode mode,               ///< define parameters to be fitted
     psImageMaskType maskVal             ///< Vale to mask
+    pmSource *source,                   ///< The input pmSource
+    psArray *modelSet,                  ///< model to be fitted
+    pmSourceFitMode mode,               ///< define parameters to be fitted
+    psImageMaskType maskVal             ///< Vale to mask
 );

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourcePlots.c

r19849	r23594
139	139	}
140	140
141		pmSourcePlotLayout *pmSourcePlotLayoutAlloc()
	141	pmSourcePlotLayout *pmSourcePlotLayoutAlloc(void)
142	142	{
143	143	pmSourcePlotLayout layout = (pmSourcePlotLayout )psAlloc(sizeof(pmSourcePlotLayout));

branches/cnb_branches/cnb_branch_20090301/psModules/src/objects/pmSourcePlots.h

r15562	r23594
29	29	// typedef bool (pmSourcePlotFunction)(pmConfig config, pmFPAview view, pmSourcePlotLayout layout);
30	30
31		pmSourcePlotLayout *pmSourcePlotLayoutAlloc();
	31	pmSourcePlotLayout *pmSourcePlotLayoutAlloc(void);
32	32	bool psMemCheckSourcePlotLayout(psPtr ptr);
33	33

branches/cnb_branches/cnb_branch_20090301/psastro

Property svn:mergeinfo changed
/branches/eam_branches/eam_branch_20090312/psastro (added) merged: 23321-23322,23335-23337,23407
/trunk/psastro merged: 23412,23431,23486,23494,23584,23591-23592

branches/cnb_branches/cnb_branch_20090301/psastro/src

Property svn:ignore

old	new
16	16	gpcModel
17	17	psastroModelFit
	18	psastroExtract

branches/cnb_branches/cnb_branch_20090301/psastro/src/Makefile.am

-              r23352
+              r23594
 # FORCE: ;
 bin_PROGRAMS = psastro psastroModel psastroModelFit gpcModel
+bin_PROGRAMS = psastro psastroExtract psastroModel psastroModelFit gpcModel
 psastro_CFLAGS = $(PSASTRO_CFLAGS) $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
 psastro_LDFLAGS = $(PSASTRO_LIBS) $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 psastro_LDADD = libpsastro.la
+psastroExtract_CFLAGS = $(PSASTRO_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
+psastroExtract_LDFLAGS = $(PSASTRO_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
+psastroExtract_LDADD = libpsastro.la
 psastroModel_CFLAGS = $(PSASTRO_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
 …
         psastroDataSave.c           \
         psastroMetadataStats.c      \
+        psastroCleanup.c
+psastroExtract_SOURCES = \
+        psastroExtract.c                    \
+        psastroExtractArguments.c           \
+        psastroExtractParseCamera.c   \
+        psastroExtractDataLoad.c      \
+        psastroExtractAnalysis.c      \
+        psastroExtractStar.c      \
+        psastroExtractStars.c      \
+        psastroExtractGhosts.c      \
+        psastroExtractFindChip.c      \
         psastroCleanup.c

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastro.h

-              r23352
+              r23594
 bool              psastroMaskUpdates (pmConfig *config);
+psPlaneDistort   *psPlaneDistortIdentity ();
+bool              psPlaneDistortIsIdentity (psPlaneDistort *distort);
+psArray          *psastroLoadRefstars (pmConfig *config);
+psArray          *psastroLoadRefstars (pmConfig *config, const char *source);
 bool              psastroChipAstrom (pmConfig *config);
 bool              psastroOneChip (pmFPA *fpa, pmChip *chip, psArray *refset, psArray *rawset, psMetadata *recipe, psMetadata *updates);
 bool              psastroOneChipGrid (pmFPA *fpa, pmChip *chip, psArray *refset, psArray *rawset, psMetadata *recipe, psMetadata *updates);
 bool              psastroOneChipFit (pmFPA *fpa, pmChip *chip, psArray *refset, psArray *rawset, psMetadata *recipe, psMetadata *updates);
 bool              psastroChooseRefstars (pmConfig *config, psArray *refs);
+bool              psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source);
 bool              psastroRefstarSubset (pmReadout *readout);
 pmLumFunc        *psastroLuminosityFunction (psArray *stars, pmLumFunc *rawFunc);
 …
 bool              psastroDumpCorners (char *filenameU, char *filenameD, pmFPA *fpa);
+char             *psastroSetMagLimit (float *minMag, float *maxRho, pmConfig *config, const char *source);
 psArray          *psastroReadGetstarCatalog (psFits *fits);
 …
 bool              psastroMetadataStats (pmConfig *config);
+bool psastroZeroPoint (pmConfig *config);
+bool psastroZeroPointReadout(pmReadout *readout, float zeropt, float exptime);
+bool psastroZeroPointFromRecipe (float *zeropt, float *exptime, pmFPA *fpa, psMetadata *recipe);
+bool              psastroZeroPoint (pmConfig *config);
+bool              psastroZeroPointReadout(pmReadout *readout, float zeropt, float exptime);
+bool              psastroZeroPointFromRecipe (float *zeropt, float *exptime, pmFPA *fpa, psMetadata *recipe);
+// psastroExtract functions
+bool              psastroExtractAnalysis (pmConfig *config);
+bool              psastroExtractStars (pmConfig *config);
+bool              psastroExtractGhosts (pmConfig *config);
+pmChip           *psastroExtractFindChip (float *xChip, float *yChip, pmFPA *fpa, float xFPA, float yFPA);
+bool              psastroExtractChipBounds (pmFPA *fpa);
+psImage          *psastroExtractStar (psImage *input, float x, float y, float dX, float dY);
+bool              psastroExtractParseCamera (pmConfig *config);
+bool              psastroExtractDataLoad (pmConfig *config);
+pmConfig         *psastroExtractArguments (int argc, char **argv);
 ///@}

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroAnalysis.c

-              r21409
+              r23594
     // load the reference stars overlapping the data stars
     psArray *refs = psastroLoadRefstars(config);
+    psArray *refs = psastroLoadRefstars(config, "PSASTRO.INPUT");
     if (!refs) {
         psError (PSASTRO_ERR_UNKNOWN, false, "failed to load reference data\n");
 …
+    }
     if (!psastroChooseRefstars (config, refs)) {
+    if (!psastroChooseRefstars (config, refs, "PSASTRO.INPUT")) {
         psError (PSASTRO_ERR_UNKNOWN, false, "failed to select reference data for chips\n");
         psFree(refs);
         return false;
+    }
+    psFree (refs);  // refs of interest are saved on readout->analysis
     // check the command-line arguments first
 …
         if (!psastroChipAstrom (config)) {
             psError (PSASTRO_ERR_UNKNOWN, false, "failed to perform single chip astrometry\n");
-            psFree(refs);
             return false;
+        }
 …
         if (!psastroMosaicAstrom (config)) {
             psError (PSASTRO_ERR_UNKNOWN, false, "failed to perform mosaic camera astrometry\n");
-            psFree(refs);
             return false;
+        }
 …
     // psastroStackAstrom (config, refs);
-    psFree (refs);
     return true;
+}

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroChooseRefstars.c

-              r21409
+              r23594
+}
 bool psastroChooseRefstars (pmConfig *config, psArray *refs) {
+bool psastroChooseRefstars (pmConfig *config, psArray *refs, const char *source) {
     bool status;
 …
     // select the input data sources
     pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, "PSASTRO.INPUT");
+    pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, source);
     if (!input) {
         psError(PSASTRO_ERR_CONFIG, true, "Can't find input data!\n");

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroLoadRefstars.c

-              r21422
+              r23594
 # define ELIXIR_MODE 1
+char *psastroSetMagLimit (float *minMag, float *maxRho, pmConfig *config);
+psArray *psastroLoadRefstars (pmConfig *config) {
+psArray *psastroLoadRefstars (pmConfig *config, const char *source) {
     int fd;
 …
     float minMag;
     float maxRho;
     char *photcode = psastroSetMagLimit (&minMag, &maxRho, config);
+    char *photcode = psastroSetMagLimit (&minMag, &maxRho, config, source);
     PS_ASSERT (photcode, NULL);
 …
   goto escape; }
 char *psastroSetMagLimit (float *minMag, float *maxRho, pmConfig *config) {
+char *psastroSetMagLimit (float *minMag, float *maxRho, pmConfig *config, const char *source) {
     bool status;
 …
     // select the input data sources
     pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, "PSASTRO.INPUT");
+    pmFPAfile *input = psMetadataLookupPtr (NULL, config->files, source);
     if (!input) {
         psLogMsg ("psastro", PS_LOG_DETAIL, "no supplied reference header data");

branches/cnb_branches/cnb_branch_20090301/psconfig

Property svn:mergeinfo set to (toggle deleted branches)
/branches/eam_branches/eam_branch_20090312/psconfig 23313-23408

branches/cnb_branches/cnb_branch_20090301/psconfig/psbuild

-              r23352
+              r23594
 $verbose = 0;
 $use_svn = 1;
+$svn_trunk = "";
 $extlibs = "none";
 …
     if ($ARGV[0] eq "-extperl") {
         $extperl = $ARGV[1];
+        shift; shift; next;
+    }
+    if ($ARGV[0] eq "-trunk") {
+        $svn_trunk = $ARGV[1];
         shift; shift; next;
+    }
 …
         print "\033]0; ** psbuild: $module[$i] ** \007";
+        if ($svn_trunk && (!-d $workdir || !-r $workdir || !-x $workdir)) {
+            # try trunk instead
+            print STDERR "$module[$i] missing from local path, trying trunk path\n";
+            $workdir = "$svn_trunk/$module[$i]";
+        }
         if (!-d $workdir || !-r $workdir || !-x $workdir) {
             print STDERR "WARNING: no directory for component $module[$i], skipping\n";

branches/cnb_branches/cnb_branch_20090301/psconfig/psconfig.csh.in

-              r18708
+              r23594
 if ("$PSCONFIG" == "none") then
   alias  psconfigure configure
   alias  psautogen autogen.sh
+  alias  psconfigure ./configure
+  alias  psautogen ./autogen.sh
   alias  psperlbuild perl Build.PL
 else

branches/cnb_branches/cnb_branch_20090301/psconfig/tagsets/ipp-2.8.dist

-              r23352
+              r23594
   YNNYY  Nebulous/nebclient     ipp-2-8          -0
   YNNYY  Nebulous               ipp-2-8          -0
+  YNNYY  Nebulous-Server        ipp-2-8          -0
   YYYYY  PS-IPP-Metadata-Config ipp-2-8          -0
   YYYYY  PS-IPP-Config          ipp-2-8          -0
 …
   YYYYY  DataStore              ipp-2-8          -0
+  YNNNN  extsrc/gpcsw           ipp-2-8          -0
 # there are externally required C libraries and perl modules (see INSTALL)

branches/cnb_branches/cnb_branch_20090301/psphot

Property svn:mergeinfo changed
/trunk/psphot merged: 23442,23445,23486-23487,23492,23535

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphot.h

-              r23352
+              r23594
 bool            psphotModelTest (pmConfig *config, const pmFPAview *view, psMetadata *recipe);
 bool            psphotInit ();
+bool            psphotInit (void);
 bool            psphotReadout (pmConfig *config, const pmFPAview *view);
 bool            psphotReadoutFindPSF(pmConfig *config, const pmFPAview *view, psArray *inSources);
 …
 // XXX test functions
 psArray        *psphotFakeSources ();
+psArray        *psphotFakeSources (void);
 // psphotReadout functions
 …
 // thread-related:
 bool            psphotSetThreads ();
+bool            psphotSetThreads (void);
 bool            psphotChooseCellSizes (int *Cx, int *Cy, pmReadout *readout, int nThreads);
 bool            psphotCoordToCell (int *group, int *cell, float x, float y, int Cx, int Cy);
 …
 bool            psphotPlotMoments (pmConfig *config, pmFPAview *view, psArray *sources);
 bool            psphotPlotPSFModel (pmConfig *config, pmFPAview *view, psArray *sources);
 bool            psphotFitInit ();
 bool            psphotFitSummary ();
+bool            psphotFitInit (int nThreads);
+bool            psphotFitSummary (void);
 bool            psphotMergeSources (psArray *oldSources, psArray *newSources);
 …
 psImage *pmPCMDataSaveImage (pmPCMData *pcm);
 int psphotKapaOpen ();
 bool psphotKapaClose ();
+int psphotKapaOpen (void);
+bool psphotKapaClose (void);
 bool psphotImageBackgroundCellHistogram (psVector *values, float mean, float sigma, int ix, int iy);
 bool psphotDiagnosticPlots (const pmConfig *config, const char *name, ...);

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotCleanup.c

-              r21341
+              r23594
+}
 psExit psphotGetExitStatus () {
+psExit psphotGetExitStatus (void) {
     psErrorCode err = psErrorCodeLast ();
     switch (err) {
       case PS_ERR_NONE:
         return PS_EXIT_SUCCESS;
+        return PS_EXIT_SUCCESS;
       case PSPHOT_ERR_SYS:
         return PS_EXIT_SYS_ERROR;

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotDetect.h

r21392	r23594
19	19	bool psphotMosaicChip(pmConfig config, const pmFPAview view, char outFile, char inFile);
20	20	void psphotCleanup (pmConfig *config);
21		psExit psphotGetExitStatus ();
	21	psExit psphotGetExitStatus (void);
22	22
23	23	#endif

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotFitSourcesLinear.c

-              r21519
+              r23594
         pmSource *source = sources->data[i];
         // turn this bit off and turn it on again if we pass this test
         source->mode &= ~PM_SOURCE_MODE_LINEAR_FIT;
+        // turn this bit off and turn it on again if we pass this test
+        source->mode &= ~PM_SOURCE_MODE_LINEAR_FIT;
         // skip non-astronomical objects (very likely defects)
 …
         if (source->type == PM_SOURCE_TYPE_SATURATED) continue;
         // do not include CRs in the full ensemble fit
+        // do not include CRs in the full ensemble fit
         if (source->mode & PM_SOURCE_MODE_CR_LIMIT) continue;
 …
         if (y > AnalysisRegion.y1) continue;
         source->mode |= PM_SOURCE_MODE_LINEAR_FIT;
+        source->mode |= PM_SOURCE_MODE_LINEAR_FIT;
         psArrayAdd (fitSources, 100, source);
+    }
     psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "built fitSources: %f sec (%ld objects)\n", psTimerMark ("psphot.linear"), sources->n);
+    if (fitSources->n == 0) {
+        return true;
+    }
     // vectors to store stats for each object
 …
     psLogMsg ("psphot.ensemble", PS_LOG_MINUTIA, "solve matrix: %f sec (%d elements)\n", psTimerMark ("psphot.linear"), sparse->Nelem);
     // XXXX **** philosophical question:
+    // XXXX **** philosophical question:
     // we measure bright objects in three passes: 1) linear fit; 2) non-linear fit; 3) linear fit:
     // should retain the chisq and errors from the intermediate non-linear fit?
     // the non-linear fit provides better values for the position errors, and for
+    // should retain the chisq and errors from the intermediate non-linear fit?
+    // the non-linear fit provides better values for the position errors, and for
     // extended sources, the shape errors
 …
     for (int i = 0; final && (i < fitSources->n); i++) {
         pmSource *source = fitSources->data[i];
         if (source->mode & PM_SOURCE_MODE_NONLINEAR_FIT) continue;
+        if (source->mode & PM_SOURCE_MODE_NONLINEAR_FIT) continue;
         pmModel *model = pmSourceGetModel (NULL, source);
         pmSourceChisq (model, source->pixels, source->maskObj, source->variance, maskVal);

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotMaskReadout.c

-              r23352
+              r23594
     psMetadataAddImageMask (recipe, PS_LIST_TAIL, "MASK.SAT", PS_META_REPLACE, "user-defined mask", maskSat);
+    psImageMaskType maskBad  = pmConfigMaskGet("BAD", config); // Mask value for bad pixels
+    psImageMaskType maskBad  = pmConfigMaskGet("LOW", config); // Mask value for low pixels
+    if (!maskBad) {
+        // XXX: for backward compatability look up old name
+        maskBad  = pmConfigMaskGet("BAD", config);
+    }
     psMetadataAddImageMask (recipe, PS_LIST_TAIL, "MASK.BAD", PS_META_REPLACE, "user-defined mask", maskBad);

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadout.c

r21366	r23594
2	2
3	3	// this should be called by every program that links against libpsphot
4		bool psphotInit () {
	4	bool psphotInit (void) {
5	5
6	6	psphotErrorRegister(); // register our error codes/messages

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadoutFindPSF.c

-              r21366
+              r23594
     if (!recipe) {
         psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE);
+        return false;
+    }
+    // set the photcode for this image
+    if (!psphotAddPhotcode(recipe, config, view, "PSPHOT.INPUT")) {
+        psError(PSPHOT_ERR_CONFIG, false, "trouble defining the photcode");
         return false;
+    }

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadoutKnownSources.c

-              r21366
+              r23594
     if (!recipe) {
         psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE);
+        return false;
+    }
+    // set the photcode for this image
+    if (!psphotAddPhotcode(recipe, config, view, "PSPHOT.INPUT")) {
+        psError(PSPHOT_ERR_CONFIG, false, "trouble defining the photcode");
         return false;
+    }

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotReadoutMinimal.c

-              r21366
+              r23594
     if (!recipe) {
         psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSPHOT_RECIPE);
+        return false;
+    }
+    // set the photcode for this image
+    if (!psphotAddPhotcode(recipe, config, view, "PSPHOT.INPUT")) {
+        psError(PSPHOT_ERR_CONFIG, false, "trouble defining the photcode");
         return false;
+    }

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotStandAlone.h

r12805	r23594
17	17	bool psphotMosaicChip(pmConfig config, const pmFPAview view, char outFile, char inFile);
18	18	void psphotCleanup (pmConfig *config);
19		psExit psphotGetExitStatus ();
	19	psExit psphotGetExitStatus (void);
20	20
21	21	#endif

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarp.h

-              r23352
+              r23594
 } pswarpTransformTileArgs;
 pswarpTransformTileArgs *pswarpTransformTileArgsAlloc();
+pswarpTransformTileArgs *pswarpTransformTileArgsAlloc(void);
 bool pswarpTransformTile (pswarpTransformTileArgs *args);
 …
 bool pswarpMatchRange (int *minX, int *minY, int *maxX, int *maxY, pmReadout *dest, pmReadout *src);
 pswarpMap *pswarpMapAlloc ();
+pswarpMap *pswarpMapAlloc (void);
 pswarpMapGrid *pswarpMapGridAlloc (int Nx, int Ny);
 …
  * define threads for this program
  */
 bool pswarpSetThreads ();
+bool pswarpSetThreads (void);
 /// Return software version

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpMapGrid.c

-              r21323
+              r23594
     // measure the map for the center of each superpixel
     for (ni = 0, i = xMin; ni < nXpts; i += nXpix, ni++) {
         for (nj = 0, j = yMin; nj < nYpts; j += nYpix, nj++) {
             pswarpMapSetLocalModel (grid->maps[ni][nj], dest, src, i, j);
+        }
+        for (nj = 0, j = yMin; nj < nYpts; j += nYpix, nj++) {
+            pswarpMapSetLocalModel (grid->maps[ni][nj], dest, src, i, j);
+        }
+    }
     grid->nXpix = nXpix;
     grid->nYpix = nYpix;
 …
     for (int i = 0; i < grid->nXpts - 1; i++) {
         for (int j = 0; j < grid->nYpts - 1; j++) {
             // measure the output coordinates for the next grid position using the current grid map
             // compare with the coordinates measured using the next grid map
             pswarpMapApply (&xRaw, &yRaw, grid->maps[i][j], grid->maps[i][j]->xo + grid->nXpix, grid->maps[i][j]->yo);
             pswarpMapApply (&xRef, &yRef, grid->maps[i+1][j], grid->maps[i][j]->xo + grid->nXpix, grid->maps[i][j]->yo);
             double posError = hypot (xRaw-xRef, yRaw-yRef);
             maxError = PS_MAX (maxError, posError);
+        }
+    }
+        for (int j = 0; j < grid->nYpts - 1; j++) {
+            // measure the output coordinates for the next grid position using the current grid map
+            // compare with the coordinates measured using the next grid map
+            pswarpMapApply (&xRaw, &yRaw, grid->maps[i][j], grid->maps[i][j]->xo + grid->nXpix, grid->maps[i][j]->yo);
+            pswarpMapApply (&xRef, &yRef, grid->maps[i+1][j], grid->maps[i][j]->xo + grid->nXpix, grid->maps[i][j]->yo);
+            double posError = hypot (xRaw-xRef, yRaw-yRef);
+            maxError = PS_MAX (maxError, posError);
+        }
+    }
     return maxError;
+}
 …
     psPlaneTransformApply (FP, fpaDest->fromTPA, TP);
     psPlaneTransformApply (V00, chipDest->fromFPA, FP);
     /** V(1,0) position */
     offset->x = ix + 1;
 …
     psPlaneTransformApply (FP, fpaDest->fromTPA, TP);
     psPlaneTransformApply (V10, chipDest->fromFPA, FP);
     /** V(0,1) position */
     offset->x = ix;
 …
     map->Yy = V01->y - V00->y;
     map->Yo = V00->y - map->Yx*ix - map->Yy*iy;
     map->xo = ix;
     map->yo = iy;
 …
+}
 pswarpMap *pswarpMapAlloc () {
+pswarpMap *pswarpMapAlloc(void) {
   pswarpMap *map = (pswarpMap *) psAlloc (sizeof(pswarpMap));
 …
     for (int i = 0; i < grid->nXpts; i++) {
         for (int j = 0; j < grid->nYpts; j++) {
             psFree (grid->maps[i][j]);
+        }
         psFree (grid->maps[i]);
+        for (int j = 0; j < grid->nYpts; j++) {
+            psFree (grid->maps[i][j]);
+        }
+        psFree (grid->maps[i]);
+    }
     psFree (grid->maps);
 …
       grid->maps[i] = psAlloc (nYpts*sizeof(void *));
       for (int j = 0; j < nYpts; j++) {
           grid->maps[i][j] = pswarpMapAlloc();
+          grid->maps[i][j] = pswarpMapAlloc();
+      }
+  }
 …
   grid->nXpix = 0;
   grid->nYpix = 0;
   return grid;
+}

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpParseCamera.c

r23352	r23594
24	24
25	25	// look for the file on the RUN metadata
26		pmFPAfile *file = pmFPAfileDefineFromRun(&status, config, filerule); // File to return
	26	pmFPAfile *file = pmFPAfileDefineFromRun(&status, bind, config, filerule); // File to return
27	27	if (!status) {
28	28	psError(PSWARP_ERR_CONFIG, false, "Failed to load file definition for %s", filerule);

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpSetMaskBits.c

-              r21368
+              r23594
     // mask for non-linear flat regions (default to DETECTOR if not defined)
     psImageMaskType badMask = pmConfigMaskGet("BAD.WARP", config);
+    psImageMaskType badMask = pmConfigMaskGet("CONV.BAD", config);
     if (!badMask) {
         badMask = 0x01;
         pmConfigMaskSet (config, "BAD.WARP", badMask);
+        pmConfigMaskSet (config, "CONV.BAD", badMask);
+    }
     maskOut |= badMask;
     // mask for non-linear flat regions (default to DETECTOR if not defined)
     psImageMaskType poorMask = pmConfigMaskGet("POOR.WARP", config);
+    psImageMaskType poorMask = pmConfigMaskGet("CONV.POOR", config);
     if (!poorMask) {
         poorMask = 0x02;
         pmConfigMaskSet (config, "POOR.WARP", poorMask);
+        pmConfigMaskSet (config, "CONV.POOR", poorMask);
+    }
     maskOut |= poorMask;

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpSetThreads.c

r21323	r23594
23	23	}
24	24
25		bool pswarpSetThreads () {
	25	bool pswarpSetThreads(void) {
26	26
27	27	psThreadTask *task = NULL;

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpTransformReadout.c

-              r21529
+              r23594
     // output mask bits
     psImageMaskType maskIn   = psMetadataLookupImageMask(&mdok, recipe, "MASK.INPUT");
+    psImageMaskType maskPoor = pmConfigMaskGet("POOR.WARP", config);
+    psImageMaskType maskBad  = pmConfigMaskGet("BAD.WARP", config);
+    psImageMaskType maskPoor = pmConfigMaskGet("CONV.POOR", config);
+    if (!maskPoor) {
+        maskPoor = pmConfigMaskGet("POOR.WARP", config);
+    }
+    psImageMaskType maskBad  = pmConfigMaskGet("CONV.BAD", config);
+    if (!maskBad) {
+        maskBad  = pmConfigMaskGet("BAD.WARP", config);
+    }
     psAssert(mdok, "MASK.INPUT was not defined");

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpTransformTile.c

r21368	r23594
24	24	}
25	25
26		pswarpTransformTileArgs *pswarpTransformTileArgsAlloc()
	26	pswarpTransformTileArgs *pswarpTransformTileArgsAlloc(void)
27	27	{
28	28	pswarpTransformTileArgs *args = psAlloc(sizeof(pswarpTransformTileArgs));

/branches/eam_branches/eam_branch_20090312/ppImage (added)	merged: 23312,23316,23320,23406,23408
/trunk/ppImage	merged: 23366,23411,23426,23534-23535

/branches/eam_branches/eam_branch_20090312/psastro (added)	merged: 23321-23322,23335-23337,23407
/trunk/psastro	merged: 23412,23431,23486,23494,23584,23591-23592

Context Navigation

Legend: