Changeset 23352

branches/cnb_branches/cnb_branch_20090301

Property svn:mergeinfo set to (toggle deleted branches)
/trunk merged eligible
/branches/eam_branches/eam_branch_20090303 23158-23228

branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/MANIFEST

-              r20990
+              r23352
 bin/neb-voladd
 bin/nebdiskd
-docs/c_api.h
 docs/database_setup.txt
 docs/design.txt
+docs/install.txt
 docs/requirements.txt
-docs/setup.txt
-docs/tmp.txt
 examples/Makefile
 examples/nebexample.c

branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/bin/neb-admin

-              r20202
+              r23352
     $dbpass,
     $dbuser,
+    $so_id_start,
+    $so_id_range,
     $limit,
     $pending,
 …
     'pendingreplicate|r'    => \$pending,
     'pendingremoval'        => \$removal,
+    'so_id_start=i'         => \$so_id_start,
+    'so_id_range=i'         => \$so_id_range,
     'limit|l=i'             => \$limit,
     'verbose|v'             => \$verbose,
 …
 # check to make sure that only one instance of neb-admin is running
+# XXX this implies we should move pending replicate elsewhere
 my $pidfile = '/var/tmp/neb-admin';
 # abort if an instance is already running
 …
     $dbh->do("INSERT INTO myvolume SELECT * FROM volume");
+    if (not defined $so_id_start) { $so_id_start = 0; }
+    if (not defined $so_id_range) { $so_id_range = 100000; }
+    my $so_id_end = $so_id_start + $so_id_range;
+    # XXX check if so_id_start is beyond MAX(so_id): if so, exit with exit status 10
+    {
+        my $query = $dbh->prepare("SELECT MAX(so_id) from storage_object");
+        $query->execute;
+        my $answer = $query->fetchrow_arrayref;
+        my $max_so_id = $$answer[0];
+        $query->finish;
+        if ($so_id_start > $max_so_id) {
+            print STDERR "at end of so_id range, reset please\n";
+            exit 10;
+        }
+    }
     my $query = $dbh->prepare(
     "        SELECT
 …
                 USING(vol_id)
             WHERE mymountedvol.available = 1
+    --        WHERE storage_object_xattr.name = 'user.copies'
+            AND storage_object_xattr.name = 'user.copies'
+            AND storage_object.so_id >= $so_id_start
+            AND storage_object.so_id <  $so_id_end
             GROUP BY so_id
             HAVING available_instances < instances OR instances < copies

branches/cnb_branches/cnb_branch_20090301/Nebulous-Server/bin/neb-voladd

-              r19797
+              r23352
 =head1 SYNOPSIS
     neb-voladd --volume <volume name> --uri <volume uri>
+    neb-voladd --vname <volume name> --vhost <host name> --uri <volume uri>
     [--db <database>] [--user <username>] [--pass <password>] [--host <hostname]
 …
 =over 4
 =item * --volume <volume name>
+=item * --vname <volume name>
 Symbolic name of volume being added.
+=item * --vhost <host name>
+Name of the host that containes the C<URI> of the volume being added.
 =item * --uri|-u <volume uri>

branches/cnb_branches/cnb_branch_20090301/Nebulous/Build.PL

-              r20131
+              r23352
 my $pkg_dir = "./nebclient";
+sub ACTION_code {
+    my $self = shift;
+    $self->SUPER::ACTION_code(@_);
+    my $old_pwd = getcwd();
+    chdir $pkg_dir;
+    unless (-e "configure") {
+        system("./autogen.sh") == 0 or die "install failed: $?";
+    }
+    chdir $old_pwd;
+}
+sub ACTION_build {
+    my $self = shift;
+    $self->SUPER::ACTION_build(@_);
+    my $old_pwd = getcwd();
+    chdir $pkg_dir;
+    unless (-e "Makefile") {
+        system("sh ./configure") == 0 or die "build failed: $?";
+    }
+    system("make") == 0 or die "build failed: $?";
+    chdir $old_pwd;
+}
+# Do not attempt to install nebclient.  It is bundled in this package for
+# testing only.
+#
+# sub ACTION_code {
+#     my $self = shift;
+#
+#     $self->SUPER::ACTION_code(@_);
+#
+#     my $old_pwd = getcwd();
+#     chdir $pkg_dir;
+#
+#     unless (-e "configure") {
+#         system("./autogen.sh") == 0 or die "install failed: $?";
+#     }
+#
+#     chdir $old_pwd;
+# }
+#
+# sub ACTION_build {
+#     my $self = shift;
+#
+#     $self->SUPER::ACTION_build(@_);
+#
+#     my $old_pwd = getcwd();
+#     chdir $pkg_dir;
+#
+#     unless (-e "Makefile") {
+#         system("sh ./configure") == 0 or die "build failed: $?";
+#     }
+#
+#     system("make") == 0 or die "build failed: $?";
+#
+#     chdir $old_pwd;
+# }
+#
 # sub ACTION_install {
 #     my $self = shift;
 …
 #     chdir $old_pwd;
 # }
 sub ACTION_clean {
     my $self = shift;
     $self->SUPER::ACTION_clean(@_);
     my $old_pwd = getcwd();
     chdir $pkg_dir;
     system("make clean") == 0 or die "install failed: $?";
     chdir $old_pwd;
+}
+#
+# sub ACTION_clean {
+#     my $self = shift;
+#
+#     $self->SUPER::ACTION_clean(@_);
+#
+#     my $old_pwd = getcwd();
+#     chdir $pkg_dir;
+#
+#     system("make clean") == 0 or die "install failed: $?";
+#
+#     chdir $old_pwd;
+# }
 EOF

branches/cnb_branches/cnb_branch_20090301/Nebulous/MANIFEST

-              r20131
+              r23352
 bin/neb-touch
 bin/neb-xattr
-docs/c_api.h
-docs/database_setup.txt
-docs/design.txt
-docs/requirements.txt
-docs/setup.txt
-docs/tmp.txt
 examples/Makefile
 examples/nebexample.c

branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/src

Property svn:ignore

-              old
+              new
+Makefile.in
 .deps
+Makefile
+soapClientLib.c
+nebulous.h
+soapClient.c
+soapStub.h
+soapC.c
+soapH.h
+soapServerLib.c
+soapServer.c
 .libs
-Makefile
-Makefile.in
-SOAP.nsmap
-libnebclient.la
-nebclient.lo
-nebulous.h
-soapC.c
-soapC.lo
-soapClient.c
-soapClient.lo
-soapClientLib.c
-soapH.h
-soapServer.c
-soapServerLib.c
-soapStub.h
-stdsoap2.lo
-xmalloc.lo

branches/cnb_branches/cnb_branch_20090301/Nebulous/nebclient/tests/tap/src

Property svn:ignore

old	new
6	6	.deps
7	7	Makefile
	8	.libs

branches/cnb_branches/cnb_branch_20090301/Nebulous/scripts/ptest.pl

-              r21545
+              r23352
 use lib "./lib";
+package main;
 use Nebulous::Client;
+#use Nebulous::Client;
 use IO::Select;
 use IO::Socket;
+use POSIX qw(:DEFAULT :sys_wait_h);
+use Hook::LexWrap;
+use Sys::Hostname;
+my $neb = Nebulous::Client->new(
+#    proxy   => 'http://localhost:80/nebulous'
+    proxy   => 'http://alala:80/nebulous'
+);
+my $print_stdout = undef;
+$| = 1;
 my $key = shift || 'foobar';
 my $kids = shift || 1;
+my $s = IO::Select->new();
+$key = "/tmp/" . $key;
+#my $s = IO::Select->new();
 foreach my $id ( 1..$kids ) {
     my ($sock_parent, $sock_child) = IO::Socket->socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC);
     $s->add($sock_parent);
+#    my ($sock_parent, $sock_child) = IO::Socket->socketpair(AF_UNIX, SOCK_STREAM, PF_UNSPEC);
+#    $s->add($sock_parent);
     my $pid = fork;
     unless ( $pid )  {
+        # child
+        my $sock_child = \*STDOUT;
         child($sock_child, $id);
+        shutdown($sock_child, 2);
         exit 0;
+    }
 …
+}
+while ($kids) {
+    foreach my $child ($s->can_read(1)) {
+        my $string = <$child>;
+        my @events = split(/\n/, $string) if $string;
+        print join("\n", @events), "\n" if scalar @events;
+$SIG{CHLD} = \&REAPER;
+sub REAPER {
+    while ((my $pid = waitpid(-1,WNOHANG)) > 0) {
+        $kids--;
+#        delete $children{$pid};
+    }
+    $SIG{CHLD} = \&REAPER;
+}
-#while ( $kids ) {
-#    wait();
-#    $kids --;
-#}
+our $my_sock;
+while ($kids) {
+#    foreach my $child ($s->can_read(0)) {
+#        my $string = do { local $/; <$child>};
+#        my @events = split(/\n/, $string) if $string;
+#        print join("\n", @events), "\n" if scalar @events;
+#    }
+}
 sub child
+{
     my ($sock_child, $id) = @_;
+    my ($sock, $id) = @_;
+    $my_sock = $sock_child;
+    unless ($print_stdout) {
+        my $filename = hostname() . "." . $$ . ".txt";
+        open my $fh, ">$filename" or die "can't open $filename: $!";
+        open STDOUT, ">&", $fh or die "can't reopen STDOUT: $!";
+        autoflush STDOUT 1;
+    }
+#    select $sock;
+#    $| = 1;
     # child
     my $fname = "${key}_$id";
-    print $sock_child "$$ : i'm a little tea pot using key: $fname\n";
-    my $fh = $neb->open_create( $fname );
-    child_die("can't create file $fname") unless $fh;
+    print $fh "fooby\n";
+    my $neb = Nebulous::Client::Bench->new(
+#    proxy   => 'http://localhost:80/nebulous'
+        proxy   => 'http://alala:80/nebulous',
+#        sock    => \*STDOUT,
+    );
-    close $fh;
+    $fh = $neb->open( $fname, 'read' ) or child_die("can't open file");
+    close $fh;
+    while (1) {
+#    print $sock "$$ : i'm a little tea pot using key: $fname\n";
+        my $fh = $neb->open_create( $fname )
+            or child_die($sock, "can't create file $fname");
+        close $fh;
+    $neb->lock( $fname, 'read' );
+    $neb->unlock( $fname, 'read' );
+    $neb->replicate( $fname );
+    $neb->cull( $fname );
+        $fh = $neb->open( $fname, 'read' )
+            or child_die("can't open file");
+        close $fh;
+    print $sock_child "$$ : half way\n";
+        $neb->lock( $fname, 'read' );
+        $neb->unlock( $fname, 'read' );
+        $neb->replicate( $fname );
+        $neb->cull( $fname );
+        $neb->find( $fname );
+        $neb->copy( $fname, $fname . "_copy" );
+        $neb->move( $fname, $fname . "_move" );
+        $neb->delete( $fname . "_copy" );
+        $neb->delete( $fname . "_move" );
+    $neb->find( $fname );
+    $neb->copy( $fname, $fname . "_copy" );
+    $neb->move( $fname, $fname . "_move" );
+    $neb->delete( $fname . "_copy" );
+    $neb->delete( $fname . "_move" );
+    print $sock_child "$$ : all done!\n";
+    $sock_child->flush;
+    sleep 10;
+#    print $sock "$$ : all done!\n";
+    }
     return 1;
 …
 sub child_die
+{
+    print $my_sock $@;
+    shutdown($my_sock, 2);
+    my $sock = shift;
+    print $sock @_;
+    shutdown($sock, 2);
     exit 1;
+}
+package Nebulous::Client::Bench;
+use base qw( Nebulous::Client );
+#sub new
+#{
+#    my $class = shift;
+#    my %p = @_;
+#
+#    my $sock = delete $p{sock};
+#    my $self = $class->SUPER::new(%p);
+#    $self->{sock} = $sock;
+#
+#    return $self;
+#}
+BEGIN {
+sub make_wrapper
+{
+    my $method = shift;
+eval "sub $method {"
+.'    my $self = shift;'
+.'    my $smark = Time::HiRes::time();'
+.'    my $ret = $self->SUPER::' . "$method" .'(@_);'
+.'    my $emark = Time::HiRes::time();'
+.'    printf "%-17s %-17s %s\n", $emark, " ' . "$method" . ' ", ($emark - $smark), "\n";'
+.'    return $ret;'
+.'}';
+}
+make_wrapper("create");
+make_wrapper("open_create");
+make_wrapper("replicate");
+make_wrapper("cull");
+make_wrapper("lock");
+make_wrapper("unlock");
+make_wrapper("setxattr");
+make_wrapper("getxattr");
+make_wrapper("listxattr");
+make_wrapper("removexattr");
+make_wrapper("find_objects");
+make_wrapper("find_instances");
+#make_wrapper("find");
+#make_wrapper("open");
+#make_wrapper("delete");
+#make_wrapper("copy");
+make_wrapper("move");
+make_wrapper("swap");
+make_wrapper("delete_instance");
+make_wrapper("stat");
+make_wrapper("mounts");
+}
+;

branches/cnb_branches/cnb_branch_20090301/Ohana

Property svn:mergeinfo set to
/trunk/Ohana merged eligible

branches/cnb_branches/cnb_branch_20090301/Ohana/Makefile.in

-              r19682
+              r23352
         mkdir -p $(DESTLIB)
         mkdir -p $(DESTBIN)
         for i in $(LIBS); do make $$i.install || exit; done
+        for i in $(LIBS); do make $$i.install || exit 1; done
         chmod +x ohana-config
         cp -f ohana-config $(DESTBIN)/
 …
 all:
         make libs || exit
         for i in $(PROGRAM); do make $$i || exit; done
+        make libs || exit 1
+        for i in $(PROGRAM); do make $$i || exit 1; done
 extras:
         for i in $(EXTRAS); do make $$i || exit; done
+        for i in $(EXTRAS); do make $$i || exit 1; done
 pantasks:
         make libs || exit
         cd src/opihi; make pclient.install && exit
         cd src/opihi; make pcontrol.install && exit
         cd src/opihi; make pantasks.install && exit
+        make libs || exit 1
+        cd src/opihi; make pclient.install && exit 1
+        cd src/opihi; make pcontrol.install && exit 1
+        cd src/opihi; make pantasks.install && exit 1
 mana:
         make libs
         make kapa2.install
         cd src/opihi; make mana.install && exit
+        cd src/opihi; make mana.install && exit 1
 dvoshell:
         make libs
         make kapa2.install
         cd src/opihi; make dvo.install && exit
+        cd src/opihi; make dvo.install && exit 1
 clean:
         @if [ "$(ARCH)" = "" ]; then echo ""; echo " *** please define ARCH ***"; echo; exit 1; fi
         for i in $(LIBS); do make $$i.clean || exit; done
         for i in $(PROGRAM); do make $$i.clean || exit; done
         for i in $(EXTRAS); do make $$i.clean || exit; done
+        for i in $(LIBS); do make $$i.clean || exit 1; done
+        for i in $(PROGRAM); do make $$i.clean || exit 1; done
+        for i in $(EXTRAS); do make $$i.clean || exit 1; done
         @rm -f `find . -name .mana`
         @rm -f `find . -name .dvo`
 …
 dist:
         @if [ "$(ARCH)" = "" ]; then echo ""; echo " *** please define ARCH ***"; echo; exit 1; fi
         for i in $(LIBS); do make $$i.dist || exit; done
         for i in $(PROGRAM); do make $$i.dist || exit; done
+        for i in $(LIBS); do make $$i.dist || exit 1; done
+        for i in $(PROGRAM); do make $$i.dist || exit 1; done
         @echo -n -e "\0033]0; *** Ohana: done $@ *** \0007" \
 install:
         @if [ "$(ARCH)" = "" ]; then echo ""; echo " *** please define ARCH ***"; echo; exit 1; fi
         for i in $(LIBS); do make $$i.install || exit; done
         for i in $(PROGRAM); do make $$i.install || exit; done
+        for i in $(LIBS); do make $$i.install || exit 1; done
+        for i in $(PROGRAM); do make $$i.install || exit 1; done
         @echo -n -e "\0033]0; *** Ohana: done $@ *** \0007" \
 install.extras:
         for i in $(EXTRAS); do make $$i.install || exit; done
+        for i in $(EXTRAS); do make $$i.install || exit 1; done
         @echo -n -e "\0033]0; *** Ohana: done $@ *** \0007" \

branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/include/skycells.h

-              r21508
+              r23352
 # include <glob.h>
 enum {SQUARES, TRIANGLES, LOCAL};
+enum {SQUARES, TRIANGLES, LOCAL, RINGS};
 enum {TETRAHEDRON, CUBE, OCTOHEDRON, DODECAHEDRON, ICOSAHEDRON};
 …
 double SCALE;
 double PADDING;
+double CELLSIZE;
 int    LEVEL;
 …
 int          sky_tessellation               PROTO((FITS_DB *db, int level, int Nmax, int mode, double scale));
 int          sky_tessellation_init          PROTO((double scale));
 int          sky_tessellation_local         PROTO((FITS_DB *db, int level, int Nmax));
 int          sky_tessellation_triangles     PROTO((FITS_DB *db, int level, int Nmax));
 int          sky_tessellation_squares       PROTO((FITS_DB *db, int level, int Nmax));
+int          sky_tessellation_rings         PROTO((FITS_DB *db, int level, int Nmax));
 int          sky_triangle_to_image          PROTO((Image *image, SkyTriangle *triangle));
 int          sky_triangle_to_rectangle      PROTO((SkyRectangle *image, SkyTriangle *triangle));
 int          sky_rectangle_local            PROTO((SkyRectangle *rectangle));
 int          sky_subdivide_image            PROTO((Image *output, SkyRectangle *input, int Nx, int Ny));
 int          sky_triangle_coords            PROTO((SkyTriangle *triangle));
+SkyRectangle *sky_rectangle_ring            PROTO((float dec, float dDEC, int *nring));
 SkyTriangle *sky_divide_triangles           PROTO((SkyTriangle *in, int *ntriangles));
 SkyTriangle *sky_base_triangles             PROTO((int *ntriangles));

branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/args_skycells.c

-              r21048
+              r23352
       MODE = LOCAL;
+    }
+    if (!strcasecmp (argv[N], "rings")) {
+      MODE = RINGS;
+    }
     remove_argument (N, &argc, argv);
+  }
 …
   LEVEL = 8;
+  if ((MODE != LOCAL) && (N = get_argument (argc, argv, "-level"))) {
+    remove_argument (N, &argc, argv);
+    LEVEL = strtol (argv[N], &ptr, 10);
+    remove_argument (N, &argc, argv);
+    if ((*ptr != 0) || (LEVEL < 0)) {
+      fprintf (stderr, "-level requires an integer (>= 0) argument\n");
+  if ((MODE == SQUARES) || (MODE == TRIANGLES)) {
+    if ((N = get_argument (argc, argv, "-level"))) {
+      remove_argument (N, &argc, argv);
+      LEVEL = strtol (argv[N], &ptr, 10);
+      remove_argument (N, &argc, argv);
+      if ((*ptr != 0) || (LEVEL < 0)) {
+        fprintf (stderr, "-level requires an integer (>= 0) argument\n");
+        help ();
+      }
+    }
+  }
+  CELLSIZE = 4.0;
+  if ((MODE == RINGS) && (N = get_argument (argc, argv, "-cellsize"))) {
+    remove_argument (N, &argc, argv);
+    CELLSIZE = strtod (argv[N], &ptr);
+    if ((*ptr != 0) || (CELLSIZE < 0.0)) {
+      fprintf (stderr, "-level requires a floating-point argument\n");
       help ();
+    }
+    remove_argument (N, &argc, argv);
+  }

branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/get2mass_full.c

-              r21508
+              r23352
 # include "addstar.h"
 # include "2mass.h"
+// XXX check to see if desired output format is PS1_V1 or later?  (use 16bit version if not?)
 // fill in the data for a JHK triplet star.  takes a pointer to the start of the line the
 …
   switch (qual) {
     case 'X': star[0].measure.photFlags |= 0x0000; break;
     case 'U': star[0].measure.photFlags |= 0x0001; break;
     case 'F': star[0].measure.photFlags |= 0x0002; break;
     case 'E': star[0].measure.photFlags |= 0x0003; break;
     case 'A': star[0].measure.photFlags |= 0x0004; break;
     case 'B': star[0].measure.photFlags |= 0x0005; break;
     case 'C': star[0].measure.photFlags |= 0x0006; break;
     case 'D': star[0].measure.photFlags |= 0x0007; break;
+    case 'A': star[0].measure.photFlags |= 0x00000001; break; // was: 0x0004
+    case 'B': star[0].measure.photFlags |= 0x00000002; break; // was: 0x0005
+    case 'C': star[0].measure.photFlags |= 0x00000004; break; // was: 0x0006
+    case 'D': star[0].measure.photFlags |= 0x00000008; break; // was: 0x0007
+    case 'E': star[0].measure.photFlags |= 0x00000010; break; // was: 0x0003
+    case 'F': star[0].measure.photFlags |= 0x00000020; break; // was: 0x0002
+    case 'U': star[0].measure.photFlags |= 0x00000040; break; // was: 0x0001
+    case 'X': star[0].measure.photFlags |= 0x00000080; break; // was: 0x0000
     default:
       fprintf (stderr, "error!\n");
 …
   switch (qual) {
     case '0': star[0].measure.photFlags |= 0x0000; break;
     case '1': star[0].measure.photFlags |= 0x0010; break;
     case '2': star[0].measure.photFlags |= 0x0020; break;
     case '3': star[0].measure.photFlags |= 0x0030; break;
     case '4': star[0].measure.photFlags |= 0x0040; break;
     case '6': star[0].measure.photFlags |= 0x0050; break;
     case '9': star[0].measure.photFlags |= 0x0060; break;
+    case '0': star[0].measure.photFlags |= 0x00000100; break; // was: 0x0000
+    case '1': star[0].measure.photFlags |= 0x00000200; break; // was: 0x0010
+    case '2': star[0].measure.photFlags |= 0x00000400; break; // was: 0x0020
+    case '3': star[0].measure.photFlags |= 0x00000800; break; // was: 0x0030
+    case '4': star[0].measure.photFlags |= 0x00001000; break; // was: 0x0040
+    case '6': star[0].measure.photFlags |= 0x00002000; break; // was: 0x0050
+    case '9': star[0].measure.photFlags |= 0x00004000; break; // was: 0x0060
     default:
       fprintf (stderr, "error!\n");
 …
   switch (qual) {
     case 'p': star[0].measure.photFlags |= 0x0000; break;
     case 'c': star[0].measure.photFlags |= 0x0100; break;
     case 'd': star[0].measure.photFlags |= 0x0200; break;
     case 's': star[0].measure.photFlags |= 0x0300; break;
     case 'b': star[0].measure.photFlags |= 0x0400; break;
     case '0': star[0].measure.photFlags |= 0x0500; break;
+    case 'p': star[0].measure.photFlags |= 0x00010000; break; // was: 0x0000
+    case 'c': star[0].measure.photFlags |= 0x00020000; break; // was: 0x0100
+    case 'd': star[0].measure.photFlags |= 0x00040000; break; // was: 0x0200
+    case 's': star[0].measure.photFlags |= 0x00080000; break; // was: 0x0300
+    case 'b': star[0].measure.photFlags |= 0x00010000; break; // was: 0x0400
+    case '0': star[0].measure.photFlags |= 0x00020000; break; // was: 0x0500
     default:
       fprintf (stderr, "error!\n");
 …
   switch (qual) {
     case '0': star[0].measure.photFlags &= ~0x0008; break;
     case '1': star[0].measure.photFlags &= ~0x0008; break;
     default:  star[0].measure.photFlags |=  0x0008; break;
+    case '0': star[0].measure.photFlags &= ~0x00300000; break; // was: ~0x0008
+    case '1': star[0].measure.photFlags |=  0x00100000; break; // was: ~0x0008
+    default:  star[0].measure.photFlags |=  0x00200000; break; // was:  0x0008
+  }
   return (TRUE);
 …
   switch (qual) {
+    case '0': star[0].measure.photFlags &= ~0x0080; break;
+    case '1': star[0].measure.photFlags &= ~0x0080; break;
+    default:
+      star[0].measure.photFlags |= 0x0080;
+    case '0': star[0].measure.photFlags &= ~0x00c00000; break; // was: ~0x0080
+    case '1': star[0].measure.photFlags |=  0x00400000; break; // was: ~0x0080
+    default:  star[0].measure.photFlags |=  0x00800000;        // was:  0x0080
       star[0].measure.extNsigma = 100.0;
       break;
 …
   switch (qual) {
     case '0': star[0].measure.photFlags &= ~0x0800; break;
     case '1': star[0].measure.photFlags &= ~0x0800; break;
     default:  star[0].measure.photFlags |=  0x0800; break;
+    case '0': star[0].measure.photFlags &= ~0x03000000; break; // was: ~0x0800
+    case '1': star[0].measure.photFlags |=  0x01000000; break; // was: ~0x0800
+    default:  star[0].measure.photFlags |=  0x02000000; break; // was:  0x0800
+  }
   return (TRUE);
 …
   switch (qual) {
     case '0': star[0].measure.photFlags &= ~0x1000; break;
     case '1': star[0].measure.photFlags &= ~0x1000; break;
     default:  star[0].measure.photFlags |=  0x1000; break;
+    case '0': star[0].measure.photFlags &= ~0x0c000000; break; // was: ~0x1000
+    case '1': star[0].measure.photFlags |=  0x04000000; break; // was: ~0x1000
+    default:  star[0].measure.photFlags |=  0x08000000; break; // was:  0x1000
+  }
   return (TRUE);
 …
   switch (qual) {
     case '0': star[0].measure.photFlags &= ~0x2000; break;
     case '1': star[0].measure.photFlags |=  0x2000; break;
+    case '0': star[0].measure.photFlags &= ~0x10000000; break; // was: ~0x2000
+    case '1': star[0].measure.photFlags |=  0x10000000; break; // was:  0x2000
     default:  abort();
+  }
 …
+}
+// unused photFlags:
+// 0x0000.8000
+// 0x0004.0000
+// 0x0008.0000
+// 0x2000.0000
+// 0x4000.0000
+// 0x8000.0000

branches/cnb_branches/cnb_branch_20090301/Ohana/src/addstar/src/sky_tessalation.c

-              r21508
+              r23352
   sky_tessellation_init (scale);
+  if (mode == SQUARES) {
+    sky_tessellation_squares (db, level, Nmax);
+    return TRUE;
+  }
+  if (mode == TRIANGLES) {
+    sky_tessellation_triangles (db, level, Nmax);
+    return TRUE;
+  }
+  if (mode == LOCAL) {
+    sky_tessellation_local (db, level, Nmax);
+    return TRUE;
+  switch (mode) {
+    case SQUARES:
+      sky_tessellation_squares (db, level, Nmax);
+      return TRUE;
+    case TRIANGLES:
+      sky_tessellation_triangles (db, level, Nmax);
+      return TRUE;
+    case LOCAL:
+      sky_tessellation_local (db, level, Nmax);
+      return TRUE;
+    case RINGS:
+      sky_tessellation_rings (db, level, Nmax);
+      return TRUE;
+    default:
+      break;
+  }
 …
   free (image);
+  return (TRUE);
+}
+// the RINGS tessellation uses the declination zones proposed by Tamas Budavari
+// we generate projects on uniform rings of constant dec height
+int sky_tessellation_rings (FITS_DB *db, int level, int Nmax) {
+  int j, nDEC, Nimage, Nring;
+  float dec, dDEC;
+  SkyRectangle *ring;
+  Image *image;
+  // The tessellation has one input parameter: the approximate cell size.  Starting with
+  // the cell size, determine the optimal projection cell height (dDEC) that results in an
+  // integer number of dec zones between -90 and +90
+  // in fact, we place a single image on each pole, so the real range of dec is 180.0 - CELLSIZE:
+  nDEC = (180.0 - CELLSIZE) / CELLSIZE;
+  dDEC = (180.0 - CELLSIZE) / nDEC;
+  nDEC += 2;
+  // a test
+  // for (dec = 0.0 + 0.5*dDEC; dec < +90.0; dec += dDEC) {
+  // generate the a collection of rectangles for each ring
+  for (dec = -90.0; dec < +90.0 + 0.5*dDEC; dec += dDEC) {
+    ring = sky_rectangle_ring (dec, dDEC, &Nring);
+    if (!ring) continue;
+    // subdivide each image (Nx x Ny subcells)
+    Nimage = NX_SUB*NY_SUB*Nring;
+    ALLOCATE (image, Image, Nimage);
+    for (j = 0; j < Nring; j++) {
+      // convert the SkyRectangles to Images for output
+      sky_subdivide_image (&image[j*NX_SUB*NY_SUB], &ring[j], NX_SUB, NY_SUB);
+    }
+    /* add the new images and save */
+    dvo_image_addrows (db, image, Nimage);
+    SetProtect (TRUE);
+    dvo_image_update (db, VERBOSE);
+    SetProtect (FALSE);
+    dvo_image_clear_vtable (db);
+    free (ring);
+    free (image);
+  }
   return (TRUE);
+}
 …
   return (TRUE);
+}
+// define the parameters of a single sky projection center
+SkyRectangle *sky_rectangle_ring (float dec, float dDEC, int *nring) {
+  int i, NX, NY, nRA;
+  SkyRectangle *ring;
+  float theta, dRA;
+  // 'dec' is a guess at the center of the cell; in fact, we need to choose decLower and
+  // decUpper to ensure complete overlap of the cells
+  // we can determine the 'lower' bound (bound closest to the equator):
+  float decLower = (dec > 0.0) ? dec - 0.5*dDEC : dec + 0.5*dDEC;
+  // solve for actual cellsize (\theta):  tan(\delta_{n+1} - \theta/2) = tan(\delta_n + \theta/2)cos(\alpha_n / 2)
+  float decUpper = (dec > 0.0) ? dec + dDEC : dec - dDEC;
+  if (fabs(dec) + 0.5*dDEC > 90.0) {
+    // onPole = TRUE;
+    theta = dDEC;
+    nRA = 1;
+    dRA = theta / cos(decLower*RAD_DEG); // make a square at the pole
+  } else {
+    // onPole = FALSE;
+    // Subdivide the 'lower' bound into an integer number of segments:
+    nRA = cos(RAD_DEG*decLower) * 360.0 / CELLSIZE; // CELLSIZE is a projection size
+    dRA = 360.0 / nRA;                         // dRA is a size in RA degrees == \alpha_n
+    // tan(decUpper - theta/2) = tan(dec + theta/2) cos(dRA / 2);
+    // we solve this equation for theta (fairly ugly: expand the tangents into sin/cos, expand the
+    // sum-of-angle sine and cosine, multiply through, convert via half-angle formulae and write
+    // as a quadratic expression in sine(theta/2)
+    float sd1 = sin(RAD_DEG*decUpper);
+    float cd1 = cos(RAD_DEG*decUpper);
+    float sd2 = sin(RAD_DEG*dec);
+    float cd2 = cos(RAD_DEG*dec);
+    float   k = cos(RAD_DEG*dRA/2.0);
+    float c1 =  (sd1*cd2 + sd2*cd1)*(1.0 - k);
+    float c2 =  (sd1*cd2 - sd2*cd1)*(1.0 + k);
+    float c3 = -(sd1*sd2 + cd1*cd2)*(1.0 + k);
+    float A = SQ(c3) + SQ(c2);
+    float B = 2*c1*c3;
+    float C = SQ(c1) - SQ(c2);
+    float arg = SQ(B) - 4.0*A*C;
+    float root;
+    if (dec >= 0.0) {
+      root = (-B + sqrt (arg)) / (2.0*A);
+      theta = +DEG_RAD*asin(root);
+    } else {
+      root = (-B - sqrt (arg)) / (2.0*A);
+      theta = -DEG_RAD*asin(root);
+    }
+    // the negative solution yields a negative cellsize
+    // float root2 = (-B - sqrt (arg)) / (2.0*A);
+    // float theta2 = DEG_RAD*asin(root2);
+    // test lines:
+    // float r1 = tan(RAD_DEG*(decUpper - 0.5*theta1));
+    // float r2 = tan(RAD_DEG*(dec + 0.5*theta1));
+    // fprintf (stdout, "%f %f  %f  %f  %f %f  %f %f  %f %f %f\n", dec, decUpper, dRA, arg, root1, root2, theta1, theta2, r1, r2, k*r2);
+  }
+  fprintf (stdout, "%f %f  %f x %f (%d)\n", dec, decUpper, dRA, theta, nRA);
+  // I think we need to return the value of dec for the next ring, but I am not sure...
+  ALLOCATE (ring, SkyRectangle, nRA);
+  for (i = 0; i < nRA; i++) {
+    memset (&ring[i], 0, sizeof(SkyRectangle));
+    memset (&ring[i].coords, 0, sizeof(Coords));
+    ring[i].coords.crval1 = i*dRA;
+    ring[i].coords.crval2 = dec;
+    ring[i].coords.pc1_1 = +1.0;
+    ring[i].coords.pc1_2 = +0.0;
+    ring[i].coords.pc2_1 = -0.0;
+    ring[i].coords.pc2_2 = +1.0;
+    // range values are in projected degrees
+    NX = cos(decLower*RAD_DEG) * dRA   * 3600.0 / SCALE;
+    NY =                         theta * 3600.0 / SCALE;
+    // crpix1,crpix2 is the projection center
+    ring[i].coords.crpix1 = 0.5*NX;
+    ring[i].coords.crpix2 = 0.5*NY;
+    ring[i].coords.cdelt1 = SCALE / 3600.0;
+    ring[i].coords.cdelt2 = SCALE / 3600.0;
+    strcpy (ring[i].coords.ctype, "DEC--TAN");
+    ring[i].NX = NX;
+    ring[i].NY = NY;
+    ring[i].photcode = 1; // this needs to be set more sensibly
+    // fprintf (stderr, "%f %f  : %f %f\n",
+    // ring[i].coords.crval1, ring[i].coords.crval2,
+    // ring[i].coords.crpix1, ring[i].coords.crpix2);
+  }
+  *nring = nRA;
+  return (ring);
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/libdvo

Property svn:ignore

old	new
1		lib
	1

branches/cnb_branches/cnb_branch_20090301/Ohana/src/libdvo/lib

Property svn:ignore

-              old
+              new
+*.linux
+*.lin64
+*.sol
+*.sun
+*.sid
+*.hp
+*.irix
+*.darwin.dylib
+*.darwin_x86.dylib
+*.a
+*.so

branches/cnb_branches/cnb_branch_20090301/Ohana/src/libfits/lib

Property svn:ignore

-              old
+              new
+*.linux
+*.lin64
+*.sol
+*.sun
+*.sid
+*.hp
+*.irix
+*.darwin.dylib
+*.darwin_x86.dylib
+*.a
+*.so

branches/cnb_branches/cnb_branch_20090301/Ohana/src/libkapa/lib

Property svn:ignore

-              old
+              new
+*.linux
+*.lin64
+*.sol
+*.sun
+*.sid
+*.hp
+*.irix
+*.darwin.dylib
+*.darwin_x86.dylib
+*.a
+*.so

branches/cnb_branches/cnb_branch_20090301/Ohana/src/libohana/lib

Property svn:ignore

-              old
+              new
+*.linux
+*.lin64
+*.sol
+*.sun
+*.sid
+*.hp
+*.irix
+*.darwin.dylib
+*.darwin_x86.dylib
+*.a
+*.so

branches/cnb_branches/cnb_branch_20090301/Ohana/src/libohana/src/CommOps.c

r8385	r23352
14	14	return (status);
15	15	}
	16	// fprintf (stderr, "resp: %s\n", command.buffer);
16	17
17	18	/* buffer contains an EOL NULL, we can just sscan it */

branches/cnb_branches/cnb_branch_20090301/Ohana/src/libtap/lib

Property svn:ignore

-              old
+              new
+*.linux
+*.lin64
+*.sol
+*.sun
+*.sid
+*.hp
+*.irix
+*.darwin.dylib
+*.darwin_x86.dylib
+*.a
+*.so

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dimm/Analysis.c

-              r2598
+              r23352
+        }
+/* find contiguous trigger pixels in row from starting point */
+int fillrow (float *buffer, int Nx, int offset, int sx, int *xs, int *xe) {
+  trigger = FALSE;
+  for (i = sx, pix = offset + i; buffer[pix] && (i < Nx); i++, pix++) {
+    addpix (pix);
+    buffer[pix] = 0;
+    trigger = TRUE;
+    *xe = i;
+  }
+  for (i = sx - 1, pix = offset + i; (i >= 0) && buffer[pix]; i--, pix--) {
+    addpix (pix);
+    buffer[pix] = 0;
+    trigger = TRUE;
+    *xs = i;
+  }
+  return (trigger);
+}
+static int Npix = 0;
+static int *Pix = (int *) NULL;
+addpix (int pix) {
+  Npix ++;
+  if (Pix == (int *) NULL) {
+    ALLOCATE (Pix, int, MAX (1, Npix));
+  } else {
+    REALLOCATE (Pix, int, MAX (1, Npix));
+  }
+  Pix[Npix - 1] = pix;
+}
+clearpix () {
+  Npix = 0;
+  REALLOCATE (Pix, int, 1);
+}
+statpix (double *x, double *y, float *buffer, int Nx) {
+  int X, Y;
+  double Sx, Sy, So;
+  So = Sx = Sy = 0;
+  for (i = 0; i < Npix; i++) {
+    Y = pix / Nx;
+    X = pix % Nx;
+    So += buffer[pix];
+    Sx += X * buffer[pix];
+    Sy += Y * buffer[pix];
+  }
+  *x = Sx / So;
+  *y = Sy / So;
+}
+/* find stars:
+   - binarize @ threshold
+   - find all contiguous blobs
+   - find geom center of each blob
+*/
+/*
+.....................
+....x................
+...xxx...............
+..........xxx..........
+.........xx..........
+.....................
+.....................
+.....................
+.....................
+.....................
+ */
 # endif

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dimm/Makefile

-              r17283
+              r23352
 $(SRC)/init.$(ARCH).o                   \
 $(SRC)/dimm.$(ARCH).o                   \
 $(SRC)/camera.$(ARCH).o         \
+$(SRC)/camera_cmds.$(ARCH).o            \
 $(SRC)/findstars.$(ARCH).o              \
 $(SRC)/telescope.$(ARCH).o   \
+$(SRC)/telescope_cmds.$(ARCH).o   \
 $(SRC)/version.$(ARCH).o
 …
 .PHONY: dimm
+# are these used or replaced?
+# $(SRC)/analysis.$(ARCH).o             \
+# these have not been finished: should be used for analysis of the extracted images
 # $(SRC)/Analysis.$(ARCH).o            \
 # $(SRC)/Image.$(ARCH).o                \

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dimm/camera_cmds.c

Property svn:mergeinfo changed
/trunk/Ohana/src/opihi/dimm/camera_cmds.c (added) merged: 22723-23289

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/dimm/telescope_cmds.c

Property svn:mergeinfo changed
/trunk/Ohana/src/opihi/dimm/telescope_cmds.c (added) merged: 22723-23289

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/lib

Property svn:ignore set to
*.a
*.so

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/CheckTasks.c

r15872	r23352
48	48	// add random offset between 0 and 5% of exec_period
49	49	// XXX this should be optional
50		fuzz = 0.~~05task[0].exec_perioddrand48() + 1e-6*task[0].last.tv_usec;~~
	50	fuzz = 0.1task[0].exec_perioddrand48();
51	51	task[0].last.tv_usec = 1e6*(fuzz - (int)fuzz);
52	52	task[0].last.tv_sec += (int) fuzz;

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pantasks/task_threads.c

-              r15791
+              r23352
 /** things related to CheckTasks **/
+void ResetTaskTimers ();
 static int CheckTasksRun = FALSE;
 void CheckTasksSetState (int state) {
+  if (state && !CheckTasksRun) {
+    ResetTaskTimers ();
+  }
   CheckTasksRun = state;
+}
 …
 // timeout.  this enforces a certain granularity in the task creation, but prevents the task
 // thread from driving the load up to silly levels.
+// reset all of the task timers, with a bit of fuzz.  this is called
+// whenever we transition from 'stop' to 'run'
+void ResetTaskTimers () {
+  Task *task;
+  struct timeval now;
+  float fuzz;
+  // get the current time
+  gettimeofday (&now, NULL);
+  // check all tasks
+  while ((task = NextTask ()) != NULL) {
+    task[0].last.tv_usec = now.tv_usec;
+    task[0].last.tv_sec  = now.tv_sec;
+    // add random offset between 0 and 10% of exec_period
+    // XXX this should be optional
+    fuzz = task[0].exec_period*drand48();
+    task[0].last.tv_usec += 1e6*(fuzz - (int)fuzz);
+    task[0].last.tv_sec += (int) fuzz;
+    // gprint (GP_LOG, "fuzz: %f, last: %d %d\n", fuzz, task[0].last.tv_sec, task[0].last.tv_usec);
+  }
+  return;
+}

branches/cnb_branches/cnb_branch_20090301/Ohana/src/opihi/pcontrol/CheckIdleHost.c

-              r22675
+              r23352
     host[0].job = (struct Job *) job;
     if (logfile) fprintf (logfile, "start needhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    // if (logfile) fprintf (logfile, "start needhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     /* take the job off the stack and unlock the stack */
 …
     host[0].job = (struct Job *) job;
     if (logfile) fprintf (logfile, "start wanthost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    // if (logfile) fprintf (logfile, "start wanthost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     /* take the job off the stack and unlock the stack */
 …
     host[0].job = (struct Job *) job;
     if (logfile) fprintf (logfile, "start  anyhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    // if (logfile) fprintf (logfile, "start  anyhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     /* take the job off the stack and unlock the stack */

branches/cnb_branches/cnb_branch_20090301/PS-IPP-Config/lib/PS/IPP/Config.pm

-              r21320
+              r23352
     unless (defined $class) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (open $file, $name) {
         carp "Unable to open ipprc file $name: $!";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
     my @contents = <$file>;        # Contents of the ipprc file
 …
     unless (defined $path) {
         carp "PATH is not set in $name\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     bless $self, $class;
     $self->load_site();
     $self->load_system();
     $self->define_camera($camera) if defined $camera;
+    $self->load_site() or return undef;
+    $self->load_system() or return undef;
+    ( $self->define_camera($camera) or return undef ) if defined $camera;
     return $self;
 …
                 unless ($found) {
                     carp "Unable to find camera configuration file $filename\n";
                     exit($PS_EXIT_CONFIG_ERROR);
+                    return undef;
+                }
+            }
 …
     carp "Unable to find configuration file: $filename";
     exit($PS_EXIT_CONFIG_ERROR);
+    return undef;
+}
 …
     unless (defined $self) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (defined $filename) {
         carp "Unable to find site configuration file\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
     my $realfile = $self->_find_config($filename); # Resolved filename, after hunting the PATH
+    ( carp "Unable to find site configuration" and return undef ) unless defined $realfile;
     # Read the file
 …
     unless (open $file, $realfile) {
         carp "Unable to open site configuration file $realfile: $!";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     unless (defined $self->{_siteConfig}) {
         carp "Failure to parse the site configuration file $realfile";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     unless (defined $self) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (defined $filename) {
         carp "Unable to find system configuration file\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
     my $realfile = $self->_find_config($filename); # Resolved filename, after hunting the PATH
+    ( carp "Unable to find system configuration" and return undef ) unless defined $realfile;
     # Read the file
 …
     unless (open $file, $realfile) {
         carp "Unable to open system configuration file $realfile: $!";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
     my @contents = <$file>;
 …
     unless (defined $self->{_systemConfig}) {
         carp "Failure to define system configuration";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     unless (defined $self and defined $name) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (defined $filename) {
         carp "Unable to find configuration file for camera $name\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
     my $realfile = $self->_find_config($filename); # Resolved filename, after hunting the PATH
+    ( carp "Unable to find camera configuration" and return undef ) unless defined $realfile;
     # Read the file
 …
     unless (open $file, $realfile) {
         carp "Unable to open camera configuration file $realfile: $!";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
     my @contents = <$file>;
 …
     unless (defined $self->{camera}) {
         carp "Failure to define camera";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     unless (defined $self and defined $name) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (defined $pathname) {
         carp "Unable to find datapath $name\n" ;
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
 # convert the database name and the table ID to a image source id
 sub source_id
+sub source_id
+{
     my $self = shift;
 …
     my $admindb = "ippadmin";
     die "dbserver not defined in configuration" unless defined($dbserver);
     die "dbuser not defined in configuration" unless defined($dbuser);
     die "dbpassword not defined in configuration" unless defined($dbpassword);
     die "dbname not defined in configuration" unless defined($dbname);
+    ( carp "dbserver not defined in configuration"  and return undef ) unless defined($dbserver);
+    ( carp "dbuser not defined in configuration"  and return undef ) unless defined($dbuser);
+    ( carp "dbpassword not defined in configuration"  and return undef ) unless defined($dbpassword);
+    ( carp "dbname not defined in configuration"  and return undef ) unless defined($dbname);
     my $dsn = "DBI:mysql:host=$dbserver;database=$admindb";
 …
     $stmt->execute();
     my $ref = $stmt->fetchrow_hashref();
     die "ippdb $dbname not found" unless ($ref);
+    ( carp "ippdb $dbname not found" and return undef ) unless ($ref);
     my $proj_id = $ref->{proj_id};
 …
     if ($@) {
         carp "Can't find Nebulous::Client.";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (defined $server) {
         carp "Unable to find NEB_SERVER in camera configuration file.";
         exit($PS_EXIT_CONFIG_ERROR);
+    }
     my $neb = Nebulous::Client->new( proxy => $server );
     unless (defined $neb) {
+        return undef;
+    }
+    my $neb = eval { Nebulous::Client->new( proxy => $server ); };
+    if ($@ or not defined $neb) {
         carp "Unable to find NEB_SERVER in camera configuration file.";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     if ($scheme) {
         $scheme = lc($scheme);
         # print "scheme: $scheme\n";
+        # print "scheme: $scheme\n";
         if ($scheme eq 'neb') {
             $self->_neb_start();
+            $self->_neb_start() or ( carp "Can't start Nebulous" and return undef );
             my $neb = $self->{nebulous}; # Nebulous handle
             if ($create_if_doesnt_exist) {
+                unless ($neb->stat( $name )) {
+                    # print "entry $name not found, creating...\n";
+                    my $uri = $neb->create( $name );
+                    unless(defined $uri) {
+                my $status = eval { $neb->stat( $name ); };
+                ( carp "Unable to stat Nebulous handle $name" and return undef ) if $@;
+                unless ($status) {
+                    # print "entry $name not found, creating...\n";
+                    my $uri = eval { $neb->create( $name ); };
+                    if ($@ or not defined $uri) {
                         carp "unable to instantiate $name.";
                         exit($PS_EXIT_DATA_ERROR);
+                        return undef;
+                    }
                     my $path = URI->new( $uri )->path;
                     # print "created path: $path\n";
+                    # print "created path: $path\n";
                     return $path;
+                }
+            }
             my $path = $neb->find( $name );
+            if (not defined $path) {
+                carp "neb entry $name not found, not created\n";
                 exit($PS_EXIT_DATA_ERROR);
+            }
             # print "found path: $path\n";
+            my $path = eval { $neb->find( $name ); };
+            if ($@ or not defined $path) {
+                carp "neb entry $name not found, not created\n";
+                return undef;
+            }
+            # print "found path: $path\n";
             return $path;
+        }
 …
         if ($scheme eq 'path' or $scheme eq 'file') {
             # guaranteed to have a scheme (path:// or file://)
             $name = $self->convert_filename_absolute( $name );
             # print "resolved path to $name\n";
+            $name = $self->convert_filename_absolute( $name ) or return undef;
+            # print "resolved path to $name\n";
+        }
+    }
 …
         if (! -e $dir) {
             my $rc = system "mkdir -p $dir";
             die "failed to create directory for $name" unless (!$rc);
+            ( carp "failed to create directory for $name" and return undef ) unless (!$rc);
         } elsif (! -d $dir ) {
+            die "parent for $name exists and is not a directory";
+        }
+        open F, ">$name" or die "failed to create $name";
+            carp "parent for $name exists and is not a directory";
+            return undef;
+        }
+        open F, ">$name" or ( carp "failed to create $name" and return undef );
         close F;
         # print "created target $name\n";
+        # print "created target $name\n";
+    }
 …
     my $name = shift;                # File name to check
     $self->file_prepare( $name );
+    $self->file_prepare( $name ) or return undef;
     my $scheme = file_scheme($name); # The scheme, e.g., file://, path://
 …
         $scheme = lc($scheme);
         if ($scheme eq 'neb') {
+            $self->_neb_start();
+            return $self->{nebulous}->open_create( $name );
+            $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+            my $fh = eval { $self->{nebulous}->open_create( $name ); };
+            if ($@ or not defined $fh) {
+                carp "Unable to open/create Nebulous handle $name";
+                return undef;
+            }
+            return $fh;
+        }
         if ($scheme eq 'path' or $scheme eq 'file') {
             # guaranteed to have a scheme (path:// or file://)
             $name = $self->convert_filename_absolute( $name );
+            $name = $self->convert_filename_absolute( $name ) or return undef;
+        }
+    }
 …
     if (-f $name) {
         carp "Unable to create file $name --- file exists.";
         exit($PS_EXIT_SYS_ERROR);
+        return undef;
+    }
 …
     unless (open $fh, '>', $name) {
         carp "Unable to create file $name --- $!";
         exit($PS_EXIT_SYS_ERROR);
+        return undef;
+    }
     return $fh;
 …
     my $name = shift;                # File name to check
     $self->file_prepare( $name );
+    $self->file_prepare( $name ) or return undef;
     my $scheme = file_scheme($name); # The scheme, e.g., file://, path://
 …
         $scheme = lc($scheme);
         if ($scheme eq 'neb') {
+            $self->_neb_start();
+            return $self->{nebulous}->open_create( $name );
+            $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+            my $fh = eval { $self->{nebulous}->open_create( $name ) };
+            if ($@ or not defined $fh) {
+                carp "Unable to open/create Nebulous handle $name";
+                return undef;
+            }
+            return $fh;
+        }
         if ($scheme eq 'path' or $scheme eq 'file') {
             # guaranteed to have a scheme (path:// or file://)
             $name = $self->convert_filename_absolute( $name );
+            $name = $self->convert_filename_absolute( $name ) or return undef;
+        }
+    }
 …
     unless (open $fh, '>>', $name) {
         carp "Unable to create file $name --- $!";
         exit($PS_EXIT_SYS_ERROR);
+        return undef;
+    }
     return $fh;
 …
     my $name = shift;                # File name to check
     $self->file_prepare( $name );
+    $self->file_prepare( $name ) or return undef;
     my $scheme = file_scheme($name); # The scheme, e.g., file://, path://
     if (defined $scheme and lc($scheme) eq 'neb') {
+        $self->_neb_start();
+        $name = $self->{nebulous}->create( $name );
+        $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+        $name = eval { $self->{nebulous}->create( $name ) };
+        if ($@ or not defined $name) {
+            carp "Unable to create Nebulous handle $name";
+            return undef;
+        }
+    }
 …
     my $scheme = file_scheme($name); # The scheme, e.g., file://, path://
     if (defined $scheme and lc($scheme) eq 'neb') {
+        $self->_neb_start();
+        return (defined $self->{nebulous}->find_instances( $name ) ? 1 : 0);
+        $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+        my $found = eval { $self->{nebulous}->find_instances( $name ); };
+        ( carp "Unable to find instances of Nebulous handle $name" and return undef ) if $@;
+        return (defined $found ? 1 : 0);
+    }
 …
     my $target = shift;                # Name of target file
     $self->file_prepare( $target );
+    $self->file_prepare( $target ) or return undef;
     my $scheme = file_scheme($target); # The scheme, e.g., file://, path://
     if (defined $scheme and lc($scheme) eq 'neb') {
+        $self->_neb_start();
+        $target = $self->{nebulous}->create( $target );
+        $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+        $target = eval { $self->{nebulous}->create( $target ); };
+        if ($@ or not defined $target) {
+            carp "Unable to create Nebulous handle";
+            return undef;
+        }
+    }
     $target = $self->file_resolve( $target );
     $source = $self->file_resolve( $source );
+    system("cp $source $target") == 0 or (carp "Can't copy file $source to $target." and
+                                          exit($PS_EXIT_DATA_ERROR));
+    system("cp $source $target") == 0 or ( carp "Can't copy file $source to $target." and return undef );
     return 1;
+}
 …
     my $preserve = shift;
     die "pathname must be defined" unless ($pathname);
+    ( carp "pathname must be defined" and return undef ) unless ($pathname);
     my $fileRef;
 …
     if ($preserve) {
         # we want to keep the file just create it in the current directory
+        # we want to keep the file just create it in the current directory
         $fileName = "./$base";
         open $fileRef, ">$fileName" or die "can't open $fileName for output";
+        open $fileRef, ">$fileName" or ( carp "can't open $fileName for output" and return undef );
     } else {
         #  we really want a tempfile, so put it in /tmp
 …
     my $scheme = file_scheme($name); # The scheme, e.g., file://, path://
     if (defined $scheme and lc($scheme) eq 'neb') {
+        $self->_neb_start();
+        $status = $self->{nebulous}->delete( $name );
+        $self->_neb_start() or ( carp "Unable to start Nebulous" and return undef );
+        $status = eval { $self->{nebulous}->delete( $name ); };
+        ( carp "Unable to delete Nebulous handle $name" and return undef ) if $@;
     } else {
         my $resolved = $self->file_resolve($name);
         if ($resolved && -e $resolved) {
+        my $resolved = $self->file_resolve($name) or return undef;
+        if (defined $resolved and -e $resolved) {
             $status = unlink($resolved);
+        }
 …
     my $name = shift;
     die "need name" unless $name;
+    ( carp "need redirection target" and return undef ) unless $name;
     my $filename = $self->file_resolve($name, 1);
     die "cannot resolve $name" unless $filename;
+    ( carp "cannot resolve $name" and return undef ) unless $filename;
     if (! open(STDOUT, ">>$filename") ) {
 …
         while (! open STDOUT, ">>$filename" ) {
             if ($try == $max_tries) {
+                die "failed to redirect stdout to $filename after trying $max_tries times";
+                carp "failed to redirect stdout to $filename after trying $max_tries times";
+                return undef;
+            }
             sleep 5;
 …
         print STDERR "   redirect stdout to $filename succeded on try $try\n";
+    }
+    open STDERR, ">>$filename" or die "failed to redirect stderr to $filename";
+    open STDERR, ">>$filename" or ( carp "failed to redirect stderr to $filename" and return undef );
+    return 1;
+}
 …
     my $name = shift;                # File name for which to prepare
     my $workdir = shift;        # Working directory
     my $template = shift;        # Template filename from which to get working directory if
+    my $template = shift;        # Template filename from which to get working directory if
     if (defined $workdir) {
 …
     # not guaranteed to have a scheme (path:// or file://) - might be /PATH/foobar
     # a relative path (PATH/foobar) is invalid here
     my $resolved = $self->convert_filename_absolute( $name );
+    my $resolved = $self->convert_filename_absolute( $name ) or return undef;
     my ( $vol, $dirs, $file ) = File::Spec->splitpath( $resolved );
     unless (-d $dirs) {
         system("mkdir -p $dirs") == 0 or (carp "Can't create directory $dirs" and exit($PS_EXIT_DATA_ERROR));
+        system("mkdir -p $dirs") == 0 or ( carp "Can't create directory $dirs" and return undef );
+    }
 …
     # not guaranteed to have a scheme (path:// or file://) - might be /PATH/foobar
     # a relative path (PATH/foobar) is invalid here
     my $resolved = $self->convert_filename_absolute( $outroot );
+    my $resolved = $self->convert_filename_absolute( $outroot ) or return undef;
     my ( $vol, $dirs, $file ) = File::Spec->splitpath( $resolved );
     unless (-d $dirs) {
         system("mkdir -p $dirs") == 0 or (carp "Can't create directory $dirs" and exit($PS_EXIT_DATA_ERROR));
+        system("mkdir -p $dirs") == 0 or ( carp "Can't create directory $dirs" and return undef );
+    }
 …
     unless (defined $self and defined $name) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     ## if this is already an absolute path (/PATH/file), just return the path
     unless (defined $scheme) {
+    unless (defined $scheme) {
         if ($name =~ m|^/|) { return $name; }
         # without a leading slash, this is an error
         carp "Relative file name provided: relative paths are not permitted.";
         exit($PS_EXIT_SYS_ERROR);
+        return undef;
+    }
 …
     if (lc($scheme) eq 'file') {
         # the above strips of the leading slash; replace it for file://
+        # the above strips of the leading slash; replace it for file://
         $name = '/' . $name;
         return $name;
 …
+    }
+    # looks like we cannot reach here without an invalid scheme.
+    # programming error?
+    # return $name;
+    carp "Programming error";
+    exit($PS_EXIT_PROG_ERROR);
+    # It's already absolute
+    return $name;
+}
 …
     my $self = shift;                # Configuration object
     my $name = shift;                # raw name
     unless (defined $self and defined $name) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+    }
+        return undef;
+    }
     # First, check to see if it's already in a relative form
     my $scheme = file_scheme($name); # The scheme, e.g., file, path
 …
             # We may as well search for a 'better' path
             # guaranteed to have a scheme (path:// or file://)
             $name = $self->convert_filename_absolute( $name );
+            $name = $self->convert_filename_absolute( $name ) or return undef;
         } elsif ($scheme eq 'neb') {
             # No chance of changing anything --- move along
 …
+        }
+    }
     $name = File::Spec->canonpath( $name); # Clean up
     my @dirs = File::Spec->splitdir( $name );
     my $path_list = metadataLookupMD($self->{_siteConfig}, 'DATAPATH'); # List of paths
     my $best_path;
 …
       $path =~ s|/*$||;
       my @path_dirs = File::Spec->splitdir( $path );
       # Check if the path is suitable
       next if scalar @path_dirs > scalar @dirs;
 …
+      }
+  }
     $name =~ s|^/||;
     $name =~ s|/$||;
 …
     unless (defined $self and defined $name and defined $type) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
             return undef;
+        }
         # rejections are saved as a recipe: REJECTIONS
         my @rejContents = `ppConfigDump -dump-recipe REJECTIONS -camera $camera -`;
         # load from resulting psMetadataConfig
         $self->{rejection} = $parser->parse( join '', @rejContents); # The rejection metadata
         unless (defined $self->{rejection}) {
             carp "Unable to parse REJECTION recipe for $camera.";
             exit($PS_EXIT_CONFIG_ERROR);
+            return undef;
+        }
+    }
 …
           unless ($item->{class} eq "metadata") {
               carp "$name within REJECTIONS is not of type METADATA";
               exit($PS_EXIT_PROG_ERROR);
+              return undef;
+          }
           my $limits = $item->{value}; # List of rejection limits
 …
           foreach my $limit (@$limits) {
               if ($limit->{name} eq 'FILTER') {
+                  if ($limit->{value} eq '*' or
+                      (defined $filter and
+                       $limit->{value} eq $filter)) {
+                  if ($limit->{value} eq '*' or (defined $filter and $limit->{value} eq $filter)) {
                       last;
+                  }
 …
+              }
+          }
           foreach my $limit (@$limits) {
               return $limit->{value} if $limit->{name} eq $name;
 …
     unless (defined $self and defined $name and defined $output) {
         carp "Programming error: required inputs left undefined";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
             return undef;
+        }
         $filerules = metadataLookup($camera, 'FILERULES');        # File rules
         unless (defined $filerules) {
 …
             return undef;
+        }
         if ($filerules->{class} eq "scalar" and $filerules->{type} eq "STR") {
             # Allow indirection to a file
             my $filename = $self->_find_config($filerules->{value}); # Resolved filename
+            ( carp "Unable to find file rules file" and return undef ) unless defined $filename;
             # Read the file
             my $file;                # File handle
             unless (open $file, $filename) {
                 carp "Unable to open filerules file $filename: $!";
                 exit($PS_EXIT_CONFIG_ERROR);
+                return undef;
+            }
             my @contents = <$file>;
 …
         unless (defined $component) {
             carp "Programming error";
             exit($PS_EXIT_PROG_ERROR);
+            return undef;
+        }
         $filename =~ s/\{CHIP\.NAME\}/$component/;
 …
     return $filename;
+}
+}
 # Return an EXTNAME From the EXTNAME.RULE table in the camera configuration
 …
     unless (defined $self and defined $name) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
         unless (defined $component) {
             carp "Programming error";
             exit($PS_EXIT_PROG_ERROR);
+            return undef;
+        }
         $extname =~ s/\{CHIP\.NAME\}/$component/;
 …
     return $extname;
+}
+}
 # Return catdir for tessellation, from TESSELLATIONS within the site configuration
 …
     unless (defined $self and defined $self->{_siteConfig} and defined $tess_id) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (defined $tessellations) {
         carp "Can't find TESSELLATIONS in site configuration.\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     if (defined $scheme and lc($scheme) eq 'neb') {
         carp "Tessellation $tess_id refers to a Nebulous path: $catdir\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     unless (defined $self and defined $self->{_siteConfig} and defined $dvodb) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (defined $catdirs) {
         carp "Can't find DVO.CATDIRS in site configuration.\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     if (defined $scheme and lc($scheme) eq 'neb') {
         carp "DVO catdir $dvodb refers to a Nebulous path: $catdir\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     unless (defined $self and defined $self->{_siteConfig} and defined $dvodb) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (defined $catdirs) {
         carp "Can't find PSASTRO.CATDIRS in site configuration.\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     if (defined $scheme and lc($scheme) eq 'neb') {
         carp "PSASTRO catdir $dvodb refers to a Nebulous path: $catdir\n";
         exit($PS_EXIT_CONFIG_ERROR);
+        return undef;
+    }
 …
     unless (defined $self) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     unless (defined $self and defined $reduction and defined $name) {
         carp "Programming error --- inputs undefined";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
             return undef;
+        }
         if ($reductionClasses->{class} eq "scalar" and $reductionClasses->{type} eq "STR") {
             # Allow indirection to a file
             my $filename = $self->_find_config($reductionClasses->{value}); # Resolved filename
+            ( carp "Unable to find reduction classes file" and return undef ) unless defined $filename;
             # Read the file
             my $file;                # File handle
             unless (open $file, $filename) {
                 carp "Unable to open reductionClasses file $filename: $!";
                 exit($PS_EXIT_CONFIG_ERROR);
+                return undef;
+            }
             my @contents = <$file>;
 …
     my $class = metadataLookupMD($reductionClasses, $reduction) or # Class of interest
+        (carp "Can't find $reduction in REDUCTION in camera configuration.\n" and
+         exit($PS_EXIT_CONFIG_ERROR));
+        ( carp "Can't find $reduction in REDUCTION in camera configuration.\n" and return undef );
     my $actual = metadataLookupStr($class, $name) or # The actual recipe name of interest
+        (carp "Can't find $name in $class in REDUCTION in camera configuration.\n" and
+         exit($PS_EXIT_CONFIG_ERROR));
+        (carp "Can't find $name in $class in REDUCTION in camera configuration.\n" and return undef );
     return $actual;
 …
+    }
     my $dvoImageExtract = can_run('dvoImageExtract') or die "Can't find dvoImageExtract";
+    my $dvoImageExtract = can_run('dvoImageExtract') or ( carp "Can't find dvoImageExtract" and return undef );
     my $tess_dir = $self->tessellation_catdir( $tess_id ); # Tessellation catdir for DVO
     unless (defined $tess_dir) {
         carp "Can't get list of tessellations.";
         return 0;
+    }
     $tess_dir = $self->convert_filename_absolute( $tess_dir );
+        return undef;
+    }
+    $tess_dir = $self->convert_filename_absolute( $tess_dir ) or return undef;
     unless ($self->file_exists( $outname )) {
         my $outnameResolved = $self->file_create( $outname ); # Resolved filename, for Nebulous
+        my $outnameResolved = $self->file_create( $outname ) or return undef; # Resolved filename, for Nebulous
         my $command = "$dvoImageExtract -D CATDIR $tess_dir $skycell_id -o $outnameResolved";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
         die "Unable to perform dvoImageExtract for $tess_id $skycell_id\n" unless ($success and $self->file_exists( $outname ));
+        ( carp "Unable to perform dvoImageExtract for $tess_id $skycell_id\n" and return undef ) unless ($success and $self->file_exists( $outname ));
+    }
 …
         unless (defined $value) {
             carp "Unable to find environment variable $name";
             exit($PS_EXIT_SYS_ERROR);
+            return undef;
+        }
         $dir =~ s/\$\{?$name\}?/$value\//;
 …
     unless (defined $mdc and defined $name) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }
 …
     return undef;
+}
 # Lookup the metadata, checking the type is STR
 …
     unless (defined $mdc and defined $name) {
         carp "Programming error";
         exit($PS_EXIT_PROG_ERROR);
+        return undef;
+    }

branches/cnb_branches/cnb_branch_20090301/archive/conv_variance/fake.c

r20210	r23352
47	47	psFree(weight);
48	48
49		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS~~, 0~~);
	49	psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
50	50
51	51	psImage *bright = psImageAlloc(IMAGE_SIZE, IMAGE_SIZE, PS_TYPE_F32);

branches/cnb_branches/cnb_branch_20090301/archive/conv_variance/phot.c

r20210	r23352
52	52
53	53	psStats *bgStats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN);
54		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS~~, 0~~);
	54	psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
55	55	psImageBackground(bgStats, NULL, image, mask, maskVal, rng);
56	56	double bg = bgStats->robustMedian;

branches/cnb_branches/cnb_branch_20090301/dbconfig/changes.txt

-              r21434
+              r23352
 ALTER TABLE magicInputSkyfile DROP PRIMARY KEY, ADD PRIMARY KEY(magic_id, diff_id, node);
 -- Version: 1.1.48
 …
     WHERE warpImfile.skycell_id IS NULL;
+-- Version: 1.1.49 (change detrend sequence)
+show create table detResidImfile;
+alter table detResidImfile drop foreign key detResidImfile_ibfk_3;

branches/cnb_branches/cnb_branch_20090301/dbconfig/config.md

r21310	r23352
2	2	pkg_name STR ippdb
3	3	pkg_namespace STR ippdb
4		pkg_version STR 1.1.48
	4	pkg_version STR 1.1.49
5	5	END

branches/cnb_branches/cnb_branch_20090301/dbconfig/det.md

-              r19620
+              r23352
     class_id    STR         64      # Primary Key
     uri         STR         255
+    recipe      STR         64
+    bg          F64         0.0
+    bg_stdev    F64         0.0
+    bg_mean_stdev   F64     0.0
+    user_1      F64         0.0
+    user_2      F64         0.0
+    user_3      F64         0.0
+    user_4      F64         0.0
+    user_5      F64         0.0
+#   XXX does it make sense to 'clean' the stacked imfiled?
+    # XXX missing path_base
+    recipe      STR         64
+    bg          F64         0.0
+    bg_stdev    F64         0.0
+    bg_mean_stdev   F64     0.0
+    user_1      F64         0.0
+    user_2      F64         0.0
+    user_3      F64         0.0
+    user_4      F64         0.0
+    user_5      F64         0.0
+    #   XXX does it make sense to 'clean' the stacked imfiled? (EAM: yes)
     data_state  STR         64      # full, cleaned, purged (only track end states; request states are in detRunSummary by iteration)
     fault       S16         0       # Key NOT NULL
 …
     fault       S16         0       # Key NOT NULL
 END
-#detMasterFrame METADATA
-#    det_id      S64         0       # Primary Key
-#    iteration   S32         0       # Primary Key
-#    comment     STR         255
-#END
+#
-## drop?
-#detMasterImfile METADATA
-#    det_id      S64         0       # Primary Key
-#    class_id    STR         64      # Primary Key
-#    uri         STR         255
-#    recipe      STR         64
-#END
 detResidImfile METADATA

branches/cnb_branches/cnb_branch_20090301/doc/pslib/psLibSDRS.tex

-              r12382
+              r23352
 while returning the type involves descending through a case statement.
+To compare the types of two pointers without knowing their types, the following function may be used:
+\begin{prototype}
+bool psMemTypeEqual (void *ptr1, ///< pointer to first psMemory object
+                     void *ptr2 ///< pointer to second psMemory object
+  );
+\end{prototype}
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detNormalizedStatImfile_failure.d

r19726	r23352
4	4	MENU ipp.detrend.dat
5	5
6		~~where~~ fault > 0
	6	WHERE fault > 0
7	7
8	8	# field size format name show link to extras

branches/cnb_branches/cnb_branch_20090301/ippMonitor/def/detStackedImfile.d

-              r18193
+              r23352
 TABLE detStackedImfile
+TABLE detStackedImfile, detRun
 TITLE detStackedImfile
 FILE  detStackedImfile.php
 MENU  ipp.imfiles.dat
+#     field         size  format  name                   show  link to     extras
+FIELD det_id,         7,  %s,    det_id
+FIELD iteration,      5,  %s,    iteration
+FIELD class_id,       8,  %s,    class_id
+FIELD fault,          5,  %d,    fault
+FIELD bg,             8,  %f,    backgnd
+FIELD bg_stdev,       8,  %f,    stdev
+FIELD bg_mean_stdev,  8,  %f,    [stdev]
+FIELD uri,           20,  %s,    uri
+WHERE detStackedImfile.det_id = detRun.det_id
+# XXX change uri to path_base in db
+ARGS  ARG1 detStackedImfile.det_id=$det_id
+ARGS  ARG1 detStackedImfile.iteration=$iteration
+ARGS  ARG1 camera=$detRun.camera
+ARGS  ARG1 basename=$detStackedImfile.uri
+#     field         size  format  name      show  link to     extras
+FIELD detStackedImfile.det_id,         7,  %s,    det_id
+FIELD detStackedImfile.iteration,      5,  %s,    iteration
+FIELD detStackedImfile.class_id,       8,  %s,    class_id,   value, detStackedImfile.php, ARG1
+FIELD detStackedImfile.fault,          5,  %d,    fault
+FIELD detStackedImfile.bg,             8,  %f,    backgnd
+FIELD detStackedImfile.bg_stdev,       8,  %f,    stdev
+FIELD detStackedImfile.bg_mean_stdev,  8,  %f,    [stdev]
+FIELD detStackedImfile.uri,           20,  %s,    uri
 # FIELD recipe, 20,   recipe
 TD_CLASS list_off $fault > 0
+TAIL PHP insert_log ('LOG.EXP');

branches/cnb_branches/cnb_branch_20090301/ippMonitor/raw/getimage.php

-              r14128
+              r23352
 putenv("PATH=$BINDIR:$PATH");
+# echo "args: $args<br>";
+if ($debug) {
+  echo "args: $args<br>";
+  echo "path: $PATH<br>";
+  echo "perl: $PERLLIB<br>";
+}
 # $basename may contain filename@filerule
 …
 # $filerule = strtok("@");
+# echo "basename: $basename<br>";
+# echo "filerule: $filerule<br>";
+if ($debug) {
+  echo "basename: $basename<br>";
+  echo "filerule: $filerule<br>";
+}
 # need to supply the camera as well...
 …
 $basename = escapeshellarg($basename);
 $basename = str_replace ('..','',$basename);
+if ($debug) {
+  exec ("which ipp_filename.pl", $output, $status);
+  echo "output:   $output[0]<br>";
+  echo "status:   $status<br>";
+  exec ("whoami", $output, $status);
+  echo "output:   $output[0]<br>";
+  echo "status:   $status<br>";
+}
 exec ("ipp_filename.pl --site=$SITE --basename $basename --filerule $filerule --camera $camera --class_id $class_id", $output, $status);

branches/cnb_branches/cnb_branch_20090301/ippScripts/Build.PL

r20931	r23352
80	80	scripts/summit_copy.pl
81	81	scripts/ipp_image_path.pl
	82	scripts/dist_component.pl
	83	scripts/dist_advancerun.pl
82	84	)],
83	85	dist_abstract => 'Scripts for running the Pan-STARRS IPP',

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/calibrate_dvo.pl

-              r20100
+              r23352
 my $caltool  = can_run('caltool')  or (warn "Can't find caltool"  and $missing_tools = 1);
 if ($missing_tools) {
+if ($missing_tools) {
     warn ("Can't find required tools");
     exit($PS_EXIT_CONFIG_ERROR);
+    exit($PS_EXIT_CONFIG_ERROR);
+}
 …
         cache_run(command => $command, verbose => 1);
     unless ($success) {
+    unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
         &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RESORT", $status, $dbname);
 …
+{
     foreach my $filter (@filters) {
         my $command = "$relphot $filter";
         $command .= "-D CATDIR $dvodb";
         $command .= "-region $RAs $RAe $DECs $DECe";
+        my $command = "$relphot $filter";
+        $command .= "-D CATDIR $dvodb";
+        $command .= "-region $RAs $RAe $DECs $DECe";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             cache_run(command => $command, verbose => 1);
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            cache_run(command => $command, verbose => 1);
+        unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
             &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELPHOT", $status, $dbname);
+        }
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELPHOT", $status, $dbname);
+        }
+    }
+}
 …
 if (0) {
     foreach my $filter (@filters) {
         my $command = "$uniphot $filter";
         $command .= "-D CATDIR $dvodb";
         $command .= "-region $RAs $RAe $DECs $DECe";
+        my $command = "$uniphot $filter";
+        $command .= "-D CATDIR $dvodb";
+        $command .= "-region $RAs $RAe $DECs $DECe";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             cache_run(command => $command, verbose => 1);
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            cache_run(command => $command, verbose => 1);
+        unless ($success) {
             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
             &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "UNIPHOT", $status, $dbname);
+        }
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "UNIPHOT", $status, $dbname);
+        }
+    }
+}
 …
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         cache_run(command => $command, verbose => 1);
+        cache_run(command => $command, verbose => 1);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
         &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.OBJECTS", $status, $dbname);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.OBJECTS", $status, $dbname);
+    }
+}
 …
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         cache_run(command => $command, verbose => 1);
+        cache_run(command => $command, verbose => 1);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
         &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.IMAGES", $status, $dbname);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die ("Unable to perform addstar -resort on region $region: $error_code", $cal_id, $region, "RELASTRO.IMAGES", $status, $dbname);
+    }
+}
 …
     my $region    = shift;
     my $last_step = shift;
     my $status    = shift;
     my $dbname    = shift;
+    my $status    = shift;
+    my $dbname    = shift;
     carp($msg);
     if (defined $cal_id && defined $region && defined $last_step && defined $status and not $no_update) {
         my $command = "$caltool -addcalrun";
         $command .= " -cal_id $cal_id";
+        $command .= " -cal_id $cal_id";
         $command .= " -region $region";
         $command .= " -last_step $last_step";
         $command .= " -state $status";
+        $command .= " -last_step $last_step";
+        $command .= " -state $status";
         $command .= " -dbname $dbname" if defined $dbname;
         system ($command);

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/camera_exp.pl

-              r22430
+              r23352
 use PS::IPP::Config 1.01 qw( :standard );
 use File::Temp qw( tempfile );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $camera;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $cam_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $cam_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $cam_id, $PS_EXIT_CONFIG_ERROR);
 …
 if ($redirect) {
     $ipprc->redirect_output($logDest);
+    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $cam_id, $PS_EXIT_SYS_ERROR );
     print "\n\n";
     print "Starting script $0 on $host\n\n";
 …
     my $cam_id = shift; # Camtool identifier
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/chip_imfile.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::Config;
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $run_state;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $exp_id, $chip_id, $class_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $chip_id, $class_id, $PS_EXIT_CONFIG_ERROR);
 …
 if ($redirect) {
     $ipprc->redirect_output($logDest);
+    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR );
     print STDOUT "\n\n";
     print STDOUT "Starting script $0 on $host\n\n";
 …
     # run_state, outputImage, and outroot are globals
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $chip_id and defined $class_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_correct_imfile.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::Config;
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     and defined $camera;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $det_id, $class_id, $PS_EXIT_UNKNOWN_ERROR ); };
+# XXX this exits with status = 0 on failure
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $det_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 if ($redirect) {
     my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id)
        or &my_die("Missing entry from camera config", $det_id, $class_id, $PS_EXIT_CONFIG_ERROR);
     $ipprc->redirect_output($logDest);
+    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $det_id, $class_id, $PS_EXIT_SYS_ERROR );
+}
 …
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $det_id and defined $class_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_apply.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::Config;
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     'iteration|n=s'     => \$iter,       # Iteration
     'class_id|i=s'      => \$class_id,   # Class ID
     'value|v=s'         => \$value,      # Value to multiple (for normalisation)
+    'value|v=s'         => \$value,      # Value to apply (for normalisation)
     'input_uri|u=s'     => \$input_uri,  # Input file
     'camera|c=s'        => \$camera,     # Camera
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $det_id, $iter, $class_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id)
         or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR);
 $ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 my $RECIPE_PPIMAGE = 'PPIMAGE_N'; # Recipe to use with ppImage
 …
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $det_id and defined $iter and defined $class_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_calc.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $det_id, $iter, $PS_EXIT_UNKNOWN_ERROR ); };
+my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $det_id, $iter, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 my $logfile = $outroot . ".log";
+$ipprc->redirect_output($logfile) if $redirect;
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $det_id, $iter, $PS_EXIT_SYS_ERROR ) if $redirect;
 use constant STATISTIC => 'bg'; # Background statistic to use from the database
 …
 my @files;                      # The input files
+{
+    my $command = "$dettool -processedimfile";
+    $command .= " -det_id $det_id"; # Command to run
+    my $command = "$dettool -residimfile";
+    $command .= " -det_id $det_id";
+    $command .= " -iteration $iter";
     $command .= " -included"; # only use the inputs for this detrend run to calculate the norm
     $command .= " -dbname $dbname" if defined $dbname;
 …
     print "Running [$command]...\n" if $verbose;
     if (not run(\@command, \$stdin, \$stdout, \$stderr)) {
         &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?",
+        &my_die("Unable to perform dettool -residimfile on detrend $det_id/$iter: $?",
                 $det_id, $iter, $PS_EXIT_SYS_ERROR);
+    }
 …
     my $iter = shift;           # Iteration
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_norm_exp.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::List qw( parse_md_list );
 use File::Temp qw( tempfile );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $det_id, $iter, $PS_EXIT_UNKNOWN_ERROR ); };
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $det_id, $iter, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 my $logfile = $outroot . ".log";
+$ipprc->redirect_output($logfile) if $redirect;
+$ipprc->define_camera($camera);
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $det_id, $iter, $PS_EXIT_SYS_ERROR ) if $redirect;
 # Recipes to use based on reduction class
 …
     my $iter = shift;           # Iteration
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_process_exp.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::List qw( parse_md_list );
 use File::Temp qw( tempfile );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $det_id, $exp_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $det_id, $exp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 if ($redirect) {
     my $logDest = $ipprc->filename("LOG.EXP", $outroot, "NONE")
         or &my_die("Missing entry in file rules", $det_id, $exp_id, $PS_EXIT_CONFIG_ERROR);
     $ipprc->redirect_output($logDest);
+    $ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $det_id, $exp_id, $PS_EXIT_SYS_ERROR );
+}
 …
     my $exp_id = shift; # Exposure tag
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_process_imfile.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::Config;
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $det_id, $exp_id, $class_id, $PS_EXIT_UNKNOWN_ERROR ); };
+# XXX this exits with status = 0 on failure
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $det_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+$ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 # Recipes to use as a function of detrend type
 …
 my $jpeg_recipe = $ipprc->reduction($reduction, uc($det_type) . '_JPEG_IMAGE'); # Recipe name for JPEG
+# The output file rule name depends on the detrend type
+my $FILERULES = { 'FLATMASK'         => undef,
+                  'DARKMASK'         => undef,
+                  'MASK'             => undef,
+                  'BIAS'             => undef,
+                  'DARK'             => undef,
+                  'DARK_PREMASK'     => undef,
+                  'SHUTTER'          => 'PPIMAGE.OUTPUT.DETREND',
+                  'FLAT_PREMASK'     => 'PPIMAGE.OUTPUT.DETREND',
+                  'DOMEFLAT_PREMASK' => 'PPIMAGE.OUTPUT.DETREND',
+                  'SKYFLAT_PREMASK'  => 'PPIMAGE.OUTPUT.DETREND',
+                  'FLAT_RAW'         => 'PPIMAGE.OUTPUT.DETREND',
+                  'DOMEFLAT_RAW'     => 'PPIMAGE.OUTPUT.DETREND',
+                  'SKYFLAT_RAW'      => 'PPIMAGE.OUTPUT.DETREND',
+                  'FLAT'             => 'PPIMAGE.OUTPUT.DETREND',
+                  'DOMEFLAT'         => 'PPIMAGE.OUTPUT.DETREND',
+                  'SKYFLAT'          => 'PPIMAGE.OUTPUT.DETREND',
+                  'FRINGE'           => undef,
+              };
 &my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri);
 …
 my $cmdflags;
+my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
+my $filerule = $FILERULES->{$det_type}; # File rule to use for output
+$filerule =  "PPIMAGE.OUTPUT" unless defined $filerule;
+my $outputImage = $ipprc->filename($filerule,        $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 my $outputBin1  = $ipprc->filename("PPIMAGE.BIN1",   $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 my $outputBin2  = $ipprc->filename("PPIMAGE.BIN2",   $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 my $outputStats = $ipprc->filename("PPIMAGE.STATS",  $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 my $traceDest   = $ipprc->filename("TRACE.IMFILE",   $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR);
 # Run ppImage
 unless ($no_op) {
 …
     $command .= " -recipe PPSTATS DETSTATS";
     $command .= " -stats $outputStats";
+    $command .= " -F PPIMAGE.OUTPUT $filerule" if $filerule ne "PPIMAGE.OUTPUT";
     $command .= " -tracedest $traceDest -log $logDest";
     $command .= " -threads $threads" if defined $threads;
 …
     my $class_id = shift; # Class identifier
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_reject_exp.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::List qw( parse_md_list );
 use Statistics::Descriptive;
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 my $ITER_LIMIT = 20;
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $det_id, $iter, $PS_EXIT_UNKNOWN_ERROR ); };
+# check for existing directory, generate if needed
+$ipprc->outroot_prepare($outroot);
+my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $det_id, $iter, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->outroot_prepare($outroot) or my_die( "Unable to prepare output root", $det_id, $iter, $PS_EXIT_SYS_ERROR );
 my $logName = "$outroot.log"; # Name for log
+$ipprc->redirect_output($logName) if $redirect;
+$ipprc->redirect_output($logName) or my_die( "Unable to redirect", $det_id, $iter, $PS_EXIT_SYS_ERROR ) if $redirect;
 # values to extract from output metadata and the stats to calculate
 …
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $det_id and defined $iter and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_exp.pl

-              r22430
+              r23352
 use File::Temp qw( tempfile );                   # tools to construct temp files
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); # option parsing
 …
 # parse the command-line options
 my ( $det_id, $iter, $exp_id, $exp_tag, $det_type, $camera, $filter, $reject, $outroot, $dbname, $reduction,
+my ( $det_id, $iter, $exp_id, $exp_tag, $det_mode, $det_type, $camera, $filter, $reject, $outroot, $dbname, $reduction,
      $verbose, $no_update, $no_op, $save_temps, $redirect );
 GetOptions(
 …
            'exp_tag|=s'        => \$exp_tag,
            'det_type|t=s'      => \$det_type,
+           'det_mode=s'        => \$det_mode,
            'camera=s'          => \$camera,
            'outroot|w=s'       => \$outroot,   # output file base name
 …
     defined $exp_tag  and
     defined $det_type and
+    defined $det_mode and
     defined $camera   and
     defined $outroot;
-# Unhandled exceptions should be passed on to my_die so they get pushed into the database
-$SIG{__DIE__} = sub { die @_ if $^S;
-                      my_die( $_[0], $det_id, $iter, $exp_id, $PS_EXIT_UNKNOWN_ERROR ); };
 # load IPP config information for the specified camera
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $det_id, $iter, $exp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 my $logDest = $ipprc->filename("LOG.EXP", $outroot) or &my_die("Missing entry from camera config", $det_id, $iter, $exp_id, $PS_EXIT_CONFIG_ERROR);
+$ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 # Recipes to use based on reduction class
 …
 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe;
+# variables used for I/O
+my ($command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
+# Get list of normalizations by class_id : stored as $norms; save to temp file for ppImage runs below
+my (%norms, $normsName);
+if ($det_mode eq 'master') {
+    # dettool command to select imfile data for this exp_id
+    $command  = "$dettool -normalizedstat";
+    $command .= " -det_id $det_id";
+    $command .= " -iteration $iter";
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("Unable to perform dettool -residimfile: $error_code\n");
+        exit($error_code);
+    }
+    if (@$stdout_buf == 0) {
+        &my_die("No normalizations were found", $det_id, $iter, $PS_EXIT_PROG_ERROR);
+    }
+    # Parse the stdout buffer into a metadata
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
+    # parse the file info in the metadata
+    my $normsMD = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
+    # write the normalizations to a file as a metadata config file in the form: class_id F32 value
+    # XXX a possible optimization: if there is only one imfile, skip normalization
+    my $normsFile;
+    ($normsFile, $normsName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.norms.XXXX", UNLINK => !$save_temps );
+    print "saving norms to $normsName\n";
+    foreach my $norm (@$normsMD) {
+        my $class_id = $norm->{class_id};
+        my $normalization = $norm->{norm};
+        $norms{$class_id} = $normalization;
+        printf $normsFile "$class_id F32 $normalization\n",
+    }
+    close $normsFile;
+}
 # Get list of imfile files
 my $cmdflags;
 my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
+my (@files);
+{
     # dettool command to select imfile data for this exp_id
 …
         exit($error_code);
+    }
+    # XXX report an error message if stdout_buf is empty
+    if (@$stdout_buf == 0) {
+        &my_die("No imfiles were found", $det_id, $iter, $PS_EXIT_PROG_ERROR);
+    }
     # Parse the stdout buffer into a metadata
 …
         &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
+    # since I can't figure out how to do input and output within PERL, I'm writing the (modified) metadata to a temp file
+    my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.stats.XXXX", UNLINK => !$save_temps );
+    print $statFile "rawResidImfile MULTI\n";
     # parse the file info in the metadata
+    $files = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
+    # since I can't figure out how to do input and output within PERL, I'm writing to a temp file
+    my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.stats.XXXX", UNLINK => !$save_temps );
+    print "saving stats to $statName\n";
+    foreach my $line (@$stdout_buf) {
+        print $statFile $line;
+    # as we parse the list of files and their stats, apply the normalization to the relevant fields
+    # also, write out the modified metadata set
+    foreach my $mdItem (@$metadata) {
+        if ($mdItem->{class} ne "metadata") {
+            carp "MD element ", $mdItem->{name}, " isn't of type METADATA --- ignored.\n";
+            next;
+        }
+        my %hash;               # Hash element
+        my $mdComponents = $mdItem->{value}; # Components of the metadata
+        # determine the class_id for this block:
+        my $class_id;
+        foreach my $data (@$mdComponents) {
+            unless ($data->{name} eq "class_id") { next; }
+            $class_id = $data->{value};
+            last;
+        }
+        # a new metadata block
+        print $statFile "rawResidImfile  METADATA\n";
+        # modify and save the data in this block:
+        foreach my $data (@$mdComponents) {
+            my $norm = 1.0;
+            if ($det_mode eq "master") {
+                $norm = $norms{$class_id};
+            }
+            # fields to modify by the normalization:
+            if ($data->{name} eq "bg")            { $data->{value} *= $norm; }
+            if ($data->{name} eq "bg_stdev")      { $data->{value} *= $norm; }
+            if ($data->{name} eq "bg_mean_stdev") { $data->{value} *= $norm; }
+            if ($data->{name} eq "bg_skewness")   { $data->{value} *= $norm; }
+            if ($data->{name} eq "bg_kurtosis")   { $data->{value} *= $norm; }
+            if ($data->{name} eq "bin_stdev")     { $data->{value} *= $norm; }
+            # write out the metadata, save on the array of hashes
+            print $statFile "  $data->{name}  $data->{type}  $data->{value}\n";
+            $hash{$data->{name}} = $data->{value};
+        }
+        print $statFile "END\n";
+        push @files, \%hash;
+    }
     close $statFile;
 …
 my ($list1File, $list1Name) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.b1.list.XXXX", UNLINK => !$save_temps );
 my ($list2File, $list2Name) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.b2.list.XXXX", UNLINK => !$save_temps );
 foreach my $file (@$files) {
+foreach my $file (@files) {
     print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
     print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
 …
 unless ($no_op) {
     # Make the jpeg for binning 1
+    # XXX EAM : supply the collection of normalizations as a metadata
     $command = "$ppImage -list $list1Name $outroot"; # Command to run
     $command .= " -recipe PPIMAGE PPIMAGE_J1";
     $command .= " -recipe JPEG $recipe";
+    $command .= " -normlist $normsName" if defined $normsName;
     $command .= " -dbname $dbname" if defined $dbname;
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 …
     # Make the jpeg for binning 2
+    # XXX EAM : supply the collection of normalizations as a metadata
     $command = "$ppImage -list $list2Name $outroot"; # Command to run
     $command .= " -recipe PPIMAGE PPIMAGE_J2";
     $command .= " -recipe JPEG $recipe";
+    $command .= " -normlist $normsName" if defined $normsName;
     $command .= " -dbname $dbname" if defined $dbname;
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 …
 my @fluxes;
 foreach my $file (@$files) {
+foreach my $file (@files) {
     my $name      = $file->{class_id};
     my $mean      = $file->{bg};        # Mean for this imfile
 …
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $det_id and defined $iter and defined $exp_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_resid_imfile.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::Config;
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $detrend;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $det_id, $iter, $exp_id, $class_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+my $logDest     = $ipprc->filename("LOG.IMFILE", $outroot, $class_id);
+if ($redirect) {
+    $ipprc->redirect_output($logDest);
+}
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or my_die( "Unable to find LOG.IMFILE", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR );
+$ipprc->redirect_output($logDest) or my_die( "Unable to find LOG.IMFILE", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 # Recipes to use as a function of detrend type and mode
 …
 };
+# The output file rule name depends on the detrend type
+my $FILERULES = { 'FLATMASK'         => 'PPIMAGE.OUTPUT.RESID',
+                  'DARKMASK'         => 'PPIMAGE.OUTPUT.RESID',
+                  'MASK'             => 'PPIMAGE.OUTPUT.RESID',
+                  'BIAS'             => 'PPIMAGE.OUTPUT.RESID',
+                  'DARK'             => 'PPIMAGE.OUTPUT.RESID',
+                  'DARK_PREMASK'     => 'PPIMAGE.OUTPUT.RESID',
+                  'SHUTTER'          => 'PPIMAGE.OUTPUT.DETREND',
+                  'FLAT_PREMASK'     => 'PPIMAGE.OUTPUT.DETREND',
+                  'DOMEFLAT_PREMASK' => 'PPIMAGE.OUTPUT.DETREND',
+                  'SKYFLAT_PREMASK'  => 'PPIMAGE.OUTPUT.DETREND',
+                  'FLAT_RAW'         => 'PPIMAGE.OUTPUT.DETREND',
+                  'DOMEFLAT_RAW'     => 'PPIMAGE.OUTPUT.DETREND',
+                  'SKYFLAT_RAW'      => 'PPIMAGE.OUTPUT.DETREND',
+                  'FLAT'             => 'PPIMAGE.OUTPUT.DETREND',
+                  'DOMEFLAT'         => 'PPIMAGE.OUTPUT.DETREND',
+                  'SKYFLAT'          => 'PPIMAGE.OUTPUT.DETREND',
+                  'FRINGE'           => 'PPIMAGE.OUTPUT.RESID',
+              };
 # outroot examples (HOST components must be set)
 # file://data/ipp004.0/gpc1/20080130
 …
 # my $outputName  = $ipprc->filename("PPIMAGE.OUTPUT", $outroot, $class_id);
+my $outputName  = $ipprc->filename("PPIMAGE.OUTPUT.RESID", $outroot, $class_id);
+my $filerule = $FILERULES->{$det_type}; # File rule to use
+my $outputName  = $ipprc->filename($filerule,        $outroot, $class_id);
 my $bin1Name    = $ipprc->filename("PPIMAGE.BIN1",   $outroot, $class_id);
 my $bin2Name    = $ipprc->filename("PPIMAGE.BIN2",   $outroot, $class_id);
 …
     $command .= " -recipe JPEG $jpeg_recipe";
     $command .= " -recipe PPSTATS RESIDUAL";
     $command .= " -F PPIMAGE.OUTPUT PPIMAGE.OUTPUT.RESID";
+    $command .= " -F PPIMAGE.OUTPUT $filerule";
     $command .= " -stats $outputStats";
     $command .= " -tracedest $traceDest -log $logDest";
 …
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $det_id and defined $iter and defined $exp_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/detrend_stack.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
-$ipprc->define_camera($camera);
 $det_type = uc($det_type);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id)
     or &my_die("Missing entry in file rules", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR);
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $det_id, $iter, $class_id, $PS_EXIT_UNKNOWN_ERROR ); };
+# optionally redirect the outputs from this script to LOG.IMFILE
+$ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 # Recipes to use as a function of detrend type
 …
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $det_id and defined $iter and defined $class_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/diff_skycell.pl

-              r22433
+              r23352
 use Data::Dumper;
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     and defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $diff_id, $skycell_id, $PS_EXIT_UNKNOWN_ERROR ); };
+my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $diff_id, $skycell_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 # XXX camera is not known here; cannot use filerules...
 # my $logDest = $ipprc->filename("LOG.EXP", $outroot);
 my $logDest = "$outroot.log";
 $ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $diff_id, $skycell_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_DIFF);
 …
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     warn($msg);
     if (defined $diff_id and defined $skycell_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/fake_imfile.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::Config;
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $exp_id, $fake_id, $class_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id)  or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+$ipprc->redirect_output($logDest) if $redirect;
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+my $logDest = $ipprc->filename("LOG.IMFILE", $outroot, $class_id) or &my_die("Missing entry from camera config", $exp_id, $fake_id, $class_id, $PS_EXIT_CONFIG_ERROR);
+$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $exp_id, $fake_id, $class_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 # Recipes to use based on reduction class
 …
     my $exit_code = shift; # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $exp_id and defined $fake_id and defined $class_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_cleanup.pl

-              r21371
+              r23352
 use Pod::Usage qw( pod2usage );
-my $ipprc = PS::IPP::Config->new(); # this is used for PATH, NEB filename conversions
 # Parse the command-line arguments
 my ($stage, $camera, $stage_id, $mode, $path_base, $dbname, $verbose, $no_op, $helplist);
 GetOptions('stage=s'        => \$stage,     # which analysis stage to clean?
            'camera|i=s'     => \$camera,    # user-supplied camera name
            'stage_id=s'     => \$stage_id,  # id for this stage (only needed for certain stages)
            'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
            'path_base=s'    => \$path_base, # basename for files
            'dbname|d=s'     => \$dbname,    # Database name
+           'camera|i=s'     => \$camera,    # user-supplied camera name
+           'stage_id=s'     => \$stage_id,  # id for this stage (only needed for certain stages)
+           'mode|m=s'       => \$mode,      # cleanup mode (clean / purge)
+           'path_base=s'    => \$path_base, # basename for files
+           'dbname|d=s'     => \$dbname,    # Database name
            'verbose'        => \$verbose,   # Print to stdout
            'no-op'          => \$no_op,     # pretend but don't actually inject
            'helplist'       => \$helplist   # give help listing
            ) or pod2usage( 2 );
 pod2usage( -msg => "remove temporary / all data files for an IPP analysis stage",
            -exitval => 2) if defined $helplist;
 pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id (stage_id) --mode (mode) [--path_base (path)] [--dbname dbname] [--no-op] [--help]",
            -exitval => 2 ) if scalar @ARGV;
+           'no-op'          => \$no_op,     # pretend but don't actually inject
+           'helplist'       => \$helplist   # give help listing
+           ) or pod2usage( 2 );
+pod2usage( -msg => "remove temporary / all data files for an IPP analysis stage",
+           -exitval => 2) if defined $helplist;
+pod2usage( -msg => "Usage: $0 --camera (name) --stage (stage) --stage_id (stage_id) --mode (mode) [--path_base (path)] [--dbname dbname] [--no-op] [--help]",
+           -exitval => 2 ) if scalar @ARGV;
 pod2usage( -msg => "Required options:--camera (name) --stage (stage) --mode (mode)",
+           -exitval => 3) unless
+           -exitval => 3) unless
     defined $camera and
     defined $stage and
     defined $mode;
+# $mode must be one of "goto_cleaned" or "goto_purged"
+unless (($mode eq "goto_cleaned") || ($mode eq "goto_purged")) {
+    die "invalid cleanup mode $mode\n";
+}
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to set up", $stage_id, $PS_EXIT_CONFIG_ERROR); # this is used for PATH, NEB filename conversions
+# $mode must be one of "goto_cleaned", "goto_scrubbed", or
+# "goto_purged" goto_cleaned and goto_scrubbed both result in
+# 'cleaned' on success ('scrubbed' allows chips without config files
+# to be cleaned; they cannot be recovered, but the small data is left
+# behind). XXX make 'scrubbed' a data_state?
+unless (($mode eq "goto_cleaned") || ($mode eq "goto_scrubbed") || ($mode eq "goto_purged")) {
+    die "invalid cleanup mode $mode\n";
+}
+my $error_state;
+if ($mode eq "goto_cleaned")  { $error_state = "error_cleaned";  }
+if ($mode eq "goto_scrubbed") { $error_state = "error_scrubbed"; }
+if ($mode eq "goto_purged")   { $error_state = "error_purged";   }
 my %stages = ( chip => 1, camera => 1, fake => 1, warp => 1, stack => 1, diff  => 1);
 …
+}
-$ipprc->define_camera($camera);
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 # choice of files to delete depends on the stage
 if ($stage eq "chip") {
     die "--stage_id required for stage chip\n" if !$stage_id;
     ### select the imfiles for this entry
 …
         &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+    }
+    # if there are no chipProcessedImfiles (@$stdout_buf == 0), the reset the state to 'new'
+    if (@$stdout_buf == 0)  {
+        my $command = "$chiptool -chip_id $stage_id -updaterun -set_state new";
+        $command .= " -dbname $dbname" if defined $dbname;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+        }
+        exit 0;
+    }
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", "chip", $stage_id, $PS_EXIT_PROG_ERROR);
 …
     # loop over all of the imfiles, determine the path_base and class_id for each
     foreach my $imfile (@$imfiles) {
         my $class_id = $imfile->{class_id};
         my $path_base = $imfile->{path_base};
+        my $class_id = $imfile->{class_id};
+        my $path_base = $imfile->{path_base};
         my $status = 1;
         # don't clean up unless the data needed to update is available
+        # modes goto_purged and goto_scrubbed will remove files even if the config is non-existent
         if ($mode eq "goto_cleaned") {
             my $config_file = $ipprc->filename("PPIMAGE.CONFIG", $path_base, $class_id);
 …
                 addFilename (\@files, "PPIMAGE.CONFIG", $path_base, $class_id);
+            }
             # actual command to delete the files
             $status = &delete_files (\@files);
+        }
         if ($status)  {
             my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
+        if ($status)  {
+            my $command = "$chiptool -chip_id $stage_id -class_id $class_id";
             if ($mode eq "goto_purged") {
                 $command .= " -topurgedimfile";
 …
                 $command .= " -tocleanedimfile";
+            }
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+            }
+        } else {
+            # if an error happens for one chip, the chipRun will stay in goto_*, but the chips will go to error_* (matching the goto_*)
+            my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -set_state $error_state";
             $command .= " -dbname $dbname" if defined $dbname;
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                     run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+            }
+        } else {
+            my $command = "$chiptool -updateprocessedimfile -chip_id $stage_id -class_id $class_id -code 1";
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+            }
+        }
+    }
+} elsif ($stage eq "camera") {
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform chiptool: $error_code", "chip", $stage_id, $error_code);
+            }
+        }
+    }
+    exit 0;
+}
+if ($stage eq "camera") {
     die "--stage_id required for stage camera\n" if !$stage_id;
     # this stage uses 'camtool'
 …
     if ($status)  {
         my $command = "$camtool -cam_id $stage_id -updaterun";
+        my $command;
         if ($mode eq "goto_cleaned") {
+            $command .= " -state cleaned";
+        } else {
+            $command .= " -state purged";
+            $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
+        }
+        if ($mode eq "goto_scrubbed") {
+            $command = "$camtool -updaterun -cam_id $stage_id -set_state cleaned";
+        }
+        if ($mode eq "goto_purged") {
+            $command = "$camtool -updaterun -cam_id $stage_id -set_state purged";
+        }
         $command .= " -dbname $dbname" if defined $dbname;
 …
+        }
     } else {
+        my $command = "$camtool -updateprocessedexp -cam_id $stage_id -code 1";
+        # since 'camera' has only a single imfile, we can just update the run
+        my $command = "$camtool -updaterun -cam_id $stage_id -set_state $error_state";
         $command .= " -dbname $dbname" if defined $dbname;
 …
+    }
     exit 0;
+} elsif ($stage eq "warp") {
+}
+if ($stage eq "warp") {
     die "--stage_id required for stage warp\n" if !$stage_id;
     # this stage uses 'warptool'
 …
+        }
         if ($status)  {
             my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
+        if ($status)  {
+            my $command = "$warptool -warp_id $stage_id -skycell_id $skycell_id";
             if ($mode eq "goto_purged") {
                 $command .= " -topurgedskyfile";
 …
                 $command .= " -tocleanedskyfile";
+            }
             $command .= " -dbname $dbname" if defined $dbname;
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            $command .= " -dbname $dbname" if defined $dbname;
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                     run(command => $command, verbose => $verbose);
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+            }
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+            }
          } else {
+            # XXX: -updateskyfile mode does not exist, need to add it
+            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -code 1";
+            my $command = "$warptool -updateskyfile -warp_id $stage_id -skycell_id $skycell_id -set_state $error_state";
             $command .= " -dbname $dbname" if defined $dbname;
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $command, verbose => $verbose);
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
                 &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+            }
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform warptool: $error_code", "warp", $stage_id, $error_code);
+            }
             exit $PS_EXIT_UNKNOWN_ERROR;
+        }
+        }
+    }
     exit 0;
 …
 # left TODO
 # fake : faketool : -pendingcleanupimfile (loop over imfiles)
 # stack: stacktool : -pendingcleanupskyfile (loop over skyfiles)
 # diff:  difftool : -pendingcleanupskyfile
+# stack: stacktool : -pendingcleanupskyfile
+# diff:  difftool : -pendingcleanupskyfile (loop over skyfiles)
 die "ipp_cleanup.pl -stage $stage not yet implemented\n";
 sub delete_files
+sub delete_files
+{
     my $files = shift; # reference to a list of files to unlink
     # this script is, of course, very dangerous.
+    # this script is, of course, very dangerous.
     foreach my $file (@$files) {
         print STDERR "unlinking $file\n";
+        print STDERR "unlinking $file\n";
         $ipprc->file_delete($file);
+    }
 …
+}
 sub addFilename
+sub addFilename
+{
     my $files      = shift; # reference to a list of files to unlink
 …
+}
+# XXX we currently do not set the error state in the db on my_die
 sub my_die
+{

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/ipp_maskscript.pl

-              r17671
+              r23352
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 my ($dbname, $det_id, $camera);
 GetOptions('dbname=s'    => \$dbname,
            'det_id=s'    => \$det_id,
            'camera|c=s'  => \$camera,
            ) or pod2usage( 2 );
+           'det_id=s'    => \$det_id,
+           'camera|c=s'  => \$camera,
+           ) or pod2usage( 2 );
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
           -msg => "USAGE: ipp_maskscript.pl --dbname (name) --det_id (id) --iter (iteration) --camera (name)",
           -exitval => 3,
           ) unless defined $dbname and defined $det_id and defined $camera;
+          -msg => "USAGE: ipp_maskscript.pl --dbname (name) --det_id (id) --iter (iteration) --camera (name)",
+          -exitval => 3,
+          ) unless defined $dbname and defined $det_id and defined $camera;
 # I could determine the camera from a query for the detrun
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die("Unable to setup", $PS_EXIT_CONFIG_ERROR); # IPP configuration
 ###  Get list of dark imfile results
 …
 # parse the output into a list
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
     &my_die("Unable to parse metadata config doc", $PS_EXIT_PROG_ERROR);
 …
     # print STDERR "contents: @contents\n";
     my $parser = PS::IPP::Metadata::Config->new;        # Parser for metadata config files
+    my $parser = PS::IPP::Metadata::Config->new;        # Parser for metadata config files
     my $statsList = $parser->parse(join "", @contents) or &my_die("Unable to parse metadata for imfile stats", $PS_EXIT_SYS_ERROR);
 …
     open (DATA, ">$component.dat");
     for (my $i = 0; $i < @{$nameX}; $i++) {
         print DATA "${$nameX}[$i] ${$nameY}[$i]\n";
+        print DATA "${$nameX}[$i] ${$nameY}[$i]\n";
+    }
     close (DATA);
 …
     my ($exp_time, $tag, $md) = @_;
     # descend through the fpa
+    # descend through the fpa
     foreach my $entry (@$md) {
         # print STDERR "name: $entry->{name}, class: $entry->{class}\n";
+        # print STDERR "name: $entry->{name}, class: $entry->{class}\n";
         # recurse on nested metadata
         if ($entry->{class} eq 'metadata') {
             my $newtag = $tag . "_" . $entry->{name};
+            my $newtag = $tag . "_" . $entry->{name};
             &parse_stats_table ($exp_time, $newtag, $entry->{value});
+        }
 …
                 push @bg_stdev_data, $entry->{value};
             } else {
                 push @bg_name,    $tag;
+                push @bg_name,    $tag;
                 push @bg_data,    $entry->{value};
                 push @bg_exptime, $exp_time;
                 # print STDERR "$tag $exp_time $entry->{value}\n";
+                push @bg_exptime, $exp_time;
+                # print STDERR "$tag $exp_time $entry->{value}\n";
+            }
             if (!$componentsHash{$tag}) {
                 push @components, $tag;
                 $componentsHash{$tag} = 1;
+            }
             next;
+        }
+            if (!$componentsHash{$tag}) {
+                push @components, $tag;
+                $componentsHash{$tag} = 1;
+            }
+            next;
+        }
+    }
     return 1;

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_definerun.pl

-              r21006
+              r23352
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
+# Look for programs we need
+my $missing_tools;
+my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $difftool  = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $warptool  = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
 # Parse the command-line arguments
 my ($exp_id, $warp_id, $min_diff_id, $label, $workdir, $dbname, $save_temps, $verbose);
 GetOptions(
            'exp_id=s'        => \$exp_id,     # exposure identifier
 …
     defined $warp_id;
+# $ipprc->define_camera($camera);
+# Look for programs we need
+my $missing_tools;
+my $magictool = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $difftool  = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $warptool  = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
+if ($missing_tools) {
+    warn("Can't find required tools.");
+    exit($PS_EXIT_CONFIG_ERROR);
+}
+my $ipprc = PS::IPP::Config->new() or my_die("Unable to set up", $PS_EXIT_CONFIG_ERROR); # IPP configuration
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_destreak.pl

-              r22430
+              r23352
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $magic_ds_id, $component, $PS_EXIT_UNKNOWN_ERROR ); };
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile;
 my ($skycell_args, $class_id, $skycell_id);
 …
     &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
+}
-$ipprc->redirect_output($logfile) if $logfile;
-$ipprc->define_camera($camera);
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 …
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     my $command = "$magicdstool -adddestreakedfile";
     $command   .= " -magic_ds_id $magic_ds_id";

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_mask.pl

-              r22430
+              r23352
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $magic_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+$ipprc->redirect_output($logfile) if $logfile;
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 …
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $magic_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_process.pl

-              r22430
+              r23352
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $node, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $node, $PS_EXIT_SYS_ERROR ) if $logfile;
 # RemoveStreaks doesn't know about nebulous. It expects to be able to append strings to outroot
 …
 # of the file names as arguments
 if ($outroot =~ 'neb:/') {
+    &my_die("RemoveStreaks does not support nebulous paths in outroot", $magic_id, $node,
+        $PS_EXIT_CONFIG_ERROR);
+    &my_die("RemoveStreaks does not support nebulous paths in outroot", $magic_id, $node, $PS_EXIT_CONFIG_ERROR);
+}
 # resolve any path:// or file:// in outroot
 $outroot = $ipprc->file_resolve($outroot);
-$ipprc->redirect_output($logfile) if $logfile;
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 …
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $magic_id and defined $node and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/magic_tree.pl

-              r22430
+              r23352
 use File::Temp qw( tempfile );
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     defined $outroot;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $magic_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+$ipprc->redirect_output($logfile) if $logfile;
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 …
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $magic_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/register_exp.pl

-              r22430
+              r23352
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
-my $ipprc = PS::IPP::Config->new();
 # Look for commands we need
 …
 ) or pod2usage( 2 );
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $exp_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->redirect_output($logfile) if $logfile;
+my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $PS_EXIT_CONFIG_ERROR );
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 …
     my $exit_code = shift;
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
     if (defined $exp_id and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/register_imfile.pl

-              r22428
+              r23352
 use Math::Trig;
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use File::Spec;
 …
 ) or pod2usage( 2 );
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->redirect_output($logfile) if $logfile;
+my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $PS_EXIT_SYS_ERROR ) if $logfile;
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 …
     print "STDOUT:\n$out1";
     print "STDERR:\n$err1";
     &my_die("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $h1->result() ) unless $result1;
+    &my_die("Unable to perform ppStats on exposure id $exp_id: " . $h1->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h1->result() or $PS_EXIT_PROG_ERROR) ) unless $result1;
     print "[Running " . join(' ', @command2) . "]\n";
 …
     print "STDOUT:\n$out2";
     print "STDERR:\n$err2";
     &my_die("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, $h2->result() ) unless $result2;
+    &my_die("Unable to perform ppStatsFromMetadata on exposure id $exp_id: " . $h2->result(), $exp_id, $tmp_exp_name, $tmp_class_id, $uri, ($h2->result() or $PS_EXIT_PROG_ERROR) ) unless $result2;
     chomp $out2;
     $cmdflags = $out2;
 …
 # determine solar-system parameters
 my $longitude = &value_for_flag ($cmdflags, "-longitude");
 my $latitude  = &value_for_flag ($cmdflags, "-latitude");
 my $elevation = &value_for_flag ($cmdflags, "-elevation");
 my $ra        = &value_for_flag ($cmdflags, "-ra");
 my $dec       = &value_for_flag ($cmdflags, "-decl");
 my $dateobs   = &value_for_flag ($cmdflags, "-dateobs");
+my $longitude = &value_for_flag($cmdflags, "-longitude");
+my $latitude  = &value_for_flag($cmdflags, "-latitude");
+my $elevation = &value_for_flag($cmdflags, "-elevation");
+my $ra        = &value_for_flag($cmdflags, "-ra");
+my $dec       = &value_for_flag($cmdflags, "-decl");
+my $dateobs   = &value_for_flag($cmdflags, "-dateobs");
 # if the needed data is available, pass it to sunmoon:
 …
     # for failed imfiles, we insert UNKNOWN for inst, telescope, class_id
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);
 …
         $command .= " -hostname $host" if defined $host;
         $command .= " -dbname $dbname" if defined $dbname;
+        system ($command);
+        print "Running: $command\n";
+        system($command);
+    }
     exit $exit_code;

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/stack_skycell.pl

-              r22430
+              r23352
 use File::Basename;
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 …
     and defined $run_state;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $stack_id, $PS_EXIT_UNKNOWN_ERROR ); };
+my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $stack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 # XXX camera is not known here; cannot use filerules...
 …
+}
 $ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $stack_id, $PS_EXIT_SYS_ERROR ) if $redirect;
 my $temp_images_exist = 0;
 …
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/summit_copy.pl

r20310	r23352
136	136	my $exit_code = shift; # Exit code to add
137	137
	138	$exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
	139
138	140	warn $msg;
139	141	unless ($no_update) {

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/warp_overlap.pl

-              r22430
+              r23352
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 # Look for programs we need
 …
 ) or pod2usage( 2 );
-$ipprc->redirect_output($logfile) if $logfile;
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage(
 …
     and defined $tess_dir;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $warp_id, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+$ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $warp_id, $PS_EXIT_SYS_ERROR ) if $logfile;
 &my_die("Tessellation identifier not provided: $tess_dir", $warp_id, $PS_EXIT_SYS_ERROR) unless $tess_dir ne "NULL";
 …
     my $warp_id = shift;        # Warp identifier
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     carp($msg);

branches/cnb_branches/cnb_branch_20090301/ippScripts/scripts/warp_skycell.pl

-              r22430
+              r23352
 use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Config 1.01 qw( :standard );
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 # Look for programs we need
 …
     and defined $run_state;
+# Unhandled exceptions should be passed on to my_die so they get pushed into the database
+$SIG{__DIE__} = sub { die @_ if $^S;
+                      my_die( $_[0], $warp_id, $skycell_id, $tess_dir, $PS_EXIT_UNKNOWN_ERROR ); };
+$ipprc->define_camera($camera);
+my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id);
+my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
+my $logDest = $ipprc->filename("LOG.EXP", $outroot, $skycell_id) or my_die( "Unable to get log filename", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR );
 $logDest .= ".update" if ($run_state eq 'update');
 $ipprc->redirect_output($logDest) if $redirect;
+$ipprc->redirect_output($logDest) or my_die( "Unable to redirect output", $warp_id, $skycell_id, $tess_dir, $PS_EXIT_SYS_ERROR ) if $redirect;
 my $source_id = $ipprc->source_id($dbname, $PS_TABLE_ID_WARP);
 …
     my $exit_code = shift;      # Exit code to add
+    $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
     warn($msg);
     if (defined $warp_id and defined $skycell_id and defined $tess_dir and not $no_update) {

branches/cnb_branches/cnb_branch_20090301/ippTasks/Makefile.am

r20879	r23352
20	20	summit.copy.pro \
21	21	replicate.pro \
	22	dist.pro \
22	23	pstamp.pro
23	24

branches/cnb_branches/cnb_branch_20090301/ippTasks/automate.pro

-              r19621
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    book setword automate $pageName pantaskState INIT.REGULAR
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/calibration.pro

-              r19465
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword calBook $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/camera.pro

-              r19559
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit default
     process_exit camPendingExp $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword camPendingExp $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword camPendingCleanup $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/chip.pro

-              r21464
+              r23352
     active true
   end
   task chip.promoteexp
+  task chip.advanceexp
     active true
   end
 …
     active false
   end
   task chip.promoteexp
+  task chip.advanceexp
     active false
   end
 …
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword chipPendingImfile $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout
 …
 # this variable will cycle through the known database names
 $chip_promote_DB = 0
 # promote exposures for which all imfiles have completed processing
+$chip_advance_DB = 0
+# advance exposures for which all imfiles have completed processing
 # sets the exposure state to full and queues warp processing if requested
 task           chip.promoteexp
+task           chip.advanceexp
   host         local
 …
   stdout NULL
   stderr $LOGDIR/chip.promoteexp.log
   task.exec
     $run = chiptool -promoteexp -limit 10
+  stderr $LOGDIR/chip.advanceexp.log
+  task.exec
+    $run = chiptool -advanceexp -limit 10
     if ($DB:n == 0)
       option DEFAULT
     else
       # save the DB name for the exit tasks
       option $DB:$chip_promote_DB
       $run = $run -dbname $DB:$chip_promote_DB
       $chip_promote_DB ++
       if ($chip_promote_DB >= $DB:n) set chip_promote_DB = 0
+      option $DB:$chip_advance_DB
+      $run = $run -dbname $DB:$chip_advance_DB
+      $chip_advance_DB ++
+      if ($chip_advance_DB >= $DB:n) set chip_advance_DB = 0
     end
     add_poll_args run
 …
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword chipPendingCleanup $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.correct.pro

-              r20398
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
     showcommand
+    showcommand timeout
   end
 end
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detPendingCorrectImfile $options:0 pantaskState CRASH
+  end
   # operation times out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.mkruns.pro

-              r16298
+              r23352
   end
+  task.exit crash
+    # send someone email?
+    echo "unable to define a new detrend run"
+  end
   # operation timed out?  is the database down?
   task.exit timeout
 …
   # default exit status
   task.exit default
+    echo "unable to define a new detrend run"
+  end
+  task.exit crash
+    # send someone email?
     echo "unable to define a new detrend run"
   end
 …
   end
+  task.exit crash
+    # send someone email?
+    echo "unable to define a new detrend run"
+  end
   # operation timed out?  is the database down?
   task.exit timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.norm.pro

-              r19465
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit detPendingNormStatImfile $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detPendingNormStatImfile $options:0 pantaskState CRASH
   end
 …
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detPendingNormImfile $options:0 pantaskState CRASH
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit detPendingNormExp $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detPendingNormExp $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit detCleanupNormStatImfile $options:0 $JOB_STATUS
+  end
+  task.exit    crash
+    showcommand crash
+    book setword detCleanupNormStatImfile $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit detCleanupNormImfile $options:0 $JOB_STATUS
+  end
+  task.exit    crash
+    showcommand crash
+    book setword detCleanupNormImfile $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword detCleanupNormExp $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.process.pro

-              r19465
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
     showcommand
+    showcommand timeout
   end
 end
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detPendingProcessedImfile $options:0 pantaskState CRASH
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detPendingProcessedExp $options:0 pantaskState CRASH
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit detCleanupProcessedImfile $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detCleanupProcessedImfile $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detCleanupProcessedExp $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.reject.pro

-              r19465
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
     showcommand failure
+    showcommand timeout
   end
 end
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detRejectExp $options:0 pantaskState CRASH
+  end
   # operation times out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.resid.pro

-              r19621
+              r23352
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detPendingResidImfile $options:0 pantaskState CRASH
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
     stderr $LOGDIR/detrend.resid.exp.log
     $run = detrend_resid_exp.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose
+    $run = detrend_resid_exp.pl --det_id $DET_ID --iteration $ITERATION --exp_id $EXP_ID --exp_tag $EXP_TAG --det_mode $MODE --det_type $DET_TYPE --camera $CAMERA --outroot $outroot --redirect-output --verbose
     add_standard_args run
 …
   task.exit    default
     process_exit detPendingResidExp $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detPendingResidExp $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit detCleanupResidImfile $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detCleanupResidImfile $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detCleanupResidExp $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/detrend.stack.pro

-              r19465
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit detPendingStackedImfile $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detPendingStackedImfile $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword detCleanupStackedImfile $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/diff.pro

-              r21312
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit diffSkyfile $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword diffSkyfile $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword diffCleanup $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/fake.pro

-              r21464
+              r23352
     active true
   end
   task fake.promoteexp
+  task fake.advanceexp
     active true
   end
 …
     active false
   end
   task fake.promoteexp
+  task fake.advanceexp
     active false
   end
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword fakePendingImfile $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout
 …
 end
 # promote exposures for which all imfiles have completed processing
+# advance exposures for which all imfiles have completed processing
 # sets the exposure state to full and queues warp processing if requested
 $fake_promote_DB = 0
 task           fake.promoteexp
+$fake_advance_DB = 0
+task           fake.advanceexp
   host         local
 …
   stdout NULL
   stderr $LOGDIR/fake.promoteexp.log
   task.exec
     $run = faketool -promoteexp -limit 10
+  stderr $LOGDIR/fake.advanceexp.log
+  task.exec
+    $run = faketool -advanceexp -limit 10
     if ($DB:n == 0)
       option DEFAULT
     else
       # save the DB name for the exit tasks
       option $DB:$fake_promote_DB
       $run = $run -dbname $DB:$fake_promote_DB
       $fake_promote_DB ++
       if ($fake_promote_DB >= $DB:n) set fake_promote_DB = 0
+      option $DB:$fake_advance_DB
+      $run = $run -dbname $DB:$fake_advance_DB
+      $fake_advance_DB ++
+      if ($fake_advance_DB >= $DB:n) set fake_advance_DB = 0
     end
     add_poll_args run
 …
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword fakePendingCleanup $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/flatcorr.pro

-              r20513
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword flatcorrBook $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/magic.pro

-              r21436
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword magicToTree $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit magicToProcess $options:0 $JOB_STATUS
+  end
+  task.exit    crash
+    showcommand crash
+    book setword magicToProcess $options:0 pantaskState CRASH
   end
 …
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword magicToDS $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/pstamp.pro

-              r21411
+              r23352
     end
+    task.exit   crash
+        showcommand crash
+    end
     task.exit   timeout
         showcommand timeout
 …
     end
+    task.exit   crash
+        showcommand crash
+    end
     task.exit   timeout
         showcommand timeout
 …
         showcommand failure
         process_exit pstampRequest $options:0 $JOB_STATUS
+    end
+    task.exit crash
+        showcommand crash
+        book setword pstampRequest $options:0 pantaskState CRASH
     end
 …
     end
+    task.exit   crash
+        showcommand crash
+    end
     task.exit   timeout
         showcommand timeout
 …
         process_exit pstampFinish $options:0 $JOB_STATUS
         showcommand failure
+    end
+    task.exit crash
+        showcommand crash
+        book setword pstampFinish $options:0 pantaskState CRASH.run
     end
 …
     end
+    task.exit   crash
+        showcommand crash
+    end
     task.exit   timeout
         showcommand timeout
 …
     end
+    task.exit crash
+        echo pstamp.job.run task.crash $JOB_ID status: $JOB_STATUS
+        process_exit pstampJob $options:0 $JOB_STATUS
+        showcommand crash
+        book setword pstampJob $options:0 pantaskState CRASH
+    end
     task.exit timeout
         echo pstamp.job.run task.timeout $JOB_ID status: $JOB_STATUS

branches/cnb_branches/cnb_branch_20090301/ippTasks/register.pro

-              r20498
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit default
     process_exit regPendingImfile $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword regPendingImfile $options:0 pantaskState CRASH
   end
 …
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword regPendingExp $options:0 pantaskState CRASH
+  end
+  task.exit    crash
+    showcommand crash
+    book setword regPendingExp $options:0 pantaskState CRASH
+  end
   # operation times out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/replicate.pro

-              r19747
+              r23352
 # the replicate process interacts with only the single Nebulous server
+# Each 'pendingreplicate' query is limited to a finite number of so_id values.
+# Each time we call replicate.load, we increment SO_ID_START by the range value.
+# If the pendingreplicate query exits with exit status 10, we start over at 0
+$SO_ID_START = 0
+$SO_ID_RANGE = 100000
 # select Nebulous objects which desire additional copies
 task           replicate.load
 …
   # modify these after the tasks are tested
   periods      -poll 10
   periods      -exec 1
+  periods      -exec 10
   periods      -timeout 1500
   npending     1
 …
       # command does not need to be dynamic, but having it so allows us to adjust the periods
       # so that we dont have to wait 10 minutes for things to start up
+      command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 7500
+      # XXX smaller limited?  7500 will be a huge book of things to do...
+      command neb-admin --host $NEB_HOST --db $NEB_DB --user $NEB_USER --pass $NEB_PASS --pendingreplicate --limit 7500 --so_id_start $SO_ID_START --so_id_range $SO_ID_RANGE
       periods      -exec 1800
 …
   # success
   task.exit $EXIT_SUCCESS
+    # advance the so_id counter
+    $SO_ID_START += $SO_ID_RANGE
     # convert 'stdout' to book format
     ipptool2book stdout replicatePending -key key -uniq -setword pantaskState INIT
 …
   end
+  # out of so_id range, reset
+  task.exit 10
+    # advance the so_id counter
+    $SO_ID_START = 0
+  end
   # locked list
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword replicatePending $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/stack.pro

-              r19597
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit stackSumSkyfile $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword stackSumSkyfile $options:0 pantaskState CRASH
   end
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword stackCleanup $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/summit.copy.pro

-              r21478
+              r23352
         showcommand failure
     end
+    task.exit crash
+        showcommand crash
+    end
     task.exit timeout
         showcommand timeout
 …
     showcommand failure
   end
+  task.exit     crash
+    showcommand crash
+  end
   task.exit     timeout
     showcommand timeout
 …
           queueprint stderr
         end
+    end
+    task.exit crash
+        showcommand crash
     end
     task.exit timeout
 …
         showcommand failure
         process_exit pzPendingExp $options:0 $JOB_STATUS
+    end
+    task.exit crash
+        showcommand crash
+        book setword pzPendingExp $options:0 pantaskState CRASH
     end
 …
         showcommand failure
     end
+    task.exit     crash
+        showcommand crash
+    end
     task.exit     timeout
         showcommand timeout
 …
         book setword pzPendingImfile $pageName filename $FILENAME
+        stdout $LOGDIR/summit.copy.log
+        stderr $LOGDIR/summit.copy.log
         $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --end_stage reg --workdir $workdir --dbname $DBNAME --timeout 120 --verbose --copies 2
         if ($COMPRESS)
 …
     end
+    task.exit crash
+        showcommand crash
+        book setword pzPendingImfile $options:0 pantaskState CRASH
+    end
     # operation timed out?
     task.exit timeout
 …
     task.exit default
         showcommand failure
+    end
+    task.exit crash
+        showcommand crash
     end
     task.exit timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTasks/warp.pro

-              r20932
+              r23352
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   task.exit    default
     process_exit warpInputExp $options:0 $JOB_STATUS
+  end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword warpInputExp $options:0 pantaskState CRASH
   end
 …
   task.exit    default
     showcommand failure
+  end
+  task.exit    crash
+    showcommand crash
   end
 …
   end
+  # locked list
+  task.exit    crash
+    showcommand crash
+    echo "hostname: $JOB_HOSTNAME"
+    book setword warpPendingSkyCell $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+  end
   # operation times out?
   task.exit    timeout
 …
   end
+  task.exit    crash
+    showcommand crash
+    book setword warpPendingCleanup $options:0 pantaskState CRASH
+  end
   # operation timed out?
   task.exit    timeout

branches/cnb_branches/cnb_branch_20090301/ippTools/share/Makefile.am

-              r21521
+              r23352
      difftool_skyfile.sql \
      difftool_todiffskyfile.sql \
+     disttool_pendingcomponent.sql \
+     disttool_processedcomponent.sql \
+     disttool_revertrun_update.sql \
+     disttool_revertrun_delete.sql \
+     disttool_toadvance.sql \
      faketool_change_exp_state.sql \
      faketool_change_imfile_data_state.sql \

branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_change_imfile_data_state.sql

-              r19527
+              r23352
     chip_id = %lld
     AND class_id = '%s'
-    -- only update if chipRun.state has the expected value
-    AND (
-        SELECT state from chipRun where chipRun.chip_id = chipProcessedImfile.chip_id
-    ) = '%s'

branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_pendingcleanupimfile.sql

r19527	r23352
15	15	((chipRun.state = 'goto_cleaned' AND chipProcessedImfile.data_state = 'full')
16	16	OR
	17	(chipRun.state = 'goto_scrubbed' AND chipProcessedImfile.data_state = 'full')
	18	OR
17	19	(chipRun.state = 'goto_purged' AND chipProcessedImfile.data_state != 'purged'))

branches/cnb_branches/cnb_branch_20090301/ippTools/share/chiptool_pendingcleanuprun.sql

r19527	r23352
7	7	USING (exp_id)
8	8	WHERE
9		(chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_purged')
	9	(chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_scrubbed' OR chipRun.state = 'goto_purged')

branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_processedimfile.sql

r18561	r23352
	1	-- is this DISTINCT needed?
1	2	SELECT DISTINCT
2	3	detRun.det_type,

branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_residimfile.sql

-              r19632
+              r23352
    detRun.det_type,
    detRun.mode,
+   detResidImfile.*,
+   rawExp.exp_time
+ FROM detResidImfile
+ JOIN detRun
+   USING(det_id, iteration)
+ JOIN rawExp
+   USING(exp_id)
+   rawExp.exp_time,
+   detResidImfile.*
+FROM detResidImfile
+JOIN detRun
+  USING(det_id, iteration)
+JOIN detInputExp
+  ON detRun.det_id = detInputExp.det_id
+  AND detRun.iteration = detInputExp.iteration
+  AND detResidImfile.exp_id = detInputExp.exp_id
+JOIN rawExp
+  ON rawExp.exp_id = detResidImfile.exp_id

branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_todetrunsummary.sql

r14078	r23352
40	40	AND detRunSummary.det_id IS NULL
41	41	AND detRunSummary.iteration IS NULL
	42	AND detResidExp.fault = 0
42	43	GROUP BY
43	44	detRun.det_id,

branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_tonormalizedstat.sql

-              r18118
+              r23352
+-- select detRun.det_id (det_id)
+-- select detRun.iteration
+-- by:
+-- find the current iteration bassed on det_id
+-- find all exp_ids in the current det_id/iteration from detInputExp
+-- sort to detInputExp.imfiles to find the largest value per det_id/iter
+-- compare imfiles to the number of detStackedImfiles by class_id
+-- and:
+-- ???
+-- det_id is not in detStackedImfile;
+-- iteration is not in detStackedImfile;
+SELECT
+    det_id,
+    det_type,
+    iteration,
+    camera,
+    workdir,
+    class_id
+FROM
+    (SELECT DISTINCT
+        detRun.det_id,
+        detRun.det_type,
+        detRun.iteration,
+        detRun.workdir,
+        rawExp.camera,
+        detStackedImfile.class_id,
+        rawImfile.class_id as rawimfile_class_id
+    FROM detRun
+    JOIN detInputExp
+        ON detRun.det_id = detInputExp.det_id
+        AND detRun.iteration = detInputExp.iteration
+    JOIN rawExp
+        ON detInputExp.exp_id = rawExp.exp_id
+    JOIN rawImfile
+        ON rawExp.exp_id = rawImfile.exp_id
+    LEFT JOIN detStackedImfile
+        ON detInputExp.det_id = detStackedImfile.det_id
+        AND detInputExp.iteration = detStackedImfile.iteration
+        AND rawImfile.class_id = detStackedImfile.class_id
+    LEFT JOIN detNormalizedStatImfile
+        ON detStackedImfile.det_id = detNormalizedStatImfile.det_id
+        AND detStackedImfile.iteration = detNormalizedStatImfile.iteration
+        AND detStackedImfile.class_id = detNormalizedStatImfile.class_id
+    WHERE
+        detRun.state = 'run'
+        AND detRun.mode = 'master'
+        AND detNormalizedStatImfile.det_id IS NULL
+        AND detNormalizedStatImfile.iteration IS NULL
+        AND detNormalizedStatImfile.class_id IS NULL
+    GROUP BY
+--        rawExp.exp_id,
+        detRun.iteration,
+        detRun.det_id
+    HAVING
+        COUNT(rawImfile.class_id) = COUNT(detStackedImfile.class_id)
+        AND SUM(detStackedImfile.fault) = 0
+    ) as tonormalizedstat
+-- a det_run + iteration is ready for normstat when: all detResidImfile entries for that iteration corresponding to all detProcessedImfile entries are available
+SELECT
+  detRun.det_id,
+  detRun.det_type,
+  detRun.iteration,
+  detRun.camera,
+  detRun.workdir,
+  detProcessedImfile.class_id,
+  COUNT(detProcessedImfile.class_id),
+  COUNT(detResidImfile.class_id)
+FROM detRun
+JOIN detProcessedImfile
+    ON  detProcessedImfile.det_id    = detRun.det_id
+LEFT JOIN detResidImfile
+    ON  detResidImfile.det_id    = detProcessedImfile.det_id
+    AND detResidImfile.exp_id    = detProcessedImfile.exp_id
+    AND detResidImfile.class_id  = detProcessedImfile.class_id
+    AND detResidImfile.iteration = detRun.iteration
+LEFT JOIN detNormalizedStatImfile
+    ON  detNormalizedStatImfile.det_id    = detRun.det_id
+    AND detNormalizedStatImfile.iteration = detRun.iteration
+    AND detNormalizedStatImfile.class_id  = detProcessedImfile.class_id
+WHERE
+    detRun.state = 'run'
+    AND detRun.mode = 'master'
+    AND detNormalizedStatImfile.det_id IS NULL
+    AND detNormalizedStatImfile.iteration IS NULL
+    AND detNormalizedStatImfile.class_id IS NULL
+GROUP BY
+    detRun.iteration,
+    detRun.det_id
+HAVING
+    COUNT(detProcessedImfile.class_id) = COUNT(detResidImfile.class_id)
+    AND SUM(detResidImfile.fault) = 0

branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_toresidexp.sql

-              r19815
+              r23352
 -- id, detrend type, and whether the exposure was included in the stack for
 -- this iteration.
+-- require the corresponding detNormalizedExp to complete before starting
 -- select detRun.det_id
 …
 -- detResidImfile.{det_id, iteration, exp_id} is not in detResidExp
 SELECT DISTINCT
+SELECT
     det_id,
     iteration,
 …
     workdir,
     exp_tag
 FROM
     (SELECT DISTINCT
+FROM
+(   SELECT
         detRun.det_id AS det_id,
         detRun.iteration,
         detRun.det_type,
         detRun.mode,
-        detRun.workdir,
         detInputExp.exp_id,
         detInputExp.include,
         rawExp.camera,
+        rawExp.exp_tag,
+        detResidImfile.class_id
+        detRun.workdir,
+        rawExp.exp_tag
+    FROM detRun
+    JOIN detNormalizedExp
+        USING(det_id, iteration)
+    JOIN detInputExp
+        USING(det_id, iteration)
+    JOIN rawExp
+        ON detInputExp.exp_id = rawExp.exp_id
+    JOIN rawImfile
+        on rawExp.exp_id = rawImfile.exp_id
+    LEFT JOIN detResidImfile
+        ON detRun.det_id = detResidImfile.det_id
+        AND detRun.iteration = detResidImfile.iteration
+        AND detInputExp.exp_id = detResidImfile.exp_id
+        AND rawImfile.class_id = detResidImfile.class_id
+    LEFT JOIN detResidExp
+        ON detResidImfile.det_id = detResidExp.det_id
+        AND detResidImfile.iteration = detResidExp.iteration
+        AND detResidImfile.exp_id = detResidExp.exp_id
+    WHERE
+        detRun.state = 'run'
+        AND detRun.mode = 'master'
+        AND detResidExp.det_id IS NULL
+        AND detResidExp.iteration IS NULL
+        AND detResidExp.exp_id IS NULL
+    GROUP BY
+        detInputExp.exp_id,
+        detRun.iteration,
+        detRun.det_id
+    HAVING
+        COUNT(rawImfile.class_id) = COUNT(detResidImfile.class_id)
+        AND SUM(detResidImfile.fault) = 0
+    UNION
+    SELECT
+        detRun.det_id AS det_id,
+        detRun.iteration,
+        detRun.det_type,
+        detRun.mode,
+        detInputExp.exp_id,
+        detInputExp.include,
+        rawExp.camera,
+        detRun.workdir,
+        rawExp.exp_tag
     FROM detRun
     JOIN detInputExp
 …
     WHERE
         detRun.state = 'run'
+        AND detRun.mode = 'verify'
         AND detResidExp.det_id IS NULL
         AND detResidExp.iteration IS NULL
 …
         COUNT(rawImfile.class_id) = COUNT(detResidImfile.class_id)
         AND SUM(detResidImfile.fault) = 0
     ) AS toresidexp
+) as temp

branches/cnb_branches/cnb_branch_20090301/ippTools/share/dettool_toresidimfile.sql

-              r19816
+              r23352
     detProcessedImfile.class_id,
     detProcessedImfile.uri,
+    detNormalizedImfile.uri AS det_uri,
+--  detNormalizedImfile.uri AS det_uri,
+    detStackedImfile.uri AS det_uri,
     detRun.det_id AS ref_det_id,
     detRun.iteration AS ref_iter,
 …
     ON detRun.det_id = detProcessedImfile.det_id
     AND detInputExp.exp_id = detProcessedImfile.exp_id
+JOIN detNormalizedImfile
+    ON detRun.det_id = detNormalizedImfile.det_id
+    AND detRun.iteration = detNormalizedImfile.iteration
+    AND detProcessedImfile.class_id = detNormalizedImfile.class_id
+JOIN detNormalizedExp
+    ON detRun.det_id = detNormalizedExp.det_id
+    AND detRun.iteration = detNormalizedExp.iteration
+JOIN detStackedImfile
+    ON detRun.det_id = detStackedImfile.det_id
+    AND detRun.iteration = detStackedImfile.iteration
+    AND detProcessedImfile.class_id = detStackedImfile.class_id
+-- EAM : replacing detNormalizedImfile with detStackedImfile to change the sequencing
+-- JOIN detNormalizedImfile
+--     ON detRun.det_id = detNormalizedImfile.det_id
+--     AND detRun.iteration = detNormalizedImfile.iteration
+--     AND detProcessedImfile.class_id = detNormalizedImfile.class_id
+-- EAM : we there is no reason to wait for all stacks to complete before continuing
+-- JOIN detNormalizedExp
+--     ON detRun.det_id = detNormalizedExp.det_id
+--     AND detRun.iteration = detNormalizedExp.iteration
 LEFT JOIN detResidImfile
     ON detRun.det_id = detResidImfile.det_id
 …
     detRun.state = 'run'
     AND detRun.mode = 'master'
+    AND detNormalizedImfile.fault = 0
+    AND detNormalizedExp.fault = 0
+    AND detStackedImfile.fault = 0
+--  AND detNormalizedImfile.fault = 0
+--  AND detNormalizedExp.fault = 0
     AND detResidImfile.det_id IS NULL
     AND detResidImfile.iteration IS NULL

branches/cnb_branches/cnb_branch_20090301/ippTools/share/pxadmin_create_tables.sql

-              r21308
+              r23352
     REFERENCES  detInputExp(det_id, iteration, exp_id),
     FOREIGN KEY (det_id, exp_id, class_id)
+    REFERENCES  detProcessedImfile(det_id, exp_id, class_id),
+    FOREIGN KEY (ref_det_id, ref_iter)
+    REFERENCES  detNormalizedExp(det_id, iteration)
+    REFERENCES  detProcessedImfile(det_id, exp_id, class_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;

branches/cnb_branches/cnb_branch_20090301/ippTools/share/warptool_change_skyfile_data_state.sql

-              r19521
+              r23352
     AND skycell_id = '%s'
     -- only update if chipRun.state has the expected value
+    AND (
+        SELECT state from warpRun where warpRun.warp_id = warpSkyfile.warp_id
+    ) = '%s'
+--    AND (
+--        SELECT state from warpRun where warpRun.warp_id = warpSkyfile.warp_id
+--    ) = '%s'

branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtool.c

-              r21443
+              r23352
 static bool pendingcleanupexpMode(pxConfig *config);
 static bool donecleanupMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
 # define MODECASE(caseName, func) \
 …
         MODECASE(CAMTOOL_MODE_PENDINGCLEANUPEXP,    pendingcleanupexpMode);
         MODECASE(CAMTOOL_MODE_DONECLEANUP,          donecleanupMode);
+        MODECASE(CAMTOOL_MODE_EXPORTRUN,            exportrunMode);
+        MODECASE(CAMTOOL_MODE_IMPORTRUN,            importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
 …
     return true;
+}
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+  int numExportTables = 2;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_S64(det_id, config->args, "-cam_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+    psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+    return false;
+  }
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "==");
+  ExportTable tables [] = {
+    {"camRun", "camtool_export_run.sql"},
+    {"camProcessedExp", "camtool_export_processed_exp.sql"},
+  };
+  for (int i=0; i < numExportTables; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
+    if (!query) {
+      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      return false;
+    }
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
+    // treat limit == 0 as "no limit"
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+    psFree(query);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+    }
+    // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+    psFree(output);
+  }
+  fclose (f);
+  return true;
+}
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+  fprintf (stdout, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+  psMetadataItem *item = psMetadataLookup (input, "camRun");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  camRunRow *camRun = camRunObjectFromMetadata (entry->data.md);
+  camRunInsertObject (config->dbh, camRun);
+  // fprintf (stdout, "---- cam run ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+  item = psMetadataLookup (input, "camProcessedImfile");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  for (int i = 0; i < item->data.list->n; i++) {
+    psMetadataItem *entry = psListGet (item->data.list, i);
+    assert (entry);
+    assert (entry->type == PS_DATA_METADATA);
+    camProcessedExpRow *camProcessedExp = camProcessedExpObjectFromMetadata (entry->data.md);
+    camProcessedExpInsertObject (config->dbh, camProcessedExp);
+    // fprintf (stdout, "---- row %d ----\n", i);
+    // psMetadataPrint (stderr, entry->data.md, 1);
+  }
+  return true;
+}

branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtool.h

r19092	r23352
39	39	CAMTOOL_MODE_PENDINGCLEANUPEXP,
40	40	CAMTOOL_MODE_DONECLEANUP,
	41	CAMTOOL_MODE_EXPORTRUN,
	42	CAMTOOL_MODE_IMPORTRUN
41	43	} camtoolMode;
42	44

branches/cnb_branches/cnb_branch_20090301/ippTools/src/camtoolConfig.c

-              r20276
+              r23352
     psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-cam_id", 0,          "export this camera ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
     psMetadata *argSets = psMetadataAlloc();
     psMetadata *modes = psMetadataAlloc();
 …
     PXOPT_ADD_MODE("-pendingcleanupexp",    "show exposures for cleanup runs",      CAMTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs);
     PXOPT_ADD_MODE("-donecleanup",          "show runs that have been cleaned",     CAMTOOL_MODE_DONECLEANUP,       donecleanupArgs);
+   PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", CAMTOOL_MODE_EXPORTRUN, exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           CAMTOOL_MODE_IMPORTRUN, importrunArgs);
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptool.c

-              r21521
+              r23352
 static bool revertprocessedimfileMode(pxConfig *config);
 static bool updateprocessedimfileMode(pxConfig *config);
 static bool promoteexpMode(pxConfig *config);
+static bool advanceexpMode(pxConfig *config);
 static bool blockMode(pxConfig *config);
 static bool maskedMode(pxConfig *config);
 …
         MODECASE(CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,   revertprocessedimfileMode);
         MODECASE(CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,   updateprocessedimfileMode);
         MODECASE(CHIPTOOL_MODE_PROMOTEEXP,              promoteexpMode);
+        MODECASE(CHIPTOOL_MODE_ADVANCEEXP,              advanceexpMode);
         MODECASE(CHIPTOOL_MODE_BLOCK,                   blockMode);
         MODECASE(CHIPTOOL_MODE_MASKED,                  maskedMode);
 …
     pxchipGetSearchArgs (config, where); // rawExp, chipRun
     PXOPT_COPY_S64(config->args,  where, "-chip_id", "chipRun.chip_id", "==");
+    PXOPT_COPY_STR(config->args,  where, "-label", "chipRun.label", "==");
+    PXOPT_COPY_STR(config->args,  where, "-label",   "chipRun.label",   "==");
+    PXOPT_COPY_STR(config->args,  where, "-state",   "chipRun.state",   "==");
     if (!psListLength(where->list)
 …
 static bool promoteexpMode(pxConfig *config)
+static bool advanceexpMode(pxConfig *config)
+{
     PS_ASSERT_PTR_NON_NULL(config, false);
 …
 bool exportrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+    PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true,  false);
+    PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+    PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+    FILE *f = fopen (outfile, "w");
+    if (f == NULL) {
+        psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+        return false;
+    }
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
+    // *** extract the chipRun in this section ***
+    psString query = pxDataGet("chiptool_export_run.sql");
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+  int numExportTables = 3;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_S64(det_id, config->args, "-chip_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+    psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+    return false;
+  }
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
+  ExportTable tables [] = {
+    {"chipRun", "chiptool_export_run.sql"},
+    {"chipImfile", "chiptool_export_imfile.sql"},
+    {"chipProcessedImfile", "chiptool_export_processed_imfile.sql"},
+  };
+  for (int i=0; i < numExportTables; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
         return false;
+      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      return false;
+    }
     if (where && psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereSQL(where, NULL);
         psStringAppend(&query, " %s", whereClause);
         psFree(whereClause);
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
     // treat limit == 0 as "no limit"
     if (limit) {
         psString limitString = psDBGenerateLimitSQL(limit);
         psStringAppend(&query, " %s", limitString);
         psFree(limitString);
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
         return false;
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
     psFree(query);
 …
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
     if (!psArrayLength(output)) {
         psTrace("chiptool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
+      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+    }
     // we must write the export table in non-simple (true) format
     if (!ippdbPrintMetadatas(f, output, "chipRun", true)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print array");
         psFree(output);
         return false;
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
     psFree(output);
+    // *** extract the chipProcessedImfile entries in this section ***
+    query = pxDataGet("chiptool_export_imfiles.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+    if (where && psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereSQL(where, NULL);
+        psStringAppend(&query, " %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+    output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("chiptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, "chipProcessedImfiles", true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+    fclose (f);
+    return true;
+}
+  }
+  fclose (f);
+  return true;
+}
 bool importrunMode(pxConfig *config)
+{
   unsigned int nFail;
+  int numImportTables = 2;
+  char tables[2] [80] = {"chipImfile", "chipProcessedImfile"};
   PS_ASSERT_PTR_NON_NULL(config, NULL);
 …
   psMetadataPrint (stderr, input, 1);
+  psMetadataItem *chipRunItem = psMetadataLookup (input, "chipRun");
+  psAssert (chipRunItem, "entry not in input?");
+  psAssert (chipRunItem->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  psMetadataItem *chipRunEntry = psListGet (chipRunItem->data.list, 0);
+  assert (chipRunEntry);
+  assert (chipRunEntry->type == PS_DATA_METADATA);
+  chipRunRow *chipRun = chipRunObjectFromMetadata (chipRunEntry->data.md);
+  chipRunInsertObject (config->dbh, chipRun);
+  // fprintf (stdout, "---- chip run ----\n");
+  // psMetadataPrint (stderr, chipRunEntry->data.md, 1);
+  psMetadataItem *item = psMetadataLookup (input, "chipProcessedImfiles");
+  psMetadataItem *item = psMetadataLookup (input, "chipRun");
   psAssert (item, "entry not in input?");
   psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  // XXX would be better to use the iterator?
+  for (int i = 0; i < item->data.list->n; i++) {
+    psMetadataItem *entry = psListGet (item->data.list, i);
+    assert (entry);
+    assert (entry->type == PS_DATA_METADATA);
+    chipProcessedImfileRow *chipProcessedImfile = chipProcessedImfileObjectFromMetadata (entry->data.md);
+    chipProcessedImfileInsertObject (config->dbh, chipProcessedImfile);
+    // fprintf (stdout, "---- row %d ----\n", i);
+    // psMetadataPrint (stderr, entry->data.md, 1);
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  chipRunRow *chipRun = chipRunObjectFromMetadata (entry->data.md);
+  chipRunInsertObject (config->dbh, chipRun);
+  // fprintf (stdout, "---- chip run ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+  for (int i = 0; i < numImportTables; i++) {
+    psMetadataItem *item = psMetadataLookup (input, tables[i]);
+    psAssert (item, "entry not in input?");
+    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+    switch (i) {
+      case 0:
+        for (int i = 0; i < item->data.list->n; i++) {
+          psMetadataItem *entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          chipImfileRow *chipImfile = chipImfileObjectFromMetadata (entry->data.md);
+          chipImfileInsertObject (config->dbh, chipImfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 1:
+        for (int i = 0; i < item->data.list->n; i++) {
+          psMetadataItem *entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          chipProcessedImfileRow *chipProcessedImfile = chipProcessedImfileObjectFromMetadata (entry->data.md);
+          chipProcessedImfileInsertObject (config->dbh, chipProcessedImfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+    }
+  }

branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptool.h

r21521	r23352
32	32	CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,
33	33	CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,
34		CHIPTOOL_MODE_~~PROMOT~~EEXP,
	34	CHIPTOOL_MODE_ADVANCEEXP,
35	35	CHIPTOOL_MODE_BLOCK,
36	36	CHIPTOOL_MODE_MASKED,

branches/cnb_branches/cnb_branch_20090301/ippTools/src/chiptoolConfig.c

-              r21521
+              r23352
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id",              0,            "search by chip ID", 0);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0,            "set state", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",  0,          "search by label", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0,        "set state", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0,        "set label", NULL);
     psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all",  0,            "allow everything to be queued without search terms", false);
-    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label",  0,          "search by label", NULL);
     // -pendingimfile
 …
     psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-code",  0,            "set fault code (required)", 0);
     // -promoteexp
     psMetadata *promoteexpArgs = psMetadataAlloc();
     psMetadataAddS64(promoteexpArgs, PS_LIST_TAIL, "-chip_id",  0,          "search by chip ID", 0);
     psMetadataAddStr(promoteexpArgs, PS_LIST_TAIL, "-label",  0,            "promote exposures for specified label", NULL);
     psMetadataAddU64(promoteexpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
+    // -advanceexp
+    psMetadata *advanceexpArgs = psMetadataAlloc();
+    psMetadataAddS64(advanceexpArgs, PS_LIST_TAIL, "-chip_id",  0,          "search by chip ID", 0);
+    psMetadataAddStr(advanceexpArgs, PS_LIST_TAIL, "-label",  0,            "advance exposures for specified label", NULL);
+    psMetadataAddU64(advanceexpArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     // -block
 …
     PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties",    CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs);
     PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile",              CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs);
     PXOPT_ADD_MODE("-promoteexp",           "promote completed exposures",          CHIPTOOL_MODE_PROMOTEEXP,           promoteexpArgs);
+    PXOPT_ADD_MODE("-advanceexp",           "advance completed exposures",          CHIPTOOL_MODE_ADVANCEEXP,           advanceexpArgs);
     PXOPT_ADD_MODE("-block",                "set a label block",                    CHIPTOOL_MODE_BLOCK,                blockArgs);
     PXOPT_ADD_MODE("-masked",               "show blocked labels",                  CHIPTOOL_MODE_MASKED,               maskedArgs);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool.c

-              r21402
+              r23352
 static bool inputMode(pxConfig *config);
 static bool rawMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
 // run
 …
     } \
     break;
+/*
+typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+} ExportTable;
+*/
 int main(int argc, char **argv)
+{
 …
         MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode);
         MODECASE(DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileMode);
+        MODECASE(DETTOOL_MODE_EXPORTRUN,               exportrunMode);
+        MODECASE(DETTOOL_MODE_IMPORTRUN,               importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
 …
       // generate a random-valued vector, return an index sorted by the random values
       psVector *randomVector = psVectorAlloc(detrendExps->n, PS_TYPE_F32); // random values
+      psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0);
+      /*
+       * change due to PAP work on random number generator?
+       * psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0);
+       */
+      psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
       for (int i = 0; i < randomVector->n; i++) {
         randomVector->data.F32[i] = psRandomUniform(rng);
 …
+}
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+  int numExportTables = 12;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_S64(det_id, config->args, "-det_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+    psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+    return false;
+  }
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "==");
+  ExportTable tables [] = {
+    {"detInputExp", "dettool_export_input_exp.sql"},
+    {"detNormalizedExp", "dettool_export_normalized_exp.sql"},
+    {"detNormalizedImfile", "dettool_export_normalized_imfile.sql"},
+    {"detNormalizedStatImfile", "dettool_export_normalized_stat_imfile.sql"},
+    {"detProcessedExp", "dettool_export_processed_exp.sql"},
+    {"detProcessedImfile", "dettool_export_processed_imfile.sql"},
+    {"detRegisteredImfile", "dettool_export_registered_imfile.sql"},
+    {"detResidExp", "dettool_export_resid_exp.sql"},
+    {"detResidImfile", "dettool_export_resid_imfile.sql"},
+    {"detRun", "dettool_export_run.sql"},
+    {"detRunSummary", "dettool_export_run_summary.sql"},
+    {"detStackedImfile", "dettool_export_stacked_imfile.sql"},
+  };
+  for (int i=0; i < numExportTables; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
+    if (!query) {
+      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      return false;
+    }
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
+    // treat limit == 0 as "no limit"
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+    psFree(query);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psTrace("chiptool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+    }
+    // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+    psFree(output);
+  }
+  fclose (f);
+  return true;
+}
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+  int numImportTables = 11;
+  char tables[11] [80] = {"detInputExp", "detNormalizedExp",
+    "detNormalizedStatImfile", "detProcessedExp", "detProcessedImfile",
+    "detRegisteredImfile", "detResidExp", "detResidImfile",
+    "detNormalizedImfile", "detRunSummary", "detStackedImfile"};
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+  fprintf (stdout, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+  psMetadataItem *item = psMetadataLookup (input, "detRun");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  detRunRow *detRun = detRunObjectFromMetadata (entry->data.md);
+  detRunInsertObject (config->dbh, detRun);
+  // fprintf (stdout, "---- det run ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+  for (int i = 0; i < numImportTables; i++) {
+    item = psMetadataLookup (input, tables[i]);
+    psAssert (item, "entry not in input?");
+    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+    switch (i) {
+      case 0:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detInputExpRow *detInputExp = detInputExpObjectFromMetadata (entry->data.md);
+          detInputExpInsertObject (config->dbh, detInputExp);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 1:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detNormalizedExpRow *detNormalizedExp = detNormalizedExpObjectFromMetadata (entry->data.md);
+          detNormalizedExpInsertObject (config->dbh, detNormalizedExp);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 2:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detNormalizedStatImfileRow *detNormalizedStatImfile = detNormalizedStatImfileObjectFromMetadata (entry->data.md);
+          detNormalizedStatImfileInsertObject (config->dbh, detNormalizedStatImfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 3:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detProcessedExpRow *detProcessedExp =  detProcessedExpObjectFromMetadata (entry->data.md);
+          detProcessedExpInsertObject (config->dbh, detProcessedExp);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 4:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detProcessedImfileRow *detProcessedImfile =  detProcessedImfileObjectFromMetadata (entry->data.md);
+          detProcessedImfileInsertObject (config->dbh, detProcessedImfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 5:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detRegisteredImfileRow *detRegisteredImfile = detRegisteredImfileObjectFromMetadata (entry->data.md);
+          detRegisteredImfileInsertObject (config->dbh, detRegisteredImfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 6:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detResidExpRow *detResidExp = detResidExpObjectFromMetadata (entry->data.md);
+          detResidExpInsertObject (config->dbh, detResidExp);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 7:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detResidImfileRow *detResidImfile = detResidImfileObjectFromMetadata (entry->data.md);
+          detResidImfileInsertObject (config->dbh, detResidImfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 8:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detNormalizedImfileRow *detNormalizedImfile = detNormalizedImfileObjectFromMetadata (entry->data.md);
+          detNormalizedImfileInsertObject (config->dbh, detNormalizedImfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 9:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detRunSummaryRow *detRunSummary = detRunSummaryObjectFromMetadata (entry->data.md);
+          detRunSummaryInsertObject (config->dbh, detRunSummary);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 10:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          detStackedImfileRow *detStackedImfile = detStackedImfileObjectFromMetadata (entry->data.md);
+          detStackedImfileInsertObject (config->dbh, detStackedImfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+    }
+  }
+  return true;
+}
 #if 0
 // XXX this function was left in commented as this method may be useful in the

branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool.h

-              r19814
+              r23352
     DETTOOL_MODE_RERUN,
     DETTOOL_MODE_REGISTER_DETREND,
+    DETTOOL_MODE_REGISTER_DETREND_IMFILE
+    DETTOOL_MODE_REGISTER_DETREND_IMFILE,
+    DETTOOL_MODE_EXPORTRUN,
+    DETTOOL_MODE_IMPORTRUN
 } dettoolMode;

branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettoolConfig.c

-              r20375
+              r23352
     psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-faulted",  0,            "only return imfiles with a fault status set", false);
     psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
+    psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-included",  0,            "use only files included in ths detrun", false);
+    // XXX is this option used?
     psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-select_state",  0,            "search for state", NULL);
 …
     psMetadataAddStr(register_detrend_imfileArgs, PS_LIST_TAIL, "-path_base",  0,            "define base output location", NULL);
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-det_id", 0,          "export this detrend ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
     psFree(now);
 …
     PXOPT_ADD_MODE("-register_detrend", "", DETTOOL_MODE_REGISTER_DETREND, register_detrendArgs);
     PXOPT_ADD_MODE("-register_detrend_imfile", "", DETTOOL_MODE_REGISTER_DETREND_IMFILE, register_detrend_imfileArgs);
+    PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", DETTOOL_MODE_EXPORTRUN, exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           DETTOOL_MODE_IMPORTRUN, importrunArgs);
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_processedimfile.c

-              r19621
+              r23352
     PS_ASSERT_PTR_NON_NULL(config, false);
-    bool hasWhere = false;
-    PXOPT_LOOKUP_BOOL(included, config->args, "-included", false);
-    PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
-    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-det_id", "detProcessedImfile.det_id", "==");
 …
     PXOPT_COPY_STR(config->args, where, "-select_mode", "detRun.mode", "==");
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
+    PXOPT_LOOKUP_BOOL(included, config->args, "-included", false);
     psString query = pxDataGet("dettool_processedimfile.sql");
     if (!query) {
 …
+    }
+    bool hasWhere = false;
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereSQL(where, NULL);
 …
     if (faulted) {
+        // list only faulted rows
         psStringAppend(&query, " %s", " detProcessedImfile.fault != 0");
     } else {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/dettool_residimfile.c

-              r19632
+              r23352
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
+    PXOPT_LOOKUP_BOOL(included, config->args, "-included", false);
     psString query = pxDataGet("dettool_residimfile.sql");
 …
+    }
     psFree(where);
+    // restrict search to included imfiles
+    if (included) {
+        if (hasWhere) {
+            psStringAppend(&query, " AND detInputExp.include = 1");
+        } else {
+            psStringAppend(&query, " WHERE detInputExp.include = 1");
+        }
+        hasWhere = true;
+    }
     if (hasWhere) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.c

-              r21402
+              r23352
 static bool donecleanupMode(pxConfig *config);
 static bool updatediffskyfileMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
 static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state);
 …
         MODECASE(DIFFTOOL_MODE_DONECLEANUP,           donecleanupMode);
         MODECASE(DIFFTOOL_MODE_UPDATEDIFFSKYFILE,     updatediffskyfileMode);
+        MODECASE(DIFFTOOL_MODE_EXPORTRUN,             exportrunMode);
+        MODECASE(DIFFTOOL_MODE_IMPORTRUN,             importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
 …
+}
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+  int numExportFiles = 3;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_S64(det_id, config->args, "-diff_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+    psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+    return false;
+  }
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
+  ExportTable tables [] = {
+    {"diffRun", "difftool_export_run.sql"},
+    {"diffInputSkyfile", "difftool_export_input_skyfile.sql"},
+    {"diffSkyfile", "difftool_export_skyfile.sql"},
+  };
+  for (int i=0; i < numExportFiles; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
+    if (!query) {
+      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      return false;
+    }
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
+    // treat limit == 0 as "no limit"
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+    psFree(query);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+    }
+    // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+    psFree(output);
+  }
+  fclose (f);
+  return true;
+}
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+  int numImportTables = 2;
+  char tables[2] [80] = {"diffInputSkyfile", "diffSkyfile"};
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+  fprintf (stdout, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+  psMetadataItem *item = psMetadataLookup (input, "diffRun");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  diffRunRow *diffRun = diffRunObjectFromMetadata (entry->data.md);
+  diffRunInsertObject (config->dbh, diffRun);
+  // fprintf (stdout, "---- diff run ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+  for (int i = 0; i < numImportTables; i++) {
+    item = psMetadataLookup (input, tables[i]);
+    psAssert (item, "entry not in input?");
+    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+    switch (i) {
+      case 0:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          diffInputSkyfileRow *diffInputSkyfile = diffInputSkyfileObjectFromMetadata (entry->data.md);
+          diffInputSkyfileInsertObject (config->dbh, diffInputSkyfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 1:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          diffSkyfileRow *diffSkyfile = diffSkyfileObjectFromMetadata (entry->data.md);
+          diffSkyfileInsertObject (config->dbh, diffSkyfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+    }
+  }
+  return true;
+}

branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftool.h

r19677	r23352
39	39	DIFFTOOL_MODE_DONECLEANUP,
40	40	DIFFTOOL_MODE_UPDATEDIFFSKYFILE,
	41	DIFFTOOL_MODE_EXPORTRUN,
	42	DIFFTOOL_MODE_IMPORTRUN
41	43	} difftoolMode;
42	44

branches/cnb_branches/cnb_branch_20090301/ippTools/src/difftoolConfig.c

-              r21308
+              r23352
     psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-code", 0,         "set fault code (required)", 0);
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-diff_id", 0,          "export this diff ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
     psFree(now);
 …
     PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     DIFFTOOL_MODE_DONECLEANUP,          donecleanupArgs);
     PXOPT_ADD_MODE("-updatediffskyfile",     "update fault code for a diffskyfile",  DIFFTOOL_MODE_UPDATEDIFFSKYFILE,          updatediffskyfileArgs);
+    PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", DIFFTOOL_MODE_EXPORTRUN, exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           DIFFTOOL_MODE_IMPORTRUN, importrunArgs);
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/faketool.c

-              r21463
+              r23352
 static bool revertprocessedimfileMode(pxConfig *config);
 static bool updateprocessedimfileMode(pxConfig *config);
 static bool promoteexpMode(pxConfig *config);
+static bool advanceexpMode(pxConfig *config);
 static bool blockMode(pxConfig *config);
 static bool maskedMode(pxConfig *config);
 …
 static bool tofullimfileMode(pxConfig *config);
 static bool topurgedimfileMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
 # define MODECASE(caseName, func) \
 …
         MODECASE(FAKETOOL_MODE_REVERTPROCESSEDIMFILE,   revertprocessedimfileMode);
         MODECASE(FAKETOOL_MODE_UPDATEPROCESSEDIMFILE,   updateprocessedimfileMode);
         MODECASE(FAKETOOL_MODE_PROMOTEEXP,              promoteexpMode);
+        MODECASE(FAKETOOL_MODE_ADVANCEEXP,              advanceexpMode);
         MODECASE(FAKETOOL_MODE_BLOCK,                   blockMode);
         MODECASE(FAKETOOL_MODE_MASKED,                  maskedMode);
 …
         MODECASE(FAKETOOL_MODE_TOFULLIMFILE,            tofullimfileMode);
         MODECASE(FAKETOOL_MODE_TOPURGEDIMFILE,          topurgedimfileMode);
+        MODECASE(FAKETOOL_MODE_EXPORTRUN,               exportrunMode);
+        MODECASE(FAKETOOL_MODE_IMPORTRUN,               importrunMode);
         default:
 …
+}
 static bool promoteexpMode(pxConfig *config)
+static bool advanceexpMode(pxConfig *config)
+{
     PS_ASSERT_PTR_NON_NULL(config, false);
 …
     return change_imfile_data_state(config, "purged", "goto_purged");
+}
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+  int numExportTables = 2;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_S64(det_id, config->args, "-fake_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+    psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+    return false;
+  }
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
+  ExportTable tables [] = {
+    {"fakeRun", "faketool_export_run.sql"},
+    {"fakeProcessedImfile", "faketool_export_processed_imfile.sql"},
+  };
+  for (int i=0; i < numExportTables; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
+    if (!query) {
+      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      return false;
+    }
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
+    // treat limit == 0 as "no limit"
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+    psFree(query);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+    }
+    // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+    psFree(output);
+  }
+    fclose (f);
+    return true;
+}
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+  fprintf (stdout, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+  psMetadataItem *item = psMetadataLookup (input, "fakeRun");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  fakeRunRow *fakeRun = fakeRunObjectFromMetadata (entry->data.md);
+  fakeRunInsertObject (config->dbh, fakeRun);
+  // fprintf (stdout, "---- fake run ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+  item = psMetadataLookup (input, "fakeProcessedImfile");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  for (int i = 0; i < item->data.list->n; i++) {
+    psMetadataItem *entry = psListGet (item->data.list, i);
+    assert (entry);
+    assert (entry->type == PS_DATA_METADATA);
+    fakeProcessedImfileRow *fakeProcessedImfile = fakeProcessedImfileObjectFromMetadata (entry->data.md);
+    fakeProcessedImfileInsertObject (config->dbh, fakeProcessedImfile);
+    // fprintf (stdout, "---- row %d ----\n", i);
+    // psMetadataPrint (stderr, entry->data.md, 1);
+  }
+  return true;
+}

branches/cnb_branches/cnb_branch_20090301/ippTools/src/faketool.h

-              r21463
+              r23352
     FAKETOOL_MODE_REVERTPROCESSEDIMFILE,
     FAKETOOL_MODE_UPDATEPROCESSEDIMFILE,
     FAKETOOL_MODE_PROMOTEEXP,
+    FAKETOOL_MODE_ADVANCEEXP,
     FAKETOOL_MODE_BLOCK,
     FAKETOOL_MODE_MASKED,
 …
     FAKETOOL_MODE_TOFULLIMFILE,
     FAKETOOL_MODE_TOPURGEDIMFILE,
+    FAKETOOL_MODE_EXPORTRUN,
+    FAKETOOL_MODE_IMPORTRUN
 } FAKETOOLMode;

branches/cnb_branches/cnb_branch_20090301/ippTools/src/faketoolConfig.c

-              r21463
+              r23352
     // -promoteexp
     psMetadata *promoteexpArgs = psMetadataAlloc();
     psMetadataAddS64(promoteexpArgs, PS_LIST_TAIL, "-fake_id", 0,      "search by fake ID", 0);
     psMetadataAddStr(promoteexpArgs, PS_LIST_TAIL, "-label",  0,       "search by label ", NULL);
     psMetadataAddU64(promoteexpArgs, PS_LIST_TAIL, "-limit",  0,       "limit exposures to promote to N items", 0);
+    // -advanceexp
+    psMetadata *advanceexpArgs = psMetadataAlloc();
+    psMetadataAddS64(advanceexpArgs, PS_LIST_TAIL, "-fake_id", 0,      "search by fake ID", 0);
+    psMetadataAddStr(advanceexpArgs, PS_LIST_TAIL, "-label",  0,       "search by label ", NULL);
+    psMetadataAddU64(advanceexpArgs, PS_LIST_TAIL, "-limit",  0,       "limit exposures to advance to N items", 0);
     // -block
 …
     psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id",  0,        "class ID to update", NULL);
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-fake_id", 0,          "export this fake ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
 …
     PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties",     FAKETOOL_MODE_UPDATEPROCESSEDIMFILE,    updateprocessedimfileArgs);
     PXOPT_ADD_MODE("-revertprocessedimfile", "undo a processed imfile",              FAKETOOL_MODE_REVERTPROCESSEDIMFILE,    revertprocessedimfileArgs);
     PXOPT_ADD_MODE("-promoteexp",            "promote completed exposoures",         FAKETOOL_MODE_PROMOTEEXP,          promoteexpArgs);
+    PXOPT_ADD_MODE("-advanceexp",            "advance completed exposoures",         FAKETOOL_MODE_ADVANCEEXP,          advanceexpArgs);
     PXOPT_ADD_MODE("-block",                 "set a label block",                    FAKETOOL_MODE_BLOCK,          blockArgs);
     PXOPT_ADD_MODE("-masked",                "show blocked labels",                  FAKETOOL_MODE_MASKED,         maskedArgs);
 …
     PXOPT_ADD_MODE("-tofullimfile",        "set imfile state to full",               FAKETOOL_MODE_TOFULLIMFILE,         tofullimfileArgs);
     PXOPT_ADD_MODE("-topurgedimfile",      "set imfile state to purged",             FAKETOOL_MODE_TOPURGEDIMFILE,       topurgedimfileArgs);
+    PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", FAKETOOL_MODE_EXPORTRUN, exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           FAKETOOL_MODE_IMPORTRUN, importrunArgs);

branches/cnb_branches/cnb_branch_20090301/ippTools/src/flatcorr.c

-              r21402
+              r23352
 static bool inputexpMode(pxConfig *config);
 static bool inputimfileMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
 static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state);
 …
         MODECASE(FLATCORR_MODE_INPUTEXP,       inputexpMode);
         MODECASE(FLATCORR_MODE_INPUTIMFILE,    inputimfileMode);
+        MODECASE(FLATCORR_MODE_EXPORTRUN,      exportrunMode);
+        MODECASE(FLATCORR_MODE_IMPORTRUN,      importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
 …
     return true;
+}
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+  int numExportTables = 3;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_S64(det_id, config->args, "-corr_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+    psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+    return false;
+  }
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-coor_id", "corr_id", "==");
+  ExportTable tables [] = {
+    {"flatcorrRun", "flatcorrtool_export_run.sql"},
+    {"flatcorrCamLink", "flatcorrtool_export_cam_link.sql"},
+    {"flatcorrChipLink", "flatcorr_export_chip_link.sql"},
+  };
+  for (int i=0; i < numExportTables; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
+    if (!query) {
+      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      return false;
+    }
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
+    // treat limit == 0 as "no limit"
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+    psFree(query);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+    }
+    // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+    psFree(output);
+  }
+    fclose (f);
+    return true;
+}
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+  psMetadataItem *item, *entry;
+  int numImportTables = 3;
+  char tables[3] [80] = {"flatcorrRun", "flatcorrCamLink", "flatcorrChipLink"};
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+  fprintf (stdout, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+  for (int i = 0; i < numImportTables; i++) {
+    item = psMetadataLookup (input, tables[i]);
+    psAssert (item, "entry not in input?");
+    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+    entry = psListGet (item->data.list, 0);
+    assert (entry);
+    assert (entry->type == PS_DATA_METADATA);
+    switch (i) {
+      case 0:
+      {
+        flatcorrRunRow *flatcorrRun = flatcorrRunObjectFromMetadata (entry->data.md);
+        flatcorrRunInsertObject (config->dbh, flatcorrRun);
+        // fprintf (stdout, "---- flatcorr run ----\n");
+        // psMetadataPrint (stderr, entry->data.md, 1);
+        break;
+      }
+      case 1:
+      {
+        flatcorrCamLinkRow *flatcorrCamLink = flatcorrCamLinkObjectFromMetadata (entry->data.md);
+        flatcorrCamLinkInsertObject (config->dbh, flatcorrCamLink);
+        // fprintf (stdout, "---- flatcorr cam link ----\n");
+        // psMetadataPrint (stderr, entry->data.md, 1);
+        break;
+      }
+      case 2:
+      {
+        flatcorrChipLinkRow *flatcorrChipLink = flatcorrChipLinkObjectFromMetadata (entry->data.md);
+        flatcorrChipLinkInsertObject (config->dbh, flatcorrChipLink);
+        // fprintf (stdout, "---- flatcorr chip link ----\n");
+        // psMetadataPrint (stderr, entry->data.md, 1);
+        break;
+      }
+    }
+  }
+  return true;
+}

branches/cnb_branches/cnb_branch_20090301/ippTools/src/flatcorr.h

-              r20397
+              r23352
     FLATCORR_MODE_UPDATERUN,
     FLATCORR_MODE_INPUTEXP,
+    FLATCORR_MODE_INPUTIMFILE
+    FLATCORR_MODE_INPUTIMFILE,
+    FLATCORR_MODE_EXPORTRUN,
+    FLATCORR_MODE_IMPORTRUN
 } flatcorrMode;

branches/cnb_branches/cnb_branch_20090301/ippTools/src/flatcorrConfig.c

-              r20397
+              r23352
     psMetadataAddU64(inputimfileArgs, PS_LIST_TAIL, "-limit",   0, "limit result set to N items", 0);
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-corr_id", 0,          "export this correction ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
     psFree(now);
 …
     PXOPT_ADD_MODE("-inputexp",       "list exposures for a correction run",               FLATCORR_MODE_INPUTEXP,       inputexpArgs);
     PXOPT_ADD_MODE("-inputimfile",    "list imfiles for a chip run",                       FLATCORR_MODE_INPUTIMFILE,    inputimfileArgs);
+    PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", FLATCORR_MODE_EXPORTRUN, exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           FLATCORR_MODE_IMPORTRUN, importrunArgs);
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxadmin.c

-              r18927
+              r23352
 bool createMode(pxConfig *config);
+bool createMirrorMode(pxConfig *config);
 bool deleteMode(pxConfig *config);
 static bool runMultipleStatments(pxConfig *config, const char *query);
 …
+            }
             break;
+        case PXADMIN_MODE_CREATE_MIRROR:
+            if (!createMirrorMode(config)) {
+                goto FAIL;
+            }
+            break;
         case PXADMIN_MODE_DELETE:
             if (!deleteMode(config)) {
 …
     psString query = pxDataGet("pxadmin_create_tables.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+    // BEGIN
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!runMultipleStatments(config, query)) {
+        if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+    // COMMIT
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    return true;
+}
+bool createMirrorMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+    psString query = pxDataGet("pxadmin_create_mirror_tables.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxadmin.h

r9392	r23352
26	26	PXADMIN_MODE_NONE = 0x0,
27	27	PXADMIN_MODE_CREATE,
	28	PXADMIN_MODE_CREATE_MIRROR,
28	29	PXADMIN_MODE_DELETE,
29	30	PXADMIN_MODE_RECREATE

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxadminConfig.c

-              r19062
+              r23352
     fprintf (stderr, "\nPan-STARRS DataBase Admin Tool\n\n");
     fprintf (stderr, "Usage: %s [mode]\n", program);
     fprintf (stderr, " [mode] : -create | -delete\n\n");
+    fprintf (stderr, " [mode] : -create | -create-mirror | -delete\n\n");
     psMetadataItem *server = pmConfigUserSite(config->modules, "DBSERVER",   PS_DATA_STRING);
 …
         config->mode = PXADMIN_MODE_CREATE;
+    }
+    if ((N = psArgumentGet(argc, argv, "-create-mirror"))) {
+        psArgumentRemove(N, &argc, argv);
+        if (config->mode) {
+            psAbort("only one mode selection is allowed");
+        }
+        config->mode = PXADMIN_MODE_CREATE_MIRROR;
+    }
     if ((N = psArgumentGet(argc, argv, "-delete"))) {
         psArgumentRemove(N, &argc, argv);
 …
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-create", 0,
             "create all IPP tables", "");
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-create-mirror", 0,
+            "mirror all IPP tables", "");
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-delete", 0,
             "delete all IPP tables", "");

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxtools.c

-              r20413
+              r23352
     PS_ASSERT_PTR_NON_NULL(state, false);
-    // XXX replace strncmp with strcmp
     if (!strcmp(state, "new")) return true;
     if (!strcmp(state, "reg")) return true;
 …
     if (!strcmp(state, "wait")) return true;
     if (!strcmp(state, "goto_cleaned")) return true;
+    if (!strcmp(state, "error_cleaned")) return true;
+    if (!strcmp(state, "goto_scrubbed")) return true;
+    if (!strcmp(state, "error_scrubbed")) return true;
     if (!strcmp(state, "cleaned")) return true;
     if (!strcmp(state, "update")) return true;
     if (!strcmp(state, "purged")) return true;
     if (!strcmp(state, "goto_purged")) return true;
+    if (!strcmp(state, "error_purged")) return true;
     return false;
+}
+// 'scrubbed' is a virtual state equivalent to cleaned, but allows files to be removed
+// even if the config files is missing

branches/cnb_branches/cnb_branch_20090301/ippTools/src/pxwarp.c

r21402	r23352
58	58	// check that state is a valid string value
59	59	if (!pxIsValidState(state)) {
60		psError(PS_ERR_UNKNOWN, false, "invalid ~~chi~~pRun state: %s", state);
	60	psError(PS_ERR_UNKNOWN, false, "invalid warpRun state: %s", state);
61	61	return false;
62	62	}

branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtool.c

-              r21402
+              r23352
 static bool updateprocessedexpMode(pxConfig *config);
 static bool cleardupexpMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
 # define MODECASE(caseName, func) \
 …
         MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP,    updateprocessedexpMode);
         MODECASE(REGTOOL_MODE_CLEARDUPEXP,           cleardupexpMode);
+        MODECASE(CHIPTOOL_MODE_EXPORTRUN,            exportrunMode);
+        MODECASE(CHIPTOOL_MODE_IMPORTRUN,            importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
 …
     return true;
+}
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+  int numExportTables = 2;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_S64(det_id, config->args, "-exp_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+    psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+    return false;
+  }
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+  ExportTable tables [] = {
+    {"rawExp", "regtool_export_raw__exp.sql"},
+    {"rawImfile", "regtool_export_raw_imfile.sql"},
+  };
+  for (int i=0; i < numExportTables; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
+    if (!query) {
+      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      return false;
+    }
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
+    // treat limit == 0 as "no limit"
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+    psFree(query);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+    }
+    // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+      psError(PS_ERR_UNKNOWN, false, "failed to print array");
+      psFree(output);
+      return false;
+    }
+    psFree(output);
+  }
+  fclose (f);
+  return true;
+}
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+  fprintf (stdout, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+  psMetadataItem *item = psMetadataLookup (input, "rawExp");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  rawExpRow *rawExp = rawExpObjectFromMetadata (entry->data.md);
+  rawExpInsertObject (config->dbh, rawExp);
+  // fprintf (stdout, "---- raw exp ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+  item = psMetadataLookup (input, "rawImfile");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  for (int i = 0; i < item->data.list->n; i++) {
+    psMetadataItem *entry = psListGet (item->data.list, i);
+    assert (entry);
+    assert (entry->type == PS_DATA_METADATA);
+    rawImfileRow *rawImfile = rawImfileObjectFromMetadata (entry->data.md);
+    rawImfileInsertObject (config->dbh, rawImfile);
+    // fprintf (stdout, "---- row %d ----\n", i);
+    // psMetadataPrint (stderr, entry->data.md, 1);
+  }
+  return true;
+}

branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtool.h

r18328	r23352
36	36	REGTOOL_MODE_UPDATEPROCESSEDEXP,
37	37	REGTOOL_MODE_CLEARDUPEXP,
	38	REGTOOL_MODE_EXPORTRUN,
	39	REGTOOL_MODE_IMPORTRUN
38	40	} regtoolMode;
39	41

branches/cnb_branches/cnb_branch_20090301/ippTools/src/regtoolConfig.c

-              r20608
+              r23352
     psMetadataAddS16(updatedprocessedexpArgs, PS_LIST_TAIL, "-code",    0,            "set fault code (required)", INT16_MAX);
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-exp_id", 0,          "export this exposure ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
     // -cleardupexp
     psMetadata *cleardupexpArgs = psMetadataAlloc();
 …
     PXOPT_ADD_MODE("-updateprocessedexp",      "", REGTOOL_MODE_UPDATEPROCESSEDEXP,      updatedprocessedexpArgs);
     PXOPT_ADD_MODE("-cleardupexp",             "", REGTOOL_MODE_CLEARDUPEXP,      cleardupexpArgs);
+    PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", REGTOOL_MODE_EXPORTRUN, exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           REGTOOL_MODE_IMPORTRUN, importrunArgs);
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/stacktool.c

-              r21402
+              r23352
 static bool donecleanupMode(pxConfig *config);
 static bool updatesumskyfileMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
 static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state);
 …
         MODECASE(STACKTOOL_MODE_DONECLEANUP,           donecleanupMode);
         MODECASE(STACKTOOL_MODE_UPDATESUMSKYFILE,      updatesumskyfileMode);
+        MODECASE(STACKTOOL_MODE_EXPORTRUN,             exportrunMode);
+        MODECASE(STACKTOOL_MODE_IMPORTRUN,             importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
 …
+}
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+  int numExportTables = 3;
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_S64(det_id, config->args, "-stack_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+      psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+      return false;
+  }
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
+  ExportTable tables [] = {
+    {"stackRun", "stacktool_export_run.sql"},
+    {"stackInputSkyfile", "stacktool_export_input_skyfile.sql"},
+    {"stackSumSkyfile", "stacktool_export_sum_skyfile.sql"},
+  };
+  for (int i=0; i < numExportTables; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
+    if (!query) {
+      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      return false;
+    }
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
+    // treat limit == 0 as "no limit"
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+    psFree(query);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psTrace("regtool", PS_LOG_INFO, "no rows found");
+      psFree(output);
+      return true;
+    }
+    psFree(output);
+  }
+    fclose (f);
+    return true;
+}
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+  int numImportTables = 2;
+  char tables[2] [80] = {"stackInputSkyfile", "stackSumSkyfile"};
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+  fprintf (stdout, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+  psMetadataItem *item = psMetadataLookup (input, "stackRun");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  stackRunRow *stackRun = stackRunObjectFromMetadata (entry->data.md);
+  stackRunInsertObject (config->dbh, stackRun);
+  // fprintf (stdout, "---- stack run ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+  for (int i = 0; i < numImportTables; i++) {
+    psMetadataItem *item = psMetadataLookup (input, tables[i]);
+    psAssert (item, "entry not in input?");
+    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+    switch (i) {
+      case 0:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          stackInputSkyfileRow *stackInputSkyfile = stackInputSkyfileObjectFromMetadata (entry->data.md);
+          stackInputSkyfileInsertObject (config->dbh, stackInputSkyfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 1:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          stackSumSkyfileRow *stackSumSkyfile = stackSumSkyfileObjectFromMetadata (entry->data.md);
+          stackSumSkyfileInsertObject (config->dbh, stackSumSkyfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+    }
+  }
+  return true;
+}

branches/cnb_branches/cnb_branch_20090301/ippTools/src/stacktool.h

r19676	r23352
38	38	STACKTOOL_MODE_DONECLEANUP,
39	39	STACKTOOL_MODE_UPDATESUMSKYFILE,
	40	STACKTOOL_MODE_EXPORTRUN,
	41	STACKTOOL_MODE_IMPORTRUN
40	42	} stacktoolMode;
41	43

branches/cnb_branches/cnb_branch_20090301/ippTools/src/stacktoolConfig.c

-              r19931
+              r23352
     psMetadataAddS16(updatesumskyfileArgs, PS_LIST_TAIL, "-code", 0,            "set fault code (required)", 0);
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-stack_id", 0,          "export this stack ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
     psFree(now);
 …
     PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     STACKTOOL_MODE_DONECLEANUP,          donecleanupArgs);
     PXOPT_ADD_MODE("-updatesumskyfile",      "update fault code for sumskyfile",     STACKTOOL_MODE_UPDATESUMSKYFILE,          updatesumskyfileArgs);
+    PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", STACKTOOL_MODE_EXPORTRUN, exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           STACKTOOL_MODE_IMPORTRUN, importrunArgs);
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptool.c

-              r21402
+              r23352
 static bool tofullskyfileMode(pxConfig *config);
 static bool updateskyfileMode(pxConfig *config);
+static bool exportrunMode(pxConfig *config);
+static bool importrunMode(pxConfig *config);
 static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile);
 …
         MODECASE(WARPTOOL_MODE_TOFULLSKYFILE,      tofullskyfileMode);
         MODECASE(WARPTOOL_MODE_UPDATESKYFILE,      updateskyfileMode);
+        MODECASE(WARPTOOL_MODE_EXPORTRUN,          exportrunMode);
+        MODECASE(WARPTOOL_MODE_IMPORTRUN,          importrunMode);
         default:
 …
     return true;
+}
+bool exportrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+    PXOPT_LOOKUP_S64(det_id, config->args, "-warp_id", true,  false);
+    PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+    PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+    FILE *f = fopen (outfile, "w");
+    if (f == NULL) {
+        psError(PS_ERR_UNKNOWN, false, "failed to open output file");
+        return false;
+    }
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
+    ExportTable tables [] = {
+      {"warpRun", "warptool_export_run.sql"},
+      {"warpImfile", "warptool_export_imfile.sql"},
+      {"warpSkyfile", "warptool_export_skyfile.sql"},
+      {"warpSkyCellMap", "warptool_export_sky_cell_map.sql"},
+    };
+    for (int i=0; i < sizeof(tables); i++) {
+      psString query = pxDataGet(tables[i].sqlFilename);
+      if (!query) {
+          psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+          return false;
+      }
+      if (where && psListLength(where->list)) {
+          psString whereClause = psDBGenerateWhereSQL(where, NULL);
+          psStringAppend(&query, " %s", whereClause);
+          psFree(whereClause);
+      }
+      // treat limit == 0 as "no limit"
+      if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+      }
+      if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+      }
+      psFree(query);
+      psArray *output = p_psDBFetchResult(config->dbh);
+      if (!output) {
+          psError(PS_ERR_UNKNOWN, false, "database error");
+          return false;
+      }
+      if (!psArrayLength(output)) {
+        psTrace("regtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+      }
+      // we must write the export table in non-simple (true) format
+      if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+      }
+      psFree(output);
+    }
+    fclose (f);
+    return true;
+}
+bool importrunMode(pxConfig *config)
+{
+  unsigned int nFail;
+  int numImportTables = 3;
+  char tables[3] [80] = {"warpImfile", "warpSkyfile", "warpSkyCellMap"};
+  PS_ASSERT_PTR_NON_NULL(config, NULL);
+  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
+  fprintf (stdout, "---- input ----\n");
+  psMetadataPrint (stderr, input, 1);
+  psMetadataItem *item = psMetadataLookup (input, "warpRun");
+  psAssert (item, "entry not in input?");
+  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+  psMetadataItem *entry = psListGet (item->data.list, 0);
+  assert (entry);
+  assert (entry->type == PS_DATA_METADATA);
+  warpRunRow *warpRun = warpRunObjectFromMetadata (entry->data.md);
+  warpRunInsertObject (config->dbh, warpRun);
+  // fprintf (stdout, "---- warp run ----\n");
+  // psMetadataPrint (stderr, entry->data.md, 1);
+  for (int i = 0; i < numImportTables; i++) {
+    item = psMetadataLookup (input, tables[i]);
+    psAssert (item, "entry not in input?");
+    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
+    switch (i) {
+      case 0:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          warpImfileRow *warpImfile = warpImfileObjectFromMetadata (entry->data.md);
+          warpImfileInsertObject (config->dbh, warpImfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 1:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          warpSkyfileRow *warpSkyfile = warpSkyfileObjectFromMetadata (entry->data.md);
+          warpSkyfileInsertObject (config->dbh, warpSkyfile);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+      case 2:
+        for (int i = 0; i < item->data.list->n; i++) {
+          entry = psListGet (item->data.list, i);
+          assert (entry);
+          assert (entry->type == PS_DATA_METADATA);
+          warpSkyCellMapRow *warpSkyCellMap = warpSkyCellMapObjectFromMetadata (entry->data.md);
+          warpSkyCellMapInsertObject (config->dbh, warpSkyCellMap);
+          // fprintf (stdout, "---- row %d ----\n", i);
+          // psMetadataPrint (stderr, entry->data.md, 1);
+        }
+        break;
+    }
+  }
+  return true;
+}

branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptool.h

r19522	r23352
49	49	WARPTOOL_MODE_TOFULLSKYFILE,
50	50	WARPTOOL_MODE_UPDATESKYFILE,
	51	WARPTOOL_MODE_EXPORTRUN,
	52	WARPTOOL_MODE_IMPORTRUN
51	53	} warptoolMode;
52	54

branches/cnb_branches/cnb_branch_20090301/ippTools/src/warptoolConfig.c

-              r20973
+              r23352
     psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-code",  0,      "new fault code", 0);
+    // -exportrun
+    psMetadata *exportrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-warp_id", 0,          "export this warp ID (required)", 0);
+    psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0,          "export to this file (required)", NULL);
+    psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit",   0,          "limit result set to N items", 0);
+    // -importrun
+    psMetadata *importrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile",  0,          "import from this file (required)", NULL);
     psFree(now);
     psMetadata *argSets = psMetadataAlloc();
 …
     PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs);
     PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs);
+    PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", WARPTOOL_MODE_EXPORTRUN, exportrunArgs);
+    PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           WARPTOOL_MODE_IMPORTRUN, importrunArgs);
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_20080925.config

r21023	r23352
189	189	FPA.RA STR RA
190	190	FPA.DEC STR DEC
191		FPA.RADECSYS STR ~~RADECSYS~~
	191	FPA.RADECSYS STR EQUINOX
192	192	FPA.OBSTYPE STR OBSTYPE
193	193	FPA.OBJECT STR OBJECT

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_20080929.config

r21023	r23352
184	184	FPA.RA STR RA
185	185	FPA.DEC STR DEC
186		FPA.RADECSYS STR ~~RADECSYS~~
	186	FPA.RADECSYS STR EQUINOX
187	187	FPA.OBSTYPE STR OBSTYPE
188	188	FPA.OBJECT STR OBJECT

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_20081011.config

r20951	r23352
174	174	FPA.RA STR RA
175	175	FPA.DEC STR DEC
176		FPA.RADECSYS STR ~~RADECSYS~~
	176	FPA.RADECSYS STR EQUINOX
177	177	FPA.OBSTYPE STR OBSTYPE
178	178	FPA.OBJECT STR OBJECT

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_20090120.config

r21539	r23352
173	173	FPA.RA STR RA
174	174	FPA.DEC STR DEC
175		FPA.RADECSYS STR ~~RADECSYS~~
	175	FPA.RADECSYS STR EQUINOX
176	176	FPA.OBSTYPE STR OBSTYPE
177	177	FPA.OBJECT STR OBJECT

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_20090220.config

r22212	r23352
173	173	FPA.RA STR RA
174	174	FPA.DEC STR DEC
175		FPA.RADECSYS STR ~~RADECSYS~~
	175	FPA.RADECSYS STR EQUINOX
176	176	FPA.OBSTYPE STR OBSTYPE
177	177	FPA.OBJECT STR OBJECT

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_mef.config

r22685	r23352
168	168	FPA.RA STR RA
169	169	FPA.DEC STR DEC
170		FPA.RADECSYS STR ~~RADECSYS~~
	170	FPA.RADECSYS STR EQUINOX
171	171	FPA.OBSTYPE STR OBSTYPE
172	172	FPA.OBJECT STR OBJECT

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_orig.config

r21023	r23352
182	182	# FPA.RA STR COMRA
183	183	# FPA.DEC STR COMDEC
184		FPA.RADECSYS STR ~~RADECSYS~~
	184	FPA.RADECSYS STR EQUINOX
185	185	FPA.OBSTYPE STR OBSTYPE
186	186	FPA.OBJECT STR OBJECT

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/format_relphot.config

r21023	r23352
105	105	FPA.RA STR RA
106	106	FPA.DEC STR DEC
107		FPA.RADECSYS STR ~~RADECSYS~~
	107	FPA.RADECSYS STR EQUINOX
108	108	FPA.OBSTYPE STR OBSTYPE
109	109	FPA.OBJECT STR OBJECT

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppImage.config

-              r21370
+              r23352
 OVERSCAN.BOXCAR         S32     3
+# Normalization class
+NORM.CLASS              STR     CHIP             # How to find the per-class normalizations
 OLDDARK                 BOOL    FALSE
 …
 # Overscan, bias, dark, shutter
 PPIMAGE_OBDS       METADATA
   BASE.FITS        BOOL    TRUE            # Save base detrended image?
   BASE.MASK.FITS   BOOL    TRUE            # Save base detrended image?
+  BASE.FITS          BOOL    TRUE            # Save base detrended image?
+  BASE.MASK.FITS     BOOL    TRUE            # Save base detrended image?
   BASE.VARIANCE.FITS BOOL    TRUE            # Save base detrended image?
+  CHIP.FITS        BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.MASK.FITS   BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.VARIANCE.FITS BOOL    FALSE           # Save chip-mosaic-ed image?
+  OVERSCAN         BOOL    TRUE            # Overscan subtraction
+  BIAS             BOOL    FALSE           # Bias subtraction
+  DARK             BOOL    TRUE            # Dark subtraction
+  SHUTTER          BOOL    FALSE           # Shutter correction
+  FLAT             BOOL    FALSE           # Flat-field normalisation
+  MASK             BOOL    FALSE           # Mask bad pixels
+  FRINGE           BOOL    FALSE           # Fringe subtraction
+  PHOTOM           BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP      BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC    BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+  CHIP.FITS          BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.MASK.FITS     BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.VARIANCE.FITS BOOL    FALSE           # Save chip-mosaic-ed image?
+  OVERSCAN           BOOL    TRUE            # Overscan subtraction
+  BIAS               BOOL    FALSE           # Bias subtraction
+  DARK               BOOL    TRUE            # Dark subtraction
+  SHUTTER            BOOL    FALSE           # Shutter correction
+  FLAT               BOOL    FALSE           # Flat-field normalisation
+  MASK               BOOL    FALSE           # Mask bad pixels
+  FRINGE             BOOL    FALSE           # Fringe subtraction
+  PHOTOM             BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP        BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC      BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS          BOOL    TRUE            # Save 2nd binned chip image?
+END
+# Overscan, bias, dark, shutter
+PPIMAGE_FLATPROC_PREMASK       METADATA
+  BASE.FITS          BOOL    TRUE            # Save base detrended image?
+  BASE.MASK.FITS     BOOL    TRUE            # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL    TRUE            # Save base detrended image?
+  CHIP.FITS          BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.MASK.FITS     BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.VARIANCE.FITS BOOL    FALSE           # Save chip-mosaic-ed image?
+  OVERSCAN           BOOL    TRUE            # Overscan subtraction
+  BIAS               BOOL    FALSE           # Bias subtraction
+  DARK               BOOL    TRUE            # Dark subtraction
+  SHUTTER            BOOL    FALSE           # Shutter correction
+  FLAT               BOOL    FALSE           # Flat-field normalisation
+  MASK               BOOL    FALSE           # Mask bad pixels
+  FRINGE             BOOL    FALSE           # Fringe subtraction
+  PHOTOM             BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP        BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC      BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS          BOOL    TRUE            # Save 2nd binned chip image?
+  DETREND.CONSTRAINTS  METADATA
+    DARK METADATA
+      DETTYPE STR DARK_PREMASK
+    END
+  END
 END

branches/cnb_branches/cnb_branch_20090301/ippconfig/gpc1/ppMerge.config

-              r18252
+              r23352
 DARK.ORDINATES  METADATA
         CELL.DARKTIME   S32     1
         CHIP.TEMP       METADATA
                 ORDER   S32     1
                 SCALE   BOOL    TRUE
                 MIN     F32     -95
                 MAX     F32     -50
         END
+#       CHIP.TEMP       METADATA
+#               ORDER   S32     1
+#               SCALE   BOOL    TRUE
+#               MIN     F32     -95
+#               MAX     F32     -50
+#       END
 END

branches/cnb_branches/cnb_branch_20090301/ippconfig/ipprc.config.in

-              r22215
+              r23352
 # Default search path for configuration files (add $HOME if desired)
 # Note: do not include $HOME in the distributed copy used by ippMonitor
+# PATH            STR     @pkgdatadir@:$HOME/.ipp:.
+PATH              STR     $HOME/ippconfig/:$PSCONFDIR/$PSCONFIG/share/ippconfig/
+PATH            STR     @pkgdatadir@
+### This is a useful setting for an individual user:
+#PATH           STR     $HOME/ippconfig/:$PSCONFDIR/$PSCONFIG/share/ippconfig/
 # load the site-specific information from here

branches/cnb_branches/cnb_branch_20090301/ippconfig/megacam/psphot.config

-              r21070
+              r23352
 SKY_CLIP_SIGMA      F32  2.0             # statistic used to measure background
 PSF_SN_LIM          F32  100             # minimum S/N for stars used for PSF model
+PSF_SN_LIM          F32  20             # minimum S/N for stars used for PSF model
 PSF_MAX_NSTARS      S32  300             # limit number of stars used for PSF model
 …
 PSF_MODEL           STR  PS_MODEL_QGAUSS
+MOMENTS_SN_MIN      F32   30.0
+PSF_MOMENTS_RADIUS  F32   10.0           # calculate initial source moments with this radius
+MOMENTS_SN_MIN      F32    2.0
 EXT_MIN_SN           F32  50.0           # fit galaxies above this S/N limit
 FULL_FIT_SN_LIM      F32  50.0
+AP_MIN_SN            F32  50.0
+# OUTPUT.FORMAT       STR SMPDATA
+OUTPUT.FORMAT       STR PS1_DEV_1
+AP_MIN_SN            F32   2.0
 PSF_SHAPE_NSIGMA     F32  3.0            # max significance for shape variation

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-mef.mdc

-              r21370
+              r23352
 PPSUB.INPUT.MASK        INPUT    @FILES        FPA        MASK
 PPSUB.INPUT.VARIANCE    INPUT    @FILES        FPA        VARIANCE
+PPSUB.INPUT.SOURCES     INPUT    @FILES        FPA        CMF
 PPSUB.REF               INPUT    @FILES        FPA        IMAGE
 PPSUB.REF.MASK          INPUT    @FILES        FPA        MASK
 PPSUB.REF.VARIANCE      INPUT    @FILES        FPA        VARIANCE
 PPSUB.SOURCES           INPUT    @FILES        FPA        CMF
+PPSUB.REF.SOURCES       INPUT    @FILES        FPA        CMF
 ## files used by ppstack
 …
 PPIMAGE.OUT.WT.SPL      OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits      VARIANCE  NONE       CHIP       TRUE      SPLIT
 PPIMAGE.OUTPUT.DETMASK  OUTPUT {OUTPUT}.fits                     IMAGE     MASK       CHIP       TRUE      MEF
+PPIMAGE.OUTPUT.DETREND  OUTPUT {OUTPUT}.fits                     IMAGE     COMP_DET   CHIP       TRUE      MEF
 PPIMAGE.OUTPUT.RESID    OUTPUT {OUTPUT}.b0.fits                  IMAGE     COMP_SUB   CHIP       TRUE      MEF
 PPIMAGE.CONFIG          OUTPUT {OUTPUT}.{CHIP.NAME}.ppImage.mdc  TEXT      NONE       CHIP       TRUE      NONE

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-simple.mdc

-              r21370
+              r23352
 PPSUB.INPUT.MASK        INPUT    @FILES        FPA        MASK
 PPSUB.INPUT.VARIANCE    INPUT    @FILES        FPA        VARIANCE
+PPSUB.INPUT.SOURCES     INPUT    @FILES        FPA        CMF
 PPSUB.REF               INPUT    @FILES        FPA        IMAGE
 PPSUB.REF.MASK          INPUT    @FILES        FPA        MASK
 PPSUB.REF.VARIANCE      INPUT    @FILES        FPA        VARIANCE
 PPSUB.SOURCES           INPUT    @FILES        FPA        CMF
+PPSUB.REF.SOURCES       INPUT    @FILES        FPA        CMF
 ## files used by ppstack
 …
 PPIMAGE.OUTPUT.VARIANCE OUTPUT {OUTPUT}.wt.fits      VARIANCE  NONE       FPA        TRUE      SIMPLE
 PPIMAGE.OUTPUT.DETMASK  OUTPUT {OUTPUT}.fits         IMAGE     NONE       FPA        TRUE      SIMPLE
+PPIMAGE.OUTPUT.DETREND  OUTPUT {OUTPUT}.fits         IMAGE     NONE       FPA        TRUE      SIMPLE
 PPIMAGE.OUTPUT.RESID    OUTPUT {OUTPUT}.fits         IMAGE     NONE       FPA        TRUE      SIMPLE
 PPIMAGE.CONFIG          OUTPUT {OUTPUT}.ppImage.mdc  TEXT      NONE       FPA        TRUE      NONE

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/filerules-split.mdc

-              r21531
+              r23352
 PPSUB.INPUT.MASK        INPUT    @FILES        FPA        MASK
 PPSUB.INPUT.VARIANCE    INPUT    @FILES        FPA        VARIANCE
+PPSUB.INPUT.SOURCES     INPUT    @FILES        FPA        CMF
 PPSUB.REF               INPUT    @FILES        FPA        IMAGE
 PPSUB.REF.MASK          INPUT    @FILES        FPA        MASK
 PPSUB.REF.VARIANCE      INPUT    @FILES        FPA        VARIANCE
 PPSUB.SOURCES           INPUT    @FILES        FPA        CMF
+PPSUB.REF.SOURCES       INPUT    @FILES        FPA        CMF
 ## files used by ppstack
 …
 PPIMAGE.OUTPUT.VARIANCE OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits      VARIANCE  COMP_WT    CHIP       TRUE      NONE
 PPIMAGE.OUTPUT.DETMASK  OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     DET_MASK   CHIP       TRUE      NONE
+PPIMAGE.OUTPUT.DETREND  OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     COMP_DET   CHIP       TRUE      NONE
 PPIMAGE.OUTPUT.RESID    OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     COMP_SUB   CHIP       TRUE      NONE
 PPIMAGE.CONFIG          OUTPUT {OUTPUT}.{CHIP.NAME}.ppImage.mdc  TEXT      NONE       CHIP       TRUE      NONE
-#PPIMAGE.OUTPUT          OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     NONE  CHIP       TRUE      NONE
-#PPIMAGE.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.mk.fits      MASK      NONE  CHIP       TRUE      NONE
-#PPIMAGE.OUTPUT.VARIANCE OUTPUT {OUTPUT}.{CHIP.NAME}.wt.fits      VARIANCE  NONE  CHIP       TRUE      NONE
-#PPIMAGE.OUTPUT.DETMASK  OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     NONE  CHIP       TRUE      NONE
 PPIMAGE.CHIP            OUTPUT {OUTPUT}.{CHIP.NAME}.ch.fits      IMAGE     COMP_IMG   CHIP       TRUE      NONE
 …
 PPIMAGE.STATS           OUTPUT {OUTPUT}.{CHIP.NAME}.stats        STATS     NONE       CHIP       TRUE      NONE
 ## note: these use the  same output naming convention since they are used for different ppMerge runs
 PPMERGE.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.fits         MASK      NONE       CHIP       TRUE      NONE
+PPMERGE.OUTPUT.MASK     OUTPUT {OUTPUT}.{CHIP.NAME}.fits         MASK      DET_MASK   CHIP       TRUE      NONE
 PPMERGE.OUTPUT.BIAS     OUTPUT {OUTPUT}.{CHIP.NAME}.fits         IMAGE     NONE       CHIP       TRUE      NONE
 PPMERGE.OUTPUT.DARK     OUTPUT {OUTPUT}.{CHIP.NAME}.fits         DARK      NONE       CHIP       TRUE      NONE

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/fitstypes.mdc

-              r21534
+              r23352
 # STDEV.NUM(F32) is the number of standard deviations to the edge (when SCALING = STDEV_NEGATIVE|STDEV_POSITIVE)
 # FLOAT(STR) is the name of a custom floating-point type
+# Compressed detrend
+COMP_FLAT       METADATA
+        BITPIX          S32     16
+        SCALING         STR     MANUAL
+        BSCALE          F32     1.0
+        BZERO           F32     32768.0
+        COMPRESSION     STR     RICE
+        TILE.X          S32     0
+        TILE.Y          S32     1
+        TILE.Z          S32     1
+        NOISE           S32     8
+END
 DET_IMAGE       METADATA

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/jpeg.mdc

-              r19743
+              r23352
                 COLORMAP        STR     -greyscale
                 SCALE.MODE      STR     FRACTION
                 SCALE.MIN       F32     0.95
                 SCALE.MAX       F32     1.05
+                SCALE.MIN       F32     0.99
+                SCALE.MAX       F32     1.01
         END
 …
                 COLORMAP        STR     -greyscale
                 SCALE.MODE      STR     FRACTION
                 SCALE.MIN       F32     0.95
                 SCALE.MAX       F32     1.05
+                SCALE.MIN       F32     0.99
+                SCALE.MAX       F32     1.01
         END
 END

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppImage.config

-              r21370
+              r23352
 NONLIN.DATA             STR     nonlin.dat      # Filename for lookup table
+# Normalization class
+NORM.CLASS              STR     FPA             # How to find the per-class normalizations
 OLDDARK         BOOL    FALSE           # Use old-style darks?
 …
   BIN1.FITS        BOOL    TRUE            # Save 1st binned chip image?
   BIN2.FITS        BOOL    TRUE            # Save 2nd binned chip image?
+END
+# used for the pre-mask version of flat-field processing
+PPIMAGE_FLATPROC_PREMASK       METADATA
+  BASE.FITS          BOOL    TRUE            # Save base detrended image?
+  BASE.MASK.FITS     BOOL    TRUE            # Save base detrended image?
+  BASE.VARIANCE.FITS BOOL    TRUE            # Save base detrended image?
+  CHIP.FITS          BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.MASK.FITS     BOOL    FALSE           # Save chip-mosaic-ed image?
+  CHIP.VARIANCE.FITS BOOL    FALSE           # Save chip-mosaic-ed image?
+  OVERSCAN           BOOL    TRUE            # Overscan subtraction
+  BIAS               BOOL    TRUE            # Bias subtraction
+  DARK               BOOL    TRUE            # Dark subtraction
+  SHUTTER            BOOL    FALSE           # Shutter correction
+  FLAT               BOOL    FALSE           # Flat-field normalisation
+  MASK               BOOL    FALSE           # Mask bad pixels
+  FRINGE             BOOL    FALSE           # Fringe subtraction
+  PHOTOM             BOOL    FALSE           # Source identification and photometry
+  ASTROM.CHIP        BOOL    FALSE           # Astrometry per chip?
+  ASTROM.MOSAIC      BOOL    FALSE           # Astrometry for mosaic?
+  BIN1.FITS          BOOL    TRUE            # Save 1st binned chip image?
+  BIN2.FITS          BOOL    TRUE            # Save 2nd binned chip image?
+  DETREND.CONSTRAINTS  METADATA
+    DARK METADATA
+      DETTYPE STR DARK_PREMASK
+    END
+  END
 END

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppStack.config

-              r21475
+              r23352
 ZP.RADIUS       F32     1.0             # Radius (pixels) for matching sources
+ZP.ITER         S32     1000            # Maximum iterations for zero point
+ZP.ITER.1       S32     5               # Iterations for zero point; pass 1
+ZP.ITER.2       S32     1000            # Iterations for zero point; pass 2
 ZP.TOL          F32     1.0e-6          # Tolerance for zero point iterations
 ZP.TRANS.ITER   S32     2               # Iterations for transparency determination
 ZP.TRANS.REJ    F32     3.0             # Rejection threshold for transparency determination
 ZP.TRANS.THRESH F32     1.0             # Threshold for transparency determination
+ZP.STAR.REJ     F32     3.0             # Rejection threshold for stars
+ZP.STAR.REJ.1   F32     20.0            # Rejection threshold for stars; pass 1
+ZP.STAR.REJ.2   F32     3.0             # Rejection threshold for stars; pass 2
 ZP.STAR.LIMIT   F32     1.0e-2          # Limit on star rejection fraction for successful iteration
+ZP.STAR.SYS     F32     0.05            # Estimated systematic error
+ZP.STAR.SYS.1   F32     0.10            # Estimated systematic error; pass 1
+ZP.STAR.SYS.2   F32     0.05            # Estimated systematic error; pass 2
+ZP.MATCH        F32     0.3             # Fraction of images to match for good star
 ZP.AIRMASS      METADATA                # Airmass terms by filter
         g       F32     0.0

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/ppSub.config

r21370	r23352
5	5	SPATIAL.ORDER S32 0 # Spatial polynomial order
6	6	REGION.SIZE F32 0 # Iso-kernel region size (pixels)
	7	SOURCE.RADIUS F32 3.0 # Source matching radius (pixels)
7	8	STAMP.SPACING F32 200 # Typical spacing between stamps (pixels)
8	9	STAMP.FOOTPRINT S32 20 # Size of stamps (pixels)

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/psphot.config

r21505	r23352
16	16	ZERO_PT F32 25.000 # zero point used by DVO
17	17
18		OUTPUT.FORMAT STR PS1_~~DEV_~~1
	18	OUTPUT.FORMAT STR PS1_V1
19	19
20	20	# these parameter govern how the background is measured

branches/cnb_branches/cnb_branch_20090301/ippconfig/recipes/reductionClasses.mdc

-              r20745
+              r23352
         # *** premask (unmasked) flat-field types
         FLAT_PREMASK_PROCESS    STR     PPIMAGE_OBDS
+        FLAT_PREMASK_PROCESS    STR     PPIMAGE_FLATPROC_PREMASK
         FLAT_PREMASK_RESID      STR     PPIMAGE_F
         FLAT_PREMASK_VERIFY     STR     PPIMAGE_F
         FLAT_PREMASK_STACK      STR     PPMERGE_FLAT
         FLAT_PREMASK_JPEG_IMAGE STR     FLAT
         FLAT_PREMASK_JPEG_RESID STRE    FLAT_RESID
+        FLAT_PREMASK_JPEG_RESID STR     FLAT_RESID
         DOMEFLAT_PREMASK_PROCESS        STR     PPIMAGE_OBDS

branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksrelease.c

-              r21156
+              r23352
 static void excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, double newMaskValue);
 static void writeImages(streakFiles *sf, bool exciseImageCube);
-static bool replicateOutputs(streakFiles *sfiles);
 int
 …
         // image data directly from psFits
         readImage(sf->inImage, sf->extnum, sf->stage, false);
+        // astrom struct is only needed for raw stage, and only the chip to cell parameters are used
+        sf->astrom = streakSetAstrometry(sf->astrom, sf->stage, NULL, NULL, false,
+            sf->inImage->header, sf->inImage->numCols, sf->inImage->numRows);
+    }
     sf->outImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header);
 …
     if (sf->inImage->image) {
         setupImageRefs(sf->outImage, sf->recImage, sf->inImage, sf->extnum, exciseAll);
+    } else if (sf->inImage->imagecube) {
+        // Image cubes should have been excised in the destreaking process
+        streaksExit("unexpected imagecube found", PS_EXIT_CONFIG_ERROR);
     }  else {
         // Image cubes are handeled specially
+        return false;
+    }
 …
     if (sf->inMask) {
         readImage(sf->inMask, sf->extnum, sf->stage, true);
+        sf->outMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
+        if (sf->recMask) {
+            sf->recMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
+        }
+        setupImageRefs(sf->outMask, sf->recMask, sf->inMask, sf->extnum, exciseAll);
+        if (sf->outChMask) {
+            sf->outChMask->header = (psMetadata *) psMemIncrRefCounter(sf->outMask->header);
+            sf->outChMask->image = (psImage *) psMemIncrRefCounter(sf->outMask->image);
+            if (sf->recChMask) {
+                sf->recChMask->header = (psMetadata *) psMemIncrRefCounter(sf->recMask->header);
+                sf->recChMask->image = (psImage *) psMemIncrRefCounter(sf->recMask->image);
+            }
+        }
+        if (sf->outMask) {
+            sf->outMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
+            if (sf->recMask) {
+                sf->recMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
+            }
+            setupImageRefs(sf->outMask, sf->recMask, sf->inMask, sf->extnum, exciseAll);
+            if (sf->outChMask) {
+                sf->outChMask->header = (psMetadata *) psMemIncrRefCounter(sf->outMask->header);
+                sf->outChMask->image = (psImage *) psMemIncrRefCounter(sf->outMask->image);
+                if (sf->recChMask) {
+                    sf->recChMask->header = (psMetadata *) psMemIncrRefCounter(sf->recMask->header);
+                    sf->recChMask->image = (psImage *) psMemIncrRefCounter(sf->recMask->image);
+                }
+            }
 #ifdef STREAKS_COMPRESS_OUTPUT
         // XXX: see note above
         copyFitsOptions(sf->outMask, sf->recMask, sf->inMask);
         psFitsSetCompression(sf->outMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
         if (sf->recMask) {
             psFitsSetCompression(sf->recMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
+        }
         if (sf->outChMask) {
             copyFitsOptions(sf->outChMask, sf->recChMask, sf->inMask);
             psFitsSetCompression(sf->outChMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
             if (sf->recChMask) {
                 psFitsSetCompression(sf->recChMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
+            }
+        }
+            // XXX: see note above
+            copyFitsOptions(sf->outMask, sf->recMask, sf->inMask);
+            psFitsSetCompression(sf->outMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
+            if (sf->recMask) {
+                psFitsSetCompression(sf->recMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
+            }
+            if (sf->outChMask) {
+                copyFitsOptions(sf->outChMask, sf->recChMask, sf->inMask);
+                psFitsSetCompression(sf->outChMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
+                if (sf->recChMask) {
+                    psFitsSetCompression(sf->recChMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
+                }
+            }
 #endif
+        }
+    }
 …
+    }
+}
-// for any of the outputs that are stored in Nebulous set their extended attributes
-// and replicate the files
-static bool
-replicateOutputs(streakFiles *sfiles)
+{
-    bool status = false;
-    // XXX: TODO: need a nebGetXatrr function, but there isn't one
-    // another option would be to take the number of copies to be
-    // created as an option. That way the system could decide
-    // whether to replicate anything other than raw Image files
-    void *xattr = NULL;
-    if (!replicate(sfiles->outImage, xattr)) {
-        psError(PM_ERR_SYS, false, "failed to replicate outImage.");
-        return false;
+    }
-#ifdef notyet
-    // XXX: don't replicate mask and weight images until we can look up
-    // the input's xattr. There may be a perl program that can getXattr
-    if (sfiles->outMask) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outMask, xattr)) {
-            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
-            return false;
+        }
+    }
-    if (sfiles->outWeight) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outWeight, xattr)) {
-            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
-            return false;
+        }
+    }
-#endif
-    return true;
+}

branches/cnb_branches/cnb_branch_20090301/magic/remove/src/streaksreplace.c

-              r21156
+              r23352
 static bool readAndCopyToOutput(streakFiles *sf, bool restoreImageCube);
 static void writeImages(streakFiles *sf, bool exciseImageCube);
-static bool replicateOutputs(streakFiles *sfiles);
 static bool readAndCopyToOutput(streakFiles *sf, bool restoreImageCube);
 …
     closeImages(sfiles);
-    // NOTE: from here on we can't just quit if something goes wrong.
-    // especially if we're working at the raw stage
     if (!replicateOutputs(sfiles)) {
         psError(PS_ERR_UNKNOWN, false, "failed to replicate output files");
 …
+    }
+}
-static bool replicateOutputs(streakFiles *sfiles)
+{
-    bool status = false;
-    // XXX: TODO: need a nebGetXatrr function, but there isn't one
-    // another option would be to take the number of copies to be
-    // created as an option. That way the system could decide
-    // whether to replicate anything other than raw Image files
-    void *xattr = NULL;
-    if (!replicate(sfiles->outImage, xattr)) {
-        psError(PM_ERR_SYS, false, "failed to replicate outImage.");
-        return false;
+    }
-#ifdef notyet
-    // XXX: don't replicate mask and weight images until we can look up
-    // the input's xattr. There may be a perl program that can getXattr
-    if (sfiles->outMask) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outMask, xattr)) {
-            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
-            return false;
+        }
+    }
-    if (sfiles->outWeight) {
-        // get xattr from input to see if we need to replicate
-        if (!replicate(sfiles->outWeight, xattr)) {
-            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
-            return false;
+        }
+    }
-#endif
-//      replicate the recovery images (if in nebulous)
-//      perhaps whether we do that or not should be configurable.
-//      Sounds like we need a recipe
-    return true;
+}

branches/cnb_branches/cnb_branch_20090301/ppArith/src/Makefile.am

-              r15571
+              r23352
 bin_PROGRAMS = ppArith
+ppArith_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PPARITH_CFLAGS)
+# PPARITH_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+# PPARITH_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+# PPARITH_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+#
+# # Force recompilation of ppArithVersion.c, since it gets the version information
+# ppArithVersion.c: FORCE
+#       touch ppArith.c
+# FORCE: ;
+ppArith_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PPARITH_CFLAGS) -DPPARITH_VERSION=$(SVN_VERSION) -DPPARITH_BRANCH=$(SVN_BRANCH) -DPPARITH_SOURCE=$(SVN_SOURCE)
 ppArith_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PPSTATS_LIBS)   $(PSPHOT_LIBS)   $(PPARITH_LIBS)

branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArith.c

r21378	r23352
34	34	}
35	35
	36	ppArithVersionPrint();
	37
36	38	if (!ppArithArguments(argc, argv, config)) {
37	39	psErrorStackPrint(stderr, "Error reading arguments.\n");

branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArith.h

-              r21244
+              r23352
  * Parse the arguments
  */
 bool ppArithArguments(int argc, char *argv[], ///< Command-line arguments
                       pmConfig *config    ///< Configuration
+bool ppArithArguments(int argc, char *argv[], ///< Command-line arguments
+                      pmConfig *config    ///< Configuration
     );
 …
  * Parse the camera input
  */
 bool ppArithCamera(pmConfig *config       ///< Configuration
+bool ppArithCamera(pmConfig *config       ///< Configuration
     );
 …
  * Loop over the FPA hierarchy
  */
 bool ppArithLoop(pmConfig *config         ///< Configuration
+bool ppArithLoop(pmConfig *config         ///< Configuration
     );
 …
  * Perform arithmetic on the readout
  */
 bool ppArithReadout(pmReadout *output,  ///< Output readout
                     const pmReadout *input1, ///< Input readout
                     const pmReadout *input2, ///< Input readout
                     const pmConfig *config, ///< Configuration
                     const pmFPAview *view ///< View of readout on which to operate
+bool ppArithReadout(pmReadout *output,  ///< Output readout
+                    const pmReadout *input1, ///< Input readout
+                    const pmReadout *input2, ///< Input readout
+                    const pmConfig *config, ///< Configuration
+                    const pmFPAview *view ///< View of readout on which to operate
     );
 /**
  * Put the program version information into a metadata
+ * Put the program version information into header
  */
 void ppArithVersionMetadata(psMetadata *metadata ///< Metadata to populate
+bool ppArithVersionHeader(psMetadata *header ///< Header to populate
     );
+/// Print version information
+void ppArithVersionPrint(void);
 ///@}

branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArithLoop.c

r21378	r23352
113	113	hdu->header = psMetadataAlloc();
114	114	}
115		ppArithVersion~~Metadata~~(hdu->header);
	115	ppArithVersionHeader(hdu->header);
116	116	lastHDU = hdu;
117	117	}

branches/cnb_branches/cnb_branch_20090301/ppArith/src/ppArithVersion.c

-              r21378
+              r23352
 #include "ppArith.h"
+static const char *cvsTag = "$Name: not supported by cvs2svn $";///< CVS tag name
+#ifndef PPARITH_VERSION
+#error "PPARITH_VERSION is not set"
+#endif
+#ifndef PPARITH_BRANCH
+#error "PPARITH_BRANCH is not set"
+#endif
+#ifndef PPARITH_SOURCE
+#error "PPARITH_SOURCE is not set"
+#endif
+#define xstr(s) str(s)
+#define str(s) #s
 psString ppArithVersion(void)
+{
+    psString version = NULL;            // Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", xstr(PPARITH_BRANCH), xstr(PPARITH_VERSION));
+    return value;
+}
+psString ppArithSource(void)
+{
+    return psStringCopy(xstr(PPARITH_SOURCE));
+}
 psString ppArithVersionLong(void)
+{
+    psString version = ppArithVersion(); // Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag
+    psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__);
+    psFree(tag);
+    psString version = ppArithVersion();  // Version, to return
+    psString source = ppArithSource();    // Source
+    psStringPrepend(&version, "ppArith ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
     return version;
+};
+bool ppArithVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "ppArith at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+    ppStatsVersionHeader(header);
+    psString version = ppArithVersion(); // Software version
+    psString source  = ppArithSource();  // Software source
+    psStringPrepend(&version, "ppArith version: ");
+    psStringPrepend(&version, "ppArith source: ");
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+    psFree(version);
+    psFree(source);
+    return true;
+}
+void ppArithVersionMetadata(psMetadata *metadata)
+void ppArithVersionPrint(void)
+{
+    PS_ASSERT_METADATA_NON_NULL(metadata,);
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("ppArith", PS_LOG_INFO, "ppArith at %s", timeString);
+    psFree(timeString);
     psString pslib = psLibVersionLong();// psLib version
 …
     psString ppArith = ppArithVersionLong(); // ppArith version
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString head = NULL;               // Head string
+    psStringAppend(&head, "ppArith processing at %s. Component information:", timeString);
+    psFree(timeString);
+    psLogMsg("ppArith", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("ppArith", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("ppArith", PS_LOG_INFO, "%s", ppStats);
+    psLogMsg("ppArith", PS_LOG_INFO, "%s", ppArith);
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, head, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, pslib, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psmodules, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppStats, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppArith, "");
-    psFree(head);
     psFree(pslib);
     psFree(psmodules);

branches/cnb_branches/cnb_branch_20090301/ppImage

Property svn:mergeinfo set to (toggle deleted branches)

/branches/cnb_branches/cnb_branch_20090215/ppImage	merged	eligible
/trunk/ppImage	merged	eligible
/branches/cnb_branch_20090215/ppImage	21495-22685
/branches/eam_branches/eam_branch_20090303/ppImage	23158-23228

branches/cnb_branches/cnb_branch_20090301/ppImage/src/Makefile.am

-              r20774
+              r23352
         ppImage.h
+ppImage_CFLAGS = $(PPIMAGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSASTRO_CFLAGS) $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
+#PPIMAGE_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+#PPIMAGE_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+#PPIMAGE_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+# Force recompilation of ppImageVersion.c, since it gets the version information
+# ppImageVersion.c: FORCE
+#       touch ppImageVersion.c
+# FORCE: ;
+ppImage_CFLAGS = $(PPIMAGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSASTRO_CFLAGS) $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPPIMAGE_VERSION=$(SVN_VERSION) -DPPIMAGE_BRANCH=$(SVN_BRANCH) -DPPIMAGE_SOURCE=$(SVN_SOURCE)
 ppImage_LDFLAGS = $(PPIMAGE_LIBS) $(PSASTRO_LIBS) $(PPSTATS_LIBS) $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 ppImage_SOURCES = \

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImage.c

r19928	r23352
21	21	exit(PS_EXIT_CONFIG_ERROR);
22	22	}
	23
	24	ppImageVersionPrint();
23	25
24	26	// define recipe options

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImage.h

-              r21364
+              r23352
     int remnanceSize;                   // Size for remnance detection
     float remnanceThresh;               // Threshold for remnance detection
+    char *normClass;                    // class to use for per-class normalization
 } ppImageOptions;
 …
 psString ppImageVersion(void);
+/// Return software source
+psString ppImageSource(void);
 /// Return long version information
 psString ppImageVersionLong(void);
+/// Update the metadata with version information for all dependencies
+void ppImageVersionMetadata(psMetadata *metadata ///< Metadata to update with version information
+    );
+/// Populate the header with version information for all dependencies
+bool ppImageVersionHeader(psMetadata *metadata ///< Header to populate
+    );
+/// Print version information
+void ppImageVersionPrint(void);

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageArguments.c

-              r19943
+              r23352
     fprintf(stderr, "\t-chip CHIPNUM: Only process this chip number.\n");
     fprintf(stderr, "\t-norm VALUE: Divide through by this value when done.\n");
+    fprintf(stderr, "\t-normlist file.mdc: normalizations by class_id.\n");
     fprintf(stderr, "\n");
     fprintf(stderr, "Input options (single file / file list):\n");
 …
     if ((argnum = psArgumentGet(argc, argv, "-visual"))) {
         psArgumentRemove(argnum, &argc, argv);
         psphotSetVisual (true);
+        pmVisualSetVisual(true);
+    }
 …
     if ((argnum = psArgumentGet(argc, argv, "-threads"))) {
         psArgumentRemove(argnum, &argc, argv);
         int nThreads = atoi(argv[argnum]);
+        int nThreads = atoi(argv[argnum]);
         psMetadataAddS32(config->arguments, PS_LIST_TAIL, "NTHREADS", 0, "number of warp threads", nThreads);
         psArgumentRemove(argnum, &argc, argv);
         // create the thread pool with number of desired threads, supplying our thread launcher function
         // XXX need to determine the number of threads from the config data
         psThreadPoolInit (nThreads);
+        // create the thread pool with number of desired threads, supplying our thread launcher function
+        // XXX need to determine the number of threads from the config data
+        psThreadPoolInit (nThreads);
+    }
 …
     // the input file is a required argument; if not found, we will exit
+    bool status = pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list");
+    if (!status) {
+        usage ();
+    }
+    pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list");
     // if these command-line options are supplied, load the file name lists into config->arguments
 …
+    }
     // Optional normalisation factor
+    // Optional normalization factor
     if ((argnum = psArgumentGet(argc, argv, "-norm"))) {
         psArgumentRemove(argnum, &argc, argv);
         float norm = atof(argv[argnum]);
         psMetadataAddF32(config->arguments, PS_LIST_TAIL, "NORMALISATION", 0,
+        psMetadataAddF32(config->arguments, PS_LIST_TAIL, "NORMALIZATION", 0,
                          "Normalisation to apply", norm);
+        psArgumentRemove(argnum, &argc, argv);
+    }
+    // Optional per-class normalization table
+    if ((argnum = psArgumentGet(argc, argv, "-normlist"))) {
+        psArgumentRemove(argnum, &argc, argv);
+        unsigned int nFail = 0;
+        psMetadata *normlist = psMetadataConfigRead (NULL, &nFail, argv[argnum], false);
+        // XXX allow this file to be in nebulous?
+        psMetadataAddMetadata(config->arguments, PS_LIST_TAIL, "NORMALIZATION.TABLE", 0, "Normalization to apply", normlist);
+        psFree (normlist);
         psArgumentRemove(argnum, &argc, argv);
+    }

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageDefineFile.c

-              r13562
+              r23352
 # include "ppImage.h"
+bool ppImageDefineFile (pmConfig *config, pmFPA *input, char *filerule, char *argname, pmFPAfileType fileType, pmDetrendType detrendType) {
+bool ppImageDefineFile(pmConfig *config, pmFPA *input, char *filerule, char *argname,
+                       pmFPAfileType fileType, pmDetrendType detrendType)
+{
+    bool status;
+    pmFPAfile *file = NULL;             // File to be defined
+    bool status;
+    pmFPAfile *file;
+    // look for the file on the argument list
+    file = pmFPAfileDefineFromArgs  (&status, config, filerule, argname);
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "failed to load find definition");
+        return false;
+    if (!file) {
+        // look for the file on the RUN metadata
+        file = pmFPAfileDefineFromRun(&status, config, filerule);
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to load file definition");
+            return false;
+        }
+    }
+    if (file) {
+        if (file->type != fileType) {
+            psError(PS_ERR_IO, true, "%s is not of type %s", filerule, pmFPAfileStringFromType (fileType));
+            return false;
+        }
+        return true;
+    if (!file) {
+        // look for the file on the argument list
+        file = pmFPAfileDefineFromArgs(&status, config, filerule, argname);
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to load file definition");
+            return false;
+        }
+    }
+    if (!file) {
+        // look for the file in the camera config table
+        file = pmFPAfileDefineFromConf(&status, config, filerule);
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to load file definition");
+            return false;
+        }
+    }
+    if (!file) {
+        // look for the file to be loaded from the detrend database
+        file = pmFPAfileDefineFromDetDB(&status, config, filerule, input, detrendType);
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to load file definition");
+            return false;
+        }
+    }
+    // look for the file in the camera config table
+    file = pmFPAfileDefineFromConf  (&status, config, filerule);
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "failed to load find definition");
+        return false;
+    }
+    if (file) {
+        if (file->type != fileType) {
+            psError(PS_ERR_IO, true, "%s is not of type %s", filerule, pmFPAfileStringFromType (fileType));
+            return false;
+        }
+        return true;
+    if (!file) {
+        return false;
+    }
+    // look for the file to be loaded from the detrend database
+    file = pmFPAfileDefineFromDetDB (&status, config, filerule, input, detrendType);
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "failed to load file definition");
+        return false;
+    if (file->type != fileType) {
+        psError(PS_ERR_IO, true, "%s is not of type %s", filerule, pmFPAfileStringFromType(fileType));
+        return false;
+    }
+    if (file) {
+        if (file->type != fileType) {
+            psError(PS_ERR_IO, true, "%s is not of type %s", filerule, pmFPAfileStringFromType (fileType));
+            return false;
+        }
+        return true;
+    }
+    return false;
+    return true;
+}

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageDetrendReadout.c

-              r21364
+              r23352
         if (!pmBiasSubtract(input, options->overscan, bias, oldDark, view)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to subtract bias.");
+            psFree(detview);
             return false;
+        }
 …
         if (!pmDarkApply(input, dark, options->maskValue)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to subtract dark.");
+            psFree(detview);
             return false;
+        }
 …
                         options->remnanceSize, options->remnanceThresh)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to mask remnance.");
+            psFree(detview);
             return false;
+        }
 …
         pmReadout *shutter = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.SHUTTER");
         if (!pmShutterCorrectionApply(input, shutter, pmConfigMaskGet("FLAT", config))) {
+            psFree(detview);
             return false;
+        }
 …
         pmReadout *flat = pmFPAfileThisReadout(config->files, detview, "PPIMAGE.FLAT");
         if (!pmFlatField(input, flat, options->flatMask)) {
+            psFree(detview);
             return false;
+        }
+    }
     // Normalisation by a (known) constant
+    // Normalization by a single (known) constant
     bool mdok;                          // Status of MD lookup
     float norm = psMetadataLookupF32(&mdok, config->arguments, "NORMALISATION");
+    float norm = psMetadataLookupF32(&mdok, config->arguments, "NORMALIZATION");
     if (mdok && isfinite(norm) && norm != 1.0) {
         pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest
 …
+    }
+# if (1)
+    // Normalization by per-class values
+    psMetadata *normlist = psMetadataLookupMetadata(&mdok, config->arguments, "NORMALIZATION.TABLE");
+    if (normlist) {
+        pmFPAfile *inputFile = psMetadataLookupPtr(&mdok, config->files, "PPIMAGE.INPUT");
+        // get the menu of class IDs
+        psMetadata *menu = psMetadataLookupMetadata(&mdok, inputFile->camera, "CLASSID");
+        if (!menu) {
+            psError(PS_ERR_IO, false, "Unable to find CLASSID metadata in camera configuration");
+            psFree(detview);
+            return false;
+        }
+        // get the rule for class_id for the desired class
+        const char *rule = psMetadataLookupStr(&mdok, menu, options->normClass);
+        if (!rule) {
+            psError(PS_ERR_IO, false, "Unable to find NORM.CLASS value %s in CLASSID in camera configuration", options->normClass);
+            psFree(detview);
+            return false;
+        }
+        // get the class_id from the rule
+        char *classID = pmFPAfileNameFromRule(rule, inputFile, view);
+        if (!classID) {
+            psError(PS_ERR_IO, false, "error converting CLASSID rule %s to name\n", rule);
+            psFree(detview);
+            return false;
+        }
+        // get normalization from the class_id
+        float norm = psMetadataLookupF32 (&mdok, normlist, classID);
+        pmHDU *hdu = pmHDUFromReadout(input); // HDU of interest
+        psString comment = NULL;        // Comment to add
+        psStringAppend(&comment, "Normalization: %f", norm);
+        psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, "");
+        psFree(comment);
+        // apply the normalization
+        psBinaryOp(input->image, input->image, "*", psScalarAlloc(norm, PS_TYPE_F32));
+        psFree (classID);
+    }
+# endif
     if (options->doFringe) {
         pmCell *fringe = pmFPAfileThisCell(config->files, detview, "PPIMAGE.FRINGE");
         if (!ppImageDetrendFringeMeasure(input, fringe, false, options)) {
+            psFree(detview);
             return false;
+        }

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageLoop.c

-              r21183
+              r23352
+    }
+    psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
+    if (dump_file) {
+        pmConfigCamerasCull(config, NULL);
+        pmConfigRecipesCull(config, "PPIMAGE,PPSTATS,PSPHOT,MASKS,PSASTRO");
+        pmConfigDump(config, input->fpa, dump_file);
+    }
+    pmConfigCamerasCull(config, NULL);
+    pmConfigRecipesCull(config, "PPIMAGE,PPSTATS,PSPHOT,MASKS,PSASTRO,JPEG");
     pmFPAview *view = pmFPAviewAlloc(0);// View for level of interest
 …
             // Put version information into the header
             pmHDU *hdu = pmHDUFromCell(cell);
+            pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell);
             if (hdu && hdu != lastHDU) {
                 ppImageVersionMetadata(hdu->header);
+                ppImageVersionHeader(hdu->header);
                 lastHDU = hdu;
+            }
 …
+                }
                 // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
                 if (!ppImageDetrendFree (config, view)) {
                     ESCAPE("Unable to free detrend images");
+                }
+                // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
+                if (!ppImageDetrendFree (config, view)) {
+                    ESCAPE("Unable to free detrend images");
+                }
+            }
 …
                 ppImageDetrendRecord(cell, config, options, view);
+            }
             // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
             if (!ppImageDetrendFree (config, view)) {
                 ESCAPE("Unable to free detrend images");
+            }
+        }
         // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
         if (!ppImageDetrendFree (config, view)) {
             ESCAPE("Unable to free detrend images");
+        }
+            // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
+            if (!ppImageDetrendFree (config, view)) {
+                ESCAPE("Unable to free detrend images");
+            }
+        }
+        // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
+        if (!ppImageDetrendFree (config, view)) {
+            ESCAPE("Unable to free detrend images");
+        }
         // Apply the fringe correction
 …
+            }
+        }
         // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
         if (!ppImageFringeFree (config, view)) {
             ESCAPE("Unable to free fringe images");
+        }
+        // free detrend images potentially in use: MASK, BIAS, DARK, SHUTTER, FLAT
+        if (!ppImageFringeFree (config, view)) {
+            ESCAPE("Unable to free fringe images");
+        }
         // measure various pixel-based statistics for this image
 …
+        }
         // these may be used by ppImageSubtractBackground.
         // if these are defined as internal files, drop them here
         status = true;
         status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL");
         status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV");
         status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND");
         if (!status) {
             psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files");
             psFree (view);
             return false;
+        }
+        // these may be used by ppImageSubtractBackground.
+        // if these are defined as internal files, drop them here
+        status = true;
+        status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL");
+        status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV");
+        status &= pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND");
+        if (!status) {
+            psError(PSPHOT_ERR_PROG, false, "trouble dropping internal files");
+            psFree (view);
+            return false;
+        }
         // binning (used for display) must take place after the background is replaced, if desired
 …
+    }
     psFree(view);
+    // Dump configuration
+    psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
+    if (dump_file) {
+        pmConfigDump(config, input->fpa, dump_file);
+    }
     // Write out summary statistics

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageOptions.c

-              r21364
+              r23352
     options->remnanceThresh  = 25.0;    // Threshold for remnance detection
+    // per-class normalization source
+    options->normClass       = NULL;    // per-class normalizations refer to this class
     return options;
+}
 …
     options->remnanceThresh = psMetadataLookupS32(NULL, recipe, "REMNANCE.THRESH");
+    // per-class normalization source (just a reference; don't free)
+    options->normClass = psMetadataLookupStr(NULL, recipe, "NORM.CLASS");
     return options;
+}

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageParseCamera.c

-              r21364
+              r23352
     bool status = false;
+    // the input image defines the camera, and all recipes and options the follow
+    pmFPAfile *input = pmFPAfileDefineFromArgs (&status, config, "PPIMAGE.INPUT", "INPUT");
+    if (!status || !input) {
+        psError(PS_ERR_IO, false, "Failed to build FPA from PPIMAGE.INPUT");
+        return NULL;
+    }
+    if (input->type != PM_FPA_FILE_IMAGE) {
+        psError(PS_ERR_IO, true, "PPIMAGE.INPUT is not of type IMAGE");
+        return NULL;
+    }
+    // if MASK or VARIANCE was supplied on command line, bind files to 'input'.
+    // the mask and variance will be mosaicked with the image
+    pmFPAfile *inputMask = pmFPAfileBindFromArgs(&status, input, config, "PPIMAGE.INPUT.MASK", "PPIMAGE.INPUT.MASK");
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "failed to load find definition");
+        return NULL;
+    }
+    if (inputMask) {
+      if (inputMask->type != PM_FPA_FILE_MASK) {
+        psError(PS_ERR_IO, true, "PPIMAGE.INPUT.MASK is not of type MASK");
+        return NULL;
+      }
+    }
+    pmFPAfile *inputVariance = pmFPAfileBindFromArgs(&status, input, config, "PPIMAGE.INPUT.VARIANCE", "PPIMAGE.INPUT.VARIANCE");
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "failed to load find definition");
+        return NULL;
+    }
+    if (inputVariance && inputVariance->type != PM_FPA_FILE_VARIANCE) {
+        psError(PS_ERR_IO, true, "PPIMAGE.INPUT.VARIANCE is not of type VARIANCE");
+        return NULL;
+    }
+    if (!ppImageDefineFile(config, NULL, "PPIMAGE.INPUT", "INPUT", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_NONE)) {
+        psError(PS_ERR_IO, false, "Can't find an input image source");
+        return NULL;
+    }
+    pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPIMAGE.INPUT"); // Input file
+    psAssert(input, "We just put it there!");
     // add recipe options supplied on command line
     psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, RECIPE_NAME);
+    psMetadata *recipe  = psMetadataLookupPtr(&status, config->recipes, RECIPE_NAME);
     // parse the options from the metadata format to the ppImageOptions structure
     ppImageOptions *options = ppImageOptionsParse (config);
+    ppImageOptions *options = ppImageOptionsParse(config);
     // the following are defined from the argument list, if given,
 …
     // not all input or output images are used in a given recipe
     if (options->doBias) {
+        if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.BIAS", "BIAS", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_BIAS)) {
+            psError (PS_ERR_IO, false, "Can't find a bias image source");
+            psFree (options);
+        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.BIAS", "BIAS",
+                               PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_BIAS)) {
+            psError(PS_ERR_IO, false, "Can't find a bias image source");
+            psFree(options);
             return NULL;
+        }
+    }
     if (options->doDark) {
+        if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.DARK", "DARK", PM_FPA_FILE_DARK, PM_DETREND_TYPE_DARK)) {
+            psError (PS_ERR_IO, false, "Can't find a dark image source");
+            psFree (options);
+        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.DARK", "DARK",
+                               PM_FPA_FILE_DARK, PM_DETREND_TYPE_DARK)) {
+            psError(PS_ERR_IO, false, "Can't find a dark image source");
+            psFree(options);
             return NULL;
+        }
+    }
     if (options->doMask) {
+        if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.MASK", "MASK", PM_FPA_FILE_MASK, PM_DETREND_TYPE_MASK)) {
+            psError (PS_ERR_IO, false, "Can't find a mask image source");
+            psFree (options);
+            return NULL;
+        }
+        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.MASK", "MASK",
+                               PM_FPA_FILE_MASK, PM_DETREND_TYPE_MASK)) {
+            psError(PS_ERR_IO, false, "Can't find a mask image source");
+            psFree(options);
+            return NULL;
+        }
+#if 0
+        // I think this is now done automatically in the pmFPAfileDefine and pmFPAfileIOChecks -- PAP.
         // XXX have ppImageDefineFile return the pmFPAfile?
         pmFPAfile *mask = psMetadataLookupPtr(&status, config->files, "PPIMAGE.MASK");
         psAssert (mask, "mask not defined?  not possible!");
+        psAssert(mask, "Just defined the mask!");
         // Need to read the names of bit masks from the mask header and set them in the
 …
             // XXX need to load the mask bit names from one of the headers
             // this grabs the first available hdu : no guarantee that it will be valid, though
             pmHDU *hdu = pmHDUGetFirst (mask->fpa);
+            pmHDU *hdu = pmHDUGetFirst(mask->fpa);
             if (!hdu) {
                 psError(PS_ERR_IO, true, "no valid HDU for PPIMAGE.INPUT.MASK");
 …
+            }
             // XXX is this consistent with the pmConfigMaskReadHeader call above?
             if (!pmConfigMaskReadHeader (config, hdu->header)) {
+            if (!pmConfigMaskReadHeader(config, hdu->header)) {
                 psError(PS_ERR_IO, false, "error in mask bits");
                 return NULL;
+            }
+        }
+#endif
+    }
     if (options->doShutter) {
+        if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.SHUTTER", "SHUTTER", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_SHUTTER)) {
+            psError (PS_ERR_IO, false, "Can't find a shutter image source");
+            psFree (options);
+        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.SHUTTER", "SHUTTER",
+                               PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_SHUTTER)) {
+            psError(PS_ERR_IO, false, "Can't find a shutter image source");
+            psFree(options);
             return NULL;
+        }
 …
     if (options->doFlat) {
+        if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.FLAT", "FLAT", PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_FLAT)) {
+            psError (PS_ERR_IO, false, "Can't find a flat image source");
+            psFree (options);
+            return NULL;
+        }
+    }
+    int nThreads = psMetadataLookupS32 (&status, config->arguments, "NTHREADS");
+        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.FLAT", "FLAT",
+                               PM_FPA_FILE_IMAGE, PM_DETREND_TYPE_FLAT)) {
+            psError(PS_ERR_IO, false, "Can't find a flat image source");
+            psFree(options);
+            return NULL;
+        }
+    }
+    int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS");
     if (nThreads > 0) {
         int nScanRows = psMetadataLookupS32 (&status, recipe, "SCAN.ROWS");
         pmDetrendSetThreadTasks (nScanRows);
+        int nScanRows = psMetadataLookupS32(&status, recipe, "SCAN.ROWS");
+        pmDetrendSetThreadTasks(nScanRows);
+    }
 …
 skip_fringe:
     if (options->doFringe) {
+        if (!ppImageDefineFile (config, input->fpa, "PPIMAGE.FRINGE", "FRINGE", PM_FPA_FILE_FRINGE, PM_DETREND_TYPE_FRINGE)) {
+        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.FRINGE", "FRINGE",
+                               PM_FPA_FILE_FRINGE, PM_DETREND_TYPE_FRINGE)) {
             psError (PS_ERR_IO, false, "Can't find a fringe image source");
             return NULL;
 …
         // define associated psphot input/output files
         if (!psphotDefineFiles (config, psphotInput)) {
+            psError(PSPHOT_ERR_CONFIG, false, "Trouble defining the additional input/output files for psphot");
+            psError(PSPHOT_ERR_CONFIG, false,
+                    "Trouble defining the additional input/output files for psphot");
             return false;
+        }
 …
     if (options->doAstromChip || options->doAstromMosaic) {
         if (!options->doPhotom) {
+            psError (PSASTRO_ERR_CONFIG, false, "photometry mode is not selected; it is required for astrometry");
+            psError(PSASTRO_ERR_CONFIG, false,
+                    "Photometry mode is not selected; it is required for astrometry");
             return false;
+        }
         pmFPAfile *psphotOutput = psMetadataLookupPtr (&status, config->files, "PSPHOT.OUTPUT");
         PS_ASSERT (psphotOutput, false);
         pmFPAfile *psastroInput = pmFPAfileDefineInput (config, psphotOutput->fpa, NULL, "PSASTRO.INPUT");
         PS_ASSERT (psastroInput, false);
+        pmFPAfile *psphotOutput = psMetadataLookupPtr(&status, config->files, "PSPHOT.OUTPUT");
+        PS_ASSERT(psphotOutput, false);
+        pmFPAfile *psastroInput = pmFPAfileDefineInput(config, psphotOutput->fpa, NULL, "PSASTRO.INPUT");
+        PS_ASSERT(psastroInput, false);
         psastroInput->mode = PM_FPA_MODE_REFERENCE;
         // define associated psphot input/output files
+        if (!psastroDefineFiles (config, psastroInput)) {
+            psError(PSPHOT_ERR_CONFIG, false, "Trouble defining the additional input/output files for psastro");
+        if (!psastroDefineFiles(config, psastroInput)) {
+            psError(PSPHOT_ERR_CONFIG, false,
+                    "Trouble defining the additional input/output files for psastro");
             return false;
+        }
         // deactivate the psastro files, reactive when needed
         pmFPAfileActivate (config->files, false, "PSASTRO.OUTPUT");
+        pmFPAfileActivate(config->files, false, "PSASTRO.OUTPUT");
+    }
 …
     // outImage is used as a carrier: input to chipImage -> require the data to remain at the CHIP level
     outImage->freeLevel = PS_MIN (outImage->freeLevel, PM_FPA_LEVEL_CHIP);
+    outImage->freeLevel = PS_MIN(outImage->freeLevel, PM_FPA_LEVEL_CHIP);
     outImage->dataLevel = outImage->freeLevel;
     outImage->fileLevel = PS_MIN (outImage->fileLevel, outImage->dataLevel);
+    outImage->fileLevel = PS_MIN(outImage->fileLevel, outImage->dataLevel);
     // outMask and outVariance must be freed at the same level as outImage (all freed by pmFPAFreeData)
 …
     // the input data is the same as the outImage data : force the free levels to match
     input->freeLevel = PS_MIN (outImage->freeLevel, input->freeLevel);
+    input->freeLevel = PS_MIN(outImage->freeLevel, input->freeLevel);
     // define the binned target files (which may just be carriers for some camera configurations)
+    pmFPAfile *bin1 = pmFPAfileDefineFromFPA (config, chipImage->fpa, options->xBin1, options->yBin1, "PPIMAGE.BIN1");
+    pmFPAfile *bin1 = pmFPAfileDefineFromFPA(config, chipImage->fpa, options->xBin1, options->yBin1,
+                                             "PPIMAGE.BIN1");
     if (!bin1) {
         psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.BIN1"));
 …
+    }
+    pmFPAfile *bin2 = pmFPAfileDefineFromFPA (config, chipImage->fpa, options->xBin2, options->yBin2, "PPIMAGE.BIN2");
+    pmFPAfile *bin2 = pmFPAfileDefineFromFPA(config, chipImage->fpa, options->xBin2, options->yBin2,
+                                             "PPIMAGE.BIN2");
     if (!bin2) {
         psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.BIN2"));
 …
     bin2->freeLevel = PM_FPA_LEVEL_FPA;
     pmFPAfile *jpg1 = pmFPAfileDefineOutput (config, byFPA1->fpa, "PPIMAGE.JPEG1");
+    pmFPAfile *jpg1 = pmFPAfileDefineOutput(config, byFPA1->fpa, "PPIMAGE.JPEG1");
     if (!jpg1) {
         psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.JPEG1"));
 …
         return NULL;
+    }
     pmFPAfile *jpg2 = pmFPAfileDefineOutput (config, byFPA2->fpa, "PPIMAGE.JPEG2");
+    pmFPAfile *jpg2 = pmFPAfileDefineOutput(config, byFPA2->fpa, "PPIMAGE.JPEG2");
     if (!jpg2) {
         psError(PS_ERR_IO, false, _("Unable to generate new file from PPIMAGE.JPEG2"));
 …
     // Chip selection: turn on only the chips specified (pass status to suppress missing-key log msg)
     char *chipLine = psMetadataLookupStr(&status, config->arguments, "CHIP_SELECTIONS");
     psArray *chips = psStringSplitArray (chipLine, ",", false);
+    psArray *chips = psStringSplitArray(chipLine, ",", false);
     if (chips->n > 0) {
         pmFPASelectChip (input->fpa, -1, true); // deselect all chips
 …
     if (psTraceGetLevel("ppImage.config") > 0) {
         // Get a look inside all the files.
         psMetadataIterator *filesIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD, NULL); // Iterator
+        psMetadataIterator *filesIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD, NULL);
         psMetadataItem *item;               // Item from iteration
         fprintf(stderr, "Files:\n");

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageReplaceBackground.c

r21183	r23352
101	101	}
102	102	}
103		psImageBinning *binning = psMetadataLookupPtr(&status, psphotRecipe, "PSPHOT.BACKGROUND.BINNING"); // Binning for model
	103	psImageBinning *binning = psMetadataLookupPtr(&status, modelRO->analysis,
	104	"PSPHOT.BACKGROUND.BINNING"); // Binning for model
104	105	if (!binning) {
105	106	psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find background binning");

branches/cnb_branches/cnb_branch_20090301/ppImage/src/ppImageVersion.c

-              r14000
+              r23352
 #include "ppImage.h"
+static const char *cvsTag = "$Name: not supported by cvs2svn $";// CVS tag name
+#ifndef PPIMAGE_VERSION
+#error "PPIMAGE_VERSION is not set"
+#endif
+#ifndef PPIMAGE_BRANCH
+#error "PPIMAGE_BRANCH is not set"
+#endif
+#ifndef PPIMAGE_SOURCE
+#error "PPIMAGE_SOURCE is not set"
+#endif
+#define xstr(s) str(s)
+#define str(s) #s
 psString ppImageVersion(void)
+{
+    psString version = NULL;            // Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", xstr(PPIMAGE_BRANCH), xstr(PPIMAGE_VERSION));
+    return value;
+}
+psString ppImageSource(void)
+{
+    return psStringCopy (xstr(PPIMAGE_SOURCE));
+}
 psString ppImageVersionLong(void)
+{
+    psString version = ppImageVersion(); // Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag
+    psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__);
+    psFree(tag);
+    psString version = ppImageVersion();  // Version, to return
+    psString source = ppImageSource(); // Source
+    psStringPrepend(&version, "ppImage ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
     return version;
+};
+bool ppImageVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "ppImage at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+    psphotVersionHeader(header);
+    psastroVersionHeader(header);
+    ppStatsVersionHeader(header);
+    psString version = ppImageVersion(); // ppImage software version
+    psString source  = ppImageSource();  // ppImage software source
+    psStringPrepend(&version, "ppImage version: ");
+    psStringPrepend(&source, "ppImage source: ");
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+    psFree(version);
+    psFree(source);
+    return true;
+}
 void ppImageVersionMetadata(psMetadata *metadata)
+void ppImageVersionPrint(void)
+{
+    PS_ASSERT_METADATA_NON_NULL(metadata,);
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("ppImage", PS_LOG_INFO, "ppImage at %s", timeString);
+    psFree(timeString);
     psString pslib = psLibVersionLong();// psLib version
 …
     psString ppImage = ppImageVersionLong(); // ppImage version
     psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
     psString timeString = psTimeToISO(time); // The time in an ISO string
     psFree(time);
     psString head = NULL;               // Head string
     psStringAppend(&head, "ppImage processing at %s. Component information:", timeString);
     psFree(timeString);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", psphot);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", psastro);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", ppStats);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", ppImage);
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, head, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, pslib, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psmodules, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psphot, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psastro, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppStats, "");
-    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppImage, "");
-    psFree(head);
     psFree(pslib);
     psFree(psmodules);

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/Makefile.am

-              r19586
+              r23352
 bin_PROGRAMS = ppMerge
+ppMerge_CFLAGS = $(PPMERGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
+# PPMERGE_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+# PPMERGE_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+# PPMERGE_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+#
+# # Force recompilation of ppMergeVersion.c, since it gets the version information
+# ppMergeVersion.c: FORCE
+#       touch ppMergeVersion.c
+# FORCE: ;
+ppMerge_CFLAGS = $(PPMERGE_CFLAGS) $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPPMERGE_VERSION=$(SVN_VERSION) -DPPMERGE_BRANCH=$(SVN_BRANCH) -DPPMERGE_SOURCE=$(SVN_SOURCE)
 ppMerge_LDFLAGS = $(PPMERGE_LIBS) $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
 …
         ppMergeLoop_Threaded.c  \
         ppMergeSetThreads.c     \
+        ppMergeMask.c
+        ppMergeMask.c           \
+        ppMergeVersion.c
 #       ppMergeLoop.c

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMerge.c

r21244	r23352
38	38	goto die;
39	39	}
	40
	41	ppMergeVersionPrint();
40	42
41	43	ppMergeType type = psMetadataLookupS32(NULL, config->arguments, "TYPE"); ///< Type of frame

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMerge.h

-              r21244
+              r23352
 /// @{
 #define TIMERNAME "ppMerge"             ///< Name for timer
 #define PPMERGE_RECIPE "PPMERGE"        ///< Recipe name
 #define THREADED 1                      ///< Compile with threads?
+#define TIMERNAME "ppMerge"             ///< Name for timer
+#define PPMERGE_RECIPE "PPMERGE"        ///< Recipe name
+#define THREADED 1                      ///< Compile with threads?
 /**
 …
  */
 typedef enum {
     PPMERGE_TYPE_UNKNOWN,               ///< Unknown type
     PPMERGE_TYPE_BIAS,                  ///< Bias frame
     PPMERGE_TYPE_DARK,                  ///< (Multi-)Dark frame
     PPMERGE_TYPE_MASK,                  ///< Mask frame
     PPMERGE_TYPE_SHUTTER,               ///< Shutter frame
+    PPMERGE_TYPE_UNKNOWN,               ///< Unknown type
+    PPMERGE_TYPE_BIAS,                  ///< Bias frame
+    PPMERGE_TYPE_DARK,                  ///< (Multi-)Dark frame
+    PPMERGE_TYPE_MASK,                  ///< Mask frame
+    PPMERGE_TYPE_SHUTTER,               ///< Shutter frame
     PPMERGE_TYPE_FLAT,                  ///< Flat-field frame (dome or sky)
     PPMERGE_TYPE_FRINGE,                ///< Fringe frame
+    PPMERGE_TYPE_FRINGE,                ///< Fringe frame
 } ppMergeType;
 …
  */
 typedef enum {
     PPMERGE_FILES_ALL,                  ///< All files
     PPMERGE_FILES_INPUT,                ///< Input files
     PPMERGE_FILES_OUTPUT                ///< Output files
+    PPMERGE_FILES_ALL,                  ///< All files
+    PPMERGE_FILES_INPUT,                ///< Input files
+    PPMERGE_FILES_OUTPUT                ///< Output files
 } ppMergeFiles;
 …
  */
 typedef struct {
     psArray *readouts;                  ///< Input readouts
     bool read;                          ///< Has the scan been read?
     bool busy;                          ///< Is the scan being processed?
     int firstScan;                      ///< First row of the chunk to be read for this group
     int lastScan;                       ///< Last row of the chunk to be read for this group
+    psArray *readouts;                  ///< Input readouts
+    bool read;                          ///< Has the scan been read?
+    bool busy;                          ///< Is the scan being processed?
+    int firstScan;                      ///< First row of the chunk to be read for this group
+    int lastScan;                       ///< Last row of the chunk to be read for this group
 } ppMergeFileGroup;
 …
  */
 bool ppMergeArguments(int argc, char *argv[], ///< Command-line arguments
                       pmConfig *config  ///< Configuration
+                      pmConfig *config  ///< Configuration
     );
 …
  * Set up camera files
  */
 bool ppMergeCamera(pmConfig *config     ///< Configuration
+bool ppMergeCamera(pmConfig *config     ///< Configuration
     );
 …
  * Measure scale and zero-points
  */
 bool ppMergeScaleZero(pmConfig *config  ///< Configuration
+bool ppMergeScaleZero(pmConfig *config  ///< Configuration
     );
 …
  * Main loop to do the merging
  */
 bool ppMergeLoop(pmConfig *config       ///< Configuration
+bool ppMergeLoop(pmConfig *config       ///< Configuration
     );
 …
  * Main loop for masks
  */
 bool ppMergeMask(pmConfig *config       ///< Configuration
+bool ppMergeMask(pmConfig *config       ///< Configuration
     );
 …
  * Read nominated input file
  */
 bool ppMergeFileReadInput(pmConfig *config, ///< Configuration
                           pmReadout *readout, ///< Readout into which to read
                           int num,      ///< Number of file in sequence
                           int rows      ///< Number of rows to read at once
+bool ppMergeFileReadInput(pmConfig *config, ///< Configuration
+                          pmReadout *readout, ///< Readout into which to read
+                          int num,      ///< Number of file in sequence
+                          int rows      ///< Number of rows to read at once
     );
 …
  * Open nominated input file
  */
 bool ppMergeFileOpenInput(pmConfig *config, ///< Configuration
                           const pmFPAview *view, ///< View to open
                           int num       ///< Number of file in sequence
+bool ppMergeFileOpenInput(pmConfig *config, ///< Configuration
+                          const pmFPAview *view, ///< View to open
+                          int num       ///< Number of file in sequence
     );
 …
 bool ppMergeSetThreads(void);
+/// Return software version
+psString ppMergeVersion(void);
+/// Return software source
+psString ppMergeSource(void);
+/// Return detailed software version information
+psString ppMergeVersionLong(void);
+/// Populate a FITS header with version information
+bool ppMergeVersionHeader(
+    psMetadata *header                  ///< Header to populate
+    );
+/// Print version information
+void ppMergeVersionPrint(void);
 ///@}
 #endif

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeLoop_Threaded.c

-              r21365
+              r23352
     assert(output && output->fpa);
     pmFPA *outFPA = output->fpa;        ///< Output FPA
+    pmHDU *lastHDU = NULL;              // Last HDU that was updated
     int cellNum = 0;                    ///< Index of cell
     if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
 …
                 // No data here
                 continue;
+            }
+            // Update the header
+            {
+                pmHDU *hdu = pmHDUGetHighest(outFPA, outChip, outCell); // HDU for file
+                if (hdu && hdu != lastHDU) {
+                    ppMergeVersionHeader(hdu->header);
+                    lastHDU = hdu;
+                }
+            }

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeMask.c

-              r21365
+              r23352
                       const pmFPAview *view, ///< View to chip
                       bool writeOut,     ///< Write output?
+                      pmHDU **lastHDU,   ///< HDU last updated
                       psRandom *rng,    ///< Random number generator
                       psMetadata *stats ///< Statistics output
 …
                     i, inView->chip, inView->cell);
+            // Update the header
+            {
+                pmHDU *hdu = pmHDUGetHighest(outCell->parent->parent, outCell->parent, outCell); // File HDU
+                if (hdu && hdu != *lastHDU) {
+                    ppMergeVersionHeader(hdu->header);
+                    *lastHDU = hdu;
+                }
+            }
             if (!pmFPAfileIOChecks(config, inView, PM_FPA_BEFORE)) {
                 psFree(inView);
 …
     pmFPAview *view = pmFPAviewAlloc(0); ///< View to component of interest
     psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); ///< Random number generator
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); ///< Random number generator
     psMetadata *stats = NULL;           ///< Statistics for output
 …
     assert(output && output->fpa);
     pmFPA *outFPA = output->fpa;        ///< Output FPA
+    pmHDU *lastHDU = NULL;              // Last HDU updated
     if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
 …
         for (int i = 0; i < iter; i++) {
             if (!mergeMask(config, view, (i == iter - 1), rng, stats)) {
+            if (!mergeMask(config, view, (i == iter - 1), &lastHDU, rng, stats)) {
                 psError(PS_ERR_UNKNOWN, false, "Unable to merge chip %d", view->chip);
                 goto PPMERGE_MASK_ERROR;

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeScaleZero.c

r21365	r23352
51	51	break;
52	52	}
53		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS~~, 0~~); ///< Random number generator
	53	psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); ///< Random number generator
54	54	pmFPAview *view = NULL; ///< View into FPA
55	55

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeVersion.c

-              r21244
+              r23352
 #include "ppMergeVersion.h"
+static const char *cvsTag = "$Name: not supported by cvs2svn $";///< CVS tag name
+#ifndef PPMERGE_VERSION
+#error "PPMERGE_VERSION is not set"
+#endif
+#ifndef PPMERGE_BRANCH
+#error "PPMERGE_BRANCH is not set"
+#endif
+#ifndef PPMERGE_SOURCE
+#error "PPMERGE_SOURCE is not set"
+#endif
+#define xstr(s) str(s)
+#define str(s) #s
 psString ppMergeVersion(void)
+{
+    psString version = NULL;            ///< Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", xstr(PPMERGE_BRANCH), xstr(PPMERGE_VERSION));
+    return value;
+}
+psString ppMergeSource(void)
+{
+    return psStringCopy(xstr(PPMERGE_SOURCE));
+}
 psString ppMergeVersionLong(void)
+{
+    psString version = ppMergeVersion(); ///< Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); ///< CVS tag
+    psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__);
+    psFree(tag);
+    psString version = ppMergeVersion();  // Version, to return
+    psString source = ppMergeSource();    // Source
+    psStringPrepend(&version, "ppMerge ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
     return version;
+};
+bool ppMergeVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "ppMerge at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+    ppStatsVersionHeader(header);
+    psString version = ppMergeVersion(); // Software version
+    psString source  = ppMergeSource();  // Software source
+    psStringPrepend(&version, "ppMerge version: ");
+    psStringPrepend(&source, "ppMerge source: ");
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+    psFree(version);
+    psFree(source);
+    return true;
+}
+void ppMergeVersionMetadata(psMetadata *metadata)
+void ppMergeVersionPrint(void)
+{
+    PS_ASSERT_METADATA_NON_NULL(metadata,);
+    psString pslib = psLibVersionLong();///< psLib version
+    psString psmodules = psModulesVersionLong(); ///< psModules version
+    psString ppStats = ppStatsVersionLong(); ///< ppStats version
+    psString ppMerge = ppMergeVersionLong(); ///< ppMerge version
+    psTime *time = psTimeGetNow(PS_TIME_TAI); ///< The time now
+    psString timeString = psTimeToISO(time); ///< The time in an ISO string
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
     psFree(time);
+    psString head = NULL;               ///< Head string
+    psStringAppend(&head, "ppMerge processing at %s. Component information:", timeString);
+    psLogMsg("ppMerge", PS_LOG_INFO, "ppMerge at %s", timeString);
     psFree(timeString);
+    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, head, "");
+    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, pslib, "");
+    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psmodules, "");
+    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppStats, "");
+    psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppMerge, "");
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString ppStats = ppStatsVersionLong(); // ppStats version
+    psString ppMerge = ppMergeVersionLong(); // ppMerge version
+    psFree(head);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", ppStats);
+    psLogMsg("ppImage", PS_LOG_INFO, "%s", ppMerge);
     psFree(pslib);
     psFree(psmodules);

branches/cnb_branches/cnb_branch_20090301/ppMerge/src/ppMergeVersion.h

-              r21244
+              r23352
 /**
+ * Return software source
+ */
+psString ppMergeSource(void);
+/**
  * Return long version information
  */

branches/cnb_branches/cnb_branch_20090301/ppSim/src/Makefile.am

-              r18011
+              r23352
 bin_PROGRAMS = ppSim ppSimSequence
+ppSim_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PSASTRO_CFLAGS) $(ppSim_CFLAGS)
+# PPSIM_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+# PPSIM_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+# PPSIM_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+#
+# # Force recompilation of ppSimVersion.c, since it gets the version information
+# ppSimVersion.c: FORCE
+#       touch ppSimVersion.c
+# FORCE: ;
+ppSim_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PSASTRO_CFLAGS) $(ppSim_CFLAGS) -DPPSIM_VERSION=$(SVN_VERSION) -DPPSIM_BRANCH=$(SVN_BRANCH) -DPPSIM_SOURCE=$(SVN_SOURCE)
 ppSim_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PSPHOT_LIBS) $(PSASTRO_LIBS)
 ppSim_SOURCES = \
 …
         ppSimMosaicChip.c         \
         ppSimRandomGaussian.c     \
+        ppSimBadPixels.c
+        ppSimBadPixels.c          \
+        ppSimVersion.c
 ppSimSequence_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSASTRO_CFLAGS) $(ppSim_CFLAGS)

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSim.h

-              r21183
+              r23352
 // Compare a value with minimum and maximum values, replacing where required.
 #define COMPARE(VALUE,MIN,MAX) {                \
         if (VALUE < MIN) { MIN = VALUE; }       \
         if (VALUE > MAX) { MAX = VALUE; }       \
+#define COMPARE(VALUE,MIN,MAX) {                \
+        if (VALUE < MIN) { MIN = VALUE; }       \
+        if (VALUE > MAX) { MAX = VALUE; }       \
+    }
 …
 float ppSimMagToFlux (float mag, float zp);
 float ppSimArgToRecipeF32(bool *status,
                           psMetadata *options,    // Target to which to add value
                           const char *recipeName, // Name for value in the recipe
                           psMetadata *arguments,  // Command-line arguments
                           const char *argName    // Argument name in the command-line arguments
+float ppSimArgToRecipeF32(bool *status,
+                          psMetadata *options,    // Target to which to add value
+                          const char *recipeName, // Name for value in the recipe
+                          psMetadata *arguments,  // Command-line arguments
+                          const char *argName    // Argument name in the command-line arguments
     );
 int ppSimArgToRecipeS32(bool *status,
                         psMetadata *options,    // Target to which to add value
                         const char *recipeName, // Name for value in the recipe
                         psMetadata *arguments,  // Command-line arguments
                         const char *argName      // Argument name in the command-line arguments
+                        psMetadata *options,    // Target to which to add value
+                        const char *recipeName, // Name for value in the recipe
+                        psMetadata *arguments,  // Command-line arguments
+                        const char *argName      // Argument name in the command-line arguments
     );
 char *ppSimArgToRecipeStr(bool *status,
                           psMetadata *options,    // Target to which to add value
                           const char *recipeName, // Name for value in the recipe
                           psMetadata *arguments,  // Command-line arguments
                           const char *argName    // Argument name in the command-line arguments
+                          psMetadata *options,    // Target to which to add value
+                          const char *recipeName, // Name for value in the recipe
+                          psMetadata *arguments,  // Command-line arguments
+                          const char *argName    // Argument name in the command-line arguments
     );
 bool ppSimArgToRecipeBool(bool *status,
                           psMetadata *options,    // Target to which to add value
                           const char *recipeName, // Name for value in the recipe
                           psMetadata *arguments,  // Command-line arguments
                           const char *argName       // Argument name in the command-line arguments
+                          psMetadata *options,    // Target to which to add value
+                          const char *recipeName, // Name for value in the recipe
+                          psMetadata *arguments,  // Command-line arguments
+                          const char *argName       // Argument name in the command-line arguments
     );
 …
 bool ppSimDefinePixels (psArray *sources, pmReadout *readout, psMetadata *recipe);
+/// Return software version
+psString ppSimVersion(void);
+/// Return software source
+psString ppSimSource(void);
+/// Return long version information
+psString ppSimVersionLong(void);
 #endif

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimBadPixels.c

r20910	r23352
74	74	}
75	75
76		psRandom *pseudoRNG = psRandomAlloc(PS_RANDOM_TAUS, seed); // Pseudo-random number generator
	76	psRandom *pseudoRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, seed); // Pseudo-random number generator
77	77
78	78	psImage *image = readout->image; // Image of interest

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimLoop.c

r21365	r23352
25	25	psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSIM_RECIPE); // Recipe
26	26
27		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS~~, 0~~); // Random number generator
	27	psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator
28	28
29	29	char *typeStr = psMetadataLookupStr(NULL, recipe, "IMAGE.TYPE"); // Type of image to simulate

branches/cnb_branches/cnb_branch_20090301/ppSim/src/ppSimSequence.c

-              r19315
+              r23352
     if (ppsim_recipe) psStringAppend (&ppSimCommand, " -recipe PPSIM %s", ppsim_recipe);
+    unsigned long seed = psMetadataLookupS32 (&status, config, "RND_SEED");
+    if (!status) seed = 0;
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, seed);
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
     psMetadataItem *item = psMetadataLookup (config, "SEQUENCE");
     if (item == NULL) {
         psLogMsg ("ppSimSequence", PS_LOG_WARN, "missing SEQUENCE description");
         exit (1);
+        exit (PS_EXIT_CONFIG_ERROR);
+    }
 …
+        }
         // determine the camera for the sequence and define the ppSim command
         if (camera == NULL) {
             camera = psMetadataLookupStr (&status, sequence, "CAMERA");
+        }
         psString injectCommandReal = NULL;
         psString ppSimCommandReal = NULL;
         psStringAppend (&injectCommandReal, "%s --camera %s", injectCommand, camera);
         psStringAppend (&ppSimCommandReal, "%s -camera %s", ppSimCommand, camera);
+        // determine the camera for the sequence and define the ppSim command
+        if (camera == NULL) {
+            camera = psMetadataLookupStr (&status, sequence, "CAMERA");
+        }
+        psString injectCommandReal = NULL;
+        psString ppSimCommandReal = NULL;
+        psStringAppend (&injectCommandReal, "%s --camera %s", injectCommand, camera);
+        psStringAppend (&ppSimCommandReal, "%s -camera %s", ppSimCommand, camera);
         if (!strcasecmp (type, "BIAS")) {

branches/cnb_branches/cnb_branch_20090301/ppStack

Property svn:mergeinfo set to (toggle deleted branches)

/branches/cnb_branches/cnb_branch_20090215/ppStack	merged	eligible
/trunk/ppStack	merged	eligible
/branches/cnb_branch_20090215/ppStack	21495-22685
/branches/eam_branches/eam_branch_20090303/ppStack	23158-23228

branches/cnb_branches/cnb_branch_20090301/ppStack/src/Makefile.am

-              r19337
+              r23352
 bin_PROGRAMS = ppStack
+ppStack_CFLAGS  = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PPSTATS_CFLAGS) $(PPSTACK_CFLAGS)
+# PPSTACK_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+# PPSTACK_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+# PPSTACK_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+#
+# # Force recompilation of ppStackVersion.c, since it gets the version information
+# ppStackVersion.c: FORCE
+#       touch ppStackVersion.c
+# FORCE: ;
+ppStack_CFLAGS  = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PPSTATS_CFLAGS) $(PPSTACK_CFLAGS) -DPPSTACK_VERSION=$(SVN_VERSION) -DPPSTACK_BRANCH=$(SVN_BRANCH) -DPPSTACK_SOURCE=$(SVN_SOURCE)
 ppStack_LDFLAGS = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PSPHOT_LIBS)   $(PPSTATS_LIBS)   $(PPSTACK_LIBS)
 …
         ppStackArguments.c      \
         ppStackCamera.c         \
+        ppStackFiles.c          \
         ppStackLoop.c           \
         ppStackPSF.c            \
         ppStackReadout.c        \
-        ppStackPhotometry.c     \
         ppStackVersion.c        \
         ppStackMatch.c          \
         ppStackSources.c        \
+        ppStackThread.c
+        ppStackThread.c         \
+        ppStackOptions.c        \
+        ppStackSetup.c          \
+        ppStackPrepare.c        \
+        ppStackConvolve.c       \
+        ppStackCombineInitial.c \
+        ppStackReject.c         \
+        ppStackCombineFinal.c   \
+        ppStackCleanup.c        \
+        ppStackPhotometry.c     \
+        ppStackFinish.c
 noinst_HEADERS =                \
+        ppStack.h
+        ppStack.h               \
+        ppStackLoop.h           \
+        ppStackOptions.h        \
+        ppStackThread.h
 CLEANFILES = *~

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStack.c

-              r21258
+              r23352
 #include "ppStack.h"
+#include "ppStackLoop.h"
 #define TIMER_NAME "PPSTACK"            // Name of timer
 …
         goto die;
+    }
+    ppStackVersionPrint();
     if (!pmModelClassInit()) {
 …
     pmConfigDone();
     psLibFinalize();
+    pmVisualClose();
     exit(exitValue);
+}

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStack.h

-              r21477
+              r23352
 #ifndef PP_STACK_H
 #define PP_STACK_H
+#ifndef PPSTACK_H
+#define PPSTACK_H
 #define PPSTACK_RECIPE "PPSTACK"        // Name of the recipe
 …
 // Mask values for inputs
 typedef enum {
+    PPSTACK_MASK_MATCH  = 0x01,         // PSF-matching failed
+    PPSTACK_MASK_CHI2   = 0x02,         // Chi^2 too deviant
+    PPSTACK_MASK_REJECT = 0x04,         // Rejection failed
+    PPSTACK_MASK_BAD    = 0x08,         // Bad image (too many pixels rejected)
+    PPSTACK_MASK_CAL    = 0x01,         // Photometric calibration failed
+    PPSTACK_MASK_MATCH  = 0x02,         // PSF-matching failed
+    PPSTACK_MASK_CHI2   = 0x04,         // Chi^2 too deviant
+    PPSTACK_MASK_REJECT = 0x08,         // Rejection failed
+    PPSTACK_MASK_BAD    = 0x10,         // Bad image (too many pixels rejected)
     PPSTACK_MASK_ALL    = 0xff          // All errors
 } ppStackMask;
+// Thread for stacking chunks
+//
+// Each input file contributes a readout, into which is read a chunk from that file
+typedef struct {
+    psArray *readouts;                  // Input readouts to read and stack
+    bool read;                          // Has the scan been read?
+    bool busy;                          // Is the scan being processed?
+    int firstScan;                      // First row of the chunk to be read for this group
+    int lastScan;                       // Last row of the chunk to be read for this group
+} ppStackThread;
+// List of files
+typedef enum {
+    PPSTACK_FILES_PREPARE,              // Files for preparation
+    PPSTACK_FILES_CONVOLVE,             // Files for convolution
+    PPSTACK_FILES_COMBINE,              // Files for combination
+    PPSTACK_FILES_PHOT                  // Files for photometry
+} ppStackFileList;
-// Allocator
-ppStackThread *ppStackThreadAlloc(psArray *readouts // Inputs readouts to read and stack
-    );
-// Data for threads
-typedef struct {
-    psArray *threads;                   // Threads doing stacking
-    int lastScan;                       // Last row that's been read
-    psArray *imageFits;                 // FITS file pointers for images
-    psArray *maskFits;                  // FITS file pointers for masks
-    psArray *varianceFits;                // FITS file pointers for variances
-} ppStackThreadData;
-// Set up thread data
-ppStackThreadData *ppStackThreadDataSetup(const psArray *cells, // Array of input cells
-                                          const psArray *imageNames, // Names of images to read
-                                          const psArray *maskNames, // Names of masks to read
-                                          const psArray *varianceNames, // Names of variance maps to read
-                                          const psArray *covariances, // Covariance matrices (already read)
-                                          const pmConfig *config // Configuration
-    );
-// Read chunk into the first available file thread
-ppStackThread *ppStackThreadRead(bool *status, // Status of read
-                                 ppStackThreadData *stack, // Stacks available for reading
-                                 pmConfig *config, // Configuration
-                                 int numChunk, // Chunk number (only for interest)
-                                 int overlap // Overlap between subsequent scans
-    );
-// Initialise the threads
-void ppStackThreadInit(void);
 // Setup command-line arguments
 …
     );
-// Loop over the inputs, doing the combination
-bool ppStackLoop(pmConfig *config       // Configuration
-    );
 // Determine target PSF for input images
 pmPSF *ppStackPSF(const pmConfig *config, // Configuration
                   int numCols, int numRows, // Size of image
+                  const psArray *psfs   // List of input PSFs
+                  const psArray *psfs,  // List of input PSFs
+                  const psVector *inputMask // Mask for inputs
     );
 …
     );
-// Perform photometry on stack
-bool ppStackPhotometry(pmConfig *config, // Configuration
-                       const pmReadout *readout, // Readout to be photometered
-                       const pmFPAview *view // View to readout
-    );
 // Return software version
 psString ppStackVersion(void);
+/// Return software source
+psString ppStackSource(void);
 // Return long description of software version
 psString ppStackVersionLong(void);
 // Supplement metadata with software version
 void ppStackVersionMetadata(psMetadata *metadata // Metadata to supplement
+// Supplement header with software version
+bool ppStackVersionHeader(psMetadata *header // Header to supplement
     );
+/// Print version information
+void ppStackVersionPrint(void);
 /// Convolve image to match specified seeing
 …
 /// Corrects the source PSF photometry to a common system.  Return the sum of the exposure times.
 float ppStackSourcesTransparency(const psArray *sourceLists, // Sources for each input
+                                 psVector *inputMask, // Indicates bad input
                                  const pmFPAview *view, // View to readout
                                  const pmConfig *config // Configuration
     );
+/// Dump memory debugging information
+void ppStackMemDump(
+    const char *name                    ///< Stage name, for inclusion in the output file name
+    );
+/// Activate/deactivate a list of files
+void ppStackFileActivation(
+    pmConfig *config,                   // Configuration
+    ppStackFileList list,               // Files to turn on/off
+    bool state                          // Activation state
+    );
+// Activate/deactivate a single element for a list
+void ppStackFileActivationSingle(
+    pmConfig *config,                   // Configuration
+    ppStackFileList list,               // Files to turn on/off
+    bool state,                         // Activation state
+    int num                             // Number of file in sequence
+    );
+/// Iterate down the hierarchy, loading files
+///
+/// We can get away with this simplistic treatment of the FPA hierarchy because we're working on skycells.
+pmFPAview *ppStackFilesIterateDown(
+    pmConfig *config                    // Configuration
+    );
+/// Iterate up the hierarchy, writing files
+///
+/// We can get away with this simplistic treatment of the FPA hierarchy because we're working on skycells.
+bool ppStackFilesIterateUp(
+    pmConfig *config                    // Configuration
+    );
+/// Write an image to a FITS file
+bool ppStackWriteImage(
+    const char *name,                   // Name of image
+    psMetadata *header,                 // Header
+    const psImage *image,               // Image
+    pmConfig *config                    // Configuration
+    );
 #endif

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackArguments.c

-              r21525
+              r23352
                       "Play safe with small numbers of pixels to combine?", false);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-radius", 0, "Radius (pixels) for matching sources", NAN);
+    psMetadataAddS32(arguments, PS_LIST_TAIL, "-zp-iter", 0, "Maximum iterations for zero point", 0);
+    psMetadataAddS32(arguments, PS_LIST_TAIL, "-zp-iter-1", 0, "Maximum iterations for zero point; pass 1", 0);
+    psMetadataAddS32(arguments, PS_LIST_TAIL, "-zp-iter-2", 0, "Maximum iterations for zero point; pass 2", 0);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-tol", 0, "Tolerance for zero point iterations", NAN);
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-zp-trans-iter", 0, "Iterations for transparency determination", 0);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-trans-rej", 0, "Rejection threshold for transparency determination", NAN);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-trans-thresh", 0, "Threshold for transparency determination", NAN);
+    psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-rej", 0, "Rejection threshold for stars", NAN);
+    psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-rej-1", 0, "Rejection threshold for stars; pass 1", NAN);
+    psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-rej-2", 0, "Rejection threshold for stars; pass 2", NAN);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-limit", 0, "Limit on star rejection fraction for successful iteration", NAN);
+    psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-sys", 0, "Estimated systematic error", NAN);
+    psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-sys-1", 0, "Estimated systematic error; pass 1", NAN);
+    psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp-star-sys-2", 0, "Estimated systematic error; pass 2", NAN);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-image", 0, "Suffix for temporary images", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-mask", 0, "Suffix for temporary masks", NULL);
 …
                       "Delete temporary files on completion?", false);
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-threads", 0, "Number of threads to use", 0);
     psMetadataAddBool(arguments, PS_LIST_TAIL, "-psphot-visual", 0, "psphot visualisation", 0);
+    psMetadataAddBool(arguments, PS_LIST_TAIL, "-visual", 0, "visualisation", false);
     if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 3) {
 …
     VALUE_ARG_RECIPE_FLOAT("-zp-radius",       "ZP.RADIUS",       F32);
+    VALUE_ARG_RECIPE_INT("-zp-iter",           "ZP.ITER",         S32, 0);
+    VALUE_ARG_RECIPE_INT("-zp-iter-1",         "ZP.ITER.1",       S32, 0);
+    VALUE_ARG_RECIPE_INT("-zp-iter-2",         "ZP.ITER.2",       S32, 0);
     VALUE_ARG_RECIPE_FLOAT("-zp-tol",          "ZP.TOL",          F32);
     VALUE_ARG_RECIPE_INT("-zp-trans-iter",     "ZP.TRANS.ITER",   S32, 0);
     VALUE_ARG_RECIPE_FLOAT("-zp-trans-rej",    "ZP.TRANS.REJ",    F32);
     VALUE_ARG_RECIPE_FLOAT("-zp-trans-thresh", "ZP.TRANS.THRESH", F32);
+    VALUE_ARG_RECIPE_FLOAT("-zp-star-rej",     "ZP.STAR.REJ",     F32);
+    VALUE_ARG_RECIPE_FLOAT("-zp-star-rej-1",   "ZP.STAR.REJ.1",   F32);
+    VALUE_ARG_RECIPE_FLOAT("-zp-star-rej-2",   "ZP.STAR.REJ.2",   F32);
     VALUE_ARG_RECIPE_FLOAT("-zp-star-limit",   "ZP.STAR.LIMIT",   F32);
+    VALUE_ARG_RECIPE_FLOAT("-zp-star-sys",     "ZP.STAR.SYS",     F32);
+    VALUE_ARG_RECIPE_FLOAT("-zp-star-sys-1",   "ZP.STAR.SYS.1",   F32);
+    VALUE_ARG_RECIPE_FLOAT("-zp-star-sys-2",   "ZP.STAR.SYS.2",   F32);
     VALUE_ARG_RECIPE_INT("-psf-instances", "PSF.INSTANCES", S32, 0);
 …
     VALUE_ARG_RECIPE_INT("-psf-order",     "PSF.ORDER",     S32, 0);
     valueArgRecipeStr(arguments, recipe, "-psf-model", "PSF.MODEL", recipe);
+    if (psMetadataLookupBool(NULL, arguments, "-visual")) {
+        pmVisualSetVisual(true);
+    }
     if (psMetadataLookupBool(NULL, arguments, "-photometry") ||
 …
     if (dump_file) {
         pmConfigCamerasCull(config, NULL);
         pmConfigRecipesCull(config, "PPSTACK,PPSUB,PPSTATS,PSPHOT,MASKS");
+        pmConfigRecipesCull(config, "PPSTACK,PPSUB,PPSTATS,PSPHOT,MASKS,JPEG");
         pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSTACK.INPUT"); // Input file

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackLoop.c

-              r21526
+              r23352
 #include <stdio.h>
-#include <unistd.h>
-#include <string.h>
-#include <libgen.h>
 #include <pslib.h>
 #include <psmodules.h>
-#include <ppStats.h>
 #include "ppStack.h"
+//#define TESTING
+#define WCS_TOLERANCE 0.001             // Tolerance for WCS
+#define PIXELS_BUFFER 1024              // Initial size of pixel lists
+// Here follows lists of files for activation/deactivation at various stages.  Each must be NULL-terminated.
+// Files required in preparation for convolution
+static char *prepareFiles[] = { "PPSTACK.INPUT.PSF", "PPSTACK.INPUT.SOURCES", "PPSTACK.TARGET.PSF", NULL };
+// Files required for the convolution
+static char *convolveFiles[] = { "PPSTACK.INPUT", "PPSTACK.INPUT.MASK", "PPSTACK.INPUT.VARIANCE", NULL };
+// Output files for the combination
+static char *combineFiles[] = { "PPSTACK.OUTPUT", "PPSTACK.OUTPUT.MASK", "PPSTACK.OUTPUT.VARIANCE",
+                                "PPSTACK.OUTPUT.JPEG1", "PPSTACK.OUTPUT.JPEG2", NULL };
+// Files for photometry
+static char *photFiles[] = { "PSPHOT.INPUT", "PSPHOT.OUTPUT", "PSPHOT.RESID", "PSPHOT.BACKMDL",
+                             "PSPHOT.BACKMDL.STDEV", "PSPHOT.BACKGND", "PSPHOT.BACKSUB",
+                             "SOURCE.PLOT.MOMENTS", "SOURCE.PLOT.PSFMODEL", "SOURCE.PLOT.APRESID",
+                             "PSPHOT.INPUT.CMF", NULL };
+static void memDump(const char *name)
+{
+    return;
+    static int num = 0;                 // Counter, to make files unique and give an idea of sequence
+    psString filename = NULL;           // Name of file
+    psStringAppend(&filename, "memdump_%s_%03d.txt", name, num);
+    FILE *memFile = fopen(filename, "w");
+    psFree(filename);
+    psMemBlock **leaks = NULL;
+    int numLeaks = psMemCheckLeaks(0, &leaks, NULL, true);
+    fprintf(memFile, "# MemBlock Size Source\n");
+    unsigned long total = 0;            // Total memory used
+    for (int i = 0; i < numLeaks; i++) {
+        psMemBlock *mb = leaks[i];
+        fprintf(memFile, "%12lu\t%12zd\t%s:%d\n", mb->id, mb->userMemorySize,
+                mb->file, mb->lineno);
+        total += mb->userMemorySize;
+    }
+    fclose(memFile);
+    psFree(leaks);
+    fprintf(stderr, "Memdump %s %d: Memory use: %ld, sbrk: %p\n", name, num, total, sbrk(0));
+    num++;
+}
+// Activate/deactivate a list of files
+static void fileActivation(pmConfig *config, // Configuration
+                           char **files, // Files to turn on/off
+                           bool state   // Activation state
+    )
+{
+    assert(config);
+    assert(files);
+    for (int i = 0; files[i] != NULL; i++) {
+        pmFPAfileActivate(config->files, state, files[i]);
+    }
+    return;
+}
+// Activate/deactivate a single element for a list
+static void fileActivationSingle(pmConfig *config, // Configuration
+                                 char **files, // Files to turn on/off
+                                 bool state,   // Activation state
+                                 int num // Number of file in sequence
+                                 )
+{
+    assert(config);
+    assert(files);
+    for (int i = 0; files[i] != NULL; i++) {
+        pmFPAfileActivateSingle(config->files, state, files[i], num); // Activated file
+    }
+    return;
+}
+// Iterate down the hierarchy, loading files; we can get away with this because we're working on skycells
+static pmFPAview *filesIterateDown(pmConfig *config // Configuration
+                                  )
+{
+    assert(config);
+    pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return NULL;
+    }
+    view->chip = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return NULL;
+    }
+    view->cell = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return NULL;
+    }
+    view->readout = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
+        return NULL;
+    }
+    return view;
+}
+// Iterate up the hierarchy, writing files; we can get away with this because we're working on skycells
+static bool filesIterateUp(pmConfig *config // Configuration
+                           )
+{
+    assert(config);
+    pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy
+    view->chip = view->cell = view->readout = 0;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+    view->readout = -1;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+    view->cell = -1;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+    view->chip = -1;
+    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
+        return false;
+    }
+    psFree(view);
+    return true;
+}
+// Write an image to a FITS file
+static bool writeImage(const char *name, // Name of image
+                       psMetadata *header, // Header
+                       const psImage *image, // Image
+                       pmConfig *config // Configuration
+                       )
+{
+    assert(name);
+    assert(image);
+    psString resolved = pmConfigConvertFilename(name, config, true, true); // Resolved file name
+    psFits *fits = psFitsOpen(resolved, "w");
+    if (!fits) {
+        psError(PS_ERR_IO, false, "Unable to open FITS file %s to write image.", resolved);
+        psFree(resolved);
+        return false;
+    }
+    if (!psFitsWriteImage(fits, header, image, 0, NULL)) {
+        psError(PS_ERR_IO, false, "Unable to write FITS image %s.", resolved);
+        psFitsClose(fits);
+        psFree(resolved);
+        return false;
+    }
+    psFitsClose(fits);
+    psFree(resolved);
+    return true;
+}
+#include "ppStackLoop.h"
 bool ppStackLoop(pmConfig *config)
 …
     psTimerStart("PPSTACK_TOTAL");
+    psTimerStart("PPSTACK_STEPS");
+    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe
+    psAssert(recipe, "We've thrown an error on this before.");
+    ppStackOptions *options = ppStackOptionsAlloc(); // Options for stacking
+    bool mdok;                          // Status of MD lookup
+    bool tempDelete = psMetadataLookupBool(&mdok, recipe, "TEMP.DELETE"); // Delete temporary files?
+    const char *tempDir = psMetadataLookupStr(NULL, recipe, "TEMP.DIR"); // Directory for temporary images
+    if (!tempDir) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find TEMP.DIR in recipe");
+    // Setup
+    psTrace("ppStack", 1, "Setup....\n");
+    if (!ppStackSetup(options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to setup.");
+        psFree(options);
+        return false;
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 0: Setup: %f sec", psTimerClear("PPSTACK_STEPS"));
+    ppStackMemDump("setup");
+    // Preparation for stacking
+    psTrace("ppStack", 1, "Preparation for stacking: merging sources, determining target PSF....\n");
+    if (!ppStackPrepare(options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to prepare for stacking.");
+        psFree(options);
+        return false;
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 1: Load Sources and Generate Target PSF: %f sec",
+             psTimerClear("PPSTACK_STEPS"));
+    ppStackMemDump("prepare");
+    // Convolve inputs
+    psTrace("ppStack", 1, "Convolving inputs to target PSF....\n");
+    if (!ppStackConvolve(options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to convolve images.");
+        psFree(options);
+        return false;
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 2: Generate Convolutions and Save: %f sec",
+             psTimerClear("PPSTACK_STEPS"));
+    ppStackMemDump("convolve");
+    // Start threading
+    ppStackThreadInit();
+    ppStackThreadData *stack = ppStackThreadDataSetup(options, config);
+    if (!stack) {
+        psError(PS_ERR_IO, false, "Unable to initialise stack threads.");
+        psFree(options);
         return false;
+    }
+    psString outputName = psStringCopy(psMetadataLookupStr(NULL, config->arguments, "OUTPUT")); // Name for temporary files
+    const char *tempName = basename(outputName);
+    if (!tempName) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to construct basename for temporary files.");
+    // Initial combination
+    psTrace("ppStack", 1, "Initial stack of convolved images....\n");
+    if (!ppStackCombineInitial(stack, options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to perform initial combination.");
+        psFree(stack);
+        psFree(options);
         return false;
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 2: Generate Convolutions and Save: %f sec",
+             psTimerClear("PPSTACK_STEPS"));
+    ppStackMemDump("convolve");
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 4: Make Initial Stack: %f sec", psTimerClear("PPSTACK_STEPS"));
+    const char *tempImage = psMetadataLookupStr(NULL, recipe, "TEMP.IMAGE"); // Suffix for temporary images
+    const char *tempMask = psMetadataLookupStr(NULL, recipe, "TEMP.MASK"); // Suffix for temporary masks
+    const char *tempVariance = psMetadataLookupStr(NULL, recipe, "TEMP.VARIANCE"); // Suffix for temp variance maps
+    if (!tempImage || !tempMask || !tempVariance) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, false,
+                "Unable to find TEMP.IMAGE, TEMP.MASK and TEMP.VARIANCE in recipe");
+    // Pixel rejection
+    psTrace("ppStack", 1, "Reject pixels....\n");
+    if (!ppStackReject(options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to reject pixels.");
+        psFree(stack);
+        psFree(options);
         return false;
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 5: Pixel Rejection: %f sec", psTimerClear("PPSTACK_STEPS"));
+    ppStackMemDump("reject");
-    float threshold = psMetadataLookupF32(NULL, recipe, "THRESHOLD.MASK"); // Threshold for mask deconvolution
-    float imageRej = psMetadataLookupF32(NULL, recipe, "IMAGE.REJ"); // Maximum fraction of image to reject
-                                                                     // before rejecting entire image
-    float poorFrac = psMetadataLookupF32(&mdok, recipe, "POOR.FRACTION"); // Fraction for "poor"
-    const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics
-    psMetadata *stats = NULL;           // Container for statistics
-    FILE *statsFile = NULL;             // File stream for statistics
-    if (statsName && strlen(statsName) > 0) {
-        psString resolved = pmConfigConvertFilename(statsName, config, true, true); // Resolved filename
-        statsFile = fopen(resolved, "w");
-        if (!statsFile) {
-            psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved);
-            psFree(resolved);
-            return false;
-        } else {
-            stats = psMetadataAlloc();
+        }
-        psFree(resolved);
+    }
-    pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSTACK.OUTPUT"); // Output file
-    if (!output) {
-        psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find output data!");
-        return false;
+    }
-    int num = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of inputs
-    psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe
-    int overlap = 2 * psMetadataLookupS32(NULL, ppsub,
-                                          "KERNEL.SIZE"); // Overlap by kernel size between consecutive scans
-    if (!pmConfigMaskSetBits(NULL, NULL, config)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to determine mask value.");
-        return false;
+    }
-    memDump("start");
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 0 : Initialization and Configuration : %f sec", psTimerClear("PPSTACK_STEPS"));
-    // Preparation iteration: Load the sources, and get a target PSF model
-    psTrace("ppStack", 1, "Determining target PSF....\n");
-    psArray *sourceLists = psArrayAlloc(num); // Individual lists of sources for matching
-    pmPSF *targetPSF = NULL;            // Target PSF
-    float sumExposure = NAN;            // Sum of exposure times
-    if (psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) {
-        pmFPAfileActivate(config->files, false, NULL);
-        fileActivation(config, prepareFiles, true);
-        pmFPAview *view = filesIterateDown(config);
-        if (!view) {
-            return false;
+        }
-        // Generate list of PSFs
-        psMetadataIterator *fileIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD,
-                                                               "^PPSTACK.INPUT$"); // Iterator
-        psMetadataItem *fileItem; // Item from iteration
-        psArray *psfs = psArrayAlloc(num); // PSFs for PSF envelope
-        int numCols = 0, numRows = 0;   // Size of image
-        int index = 0;                  // Index for file
-        while ((fileItem = psMetadataGetAndIncrement(fileIter))) {
-            assert(fileItem->type == PS_DATA_UNKNOWN);
-            pmFPAfile *inputFile = fileItem->data.V; // An input file
-            pmChip *chip = pmFPAviewThisChip(view, inputFile->fpa); // The chip: holds the PSF
-            pmPSF *psf = psMetadataLookupPtr(NULL, chip->analysis, "PSPHOT.PSF"); // PSF
-            if (!psf) {
-                psError(PS_ERR_UNKNOWN, false, "Unable to find PSF.");
-                psFree(view);
-                psFree(sourceLists);
-                psFree(fileIter);
-                psFree(psfs);
-                return false;
+            }
-            psfs->data[index] = psMemIncrRefCounter(psf);
-            psMetadataRemoveKey(chip->analysis, "PSPHOT.PSF");
-            pmCell *cell = pmFPAviewThisCell(view, inputFile->fpa); // Cell of interest
-            pmHDU *hdu = pmHDUFromCell(cell);
-            assert(hdu && hdu->header);
-            int naxis1 = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); // Number of columns
-            int naxis2 = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); // Number of rows
-            if (naxis1 <= 0 || naxis2 <= 0) {
-                psError(PS_ERR_UNKNOWN, false, "Unable to determine size of image from PSF.");
-                psFree(view);
-                psFree(sourceLists);
-                psFree(fileIter);
-                psFree(psfs);
-                return false;
+            }
-            if (numCols == 0 && numRows == 0) {
-                numCols = naxis1;
-                numRows = naxis2;
+            }
-            pmReadout *ro = pmFPAviewThisReadout(view, inputFile->fpa); // Readout with sources
-            psArray *sources = psMetadataLookupPtr(NULL, ro->analysis, "PSPHOT.SOURCES"); // Sources
-            if (!sources) {
-                psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find sources in readout.");
-                return NULL;
+            }
-            sourceLists->data[index] = psMemIncrRefCounter(sources);
-            // Re-do photometry if we don't trust the source lists
-            if (psMetadataLookupBool(NULL, recipe, "PHOT")) {
-                psTrace("ppStack", 2, "Photometering input %d of %d....\n", index, num);
-                pmFPAfileActivate(config->files, false, NULL);
-                fileActivationSingle(config, convolveFiles, true, index);
-                pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", index); // File
-                pmFPAview *view = filesIterateDown(config);
-                if (!view) {
-                    psFree(sourceLists);
-                    psFree(targetPSF);
-                    return false;
+                }
-                pmReadout *ro = pmFPAviewThisReadout(view, file->fpa); // Readout of interest
-                if (!ppStackInputPhotometry(ro, sources, config)) {
-                    psError(PS_ERR_UNKNOWN, false, "Unable to do photometry on input sources");
-                    psFree(sourceLists);
-                    psFree(targetPSF);
-                    return false;
+                }
-                psFree(view);
-                if (!filesIterateUp(config)) {
-                    psFree(sourceLists);
-                    psFree(targetPSF);
-                    return false;
+                }
-                pmFPAfileActivate(config->files, false, NULL);
-                fileActivation(config, prepareFiles, true);
+            }
-            index++;
+        }
-        psFree(fileIter);
-        // Zero point calibration
-        sumExposure = ppStackSourcesTransparency(sourceLists, view, config);
-        if (!isfinite(sumExposure) || sumExposure <= 0) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to calculate transparency differences");
-            psFree(sourceLists);
-            psFree(targetPSF);
-            return false;
+        }
-        // Generate target PSF
-        targetPSF = ppStackPSF(config, numCols, numRows, psfs);
-        psFree(psfs);
-        if (!targetPSF) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to determine output PSF.");
-            psFree(sourceLists);
-            psFree(view);
-            return false;
+        }
-        psMetadataAddPtr(config->arguments, PS_LIST_TAIL, "PSF.TARGET", PS_DATA_UNKNOWN,
-                         "Target PSF for stack", targetPSF);
-        pmChip *outChip = pmFPAviewThisChip(view, output->fpa); // Output chip
-        psMetadataAddPtr(outChip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN,
-                         "Target PSF", targetPSF);
-        outChip->data_exists = true;
-        psFree(view);
-        if (!filesIterateUp(config)) {
-            return false;
+        }
+    }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 1 : Load Sources and Generate Target PSF : %f sec", psTimerClear("PPSTACK_STEPS"));
-    memDump("psf");
-    psArray *imageNames = psArrayAlloc(num);
-    psArray *maskNames = psArrayAlloc(num);
-    psArray *varianceNames = psArrayAlloc(num);
-    for (int i = 0; i < num; i++) {
-        psString imageName = NULL, maskName = NULL, varianceName = NULL; // Names for convolved images
-        psStringAppend(&imageName, "%s/%s.%d.%s", tempDir, tempName, i, tempImage);
-        psStringAppend(&maskName, "%s/%s.%d.%s", tempDir, tempName, i, tempMask);
-        psStringAppend(&varianceName, "%s/%s.%d.%s", tempDir, tempName, i, tempVariance);
-        psTrace("ppStack", 5, "Temporary files: %s %s %s\n", imageName, maskName, varianceName);
-        imageNames->data[i] = imageName;
-        maskNames->data[i] = maskName;
-        varianceNames->data[i] = varianceName;
+    }
-    // Free the outputName that we grabbed above to set up tempName.
-    psFree(outputName);
-    // Generate convolutions and write them to disk
-    psTrace("ppStack", 1, "Convolving inputs to target PSF....\n");
-    psArray *cells = psArrayAlloc(num); // Cells for convolved images --- a handle for reading again
-    psArray *subKernels = psArrayAlloc(num); // Subtraction kernels --- required in the stacking
-    psArray *subRegions = psArrayAlloc(num); // Subtraction regions --- required in the stacking
-    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator
-    int numGood = 0;                    // Number of good frames
-    int numCols = 0, numRows = 0;       // Size of image
-    psVector *inputMask = psVectorAlloc(num, PS_TYPE_VECTOR_MASK); // Mask for inputs
-    psVectorInit(inputMask, 0);
-    psVector *matchChi2 = psVectorAlloc(num, PS_TYPE_F32); // chi^2 for stamps when matching
-    psVectorInit(matchChi2, NAN);
-    psVector *weightings = psVectorAlloc(num, PS_TYPE_F32); // Combination weightings for images (1/noise^2)
-    psVectorInit(weightings, NAN);
-    psList *fpaList = psListAlloc(NULL); // List of input FPAs, for concept averaging
-    psList *cellList = psListAlloc(NULL); // List of input cells, for concept averaging
-    psArray *covariances = psArrayAlloc(num); // Covariance matrices
-    for (int i = 0; i < num; i++) {
-        psTrace("ppStack", 2, "Convolving input %d of %d to target PSF....\n", i, num);
-        pmFPAfileActivate(config->files, false, NULL);
-        fileActivationSingle(config, convolveFiles, true, i);
-        pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File of interest
-        pmFPAview *view = filesIterateDown(config);
-        if (!view) {
-            psFree(sourceLists);
-            psFree(targetPSF);
-            psFree(rng);
-            psFree(inputMask);
-            psFree(matchChi2);
-            psFree(fpaList);
-            psFree(cellList);
-            psFree(covariances);
-            return false;
+        }
-        pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); // Input readout
-        psFree(view);
-        if (numCols == 0 && numRows == 0) {
-            numCols = readout->image->numCols;
-            numRows = readout->image->numRows;
-        } else if (numCols != readout->image->numCols || numRows != readout->image->numRows) {
-            psError(PS_ERR_UNKNOWN, true, "Sizes of input images don't match: %dx%d vs %dx%d",
-                    readout->image->numCols, readout->image->numRows, numCols, numRows);
-            psFree(sourceLists);
-            psFree(targetPSF);
-            psFree(rng);
-            psFree(inputMask);
-            psFree(matchChi2);
-            psFree(fpaList);
-            psFree(cellList);
-            psFree(covariances);
-            return false;
+        }
-        // Background subtraction, scaling and normalisation is performed automatically by the image matching
-        psArray *regions = NULL, *kernels = NULL; // Regions and kernels used in subtraction
-        psTimerStart("PPSTACK_MATCH");
-        if (!ppStackMatch(readout, &regions, &kernels, &matchChi2->data.F32[i], &weightings->data.F32[i],
-                          sourceLists->data[i], targetPSF, rng, config)) {
-            psErrorStackPrint(stderr, "Unable to match image %d --- ignoring.", i);
-            inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_MATCH;
-            psErrorClear();
-            continue;
+        }
-        covariances->data[i] = psMemIncrRefCounter(readout->covariance);
-        if (stats) {
-            pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, readout->analysis,
-                                                                PM_SUBTRACTION_ANALYSIS_KERNEL);// Conv kernel
-            psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_MATCH", PS_META_DUPLICATE_OK,
-                             "Time to match PSF", psTimerMark("PPSTACK_MATCH"));
-            psMetadataAddF32(stats, PS_LIST_TAIL, "STAMP.MEAN", PS_META_DUPLICATE_OK,
-                             "Mean deviation for stamps", kernels->mean);
-            psMetadataAddF32(stats, PS_LIST_TAIL, "STAMP.RMS", PS_META_DUPLICATE_OK,
-                             "RMS deviation for stamps", kernels->rms);
-            psMetadataAddF32(stats, PS_LIST_TAIL, "STAMP.NUM", PS_META_DUPLICATE_OK,
-                             "Number of stamps", kernels->numStamps);
-            float deconv = psMetadataLookupF32(NULL, readout->analysis,
-                                               PM_SUBTRACTION_ANALYSIS_DECONV_MAX); // Deconvolution fraction
-            psMetadataAddF32(stats, PS_LIST_TAIL, "KERNEL.DECONV", PS_META_DUPLICATE_OK,
-                             "Deconvolution fraction for kernel", deconv);
-            psMetadataAddF32(stats, PS_LIST_TAIL, "PPSTACK.WEIGHTING", PS_META_DUPLICATE_OK,
-                             "Weighting for image", weightings->data.F32[i]);
+        }
-        psLogMsg("ppStack", PS_LOG_INFO, "Time to match image %d: %f sec", i, psTimerClear("PPSTACK_MATCH"));
-        subRegions->data[i] = regions;
-        subKernels->data[i] = kernels;
-        // Write the temporary convolved files
-        pmHDU *hdu = readout->parent->parent->parent->hdu; // HDU for convolved image
-        assert(hdu);
-        writeImage(imageNames->data[i], hdu->header, readout->image, config);
-        psMetadata *maskHeader = psMetadataCopy(NULL, hdu->header); // Copy of header, for mask
-        pmConfigMaskWriteHeader(config, maskHeader);
-        writeImage(maskNames->data[i], maskHeader, readout->mask, config);
-        psFree(maskHeader);
-        psImageCovarianceTransfer(readout->variance, readout->covariance);
-        writeImage(varianceNames->data[i], hdu->header, readout->variance, config);
-#ifdef TESTING
+        {
-            psString name = NULL;
-            psStringAppend(&name, "covariance_%d.fits", i);
-            writeImage(name, hdu->header, readout->covariance->image, config);
-            psFree(name);
+        }
-#endif
-        pmCell *inCell = readout->parent; // Input cell
-        psListAdd(cellList, PS_LIST_TAIL, inCell);
-        psListAdd(fpaList, PS_LIST_TAIL, inCell->parent->parent);
-        cells->data[i] = psMemIncrRefCounter(inCell);
-        if (!filesIterateUp(config)) {
-            psFree(fpaList);
-            psFree(cellList);
-            psFree(covariances);
-            return false;
+        }
-        numGood++;
-        memDump("match");
+    }
-    psFree(sourceLists);
-    psFree(targetPSF);
-    psFree(rng);
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 2 : Generate Convolutions and Save : %f sec", psTimerClear("PPSTACK_STEPS"));
-    if (numGood == 0) {
-        psError(PS_ERR_UNKNOWN, false, "No good images to combine.");
-        psFree(subKernels);
-        psFree(subRegions);
-        psFree(cells);
-        psFree(inputMask);
-        psFree(matchChi2);
-        psFree(fpaList);
-        psFree(cellList);
-        psFree(covariances);
-        return false;
+    }
-    // Update concepts for output
+    {
-        pmFPAview view;                 // View for output
-        view.chip = view.cell = view.readout = 0;
-        pmCell *outCell = pmFPAfileThisCell(config->files, &view, "PPSTACK.OUTPUT"); // Output cell
-        pmFPA *outFPA = outCell->parent->parent; // Output FPA
-        pmConceptsAverageFPAs(outFPA, fpaList);
-        pmConceptsAverageCells(outCell, cellList, NULL, NULL, true);
-        psFree(fpaList);
-        psFree(cellList);
-        // Update the value of a concept
-#define UPDATE_CONCEPT(SOURCE, NAME, VALUE) { \
-            psMetadataItem *item = psMetadataLookup(SOURCE->concepts, NAME); \
-            psAssert(item, "Concept should be present"); \
-            psAssert(item->type == PS_TYPE_F32, "Concept should be F32"); \
-            item->data.F32 = VALUE; \
+        }
-        UPDATE_CONCEPT(outFPA,  "FPA.EXPOSURE",  sumExposure);
-        UPDATE_CONCEPT(outCell, "CELL.EXPOSURE", sumExposure);
-        UPDATE_CONCEPT(outCell, "CELL.DARKTIME", NAN);
+    }
-    // Reject images out-of-hand on the basis of their match chi^2
+    {
-        psVector *values = psVectorAllocEmpty(num, PS_TYPE_F32); // Values to sort
-        for (int i = 0; i < num; i++) {
-            if (inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_ALL) {
-                continue;
+            }
-            values->data.F32[values->n++] = matchChi2->data.F32[i];
+        }
-        assert(values->n == numGood);
-        if (!psVectorSortInPlace(values)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to sort vector.");
-            psFree(subKernels);
-            psFree(subRegions);
-            psFree(cells);
-            psFree(inputMask);
-            psFree(matchChi2);
-            psFree(values);
-            psFree(covariances);
-            return false;
+        }
-        float median = numGood % 2 ? values->data.F32[numGood / 2] :
-.5 * (values->data.F32[numGood / 2 - 1] + values->data.F32[numGood / 2]);
-        float rms = 0.74 * (values->data.F32[numGood * 3 / 4] -
-                            values->data.F32[numGood / 4]); // Estimated RMS from interquartile range
-        psFree(values);
-        float rej = psMetadataLookupF32(NULL, recipe, "MATCH.REJ"); // Rejection threshold (stdevs)
-        if (isfinite(rej)) {
-            float thresh = median + rej * rms; // Threshold for rejection
-            psLogMsg("ppStack", PS_LOG_INFO, "chi^2 rejection threshold = %f + %f * %f = %f",
-                     median, rej, rms, thresh);
-            int numRej = 0;                 // Number rejected
-            numGood = 0;                    // Number of good images
-            for (int i = 0; i < num; i++) {
-              if (inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] & PPSTACK_MASK_ALL) {
-                    continue;
+                }
-                if (matchChi2->data.F32[i] > thresh) {
-                    numRej++;
-                    inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] |= PPSTACK_MASK_CHI2;
-                    psLogMsg("ppStack", PS_LOG_INFO, "Rejecting image %d because of large matching chi^2: %f",
-                             i, matchChi2->data.F32[i]);
-                } else {
-                    psLogMsg("ppStack", PS_LOG_INFO, "Image %d has matching chi^2: %f",
-                             i, matchChi2->data.F32[i]);
-                    numGood++;
+                }
+            }
+        }
+    }
-    if (numGood == 0) {
-        psError(PS_ERR_UNKNOWN, false, "No good images to combine.");
-        psFree(subKernels);
-        psFree(subRegions);
-        psFree(cells);
-        psFree(inputMask);
-        psFree(matchChi2);
-        psFree(covariances);
-        return false;
+    }
-#ifdef TESTING
-    psTraceSetLevel("psModules.imcombine", 7);
-#endif
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 3 : Basic Image Rejection  : %f sec", psTimerClear("PPSTACK_STEPS"));
-    // Start threading
-    ppStackThreadInit();
-    ppStackThreadData *stack = ppStackThreadDataSetup(cells, imageNames, maskNames, varianceNames,
-                                                      covariances, config);
-    if (!stack) {
-        psError(PS_ERR_IO, false, "Unable to initialise stack threads.");
-        psFree(subKernels);
-        psFree(subRegions);
-        psFree(inputMask);
-        psFree(matchChi2);
-        psFree(cells);
-        psFree(covariances);
-        return false;
+    }
-    psTimerStart("PPSTACK_INITIAL");
-    memDump("preinitial");
-    // Stack the convolved files
-    psTrace("ppStack", 1, "Initial stack of convolved images....\n");
-    pmReadout *outRO = NULL;            // Output readout
-    pmFPAview *view = NULL;             // View to readout
-    psArray *inspect = NULL;            // Array of arrays of pixels to inspect
+    {
-        int row0, col0;                 // Offset for readout
-        int numCols, numRows;           // Size of readout
-        ppStackThread *thread = stack->threads->data[0]; // Representative thread
-        if (!pmReadoutStackSetOutputSize(&col0, &row0, &numCols, &numRows, thread->readouts)) {
-            psError(PS_ERR_UNKNOWN, false, "problem setting output readout size.");
-            psFree(subKernels);
-            psFree(subRegions);
-            psFree(stack);
-            psFree(inputMask);
-            psFree(matchChi2);
-            psFree(cells);
-            psFree(covariances);
-            return false;
+        }
-        pmFPAfileActivate(config->files, false, NULL);
-        fileActivation(config, combineFiles, true);
-        view = filesIterateDown(config);
-        if (!view) {
-            psFree(subKernels);
-            psFree(subRegions);
-            psFree(stack);
-            psFree(inputMask);
-            psFree(matchChi2);
-            psFree(cells);
-            psFree(covariances);
-            return false;
+        }
-        pmCell *outCell = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT"); // Output cell
-        outRO = pmReadoutAlloc(outCell); // Output readout
-        psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits to mask for bad
-        psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels
-        if (!pmReadoutStackDefineOutput(outRO, col0, row0, numCols, numRows, true, true, maskBad)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to prepare output.");
-            psFree(subKernels);
-            psFree(subRegions);
-            psFree(stack);
-            psFree(inputMask);
-            psFree(matchChi2);
-            psFree(view);
-            psFree(outRO);
-            psFree(cells);
-            psFree(covariances);
-            return false;
+        }
-        psFree(cells);
-        bool status;                    // Status of read
-        int numChunk;                   // Number of chunks
-        for (numChunk = 0; true; numChunk++) {
-            ppStackThread *thread = ppStackThreadRead(&status, stack, config, numChunk, overlap);
-            if (!status) {
-                // Something went wrong
-                psError(PS_ERR_UNKNOWN, false, "Unable to read chunk %d", numChunk);
-                psFree(subKernels);
-                psFree(subRegions);
-                psFree(stack);
-                psFree(inputMask);
-                psFree(matchChi2);
-                psFree(view);
-                psFree(outRO);
-                psFree(covariances);
-                return false;
+            }
-            if (!thread) {
-                // Nothing more to read
-                break;
+            }
-            // call: ppStackReadoutInitial(config, outRO, readouts, subRegions, subKernels)
-            psThreadJob *job = psThreadJobAlloc("PPSTACK_INITIAL_COMBINE"); // Job to start
-            psArrayAdd(job->args, 1, thread);
-            psArrayAdd(job->args, 1, config);
-            psArrayAdd(job->args, 1, outRO);
-            psArrayAdd(job->args, 1, inputMask);
-            psArrayAdd(job->args, 1, weightings);
-            psArrayAdd(job->args, 1, matchChi2);
-            if (!psThreadJobAddPending(job)) {
-                psFree(job);
-                psFree(subKernels);
-                psFree(subRegions);
-                psFree(stack);
-                psFree(inputMask);
-                psFree(weightings);
-                psFree(matchChi2);
-                psFree(view);
-                psFree(outRO);
-                psFree(covariances);
-                return false;
+            }
-            psFree(job);
+        }
-        if (!psThreadPoolWait(false)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to do initial combination.");
-            psFree(subKernels);
-            psFree(subRegions);
-            psFree(stack);
-            psFree(inputMask);
-            psFree(matchChi2);
-            psFree(view);
-            psFree(outRO);
-            psFree(covariances);
-            return false;
+        }
-        // Harvest the jobs, gathering the inspection lists
-        inspect = psArrayAlloc(num);
-        for (int i = 0; i < num; i++) {
-            if (inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) {
-                continue;
+            }
-            inspect->data[i] = psArrayAllocEmpty(numChunk);
+        }
-        psThreadJob *job;               // Completed job
-        while ((job = psThreadJobGetDone())) {
-            psAssert(strcmp(job->type, "PPSTACK_INITIAL_COMBINE") == 0,
-                     "Job has incorrect type: %s", job->type);
-            psArray *results = job->results; // Results of job
-            for (int i = 0; i < num; i++) {
-                if (inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) {
-                    continue;
+                }
-                inspect->data[i] = psArrayAdd(inspect->data[i], 1, results->data[i]);
+            }
-            psFree(job);
+        }
-        memDump("initial");
+    }
-#ifdef TESTING
-    writeImage("combined_initial.fits", NULL, outRO->image, config);
-#endif
-    if (stats) {
-        psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_INITIAL", 0,
-                         "Time to make initial stack", psTimerMark("PPSTACK_INITIAL"));
+    }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 4 : Make Initial Stack : %f sec", psTimerClear("PPSTACK_STEPS"));
-    psLogMsg("ppStack", PS_LOG_INFO, "Time to make initial stack: %f sec", psTimerClear("PPSTACK_INITIAL"));
-    psTimerStart("PPSTACK_REJECT");
-    // Pixel rejection
-    psArray *rejected = psArrayAlloc(num);
+    {
-        int numRejected = 0;        // Number of inputs rejected completely
-        // Count images rejected out of hand
-        for (int i = 0; i < num; i++) {
-            if (inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) {
-                numRejected++;
+            }
+        }
-        for (int i = 0; i < num; i++) {
-            if (inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) {
-                continue;
+            }
-            psThreadJob *job = psThreadJobAlloc("PPSTACK_INSPECT"); // Job to start
-            psArrayAdd(job->args, 1, inspect);
-            PS_ARRAY_ADD_SCALAR(job->args, i, PS_TYPE_S32);
-            if (!psThreadJobAddPending(job)) {
-                psFree(job);
-                psFree(subKernels);
-                psFree(subRegions);
-                psFree(stack);
-                psFree(inputMask);
-                psFree(view);
-                psFree(outRO);
-                psFree(inspect);
-                psFree(rejected);
-                psFree(covariances);
-                psFree(matchChi2);
-                return false;
+            }
-            psFree(job);
+        }
-        if (!psThreadPoolWait(true)) {
-            psError(PS_ERR_UNKNOWN, false, "Unable to concatenate inspection lists.");
-            psFree(subKernels);
-            psFree(subRegions);
-            psFree(stack);
-            psFree(inputMask);
-            psFree(view);
-            psFree(outRO);
-            psFree(inspect);
-            psFree(rejected);
-            psFree(covariances);
-            psFree(matchChi2);
-            return false;
+        }
-        if (psMetadataLookupS32(NULL, config->arguments, "-threads") > 0) {
-            pmStackRejectThreadsInit();
+        }
-        int stride = psMetadataLookupS32(NULL, ppsub, "STRIDE"); // Size of convolution patches
-        // Reject bad pixels
-        for (int i = 0; i < num; i++) {
-            if (inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) {
-                continue;
+            }
-            psTimerStart("PPSTACK_REJECT");
-#ifdef TESTING
+            {
-                psImage *mask = psPixelsToMask(NULL, inspect->data[i],
-                                               psRegionSet(0, numCols - 1, 0, numRows - 1),
-xff); // Mask image
-                psString name = NULL;           // Name of image
-                psStringAppend(&name, "inspect_%03d.fits", i);
-                psFits *fits = psFitsOpen(name, "w");
-                psFree(name);
-                psFitsWriteImage(fits, NULL, mask, 0, NULL);
-                psFree(mask);
-                psFitsClose(fits);
+            }
-#endif
-            psPixels *reject = pmStackReject(inspect->data[i], numCols, numRows, threshold, poorFrac, stride,
-                                             subRegions->data[i], subKernels->data[i]); // Rejected pixels
-#ifdef TESTING
+            {
-                psImage *mask = psPixelsToMask(NULL, reject, psRegionSet(0, numCols - 1, 0, numRows - 1),
-xff); // Mask image
-                psString name = NULL;           // Name of image
-                psStringAppend(&name, "reject_%03d.fits", i);
-                psFits *fits = psFitsOpen(name, "w");
-                psFree(name);
-                psFitsWriteImage(fits, NULL, mask, 0, NULL);
-                psFree(mask);
-                psFitsClose(fits);
+            }
-#endif
-            psFree(inspect->data[i]);
-            inspect->data[i] = NULL;
-            if (!reject) {
-                psWarning("Rejection on image %d didn't work --- reject entire image.", i);
-                numRejected++;
-                inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_REJECT;
-            } else {
-                float frac = reject->n / (float)(numCols * numRows); // Pixel fraction
-                psLogMsg("ppStack", PS_LOG_INFO, "%ld pixels rejected from image %d (%.1f%%)",
-                         reject->n, i, frac * 100.0);
-                if (frac > imageRej) {
-                    psWarning("Image %d rejected completely because rejection fraction (%.3f) "
-                              "exceeds limit (%.3f)", i, frac, imageRej);
-                    psFree(reject);
-                    // reject == NULL means reject image completely
-                    reject = NULL;
-                    inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i] = PPSTACK_MASK_BAD;
-                    numRejected++;
+                }
+            }
-            // Images without a list of rejected pixels (the list may be empty) are rejected completely
-            rejected->data[i] = reject;
-            if (stats) {
-                psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_REJECT", PS_META_DUPLICATE_OK,
-                                 "Time to perform rejection", psTimerMark("PPSTACK_REJECT"));
-                psMetadataAddS32(stats, PS_LIST_TAIL, "REJECT_PIXELS", PS_META_DUPLICATE_OK,
-                                 "Number of pixels rejected", reject ? reject->n : 0);
+            }
-            psLogMsg("ppStack", PS_LOG_INFO, "Time to perform rejection on image %d: %f sec", i,
-                     psTimerClear("PPSTACK_REJECT"));
+        }
-        psFree(inspect);
-        psFree(subKernels);
-        psFree(subRegions);
-        if (numRejected >= num - 1) {
-            psError(PS_ERR_UNKNOWN, true, "All inputs completely rejected; unable to proceed.");
-            psFree(stack);
-            psFree(rejected);
-            psFree(inputMask);
-            psFree(view);
-            psFree(outRO);
-            psFree(covariances);
-            psFree(matchChi2);
-            return false;
+        }
-        if (stats) {
-            psMetadataAddS32(stats, PS_LIST_TAIL, "REJECT_IMAGES", 0,
-                             "Number of images rejected completely", numRejected);
+        }
+    }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 5 : Pixel Rejection : %f sec", psTimerClear("PPSTACK_STEPS"));
-    psTimerStart("PPSTACK_FINAL");
-    memDump("reject");
     // Final combination
     psTrace("ppStack", 2, "Final stack of convolved images....\n");
+    {
+        stack->lastScan = 0;            // Reset read
+        bool status;                    // Status of read
+        for (int numChunk = 0; true; numChunk++) {
+            ppStackThread *thread = ppStackThreadRead(&status, stack, config, numChunk, 0);
+            if (!status) {
+                // Something went wrong
+                psError(PS_ERR_UNKNOWN, false, "Unable to read chunk %d", numChunk);
+                psFree(stack);
+                psFree(rejected);
+                psFree(inputMask);
+                psFree(view);
+                psFree(outRO);
+                psFree(covariances);
+                psFree(matchChi2);
+                return false;
+            }
+            if (!thread) {
+                // Nothing more to read
+                break;
+            }
+    if (!ppStackCombineFinal(stack, options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to perform final combination.");
+        psFree(stack);
+        psFree(options);
+        return false;
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 6: Final Stack: %f sec", psTimerClear("PPSTACK_STEPS"));
+    ppStackMemDump("final");
-            // call: ppStackReadoutFinal(config, outRO, readouts, rejected)
-            psThreadJob *job = psThreadJobAlloc("PPSTACK_FINAL_COMBINE"); // Job to start
-            psArrayAdd(job->args, 1, thread);
-            psArrayAdd(job->args, 1, config);
-            psArrayAdd(job->args, 1, outRO);
-            psArrayAdd(job->args, 1, inputMask);
-            psArrayAdd(job->args, 1, rejected);
-            psArrayAdd(job->args, 1, weightings);
-            psArrayAdd(job->args, 1, matchChi2);
-            if (!psThreadJobAddPending(job)) {
-                psFree(job);
-                psFree(stack);
-                psFree(rejected);
-                psFree(inputMask);
-                psFree(view);
-                psFree(outRO);
-                psFree(covariances);
-                psFree(matchChi2);
-                return false;
+            }
-            psFree(job);
+        }
+        if (!psThreadPoolWait(true)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to do final combination.");
+            psFree(stack);
+            psFree(inputMask);
+            psFree(rejected);
+            psFree(view);
+            psFree(outRO);
+            psFree(covariances);
+            psFree(matchChi2);
+            return false;
+        }
+    // Clean up
+    psTrace("ppStack", 2, "Cleaning up after combination....\n");
+    if (!ppStackCombineFinal(stack, options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to clean up.");
+        psFree(stack);
+        psFree(options);
+        return false;
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 7: WCS & JPEGS: %f sec", psTimerClear("PPSTACK_STEPS"));
+    ppStackMemDump("cleanup");
-    memDump("final");
-#ifdef TESTING
-    writeImage("combined_final.fits", NULL, outRO->image, config);
-#endif
-    // Sum covariance matrices
-    for (int i = 0; i < num; i++) {
-        if (inputMask->data.U8[i]) {
-            psFree(covariances->data[i]);
-            covariances->data[i] = NULL;
+        }
+    }
-    outRO->covariance = psImageCovarianceSum(covariances);
-    psFree(covariances);
-    psFree(matchChi2);
-    if (stats) {
-        psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_FINAL", 0,
-                         "Time to make final stack", psTimerMark("PPSTACK_FINAL"));
+    }
-    psLogMsg("ppStack", PS_LOG_INFO, "Stage 6 : Final Stack : %f sec", psTimerClear("PPSTACK_STEPS"));
-    psLogMsg("ppStack", PS_LOG_INFO, "Time to make final stack: %f sec", psTimerClear("PPSTACK_FINAL"));
-    psTrace("ppStack", 2, "Cleaning up after combination....\n");
-#if 0
-    // Ensure masked regions really look masked
+    {
-        psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits for bad
-        psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels
-        if (!pmReadoutMaskApply(outRO, maskBad)) {
-            psWarning("Unable to apply mask");
+        }
+    }
-#endif
-    // Close up
-    bool wcsDone = false;           // Have we done the WCS?
-    for (int i = 0; i < num; i++) {
-        if (inputMask->data.PS_TYPE_VECTOR_MASK_DATA[i]) {
-            continue;
+        }
-        ppStackThread *thread = stack->threads->data[0]; // Representative stack
-        pmReadout *inRO = thread->readouts->data[i]; // Template readout
-        if (inRO && !wcsDone) {
-            // Copy astrometry over
-            wcsDone = true;
-            pmHDU *inHDU = pmHDUFromCell(inRO->parent); // Template HDU
-            pmHDU *outHDU = pmHDUFromCell(outRO->parent); // Output HDU
-            pmChip *outChip = outRO->parent->parent; // Output chip
-            pmFPA *outFPA = outChip->parent; // Output FPA
-            if (!outHDU || !inHDU) {
-                psWarning("Unable to find HDU at FPA level to copy astrometry.");
-            } else {
-                if (!pmAstromReadWCS(outFPA, outChip, inHDU->header, 1.0)) {
-                    psErrorClear();
-                    psWarning("Unable to read WCS astrometry from input FPA.");
-                    wcsDone = false;
-                } else {
-                    if (!outHDU->header) {
-                        outHDU->header = psMetadataAlloc();
+                    }
-                    if (!pmAstromWriteWCS(outHDU->header, outFPA, outChip, WCS_TOLERANCE)) {
-                        psErrorClear();
-                        psWarning("Unable to write WCS astrometry to output FPA.");
-                        wcsDone = false;
+                    }
+                }
+            }
+        }
-        if (tempDelete) {
-            psString imageResolved = pmConfigConvertFilename(imageNames->data[i], config, false, false);
-            psString maskResolved = pmConfigConvertFilename(maskNames->data[i], config, false, false);
-            psString varianceResolved = pmConfigConvertFilename(varianceNames->data[i], config, false, false);
-            if (unlink(imageResolved) == -1 || unlink(maskResolved) == -1 ||
-                unlink(varianceResolved) == -1) {
-                psWarning("Unable to delete temporary files for image %d", i);
+            }
-            psFree(imageResolved);
-            psFree(maskResolved);
-            psFree(varianceResolved);
+        }
+    }
-    psFree(imageNames);
-    psFree(maskNames);
-    psFree(varianceNames);
-    psFree(inputMask);
     psFree(stack);
-    memDump("cleanup");
+    // Generate binned JPEGs
+    {
+        int bin1 = psMetadataLookupS32(NULL, recipe, "BIN1"); // First binning level
+        int bin2 = psMetadataLookupS32(NULL, recipe, "BIN2"); // Second binning level
+        // Target cells
+        pmCell *cell1 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG1");
+        pmCell *cell2 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG2");
+        psImageMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask
+        pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts
+        if (!pmReadoutRebin(ro1, outRO, maskValue, bin1, bin1) || !pmReadoutRebin(ro2, ro1, 0, bin2, bin2)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to bin output.");
+            psFree(ro1);
+            psFree(ro2);
+            psFree(outRO);
+            return false;
+        }
+        psFree(ro1);
+        psFree(ro2);
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 7 : WCS & JPEGS : %f sec", psTimerClear("PPSTACK_STEPS"));
+    if (psMetadataLookupBool(&mdok, recipe, "PHOTOMETRY")) {
+        psTrace("ppStack", 1, "Photometering stacked image....\n");
+        psTimerStart("PPSTACK_PHOT");
+        fileActivation(config, combineFiles, false);
+        fileActivation(config, photFiles, true);
+        pmFPAview *photView = filesIterateDown(config);
+        if (!ppStackPhotometry(config, outRO, photView)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry on output.");
+            psFree(outRO);
+            psFree(photView);
+            return false;
+        }
+        psFree(photView);
+        fileActivation(config, combineFiles, true);
+        if (stats) {
+            pmFPAfile *photFile = psMetadataLookupPtr(NULL, config->files, "PSPHOT.INPUT"); // File
+            pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout with the sources
+            psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources
+            psMetadataAddS32(stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected", sources->n);
+            psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_PHOT", 0,
+                             "Time to do photometry", psTimerMark("PPSTACK_PHOT"));
+        }
+        psLogMsg("ppStack", PS_LOG_INFO, "Time to do photometry: %f sec", psTimerClear("PPSTACK_PHOT"));
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 8 : Photometry Analysis : %f sec", psTimerClear("PPSTACK_STEPS"));
+    psThreadPoolFinalize();
+    memDump("phot");
+    // Statistics on output
+    if (stats) {
+        psTrace("ppStack", 1, "Gathering statistics on stacked image....\n");
+        psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits for bad
+        psImageMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels
+        ppStatsFPA(stats, outRO->parent->parent->parent, view, maskBad, config);
+    }
+    memDump("stats");
+    // Put version information into the header
+    pmHDU *hdu = pmHDUFromCell(outRO->parent);
+    if (!hdu) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to find HDU for output.");
+        psFree(outRO);
+        psFree(view);
+    // Photometry
+    psTrace("ppStack", 1, "Photometering stacked image....\n");
+    if (!ppStackPhotometry(options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry.");
+        psFree(options);
         return false;
+    }
+    if (!hdu->header) {
+        hdu->header = psMetadataAlloc();
+    }
+    ppStackVersionMetadata(hdu->header);
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 8: Photometry Analysis: %f sec", psTimerClear("PPSTACK_STEPS"));
+    ppStackMemDump("photometry");
-    psFree(outRO);
-    psFree(view);
+    // Write out summary statistics
+    if (stats) {
+        psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_STACK", 0,
+                         "Time to do photometry", psTimerClear("PPSTACK_TOTAL"));
+        const char *statsMDC = psMetadataConfigFormat(stats);
+        if (!statsMDC || strlen(statsMDC) == 0) {
+            psError(PS_ERR_IO, false, "Unable to get statistics MDC file.\n");
+        } else {
+            fprintf(statsFile, "%s", statsMDC);
+        }
+        psFree((void *)statsMDC);
+        fclose(statsFile);
+        psFree(stats);
+    }
+    // Write out the output files
+    if (!filesIterateUp(config)) {
+    // Finish up
+    psTrace("ppStack", 1, "Finishing up....\n");
+    if (!ppStackFinish(options, config)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to finish up.");
+        psFree(options);
         return false;
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 9: Final output: %f sec", psTimerClear("PPSTACK_STEPS"));
+    ppStackMemDump("finish");
+    psLogMsg("ppStack", PS_LOG_INFO, "Stage 9 : Final Output : %f sec", psTimerClear("PPSTACK_STEPS"));
+    memDump("finish");
+    psFree(options);
     return true;
+}

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackMatch.c

-              r21525
+              r23352
     psImage *binned = psphotBackgroundModel(ro, config); // Binned background model
     psImageBinning *binning = psMetadataLookupPtr(NULL, psphotRecipe,
+    psImageBinning *binning = psMetadataLookupPtr(NULL, ro->analysis,
                                                   "PSPHOT.BACKGROUND.BINNING"); // Binning for model
     psAssert(binning, "Need binning parameters");
 …
                 psString name = NULL;
                 psStringAppend(&name, "fake_%03d.fits", numInput);
+                pmStackVisualPlotTestImage(fake->image, name);
                 psFits *fits = psFitsOpen(name, "w");
                 psFree(name);
 …
                 psString name = NULL;
                 psStringAppend(&name, "real_%03d.fits", numInput);
+                pmStackVisualPlotTestImage(readout->image, name);
                 psFits *fits = psFitsOpen(name, "w");
                 psFree(name);
 …
                 psString name = NULL;
                 psStringAppend(&name, "conv_%03d.fits", numInput);
+                pmStackVisualPlotTestImage(output->image, name);
                 psFits *fits = psFitsOpen(name, "w");
                 psFree(name);
 …
                 psString name = NULL;
                 psStringAppend(&name, "diff_%03d.fits", numInput);
+                pmStackVisualPlotTestImage(fake->image, name);
                 psFits *fits = psFitsOpen(name, "w");
                 psFree(name);
 …
         psString name = NULL;
         psStringAppend(&name, "convolved_%03d.fits", numInput);
+        pmStackVisualPlotTestImage(output->image, name);
         psFits *fits = psFitsOpen(name, "w");
         psFree(name);

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackPSF.c

-              r18918
+              r23352
 #include "ppStack.h"
+pmPSF *ppStackPSF(const pmConfig *config, int numCols, int numRows, const psArray *psfs)
+pmPSF *ppStackPSF(const pmConfig *config, int numCols, int numRows,
+                  const psArray *psfs, const psVector *inputMask)
+{
     // Get the recipe values
 …
     const char *psfModel = psMetadataLookupStr(NULL, recipe, "PSF.MODEL"); // Model for PSF
     int psfOrder = psMetadataLookupS32(NULL, recipe, "PSF.ORDER"); // Spatial order for PSF
+    for (int i = 0; i < psfs->n; i++) {
+        if (inputMask->data.U8[i]) {
+            psFree(psfs->data[i]);
+            psfs->data[i] = NULL;
+        }
+    }
     // Solve for the target PSF

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackPhotometry.c

-              r21259
+              r23352
 #include "ppStack.h"
+#include "ppStackLoop.h"
+#define PHOT_SOURCE_MASK (PM_SOURCE_MODE_FAIL | PM_SOURCE_MODE_SATSTAR | PM_SOURCE_MODE_BLEND | \
+                          PM_SOURCE_MODE_BADPSF | PM_SOURCE_MODE_DEFECT | PM_SOURCE_MODE_SATURATED | \
+                          PM_SOURCE_MODE_CR_LIMIT | PM_SOURCE_MODE_EXT_LIMIT) // Mask to apply to sources
+bool ppStackInputPhotometry(const pmReadout *ro, const psArray *sources, const pmConfig *config)
+bool ppStackPhotometry(ppStackOptions *options, pmConfig *config)
+{
     PM_ASSERT_READOUT_NON_NULL(ro, false);
     PS_ASSERT_ARRAY_NON_NULL(sources, false);
+    psAssert(options, "Require options");
+    psAssert(config, "Require configuration");
     psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe
 …
     bool mdok;                          // Status of MD lookup
+    float zpRadius = psMetadataLookupS32(&mdok, recipe, "PHOT.RADIUS"); // Radius for PHOT measurement
+    if (!mdok) {
+        psError(PS_ERR_UNKNOWN, true, "Unable to find PHOT.RADIUS in recipe");
+        return false;
+    }
+    float zpSigma = psMetadataLookupF32(&mdok, recipe, "PHOT.SIGMA"); // Gaussian sigma for photometry
+    if (!mdok) {
+        psError(PS_ERR_UNKNOWN, true, "Unable to find PHOT.SIGMA in recipe");
+        return false;
+    }
+    float zpFrac = psMetadataLookupF32(&mdok, recipe, "PHOT.FRAC"); // Fraction of good pixels for photometry
+    if (!mdok) {
+        psError(PS_ERR_UNKNOWN, true, "Unable to find PHOT.FRAC in recipe");
+        return false;
+    if (!psMetadataLookupBool(&mdok, recipe, "PHOTOMETRY")) {
+        // Nothing to do
+        return true;
+    }
+    psString maskValStr = psMetadataLookupStr(&mdok, recipe, "MASK.VAL"); // Name of bits to mask going in
+    if (!mdok || !maskValStr) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to find MASK.VAL in recipe");
+        return false;
+    }
+    psImageMaskType maskVal = pmConfigMaskGet(maskValStr, config); // Bits to mask
+    psTimerStart("PPSTACK_PHOT");
+    psImage *image = ro->image, *mask = ro->mask; // Image and mask from readout
+    ppStackFileActivation(config, PPSTACK_FILES_COMBINE, false);
+    ppStackFileActivation(config, PPSTACK_FILES_PHOT, true);
+    pmFPAview *photView = ppStackFilesIterateDown(config); // View to readout
+    ppStackFileActivation(config, PPSTACK_FILES_COMBINE, true);
+    // Measure background
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator
+    psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics
+    if (!psImageBackground(stats, NULL, image, mask, maskVal, rng)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to measure background for image");
+        psFree(stats);
+        psFree(rng);
+        return false;
+    }
+    float bg = stats->robustMedian; // Background level
+    psFree(stats);
+    psFree(rng);
+    pmFPAfile *photFile = psMetadataLookupPtr(NULL, config->files, "PSPHOT.INPUT"); // File for photometry
+    pmFPACopy(photFile->fpa, options->outRO->parent->parent->parent);
+    // Photometer sources
+    int numCols = image->numCols, numRows = image->numRows; // Size of image
+    int numSources = sources->n; // Number of sources
+    int numGood = 0;            // Number of good sources
+    float maxMag = -INFINITY;   // Maximum magnitude
+    for (int j = 0; j < numSources; j++) {
+        pmSource *source = sources->data[j]; // Source of interest
+        if (source->mode & PHOT_SOURCE_MASK || !isfinite(source->psfMag)) {
+            source->psfMag = NAN;
+            continue;
+        }
+        float x = source->peak->xf, y = source->peak->yf; // Coordinates of source
+        int xCentral = x + 0.5, yCentral = y + 0.5; // Central pixel
+        // Range of integration
+        int xMin = PS_MAX(0, xCentral - zpRadius), xMax = PS_MIN(numCols - 1, xCentral + zpRadius);
+        int yMin = PS_MAX(0, yCentral - zpRadius), yMax = PS_MIN(numRows - 1, yCentral + zpRadius);
+        // Integrate
+        double sumImage = 0.0, sumKernel = 0.0; // Sums from integration
+        int numBadPix = 0;         // Number of bad pixels
+        for (int v = yMin; v <= yMax; v++) {
+            float dy2 = PS_SQR(y - v); // Distance from centroid
+            for (int u = xMin; u <= xMax; u++) {
+                if (mask->data.PS_TYPE_IMAGE_MASK_DATA[v][u] & maskVal) {
+                    numBadPix++;
+                    continue;
+                }
+                float dx2 = PS_SQR(x - u); // Distance from centroid
+                double kernel = exp(-0.5 * (dx2 + dy2) / PS_SQR(zpSigma)); // Kernel value
+                sumImage += (image->data.F32[v][u] - bg) * kernel;
+                sumKernel += kernel;
+            }
+        }
+        if (numBadPix > zpFrac * M_PI * PS_SQR(zpRadius) || !isfinite(sumImage)) {
+            source->psfMag = NAN;
+        } else {
+            source->psfMag = - 2.5 * log10(sumImage / sumKernel * M_PI * PS_SQR(zpRadius));
+            if (isfinite(source->psfMag)) {
+                numGood++;
+                maxMag = PS_MAX(maxMag, source->psfMag);
+            }
+        }
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Photometered %d good sources in image; max mag %f", numGood, maxMag);
+    return true;
+}
+bool ppStackPhotometry(pmConfig *config, const pmReadout *readout, const pmFPAview *view)
+{
+    pmFPAfile *photFile = psMetadataLookupPtr(NULL, config->files, "PSPHOT.INPUT");
+    pmFPACopy(photFile->fpa, readout->parent->parent->parent);
+    if (psMetadataLookupBool(NULL, config->arguments, "-psphot-visual")) {
+        psphotSetVisual(true);
+    if (psMetadataLookupBool(NULL, config->arguments, "-visual")) {
+        pmVisualSetVisual(true);
+    }
+    psMetadata *psphot = psMetadataLookupMetadata(NULL, config->recipes, PSPHOT_RECIPE); // Recipe
+#if 0
     // Need to ensure aperture residual is not calculated
-    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PSPHOT_RECIPE); // Recipe
     psMetadataItem *item = psMetadataLookup(recipe, "MEASURE.APTREND"); // Item determining aptrend
     if (!item) {
 …
         item->data.B = false;
+    }
+#endif
     // set maskValue and markValue in the psphot recipe
     psImageMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask
     psImageMaskType markValue = pmConfigMaskGet("MARK.VALUE", config); // Bits to use for marking
+    psMetadataAddImageMask (recipe, PS_LIST_TAIL, "MASK.PSPHOT", PS_META_REPLACE, "Bits to mask", maskValue);
+    psMetadataAddImageMask (recipe, PS_LIST_TAIL, "MARK.PSPHOT", PS_META_REPLACE, "Bits to use for mark", markValue);
+    psMetadataAddImageMask(psphot, PS_LIST_TAIL, "MASK.PSPHOT", PS_META_REPLACE,
+                            "Bits to mask", maskValue);
+    psMetadataAddImageMask(psphot, PS_LIST_TAIL, "MARK.PSPHOT", PS_META_REPLACE,
+                           "Bits to use for mark", markValue);
+    // XXX replace with psphotReadoutKnownSources (config, view)
+    // XXX this function requires that we construct the input source list and place it on PSPHOT.INPUT.CMF
+    pmCell *sourcesCell = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT");
+    psArray *inSources = psMetadataLookupPtr (NULL, sourcesCell->analysis, "PSPHOT.SOURCES");
+    pmCell *sourcesCell = pmFPAfileThisCell(config->files, photView, "PPSTACK.OUTPUT");
+    psArray *inSources = psMetadataLookupPtr(NULL, sourcesCell->analysis, "PSPHOT.SOURCES");
     if (!inSources) {
         psError(PS_ERR_UNKNOWN, false, "Unable to find input sources");
+        psFree(photView);
         return false;
+    }
     if (!psphotReadoutKnownSources(config, view, inSources)) {
+    if (!psphotReadoutKnownSources(config, photView, inSources)) {
         // Clear the error, so that the output files are written.
         psWarning("Unable to perform photometry on stacked image.");
         psErrorStackPrint(stderr, "Error stack from photometry:");
         psErrorClear();
+        psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry on stacked image.");
+        psFree(photView);
+        return false;
+    }
+    psFree(photView);
     if (!pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL") ||
 …
     pmFPAfileActivate(config->files, false, "PSPHOT.INPUT");
+    if (options->stats) {
+        pmReadout *photRO = pmFPAviewThisReadout(photView, photFile->fpa); // Readout with the sources
+        psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources
+        psMetadataAddS32(options->stats, PS_LIST_TAIL, "NUM_SOURCES", 0,
+                         "Number of sources detected", sources->n);
+        psMetadataAddF32(options->stats, PS_LIST_TAIL, "TIME_PHOT", 0,
+                         "Time to do photometry", psTimerMark("PPSTACK_PHOT"));
+    }
+    psLogMsg("ppStack", PS_LOG_INFO, "Time to do photometry: %f sec", psTimerClear("PPSTACK_PHOT"));
     return true;
+}

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackReadout.c

-              r21477
+              r23352
 #include <psphot.h>
+#include "ppStackThread.h"
 #include "ppStack.h"
 …
     psArray *args = job->args;          // Arguments
     ppStackThread *thread = args->data[0]; // Thread
+    pmConfig *config = args->data[1];   // Configuration
+    pmReadout *outRO = args->data[2];   // Output readout
+    psVector *mask = args->data[3];     // Mask for inputs
+    psVector *weightings = args->data[4]; // Weightings (1/noise^2) for each image
+    psVector *addVariance = args->data[5]; // Additional variance when rejecting
+    ppStackOptions *options = args->data[1]; // Options
+    pmConfig *config = args->data[2];   // Configuration
+    pmReadout *outRO = options->outRO;  // Output readout
+    psVector *mask = options->inputMask; // Mask for inputs
+    psVector *weightings = options->weightings; // Weightings (1/noise^2) for each image
+    psVector *addVariance = options->matchChi2; // Additional variance when rejecting
     psArray *inspect = ppStackReadoutInitial(config, outRO, thread->readouts, mask,
 …
     psArray *args = job->args;          // Arguments
     ppStackThread *thread = args->data[0]; // Thread
+    pmConfig *config = args->data[1];   // Configuration
+    pmReadout *outRO = args->data[2];   // Output readout
+    psVector *mask = args->data[3];     // Mask for inputs
+    psArray *rejected = args->data[4];  // Rejected pixels
+    psVector *weightings = args->data[5];  // Weighting (1/noise^2) for each image
+    psVector *addVariance = args->data[6];  // Weighting (1/noise^2) for each image
+    ppStackOptions *options = args->data[1]; // Options
+    pmConfig *config = args->data[2];   // Configuration
+    pmReadout *outRO = options->outRO;  // Output readout
+    psVector *mask = options->inputMask; // Mask for inputs
+    psArray *rejected = options->rejected; // Rejected pixels
+    psVector *weightings = options->weightings; // Weightings (1/noise^2) for each image
+    psVector *addVariance = options->matchChi2; // Additional variance when rejecting
     bool status = ppStackReadoutFinal(config, outRO, thread->readouts, mask, rejected,
 …
+        }
+        pmStackData *data = pmStackDataAlloc(ro, weightings->data.F32[i], addVariance->data.F32[i]);
+        pmStackData *data = pmStackDataAlloc(ro, weightings->data.F32[i],
+                                             addVariance ? addVariance->data.F32[i] : NAN);
         data->reject = psMemIncrRefCounter(rejected->data[i]);
         stack->data[i] = data;

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackSources.c

-              r21260
+              r23352
 #define FAKE_ROWS 4913
+float ppStackSourcesTransparency(const psArray *sourceLists, const pmFPAview *view, const pmConfig *config)
+#ifdef TESTING
+// Dump matches to a file
+static void dumpMatches(const char *filename, // File to which to dump
+                        int num,        // Number of inputs
+                        psArray *matches, // Star matches
+                        psVector *zp,   // Zero points
+                        psVector *trans // Transparencies
+                        )
+{
+    FILE *outMatches = fopen(filename, "w"); // Output matches
+    psVector *mag = psVectorAlloc(num, PS_TYPE_F32); // Magnitudes for each star
+    psVector *magErr = psVectorAlloc(num, PS_TYPE_F32); // Errors for each star
+    for (int i = 0; i < matches->n; i++) {
+        pmSourceMatch *match = matches->data[i]; // Match of interest
+        psVectorInit(mag, NAN);
+        psVectorInit(magErr, NAN);
+        for (int j = 0; j < match->num; j++) {
+            if (match->mask->data.PS_TYPE_VECTOR_MASK_DATA[j]) {
+                continue;
+            }
+            int index = match->image->data.U32[j]; // Image index
+            mag->data.F32[index] = match->mag->data.F32[j] - zp->data.F32[index];
+            if (trans) {
+                mag->data.F32[index] -= trans->data.F32[index];
+            }
+            magErr->data.F32[index] = match->magErr->data.F32[j];
+        }
+        for (int j = 0; j < num; j++) {
+            fprintf(outMatches, "%f (%f) ", mag->data.F32[j], magErr->data.F32[j]);
+        }
+        fprintf(outMatches, "\n");
+    }
+    psFree(mag);
+    psFree(magErr);
+    fclose(outMatches);
+    return;
+}
+#endif
+float ppStackSourcesTransparency(const psArray *sourceLists, psVector *inputMask,
+                                 const pmFPAview *view, const pmConfig *config)
+{
     PS_ASSERT_ARRAY_NON_NULL(sourceLists, NAN);
+    PS_ASSERT_VECTOR_NON_NULL(inputMask, NAN);
+    PS_ASSERT_VECTOR_TYPE(inputMask, PS_TYPE_U8, NAN);
+    PS_ASSERT_VECTOR_SIZE(inputMask, sourceLists->n, NAN);
     PS_ASSERT_PTR_NON_NULL(view, NAN);
     PS_ASSERT_PTR_NON_NULL(config, NAN);
 #ifdef TESTING
+#if defined(TESTING) && 0
+    {
         // Deliberately induce a major transparency difference
 …
     float radius = psMetadataLookupF32(NULL, recipe, "ZP.RADIUS"); // Radius (pixels) for matching sources
+    int iter = psMetadataLookupS32(NULL, recipe, "ZP.ITER"); // Maximum iterations
+    int iter1 = psMetadataLookupS32(NULL, recipe, "ZP.ITER.1"); // Maximum iterations for pass 1
+    int iter2 = psMetadataLookupS32(NULL, recipe, "ZP.ITER.2"); // Maximum iterations for pass 2
     float tol = psMetadataLookupF32(NULL, recipe, "ZP.TOL"); // Tolerance for zero point iterations
     int transIter = psMetadataLookupS32(NULL, recipe, "ZP.TRANS.ITER"); // Iterations for transparency
     float transRej = psMetadataLookupF32(NULL, recipe, "ZP.TRANS.REJ");// Rejection threshold for transparency
     float transThresh = psMetadataLookupF32(NULL, recipe, "ZP.TRANS.THRESH"); // Threshold for transparency
+    float starRej = psMetadataLookupF32(NULL, recipe, "ZP.STAR.REJ"); // Rejection threshold for stars
+    float starRej1 = psMetadataLookupF32(NULL, recipe, "ZP.STAR.REJ.1"); // Rejection threshold for stars
+    float starSys1 = psMetadataLookupF32(NULL, recipe, "ZP.STAR.SYS.1"); // Estimated systematic error
+    float starRej2 = psMetadataLookupF32(NULL, recipe, "ZP.STAR.REJ.2"); // Rejection threshold for stars
+    float starSys2 = psMetadataLookupF32(NULL, recipe, "ZP.STAR.SYS.2"); // Estimated systematic error
     float starLimit = psMetadataLookupF32(NULL, recipe, "ZP.STAR.LIMIT"); // Limit on star rejection fraction
+    float starSys = psMetadataLookupF32(NULL, recipe, "ZP.STAR.SYS"); // Estimated systematic error
+    float fracMatch = psMetadataLookupF32(NULL, recipe, "ZP.MATCH"); // Fraction of images to match for star
     psMetadata *airmassZP = psMetadataLookupMetadata(NULL, recipe, "ZP.AIRMASS"); // Airmass terms
 …
         pmCell *cell = pmFPAviewThisCell(view, file->fpa); // Cell of interest
 #ifdef TESTING
+#if defined(TESTING) && 0
         pmReadout *fake = pmReadoutAlloc(NULL); // Fake readout
         pmPSF *psf = psMetadataLookupPtr(NULL, config->arguments, "PSF.TARGET"); // PSF for fake image
+        pmReadoutFakeFromSources(fake, FAKE_COLS, FAKE_ROWS, sourceLists->data[i], NULL, NULL, psf, 5, 0, false, true);
+        pmReadoutFakeFromSources(fake, FAKE_COLS, FAKE_ROWS, sourceLists->data[i],
+                                 NULL, NULL, psf, 5, 0, false, true);
         psString name = NULL;
         psStringAppend(&name, "start_%03d.fits", i);
+        pmStackVisualPlotTestImage(fake->image, name);
         psFits *fits = psFitsOpen(name, "w");
         psFree(name);
 …
+    }
     psArray *matches = pmSourceMatchSources(sourceLists, radius); // List of matches
+    psArray *matches = pmSourceMatchSources(sourceLists, radius, true); // List of matches
     if (!matches) {
         psError(PS_ERR_UNKNOWN, false, "Unable to match sources");
 …
         return NAN;
+    }
+    psVector *trans = pmSourceMatchRelphot(matches, zp, iter, tol, starLimit, transIter, transRej,
+                                           transThresh, starRej, starSys); // Transparencies for each image
+#ifdef TESTING
+    {
+        // Dump the corrected magnitudes
+        FILE *outMatches = fopen("source_match.dat", "w"); // Output matches
+        psVector *mag = psVectorAlloc(num, PS_TYPE_F32); // Magnitudes for each star
+        psVector *magErr = psVectorAlloc(num, PS_TYPE_F32); // Errors for each star
+        for (int i = 0; i < matches->n; i++) {
+            pmSourceMatch *match = matches->data[i]; // Match of interest
+            psVectorInit(mag, NAN);
+            psVectorInit(magErr, NAN);
+            for (int j = 0; j < match->num; j++) {
+                if (match->mask->data.PS_TYPE_VECTOR_MASK_DATA[j]) {
+                    continue;
+                }
+                int index = match->image->data.U32[j]; // Image index
+                mag->data.F32[index] = match->mag->data.F32[j] - zp->data.F32[index] - trans->data.F32[index];
+                magErr->data.F32[index] = match->magErr->data.F32[j];
+            }
+            for (int j = 0; j < num; j++) {
+                fprintf(outMatches, "%f (%f) ", mag->data.F32[j], magErr->data.F32[j]);
+            }
+            fprintf(outMatches, "\n");
+        }
+        psFree(mag);
+        psFree(magErr);
+        fclose(outMatches);
+    }
+#endif
+#ifdef TESTING
+    dumpMatches("source_match.dat", num, matches, zp, NULL);
+#endif
+    psVector *trans = pmSourceMatchRelphot(matches, zp, tol, iter1, starRej1, starSys1,
+                                           iter2, starRej2, starSys2, starLimit,
+                                           transIter, transRej, transThresh); // Transparencies for each image
+    if (!trans) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to measure transparencies");
+        return NAN;
+    }
+#ifdef TESTING
+    dumpMatches("source_mags.dat", num, matches, zp, trans);
+#endif
+    for (int i = 0; i < trans->n; i++) {
+        if (!isfinite(trans->data.F32[i])) {
+            inputMask->data.U8[i] = PPSTACK_MASK_CAL;
+        }
+    }
     // Save best matches SOMEWHERE for future photometry
     // XXX this is a really poor output location; clean up the pmFPAfiles used in ppStack
     pmCell *sourcesCell = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT");
     psArray *sourcesBest = psArrayAllocEmpty (100);
     // XXX something of a hack: require at 2 detections or 1/2 of the max possible
     int minMatches = PS_MAX (2, 0.5*num);
+    psArray *sourcesBest = psArrayAllocEmpty(matches->n);
+    // XXX something of a hack: require at least 2 detections or the nominated fraction of the max possible
+    int minMatches = PS_MAX(2, fracMatch * num);// Minimum number of matches required
     for (int i = 0; i < matches->n; i++) {
+        pmSourceMatch *match = matches->data[i]; // Match of interest
+        if (match->num < minMatches) continue;
+        // We need to grab a single instance of this source: just take the first available
+        int nImage = match->image->data.S32[0];
+        int nIndex = match->index->data.S32[0];
+        psArray *sources = sourceLists->data[nImage];
+        pmSource *source = sources->data[nIndex];
+        // stick this sample source on sourcesBest
+        psArrayAdd (sourcesBest, 100, source);
+    }
+    psMetadataAdd (sourcesCell->analysis, PS_LIST_TAIL, "PSPHOT.SOURCES", PS_DATA_ARRAY | PS_META_REPLACE, "psphot sources", sourcesBest);
+    psLogMsg("ppStack", PS_LOG_INFO, "Selected %ld sources for photometry analysis\n", sourcesBest->n);
+    psFree (sourcesBest);
+        pmSourceMatch *match = matches->data[i]; // Match of interest
+        if (match->num < minMatches) {
+            continue;
+        }
+        // We need to grab a single instance of this source: just take the first available
+        int image = match->image->data.S32[0]; // Index of image
+        int index = match->index->data.S32[0]; // Index of source within image
+        psArray *sources = sourceLists->data[image]; // Sources for image
+        pmSource *source = sources->data[index]; // Source of interest
+        psArrayAdd(sourcesBest, sourcesBest->n, source);
+    }
+    psMetadataAdd(sourcesCell->analysis, PS_LIST_TAIL, "PSPHOT.SOURCES", PS_DATA_ARRAY | PS_META_REPLACE,
+                  "psphot sources", sourcesBest);
+    psLogMsg("ppStack", PS_LOG_INFO, "Selected %ld sources for photometry analysis", sourcesBest->n);
+    psFree(sourcesBest);
     psFree(matches);
-    if (!trans) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to measure transparencies");
-        return NAN;
+    }
     // M = m + c0 + c1 * airmass - 2.5log(t) + transparency
 …
     // We don't need to know the magnitude zero point for the filter, since it cancels out
     for (int i = 0; i < num; i++) {
+        if (!isfinite(trans->data.F32[i])) {
+            continue;
+        }
         psArray *sources = sourceLists->data[i]; // Sources of interest
         float magCorr = airmassTerm - 2.5*log10(sumExpTime) - zp->data.F32[i] - trans->data.F32[i];

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackThread.c

-              r21477
+              r23352
 #include "ppStack.h"
+#include "ppStackOptions.h"
+#include "ppStackThread.h"
 #define THREAD_WAIT 10000               // Microseconds to wait if thread is not available
 …
     psFree(stack->threads);
     for (int i = 0; i < stack->imageFits->n; i++) {
+        psFitsClose(stack->imageFits->data[i]);
+        psFitsClose(stack->maskFits->data[i]);
+        psFitsClose(stack->varianceFits->data[i]);
+        if (stack->imageFits->data[i]) {
+            psFitsClose(stack->imageFits->data[i]);
+        }
+        if (stack->maskFits->data[i]) {
+            psFitsClose(stack->maskFits->data[i]);
+        }
+        if (stack->varianceFits->data[i]) {
+            psFitsClose(stack->varianceFits->data[i]);
+        }
         stack->imageFits->data[i] = stack->maskFits->data[i] = stack->varianceFits->data[i] = NULL;
+    }
 …
+}
+ppStackThreadData *ppStackThreadDataSetup(const psArray *cells, const psArray *imageNames,
+                                          const psArray *maskNames, const psArray *varianceNames,
+                                          const psArray *covariances, const pmConfig *config)
+{
+ppStackThreadData *ppStackThreadDataSetup(const ppStackOptions *options, const pmConfig *config)
+{
+    psAssert(options, "Require options");
+    psAssert(config, "Require configuration");
+    const psArray *cells = options->cells; // Array of input cells
+    const psArray *imageNames = options->imageNames; // Names of images to read
+    const psArray *maskNames = options->maskNames; // Names of masks to read
+    const psArray *varianceNames = options->varianceNames; // Names of variance maps to read
+    const psArray *covariances = options->covariances; // Covariance matrices (already read)
     PS_ASSERT_ARRAY_NON_NULL(cells, NULL);
     PS_ASSERT_ARRAYS_SIZE_EQUAL(cells, imageNames, NULL);
 …
+    {
         psThreadTask *task = psThreadTaskAlloc("PPSTACK_INITIAL_COMBINE", 6);
+        psThreadTask *task = psThreadTaskAlloc("PPSTACK_INITIAL_COMBINE", 3);
         task->function = &ppStackReadoutInitialThread;
         psThreadTaskAdd(task);
 …
+    {
         psThreadTask *task = psThreadTaskAlloc("PPSTACK_FINAL_COMBINE", 7);
+        psThreadTask *task = psThreadTaskAlloc("PPSTACK_FINAL_COMBINE", 3);
         task->function = &ppStackReadoutFinalThread;
         psThreadTaskAdd(task);

branches/cnb_branches/cnb_branch_20090301/ppStack/src/ppStackVersion.c

-              r18419
+              r23352
 #include <psmodules.h>
 #include <ppStats.h>
+#include <psphot.h>
 #include "ppStack.h"
+static const char *cvsTag = "$Name: not supported by cvs2svn $";// CVS tag name
+#ifndef PPSTACK_VERSION
+#error "PPSTACK_VERSION is not set"
+#endif
+#ifndef PPSTACK_BRANCH
+#error "PPSTACK_BRANCH is not set"
+#endif
+#ifndef PPSTACK_SOURCE
+#error "PPSTACK_SOURCE is not set"
+#endif
+#define xstr(s) str(s)
+#define str(s) #s
 psString ppStackVersion(void)
+{
+    psString version = NULL;            // Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", xstr(PPSTACK_BRANCH), xstr(PPSTACK_VERSION));
+    return value;
+}
+psString ppStackSource(void)
+{
+    return psStringCopy(xstr(PPSTACK_SOURCE));
+}
 psString ppStackVersionLong(void)
+{
+    psString version = ppStackVersion(); // Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag
+    psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__);
+    psFree(tag);
+    psString version = ppStackVersion();  // Version, to return
+    psString source = ppStackSource();    // Source
+    psStringPrepend(&version, "ppStack ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
     return version;
+}
+};
 void ppStackVersionMetadata(psMetadata *metadata)
+bool ppStackVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(metadata,);
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString ppStats = ppStatsVersionLong(); // ppStats version
+    psString ppStack = ppStackVersionLong(); // ppStack version
+    PS_ASSERT_METADATA_NON_NULL(header, false);
     psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
     psString timeString = psTimeToISO(time); // The time in an ISO string
     psFree(time);
+    psString head = NULL;               // Head string
+    psStringAppend(&head, "ppStack processing at %s. Component information:", timeString);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "ppStack at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+    psphotVersionHeader(header);
+    ppStatsVersionHeader(header);
+    psString version = ppStackVersion(); // Software version
+    psString source  = ppStackSource();  // Software source
+    psStringPrepend(&version, "ppStack version: ");
+    psStringPrepend(&source, "ppStack source: ");
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+    psFree(version);
+    psFree(source);
+    return true;
+}
+void ppStackVersionPrint(void)
+{
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("ppStack", PS_LOG_INFO, "ppStack at %s", timeString);
     psFree(timeString);
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, "", head);
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, "", pslib);
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, "", psmodules);
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, "", ppStats);
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, "", ppStack);
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString psphot = psphotVersionLong(); // psphot version
+    psString ppStats = ppStatsVersionLong(); // psastro version
+    psString ppStack = ppStackVersionLong(); // ppStack version
+    psFree(head);
+    psLogMsg("ppStack", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("ppStack", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("ppStack", PS_LOG_INFO, "%s", psphot);
+    psLogMsg("ppStack", PS_LOG_INFO, "%s", ppStats);
+    psLogMsg("ppStack", PS_LOG_INFO, "%s", ppStack);
     psFree(pslib);
     psFree(psmodules);
+    psFree(psphot);
     psFree(ppStats);
     psFree(ppStack);

branches/cnb_branches/cnb_branch_20090301/ppStats/src/Makefile.am

-              r19626
+              r23352
 lib_LTLIBRARIES = libppStats.la
+libppStats_la_CFLAGS = $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
+# PPSTATS_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+# PPSTATS_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+# PPSTATS_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+#
+# # Force recompilation of ppStatsVersion.c, since it gets the version information
+# ppStatsVersion.c: FORCE
+#       touch ppStatsVersion.c
+# FORCE: ;
+libppStats_la_CFLAGS = $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPPSTATS_VERSION=$(SVN_VERSION) -DPPSTATS_BRANCH=$(SVN_BRANCH) -DPPSTATS_SOURCE=$(SVN_SOURCE)
 libppStats_la_LDFLAGS = $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)

branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStats.h

-              r21183
+              r23352
     psList *summary;                    // Summary statistics to calculate
     // Options for input data
     bool doFirstReadout3D;              // for 3D data, use the first readout?
+    bool doFirstReadout3D;              // for 3D data, use the first readout?
     float sample;                       // Fraction of cell to sample for statistics
     psImageMaskType maskVal;            // Mask value for images
+    psImageMaskType maskVal;            // Mask value for images
     psList *chips;                      // Chips to look at
     psList *cells;                      // Cells to look at
 …
 psString ppStatsVersion(void);
+/// Return source information
+psString ppStatsSource(void);
 /// Return long version information
 psString ppStatsVersionLong(void);
+/// Populate header with version information
+bool ppStatsVersionHeader(
+    psMetadata *header                  ///< Header to populate
+    );
 void p_ppStatsGetMetadata(psMetadata *target, // Target for metadata

branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsFromMetadataPrint.c

-              r20305
+              r23352
             VALUE_NUMERICAL_CASE(F64, "lf",   F64);
           case PS_DATA_STRING:
+            fprintf(f, "%s '%s' ", entry->flag, entry->value->data.str);
+            if (entry->value->data.str) {
+                fprintf(f, "%s '%s' ", entry->flag, entry->value->data.str);
+            }
             break;
           case PS_DATA_BOOL:
 …
           case PS_DATA_TIME: {
               psTime *t = (psTime*)entry->value->data.V;
+              psString str = psTimeToISO(t);
+              fprintf(f, "%s %.19sZ ", entry->flag, str);
+              psFree(str);
+              if (t) {
+                  psString str = psTimeToISO(t);
+                  fprintf(f, "%s %.19sZ ", entry->flag, str);
+                  psFree(str);
+              }
               break;
+          }

branches/cnb_branches/cnb_branch_20090301/ppStats/src/ppStatsVersion.c

-              r13993
+              r23352
 #include "ppStatsInternal.h"
+static const char *cvsTag = "$Name: not supported by cvs2svn $";// CVS tag name
+#ifndef PPSTATS_VERSION
+#error "PPSTATS_VERSION is not set"
+#endif
+#ifndef PPSTATS_BRANCH
+#error "PPSTATS_BRANCH is not set"
+#endif
+#ifndef PPSTATS_SOURCE
+#error "PPSTATS_SOURCE is not set"
+#endif
+#define xstr(s) str(s)
+#define str(s) #s
 psString ppStatsVersion(void)
+{
+    psString version = NULL;            // Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", xstr(PPSTATS_BRANCH), xstr(PPSTATS_VERSION));
+    return value;
+}
+psString ppStatsSource(void)
+{
+    return psStringCopy(xstr(PPSTATS_SOURCE));
+}
 psString ppStatsVersionLong(void)
+{
+    psString version = ppStatsVersion(); // Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag
+    psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__);
+    psFree(tag);
+    psString version = ppStatsVersion();  // Version, to return
+    psString source = ppStatsSource();    // Source
+    psStringPrepend(&version, "ppStats ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
     return version;
+};
+bool ppStatsVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+    psString version = ppStatsVersion(); // Software version
+    psString source = ppStatsSource();   // Software source
+    psStringPrepend(&version, "ppStats version: ");
+    psStringPrepend(&source, "ppStats source: ");
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+    psFree(version);
+    psFree(source);
+    return true;
+}

branches/cnb_branches/cnb_branch_20090301/ppSub

Property svn:mergeinfo set to (toggle deleted branches)
/branches/cnb_branches/cnb_branch_20090215/ppSub merged eligible
/trunk/ppSub merged eligible
/branches/cnb_branch_20090215/ppSub 21495-22685
/branches/eam_branches/eam_branch_20090303/ppSub 23158-23228

branches/cnb_branches/cnb_branch_20090301/ppSub/src/Makefile.am

-              r21257
+              r23352
 bin_PROGRAMS = ppSub ppSubKernel
+ppSub_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PPSUB_CFLAGS)
+# PPSUB_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+# PPSUB_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+# PPSUB_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+#
+# # Force recompilation of ppSubVersion.c, since it gets the version information
+# ppSubVersion.c: FORCE
+#       touch ppSubVersion.c
+# FORCE: ;
+ppSub_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PPSUB_CFLAGS) -DPPSUB_VERSION=$(SVN_VERSION) -DPPSUB_BRANCH=$(SVN_BRANCH) -DPPSUB_SOURCE=$(SVN_SOURCE)
 ppSub_LDFLAGS  = $(PSLIB_LIBS)   $(PSMODULE_LIBS)   $(PPSTATS_LIBS)   $(PSPHOT_LIBS)   $(PPSUB_LIBS)

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSub.c

-              r21524
+              r23352
         goto die;
+    }
+    ppSubVersionPrint();
     if (!pmModelClassInit()) {
 …
     psTimerStop();
     pmSubtractionVisualClose(); //close plot windows, if -visual is set
+    pmVisualClose(); //close plot windows, if -visual is set
     psFree(config);
     pmModelClassCleanup();

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSub.h

-              r21524
+              r23352
     );
 /// Put the program version information into a metadata
 void ppSubVersionMetadata(psMetadata *metadata ///< Metadata to populate
+/// Put the program version information into a header
+bool ppSubVersionHeader(psMetadata *header ///< Header to populate
     );
+/// Print version information
+void ppSubVersionPrint(void);

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubArguments.c

-              r21524
+              r23352
+{
     fprintf(stderr, "\nPan-STARRS PSF-matched image subtraction\n\n");
     fprintf(stderr, "Usage: %s INPUT.fits REFERENCE.fits OUTPUT_ROOT \n"
             "\t[-psf REFERENCE.psf.fits] [-sources REFERENCE.cmf]\n\n"
+    fprintf(stderr, "Usage: %s OUTPUT_ROOT \n"
+            "\t[-psf REFERENCE.psf.fits]\n\n"
             "This subtracts the convolved REFERENCE from the INPUT, by default.\n",
             program);
 …
 bool ppSubArgumentsSetup(int argc, char *argv[], pmConfig *config)
+{
     int argnum;                         // Argument Number
+    //    int argnum;                         // Argument Number
     assert(config);
 …
+    }
-    if ((argnum = psArgumentGet(argc, argv, "-psphot-visual"))) {
-        psArgumentRemove(argnum, &argc, argv);
-        psphotSetVisual(true);
+    }
-    pmConfigFileSetsMD(config->arguments, &argc, argv, "PPSUB.SOURCES", "-sources", NULL);
     psMetadata *arguments = config->arguments; // Command-line arguments
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-inimage", 0, "Input image", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-inmask", 0, "Input mask image", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-invariance", 0, "Input variance image", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-insources", 0, "Input source list", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-refimage", 0, "Reference image", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-refmask", 0, "Reference mask image", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-refvariance", 0, "Reference variance image", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-refsources", 0, "Reference source list", NULL);
+    psMetadataAddStr(arguments, PS_LIST_TAIL, "-kernel", 0, "Precalculated kernel to apply", NULL);
     psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "Statistics file", NULL);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-region", 0, "Size of iso-kernel region", NAN);
 …
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-spacing", 0, "Typical stamp spacing (pixels)", NAN);
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-footprint", 0, "Stamp footprint half-size (pixels)", -1);
+    psMetadataAddF32(arguments, PS_LIST_TAIL, "-source-radius", 0, "Source matching radius (pixels)", NAN);
     psMetadataAddS32(arguments, PS_LIST_TAIL, "-stride", 0, "Size of convolution patches (pixels)", -1);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-threshold", 0, "Minimum threshold for stamps (ADU)", NAN);
 …
     psMetadataAddBool(arguments, PS_LIST_TAIL, "-visual", 0, "Show diagnostic plots", NULL);
     if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 4) {
+    if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 2) {
         usage(argv[0], arguments, config);
+    }
-    fileList("INPUT", argv[1], "Name of the input image",     config);
-    fileList("REF",   argv[2], "Name of the reference image", config);
     psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[3]);
+    const char *inImage = psMetadataLookupStr(NULL, arguments, "-inimage"); // Name of input image
+    if (inImage && strlen(inImage) > 0) {
+        fileList("INPUT", inImage, "Name of the input image", config);
+    }
     const char *inMask = psMetadataLookupStr(NULL, arguments, "-inmask"); // Name of input mask
     if (inMask && strlen(inMask) > 0) {
 …
         fileList("INPUT.VARIANCE", inVariance, "Name of the input variance image", config);
+    }
+    const char *inSources = psMetadataLookupStr(NULL, arguments, "-insources"); // Name of input source list
+    if (inSources && strlen(inSources) > 0) {
+        fileList("INPUT.SOURCES", inSources, "Name of the input source list", config);
+    }
+    const char *refImage = psMetadataLookupStr(NULL, arguments, "-refimage"); // Name of reference image
+    if (refImage && strlen(refImage) > 0) {
+        fileList("INPUT", refImage, "Name of the reference image", config);
+    }
     const char *refMask = psMetadataLookupStr(NULL, arguments, "-refmask"); // Name of reference mask
     if (refMask && strlen(refMask) > 0) {
 …
     if (refVariance && strlen(refVariance) > 0) {
         fileList("REF.VARIANCE", refVariance, "Name of the reference variance image", config);
+    }
+    const char *refSources = psMetadataLookupStr(NULL, arguments, "-refsources"); // Name of ref source list
+    if (refSources && strlen(refSources) > 0) {
+        fileList("REF.SOURCES", refSources, "Name of the reference source list", config);
+    }
+    const char *kernel = psMetadataLookupStr(NULL, arguments, "-kernel"); // Name of kernel
+    if (kernel && strlen(kernel) > 0) {
+        fileList("KERNEL", kernel, "Name of the kernel to apply", config);
+    }
 …
+    }
+    VALUE_ARG_RECIPE_FLOAT("-region",     "REGION.SIZE",     F32);
+    VALUE_ARG_RECIPE_INT("-size",         "KERNEL.SIZE",     S32, 0);
+    VALUE_ARG_RECIPE_INT("-order",        "SPATIAL.ORDER",   S32, -1);
+    VALUE_ARG_RECIPE_FLOAT("-spacing",    "STAMP.SPACING",   F32);
+    VALUE_ARG_RECIPE_INT("-rings-order",  "RINGS.ORDER",     S32, -1);
+    VALUE_ARG_RECIPE_INT("-inner",        "INNER",           S32, -1);
+    VALUE_ARG_RECIPE_INT("-spam-binning", "SPAM.BINNING",    S32, 0);
+    VALUE_ARG_RECIPE_INT("-footprint",    "STAMP.FOOTPRINT", S32, -1);
+    VALUE_ARG_RECIPE_INT("-stride",       "STRIDE",          S32, -1);
+    VALUE_ARG_RECIPE_FLOAT("-threshold",  "STAMP.THRESHOLD", F32);
+    VALUE_ARG_RECIPE_INT("-iter",         "ITER",            S32, -1);
+    VALUE_ARG_RECIPE_FLOAT("-rej",        "REJ",             F32);
+    VALUE_ARG_RECIPE_FLOAT("-sys",        "SYS",             F32);
+    VALUE_ARG_RECIPE_FLOAT("-badfrac",    "BADFRAC",         F32);
+    VALUE_ARG_RECIPE_FLOAT("-penalty",    "PENALTY",         F32);
+    VALUE_ARG_RECIPE_FLOAT("-poor-frac",  "POOR.FRACTION",   F32);
+    VALUE_ARG_RECIPE_INT("-bin1",         "BIN1",            S32, 0);
+    VALUE_ARG_RECIPE_INT("-bin2",         "BIN2",            S32, 0);
+    VALUE_ARG_RECIPE_FLOAT("-region",        "REGION.SIZE",     F32);
+    VALUE_ARG_RECIPE_INT("-size",            "KERNEL.SIZE",     S32, 0);
+    VALUE_ARG_RECIPE_INT("-order",           "SPATIAL.ORDER",   S32, -1);
+    VALUE_ARG_RECIPE_FLOAT("-spacing",       "STAMP.SPACING",   F32);
+    VALUE_ARG_RECIPE_INT("-rings-order",     "RINGS.ORDER",     S32, -1);
+    VALUE_ARG_RECIPE_INT("-inner",           "INNER",           S32, -1);
+    VALUE_ARG_RECIPE_INT("-spam-binning",    "SPAM.BINNING",    S32, 0);
+    VALUE_ARG_RECIPE_INT("-footprint",       "STAMP.FOOTPRINT", S32, -1);
+    VALUE_ARG_RECIPE_FLOAT("-source-radius", "SOURCE.RADIUS",   F32);
+    VALUE_ARG_RECIPE_INT("-stride",          "STRIDE",          S32, -1);
+    VALUE_ARG_RECIPE_FLOAT("-threshold",     "STAMP.THRESHOLD", F32);
+    VALUE_ARG_RECIPE_INT("-iter",            "ITER",            S32, -1);
+    VALUE_ARG_RECIPE_FLOAT("-rej",           "REJ",             F32);
+    VALUE_ARG_RECIPE_FLOAT("-sys",           "SYS",             F32);
+    VALUE_ARG_RECIPE_FLOAT("-badfrac",       "BADFRAC",         F32);
+    VALUE_ARG_RECIPE_FLOAT("-penalty",       "PENALTY",         F32);
+    VALUE_ARG_RECIPE_FLOAT("-poor-frac",     "POOR.FRACTION",   F32);
+    VALUE_ARG_RECIPE_INT("-bin1",            "BIN1",            S32, 0);
+    VALUE_ARG_RECIPE_INT("-bin2",            "BIN2",            S32, 0);
     valueArgRecipeStr(arguments, recipe, "-mask-in",   "MASK.IN",  recipe);
 …
     if (psMetadataLookupBool(NULL, arguments, "-visual")) {
         pmSubtractionSetVisual(true);
+        pmVisualSetVisual(true);
+    }
 …
     psTrace("ppSub", 1, "Done reading command-line arguments\n");
-    // Dump configuration, now that's it's settled
-    psBool status;
-    psString dump_file =  psMetadataLookupStr(&status, config->arguments, "-dumpconfig");
-    if (dump_file) {
-        pmConfigCamerasCull(config, NULL);
-        pmConfigRecipesCull(config, "PPSUB,PPSTATS,PSPHOT,MASKS");
-        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); // Input file
-        pmConfigDump(config, input->fpa, dump_file);
+    }
     // XXX move this to ppSubArguments
     int threads = psMetadataLookupS32(NULL, config->arguments, "-threads"); // Number of threads

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubBackground.c

r21524	r23352
53	53	}
54	54	}
55		psImageBinning *binning = psMetadataLookupPtr(&mdok, ~~psphotRecipe~~,
	55	psImageBinning *binning = psMetadataLookupPtr(&mdok, modelRO->analysis,
56	56	"PSPHOT.BACKGROUND.BINNING"); // Binning for model
57	57	psImage *modelImage = modelRO->image; // Background model

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubCamera.c

-              r21524
+              r23352
 #include "ppSub.h"
+// Define an input file
+static pmFPAfile *defineInputFile(pmConfig *config,// Configuration
+                                  pmFPAfile *bind,    // File to which to bind, or NULL
+                                  char *filerule,     // Name of file rule
+                                  char *argname,      // Argument name
+                                  pmFPAfileType fileType // Type of file
+    )
+{
+    bool status;
+    // look for the file on the RUN metadata
+    pmFPAfile *file = pmFPAfileDefineFromRun(&status, config, filerule); // File to return
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "Failed to load file definition for %s", filerule);
+        return NULL;
+    }
+    if (!file) {
+        // look for the file on the argument list
+        if (bind) {
+            file = pmFPAfileBindFromArgs(&status, bind, config, filerule, argname);
+        } else {
+            file = pmFPAfileDefineFromArgs(&status, config, filerule, argname);
+        }
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "Failed to load file definition for %s", filerule);
+            return false;
+        }
+    }
+    if (!file) {
+        return NULL;
+    }
+    if (file->type != fileType) {
+        psError(PS_ERR_UNKNOWN, true, "%s is not of type %s", filerule, pmFPAfileStringFromType(fileType));
+        return NULL;
+    }
+    return file;
+}
+// Define an output file
+static pmFPAfile *defineOutputFile(pmConfig *config, // Configuration
+                                   pmFPAfile *template,    // File to use as basis for definition
+                                   char *filerule,     // Name of file rule
+                                   pmFPAfileType fileType // Type of file
+    )
+{
+    pmFPAfile *file = pmFPAfileDefineFromFile(config, template, 1, 1, filerule);
+    if (!file) {
+        psError(PS_ERR_IO, false, _("Unable to generate output file from %s"), filerule);
+        return NULL;
+    }
+    if (file->type != fileType) {
+        psError(PS_ERR_IO, true, "%s is not of type %s", filerule, pmFPAfileStringFromType(fileType));
+        return NULL;
+    }
+    return file;
+}
+// Define an output file that will be used in a calculation
+// This means it might already be available in the RUN metadata
+static pmFPAfile *defineCalcFile(pmConfig *config, // Configuration
+                                 pmFPAfile *bind,    // File to which to bind, or NULL
+                                 char *filerule,     // Name of file rule
+                                 pmFPAfileType fileType // Type of file
+    )
+{
+    bool status;
+    // look for the file on the RUN metadata
+    pmFPAfile *file = pmFPAfileDefineFromRun(&status, config, filerule); // File to return
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "Failed to load file definition for %s", filerule);
+        return NULL;
+    }
+    if (file) {
+        // It's an input
+        file->save = false;
+    }
+    // define new version of file
+    if (!file) {
+        pmFPAfileDefineOutput(config, bind ? bind->fpa : NULL, filerule);
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "Failed to load file definition for %s", filerule);
+            return false;
+        }
+        if (file) {
+            // It's an output
+            file->save = true;
+        }
+    }
+    if (!file) {
+        return NULL;
+    }
+    if (file->type != fileType) {
+        psError(PS_ERR_UNKNOWN, true, "%s is not of type %s", filerule, pmFPAfileStringFromType(fileType));
+        return NULL;
+    }
+    return file;
+}
 bool ppSubCamera(pmConfig *config)
+{
     bool status = false;                // result from pmFPAfileDefine operations
+    psAssert(config, "Require configuration");
     // Input image
     pmFPAfile *input = pmFPAfileDefineFromArgs(&status, config, "PPSUB.INPUT", "INPUT");
     if (!status || !input) {
+    pmFPAfile *input = defineInputFile(config, NULL, "PPSUB.INPUT", "INPUT", PM_FPA_FILE_IMAGE);
+    if (!input) {
         psError(PS_ERR_IO, false, "Failed to build FPA from PPSUB.INPUT");
         return false;
+    }
+    if (input->type != PM_FPA_FILE_IMAGE) {
+        psError(PS_ERR_IO, true, "PPSUB.INPUT is not of type IMAGE");
+        return false;
+    }
+    // Input mask
+    pmFPAfile *inputMask = pmFPAfileBindFromArgs(&status, input, config, "PPSUB.INPUT.MASK", "INPUT.MASK");
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "Failed to load file definition PPSUB.INPUT.MASK");
+        return NULL;
+    }
+    if (inputMask && inputMask->type != PM_FPA_FILE_MASK) {
+        psError(PS_ERR_IO, true, "PPSUB.INPUT.MASK is not of type MASK");
+        return false;
+    }
+    // Input variance map
+    pmFPAfile *inputVariance = pmFPAfileBindFromArgs(&status, input, config, "PPSUB.INPUT.VARIANCE", "INPUT.VARIANCE");
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "Failed to load file definition PPSUB.INPUT.VARIANCE");
+        return NULL;
+    }
+    if (inputVariance && inputVariance->type != PM_FPA_FILE_VARIANCE) {
+        psError(PS_ERR_IO, true, "PPSUB.INPUT.VARIANCE is not of type VARIANCE");
+        return false;
+    }
+    defineInputFile(config, input, "PPSUB.INPUT.MASK", "INPUT.MASK", PM_FPA_FILE_MASK);
+    pmFPAfile *inVar = defineInputFile(config, input, "PPSUB.INPUT.VARIANCE", "INPUT.VARIANCE",
+                                       PM_FPA_FILE_VARIANCE);
+    defineInputFile(config, input, "PPSUB.INPUT.SOURCES", "INPUT.SOURCES", PM_FPA_FILE_CMF);
     // Reference image
+    status = false;
+    pmFPAfile *ref = pmFPAfileDefineFromArgs(&status, config, "PPSUB.REF", "REF");
+    pmFPAfile *ref = defineInputFile(config, NULL, "PPSUB.REF", "REF", PM_FPA_FILE_IMAGE);
     if (!ref) {
         psError(PS_ERR_IO, false, "Failed to build FPA from PPSUB.REF");
         return false;
+    }
+    if (ref->type != PM_FPA_FILE_IMAGE) {
+        psError(PS_ERR_IO, true, "PPSUB.REF is not of type IMAGE");
+        return false;
+    }
+    // Reference mask
+    pmFPAfile *refMask = pmFPAfileBindFromArgs(&status, ref, config, "PPSUB.REF.MASK", "REF.MASK");
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "Failed to load file definition PPSUB.REF.MASK");
+        return NULL;
+    }
+    if (refMask && refMask->type != PM_FPA_FILE_MASK) {
+        psError(PS_ERR_IO, true, "PPSUB.REF.MASK is not of type MASK");
+        return false;
+    }
+    // Reference variance map
+    pmFPAfile *refVariance = pmFPAfileBindFromArgs(&status, ref, config, "PPSUB.REF.VARIANCE", "REF.VARIANCE");
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "Failed to load file definition PPSUB.REF.VARIANCE");
+        return NULL;
+    }
+    if (refVariance && refVariance->type != PM_FPA_FILE_VARIANCE) {
+        psError(PS_ERR_IO, true, "PPSUB.REF.VARIANCE is not of type VARIANCE");
+        return false;
+    }
+    defineInputFile(config, ref, "PPSUB.REF.MASK", "REF.MASK", PM_FPA_FILE_MASK);
+    pmFPAfile *refVar = defineInputFile(config, ref, "PPSUB.REF.VARIANCE", "REF.VARIANCE",
+                                        PM_FPA_FILE_VARIANCE);
+    defineInputFile(config, ref, "PPSUB.REF.SOURCES", "REF.SOURCES", PM_FPA_FILE_CMF);
     // Output image
+    pmFPAfile *output = pmFPAfileDefineFromFile(config, input, 1, 1, "PPSUB.OUTPUT");
+    if (!output) {
+        psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.OUTPUT"));
+        return false;
+    }
+    if (output->type != PM_FPA_FILE_IMAGE) {
+        psError(PS_ERR_IO, true, "PPSUB.OUTPUT is not of type IMAGE");
+    pmFPAfile *output = defineOutputFile(config, input, "PPSUB.OUTPUT", PM_FPA_FILE_IMAGE);
+    pmFPAfile *outMask = defineOutputFile(config, output, "PPSUB.OUTPUT.MASK", PM_FPA_FILE_MASK);
+    if (!output || !outMask) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
         return false;
+    }
     output->save = true;
-    // Output mask
-    pmFPAfile *outMask = pmFPAfileDefineOutput(config, output->fpa, "PPSUB.OUTPUT.MASK");
-    if (!outMask) {
-        psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.OUTPUT.MASK"));
-        return false;
+    }
-    if (outMask->type != PM_FPA_FILE_MASK) {
-        psError(PS_ERR_IO, true, "PPSUB.OUTPUT.MASK is not of type MASK");
-        return false;
+    }
     outMask->save = true;
+    // Output variance
+    if (inputVariance && refVariance) {
+        pmFPAfile *outVariance = pmFPAfileDefineOutput(config, output->fpa, "PPSUB.OUTPUT.VARIANCE");
+        if (!outVariance) {
+            psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.OUTPUT.VARIANCE"));
+            return false;
+        }
+        if (outVariance->type != PM_FPA_FILE_VARIANCE) {
+            psError(PS_ERR_IO, true, "PPSUB.OUTPUT.VARIANCE is not of type VARIANCE");
+            return false;
+        }
+        outVariance->save = true;
+    }
+    pmFPAfile *outVar = NULL;
+    if (inVar && refVar) {
+        outVar = defineOutputFile(config, output, "PPSUB.OUTPUT.VARIANCE", PM_FPA_FILE_VARIANCE);
+        if (!outVar) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
+            return false;
+        }
+        outVar->save = true;
+    }
     // Convolved input image
+    pmFPAfile *inConv = pmFPAfileDefineFromFile(config, input, 1, 1, "PPSUB.INPUT.CONV");
+    if (!inConv) {
+        psError(PS_ERR_IO, false, _("Unable to generate output file for PPSUB.INPUT.CONV"));
+        return false;
+    }
+    if (output->type != PM_FPA_FILE_IMAGE) {
+        psError(PS_ERR_IO, true, "PPSUB.INPUT.CONV is not of type IMAGE");
+        return false;
+    }
+    // XXX should be based on recipe : inConv->save = true;
+    // Convolved input mask
+    pmFPAfile *inConvMask = pmFPAfileDefineOutput(config, inConv->fpa, "PPSUB.INPUT.CONV.MASK");
+    if (!inConvMask) {
+        psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.INPUT.CONV.MASK"));
+        return false;
+    }
+    if (inConvMask->type != PM_FPA_FILE_MASK) {
+        psError(PS_ERR_IO, true, "PPSUB.INPUT.CONV.MASK is not of type MASK");
+        return false;
+    }
+    // XXX should be based on recipe : inConvMask->save = true;
+    // Convolved input variance
+    if (inputVariance) {
+        pmFPAfile *inConvVariance = pmFPAfileDefineOutput(config, inConv->fpa, "PPSUB.INPUT.CONV.VARIANCE");
+        if (!inConvVariance) {
+            psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.OUTPUT.VARIANCE"));
+            return false;
+        }
+        if (inConvVariance->type != PM_FPA_FILE_VARIANCE) {
+            psError(PS_ERR_IO, true, "PPSUB.INPUT.CONV.VARIANCE is not of type VARIANCE");
+            return false;
+        }
+        // XXX should be based on recipe : inConvVariance->save = true;
+    pmFPAfile *inConvImage = defineOutputFile(config, input, "PPSUB.INPUT.CONV", PM_FPA_FILE_IMAGE);
+    pmFPAfile *inConvMask = defineOutputFile(config, input, "PPSUB.INPUT.CONV.MASK", PM_FPA_FILE_MASK);
+    if (!inConvImage || !inConvMask) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
+        return false;
+    }
+    if (inVar) {
+        pmFPAfile *inConvVar = defineOutputFile(config, input, "PPSUB.INPUT.CONV.VARIANCE",
+                                                PM_FPA_FILE_VARIANCE);
+        if (!inConvVar) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
+            return false;
+        }
+    }
     // Convolved ref image
+    pmFPAfile *refConv = pmFPAfileDefineFromFile(config, ref, 1, 1, "PPSUB.REF.CONV");
+    if (!refConv) {
+        psError(PS_ERR_IO, false, _("Unable to generate output file for PPSUB.REF.CONV"));
+        return false;
+    }
+    if (output->type != PM_FPA_FILE_IMAGE) {
+        psError(PS_ERR_IO, true, "PPSUB.REF.CONV is not of type IMAGE");
+        return false;
+    }
+    // XXX should be based on recipe : refConv->save = true;
+    // Convolved ref mask
+    pmFPAfile *refConvMask = pmFPAfileDefineOutput(config, refConv->fpa, "PPSUB.REF.CONV.MASK");
+    if (!refConvMask) {
+        psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.REF.CONV.MASK"));
+        return false;
+    }
+    if (refConvMask->type != PM_FPA_FILE_MASK) {
+        psError(PS_ERR_IO, true, "PPSUB.REF.CONV.MASK is not of type MASK");
+        return false;
+    }
+    // XXX should be based on recipe : refConvMask->save = true;
+    // Convolved ref variance
+    if (refVariance) {
+        pmFPAfile *refConvVariance = pmFPAfileDefineOutput(config, refConv->fpa, "PPSUB.REF.CONV.VARIANCE");
+        if (!refConvVariance) {
+            psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.OUTPUT.VARIANCE"));
+            return false;
+        }
+        if (refConvVariance->type != PM_FPA_FILE_VARIANCE) {
+            psError(PS_ERR_IO, true, "PPSUB.REF.CONV.VARIANCE is not of type VARIANCE");
+            return false;
+        }
+        // XXX should be based on recipe : refConvVariance->save = true;
+    }
+    pmFPAfile *refConvImage = defineOutputFile(config, input, "PPSUB.REF.CONV", PM_FPA_FILE_IMAGE);
+    pmFPAfile *refConvMask = defineOutputFile(config, input, "PPSUB.REF.CONV.MASK", PM_FPA_FILE_MASK);
+    if (!refConvImage || !refConvMask) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
+        return false;
+    }
+    if (refVar) {
+        pmFPAfile *refConvVar = defineOutputFile(config, input, "PPSUB.REF.CONV.VARIANCE",
+                                                 PM_FPA_FILE_VARIANCE);
+        if (!refConvVar) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to define output files");
+            return false;
+        }
+    }
     // Output JPEGs
 …
     // Output subtraction kernel
+    pmFPAfile *outKernels = pmFPAfileDefineOutput(config, output->fpa, "PPSUB.OUTPUT.KERNELS");
+    if (!outKernels) {
+        psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.OUTPUT.KERNELS"));
+        return false;
+    }
+    if (outKernels->type != PM_FPA_FILE_SUBKERNEL) {
+        psError(PS_ERR_IO, true, "PPSUB.OUTPUT.KERNELS is not of type SUBKERNEL");
+        return false;
+    }
+    outKernels->save = true;
+#if 0
+    if (!pmFPAAddSourceFromFormat(output->fpa, "Subtraction", output->format)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to generate output FPA.");
+        return false;
+    }
+#endif
+    pmFPAfile *sources = pmFPAfileDefineFromArgs(&status, config, "PPSUB.SOURCES", "PPSUB.SOURCES");
+    if (!status) {
+        psError(PS_ERR_IO, false, "Failed to load file definition PPSUB.SOURCES");
+        return false;
+    }
+    if (sources && sources->type != PM_FPA_FILE_CMF) {
+        psError(PS_ERR_IO, true, "PPSUB.SOURCES is not of type CMF");
+    pmFPAfile *kernel = defineCalcFile(config, output, "PPSUB.OUTPUT.KERNELS", PM_FPA_FILE_SUBKERNEL);
+    if (!kernel) {
+        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to define file PPSUB.OUTPUT.KERNELS");
         return false;
+    }

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubLoop.c

-              r21524
+              r23352
 bool ppSubLoop(pmConfig *config)
+{
+    psAssert(config, "Require configuration.");
+    pmConfigCamerasCull(config, NULL);
+    pmConfigRecipesCull(config, "PPSUB,PPSTATS,PSPHOT,MASKS,JPEG");
     bool mdok;                          // Status of MD lookup
     const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics
 …
     pmFPAview *view = pmFPAviewAlloc(0); // Pointer into FPA hierarchy
-    pmHDU *lastHDU = NULL;              // Last HDU that was updated
     // Iterate over the FPA hierarchy
 …
             if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
                 return false;
+            }
-            // Put version information into the header
-            pmHDU *hdu = pmHDUFromCell(inCell);
-            if (hdu && hdu != lastHDU) {
-                if (!hdu->header) {
-                    hdu->header = psMetadataAlloc();
+                }
-                ppSubVersionMetadata(hdu->header);
-                lastHDU = hdu;
+            }
 …
+    }
+    psString dump_file = psMetadataLookupStr(&mdok, config->arguments, "-dumpconfig");
+    if (dump_file) {
+        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); // Input file
+        pmConfigDump(config, input->fpa, dump_file);
+    }
     return true;
+}

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubMakePSF.c

-              r21524
+              r23352
     // Extract the loaded sources from the associated readout, and generate PSF
     // Here, we assume the image is background-subtracted
+    pmReadout *sourcesRO = pmFPAfileThisReadout(config->files, view, "PPSUB.SOURCES");
+    psArray *sources = psMetadataLookupPtr(&mdok, sourcesRO->analysis, "PSPHOT.SOURCES");
+    psArray *sources = psMetadataLookupPtr(&mdok, minuend->analysis, "PSPHOT.SOURCES");
     if (!psphotReadoutFindPSF(config, view, sources)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry on subtracted image.");

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubMatchPSFs.c

-              r21524
+              r23352
     bool mdok;                          // Status of MD lookup
+    // Load pre-calculated kernel, if available
+    pmReadout *kernelRO = pmFPAfileThisReadout(config->files, view, "PPSUB.OUTPUT.KERNEL"); // RO with kernel
     // Sources in image, used for stamps: these must be loaded from previous analysis stages
+    pmReadout *sourcesRO = pmFPAfileThisReadout(config->files, view, "PPSUB.SOURCES"); // Readout with sources
+    psArray *sources = NULL;            // Sources in image; used for stamps
+    if (sourcesRO) {
+        sources = psMetadataLookupPtr(&mdok, sourcesRO->analysis, "PSPHOT.SOURCES");
+    psArray *inSources = psMetadataLookupPtr(&mdok, inRO->analysis, "PSPHOT.SOURCES"); // Input source list
+    psArray *refSources = psMetadataLookupPtr(&mdok, refRO->analysis, "PSPHOT.SOURCES"); // Ref source list
+    psArray *sources = NULL;            // Merged list of sources
+    if (inSources && refSources) {
+        float radius = psMetadataLookupF32(NULL, recipe, "SOURCE.RADIUS"); // Matching radius
+        psArray *lists = psArrayAlloc(2); // Source lists
+        lists->data[0] = psMemIncrRefCounter(inSources);
+        lists->data[1] = psMemIncrRefCounter(refSources);
+        sources = pmSourceMatchMerge(lists, radius);
+        psFree(lists);
+        if (!sources) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to merge source lists");
+            return false;
+        }
+    } else if (inSources) {
+        sources = psMemIncrRefCounter(inSources);
+    } else if (refSources) {
+        sources = psMemIncrRefCounter(refSources);
+    }
+    psMetadataAddArray(inConv->analysis, PS_LIST_TAIL, "PSPHOT.SOURCES", PS_META_REPLACE,
+                       "Merged source list", sources);
+    psMetadataAddArray(refConv->analysis, PS_LIST_TAIL, "PSPHOT.SOURCES", PS_META_REPLACE,
+                       "Merged source list", sources);
+    psFree(sources);                    // Drop reference
     int footprint = psMetadataLookupS32(NULL, recipe, "STAMP.FOOTPRINT"); // Stamp half-size
 …
     // Match the PSFs
+    if (!pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, stride, regionSize, spacing, threshold,
+                            sources, stampsName, type, size, order, widths, orders, inner, ringsOrder,
+                            binning, penalty, optimum, optWidths, optOrder, optThresh, iter, rej, sys,
+                            maskVal, maskBad, maskPoor, poorFrac, badFrac, subMode)) {
+        psError(PS_ERR_UNKNOWN, false, "Unable to match images.");
+        return false;
+    if (kernelRO) {
+        if (!pmSubtractionMatchPrecalc(inConv, refConv, inRO, refRO, kernelRO->analysis,
+                                       stride, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to convolve images.");
+            return false;
+        }
+    } else {
+        if (!pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, stride, regionSize, spacing,
+                                threshold, sources, stampsName, type, size, order, widths, orders, inner,
+                                ringsOrder, binning, penalty, optimum, optWidths, optOrder, optThresh, iter,
+                                rej, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac, subMode)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to match images.");
+            return false;
+        }
+    }
 …
     // XXX drop the pixels associated with inRO and refRO (now that we have inConv and refConf)
+#ifdef TESTING
     psphotSaveImage (NULL, inRO->image, "inRO.fits");
     psphotSaveImage (NULL, refRO->image, "refRO.fits");
     psphotSaveImage (NULL, inConv->image, "inConv.fits");
     psphotSaveImage (NULL, refConv->image, "refConv.fits");
+#endif
     return true;

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubReadoutUpdate.c

r21524	r23352
44	44	psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.INPUT", 0,
45	45	"Subtraction input", inFile->filename);
	46	ppSubVersionHeader(outHDU->header);
46	47
47	48	// Statistics on the matching

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVarianceFactors.c

r21524	r23352
65	65
66	66	psStats *vfStats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); // Statistics
67		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS~~, 0~~); // Random number generator
	67	psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator
68	68
69	69	// mask these values when examining the background

branches/cnb_branches/cnb_branch_20090301/ppSub/src/ppSubVersion.c

-              r21524
+              r23352
 #include <psmodules.h>
 #include <ppStats.h>
+#include <psphot.h>
 #include "ppSub.h"
+static const char *cvsTag = "$Name: not supported by cvs2svn $";///< CVS tag name
+#ifndef PPSUB_VERSION
+#error "PPSUB_VERSION is not set"
+#endif
+#ifndef PPSUB_BRANCH
+#error "PPSUB_BRANCH is not set"
+#endif
+#ifndef PPSUB_SOURCE
+#error "PPSUB_SOURCE is not set"
+#endif
+#define xstr(s) str(s)
+#define str(s) #s
 psString ppSubVersion(void)
+{
+    psString version = NULL;            // Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", xstr(PPSUB_BRANCH), xstr(PPSUB_VERSION));
+    return value;
+}
+psString ppSubSource(void)
+{
+    return psStringCopy(xstr(PPSUB_SOURCE));
+}
 psString ppSubVersionLong(void)
+{
+    psString version = ppSubVersion(); // Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag
+    psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__);
+    psFree(tag);
+    psString version = ppSubVersion();  // Version, to return
+    psString source = ppSubSource();    // Source
+    psStringPrepend(&version, "ppSub ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
     return version;
+}
+};
 void ppSubVersionMetadata(psMetadata *metadata)
+bool ppSubVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(metadata,);
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString ppStats = ppStatsVersionLong(); // ppStats version
+    psString ppSub = ppSubVersionLong(); // ppSub version
+    PS_ASSERT_METADATA_NON_NULL(header, false);
     psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
     psString timeString = psTimeToISO(time); // The time in an ISO string
     psFree(time);
+    psString head = NULL;               // Head string
+    psStringAppend(&head, "ppSub processing at %s. Component information:", timeString);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "ppSub at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+    psphotVersionHeader(header);
+    ppStatsVersionHeader(header);
+    psString version = ppSubVersion(); // Software version
+    psString source  = ppSubSource();  // Software source
+    psStringPrepend(&version, "ppSub version: ");
+    psStringPrepend(&source, "ppSub source: ");
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+    psFree(version);
+    psFree(source);
+    return true;
+}
+void ppSubVersionPrint(void)
+{
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("ppSub", PS_LOG_INFO, "ppSub at %s", timeString);
     psFree(timeString);
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, head, "");
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, pslib, "");
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psmodules, "");
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppStats, "");
     psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppSub, "");
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString psphot = psphotVersionLong(); // psphot version
+    psString ppStats = ppStatsVersionLong(); // ppStats version
+    psString ppSub = ppSubVersionLong(); // ppSub version
+    psFree(head);
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", psphot);
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", ppStats);
+    psLogMsg("ppSub", PS_LOG_INFO, "%s", ppSub);
     psFree(pslib);
     psFree(psmodules);
+    psFree(psphot);
     psFree(ppStats);
     psFree(ppSub);

branches/cnb_branches/cnb_branch_20090301/ppSub/test/fake.c

r18591	r23352
89	89	int main(int argc, char *argv[])
90	90	{
91		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS~~, 0~~); // Random number generator
	91	psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS); // Random number generator
92	92
93	93	// Images to generate

branches/cnb_branches/cnb_branch_20090301/psLib/configure.ac

-              r21400
+              r23352
 TMP_CPPFLAGS=${CPPFLAGS}
+AC_ARG_WITH(cfitsio,
+[AS_HELP_STRING(--with-cfitsio=DIR,Specify location of CFITSIO.)],
+[CFITSIO_CFLAGS="-I$withval/include"
+ CFITSIO_LDFLAGS="-L$withval/lib"])
+AC_ARG_WITH(cfitsio-include,
+[AS_HELP_STRING(--with-cfitsio-include=DIR,Specify CFITSIO include directory.)],
+[CFITSIO_CFLAGS="-I$withval"])
+AC_ARG_WITH(cfitsio-lib,
+[AS_HELP_STRING(--with-cfitsio-lib=DIR,Specify CFITSIO library directory.)],
+[CFITSIO_LDFLAGS="-L$withval"])
+PKG_CHECK_MODULES([CFITSIO], [cfitsio], [], AC_MSG_ERROR([CFITSIO package not found.  Obtain CFITSIO at  http://heasarc.gsfc.nasa.gov/docs/software/fitsio]))
 PSLIB_CFLAGS="${PSLIB_CFLAGS=} ${CFITSIO_CFLAGS}"
 PSLIB_LIBS="${PSLIB_LIBS=} $CFITSIO_LDFLAGS -lcfitsio -lm"
+PSLIB_LIBS="${PSLIB_LIBS=} ${CFITSIO_LIBS}"
 CFLAGS="${CFLAGS=} ${CFITSIO_CFLAGS}"
+LDFLAGS="${LDFLAGS=} ${CFITSIO_LDFLAGS}"
+dnl Solaris needs to suck in these symbols from unusual locations
+AC_SEARCH_LIBS([gethostbyname], [nsl])
+AC_SEARCH_LIBS([socket], [socket])
+AC_CHECK_HEADERS([fitsio.h],[],
+   [AC_MSG_ERROR([CFITSIO headers not found.  Obtain CFITSIO at  http://heasarc.gsfc.nasa.gov/docs/software/fitsio or use --with-cfitsio to specify location.])]
+)
+TMP_LIBS=${LIBS}
+AC_CHECK_LIB(cfitsio,ffopen,[],
+  [AC_MSG_ERROR([CFITSIO library not found.  Obtain it at  http://heasarc.gsfc.nasa.gov/docs/software/fitsio or use --with-cfitsio to specify location.])],[-lm]
+)
+LDFLAGS="${LDFLAGS=} ${CFITSIO_LIBS}"
 dnl Now check if CFITSIO supports fits_open_diskfile, i.e., is at least version 2.501
 AC_CHECK_LIB(cfitsio,ffdkopn,
+AC_CHECK_FUNC(ffdkopn,
   [CFITSIO_DISKFILE=1],
   [AC_MSG_WARN([The CFITSIO library version is rather old.  Suggested version is 2.501 or greater.])
    CFITSIO_DISKFILE=0],[-lm]
+   CFITSIO_DISKFILE=0]
+)
 AC_DEFINE_UNQUOTED([CFITSIO_DISKFILE],${CFITSIO_DISKFILE},[Define to 1 if you have fits_open_diskfile in CFITSIO])
 AC_SUBST([CFITSIO_CFLAGS])
+AC_SUBST([CFITSIO_LIBS])
 dnl restore the LIBS/CFLAGS/LDFLAGS
 …
 TMP_CPPFLAGS=${CPPFLAGS}
+AC_ARG_WITH(fftw3,
+[AS_HELP_STRING(--with-fftw3=DIR,Specify location of FFTW version 3.)],
+[FFTW3_CFLAGS="-I$withval/include"
+ FFTW3_LDFLAGS="-L$withval/lib"])
+AC_ARG_WITH(fftw3-include,
+[AS_HELP_STRING(--with-fftw3-include=DIR,Specify FFTW version 3 include directory.)],
+[FFTW3_CFLAGS="-I$withval"])
+AC_ARG_WITH(fftw3-lib,
+[AS_HELP_STRING(--with-fftw3-lib=DIR,Specify FFTW version 3 library directory.)],
+[FFTW3_LDFLAGS="-L$withval"])
+PKG_CHECK_MODULES([FFTW3], [fftw3f], [], AC_MSG_ERROR([FFTW version 3 (--enable-float) library not found.  Obtain it at http://www.fftw.org/]))
 PSLIB_CFLAGS="${PSLIB_CFLAGS=} ${FFTW3_CFLAGS}"
 PSLIB_LIBS="${PSLIB_LIBS=} $FFTW3_LDFLAGS -lfftw3f"
+PSLIB_LIBS="${PSLIB_LIBS=} ${FFTW3_LIBS}"
 CFLAGS="${CFLAGS} ${FFTW3_CFLAGS}"
 CPPFLAGS=${CFLAGS}
+LDFLAGS="${LDFLAGS} ${FFTW3_LDFLAGS}"
+AC_CHECK_LIB(fftw3f,fftwf_plan_dft_2d,[],
+  [AC_MSG_ERROR([FFTW version 3 (--enable-float) library not found.  Obtain it at http://www.fftw.org/ or use --with-fftw3 to specify location.])])
+FFTW_THREADS=0
+AC_CHECK_LIB(fftw3f,fftwf_init_threads,[FFTW_THREADS=1],
+  [AC_CHECK_LIB(fftw3f_threads,fftwf_init_threads, [
+    FFTW_THREADS=1
+    PSLIB_LIBS="${PSLIB_LIBS=} -lfftw3f_threads"],
+   AC_MSG_WARN([FFTW version 3 not compiled with thread support (--enable-threads)]),[-lm]
+   )],[-lm]
+)
+dnl AC_CHECK_LIB(fftw3f,fftwf_plan_dft_2d,[],
+dnl   [AC_MSG_ERROR([FFTW version 3 (--enable-float) library not found.  Obtain it at http://www.fftw.org/ or use --with-fftw3 to specify location.])],[-lm]
+dnl )
+AC_CHECK_HEADERS([fftw3.h],[],
+  [AC_MSG_ERROR([FFTW version 3 (--enable-float) headers not found.  Obtain it at http://www.fftw.org/ or use --with-fftw3 to specify location.])]
+)
+LDFLAGS="${LDFLAGS} ${FFTW3_LIBS}"
+AC_CHECK_FUNC(fftwf_plan_dft_2d,[],
+  [AC_MSG_ERROR([FFTW version 3 (--enable-float) library not found.  Obtain it at http://www.fftw.org/])])
+AC_CHECK_FUNC(fftwf_init_threads,[FFTW_THREADS=1],[FFTW_THREADS=0])
 AC_DEFINE_UNQUOTED([HAVE_FFTW_THREADS],${FFTW_THREADS},[Define to 1 if you have FFTW compiled with thread support])
 AC_SUBST([FFTW3_CFLAGS])
+AC_SUBST([FFTW3_LIBS])
 dnl restore the CFLAGS/LDFLAGS
 …
 TMP_CPPFLAGS=${CPPFLAGS}
+AC_ARG_WITH(gsl-config,
+[AS_HELP_STRING(--with-gsl-config=FILE,Specify location of gsl-config.)],
+[GSL_CONFIG=$withval],
+[GSL_CONFIG=`which gsl-config`])
+AC_CHECK_FILE($GSL_CONFIG,[],
+    [AC_MSG_ERROR([GSL is required.  Obtain it at http://www.gnu.org/software/gsl or use --with-gsl-config to specify location.])])
+AC_MSG_CHECKING([GSL cflags])
+GSL_CFLAGS="`${GSL_CONFIG} --cflags`"
+AC_MSG_RESULT([${GSL_CFLAGS}])
+AC_MSG_CHECKING([GSL ldflags])
+GSL_LDFLAGS="`${GSL_CONFIG} --libs`"
+AC_MSG_RESULT([${GSL_LDFLAGS}])
+PKG_CHECK_MODULES([GSL], [gsl], [], AC_MSG_ERROR([GSL is required.  Obtain it at http://www.gnu.org/software/gsl]))
 PSLIB_CFLAGS="${PSLIB_CFLAGS=} ${GSL_CFLAGS}"
 PSLIB_LIBS="${PSLIB_LIBS=} ${GSL_LDFLAGS}"
+PSLIB_LIBS="${PSLIB_LIBS=} ${GSL_LIBS}"
 AC_SUBST([GSL_CFLAGS])
+AC_SUBST([GSL_LIBS])
 dnl restore the CFLAGS/LDFLAGS
 …
 LDFLAGS="${LDFLAGS} ${JPEG_LDFLAGS}"
 AC_CHECK_HEADERS([jpeglib.h],[PSLIB_CFLAGS="$PSLIB_CFLAGS $JPEG_CFLAGS"
+AC_CHECK_HEADERS([jpeglib.h],[PSLIB_CFLAGS="${PSLIB_CFLAGS=} ${JPEG_CFLAGS}"
                               AC_SUBST(JPEG_CFLAGS)],
   [AC_MSG_ERROR([libjpeg headers not found.  Obtain libjpeg from http://www.ijg.org/ or use --with-jpeg to specify location.])]
+)
 AC_CHECK_LIB(jpeg,jpeg_CreateCompress,[PSLIB_LIBS="$PSLIB_LIBS $JPEG_LDFLAGS -ljpeg"],
+AC_CHECK_LIB(jpeg,jpeg_CreateCompress,[PSLIB_LIBS="${PSLIB_LIBS=} ${JPEG_LDFLAGS} -ljpeg"],
   [AC_MSG_ERROR([libjpeg library not found.  Obtain libjpeg from http://www.ijg.org/ or use --with-jpeg to specify location.])]
+)
 …
 LDFLAGS=${TMP_LDFLAGS}
 CPPFLAGS=${TMP_CPPFLAGS}
-dnl ------------------- XML2 options ---------------------
-dnl AC_ARG_WITH(xml2-config,
-dnl [AS_HELP_STRING(--with-xml2-config=FILE,Specify location of xml2-config.)],
-dnl [XML_CONFIG=$withval],
-dnl [XML_CONFIG=`which xml2-config`])
-dnl AC_CHECK_FILE($XML_CONFIG,[],
-dnl     [AC_MSG_ERROR([GNOME XML C parser is required.  Obtain it at http://www.xmlsoft.org or use --with-xml2-config to specify location.])])
-dnl
-dnl AC_MSG_CHECKING([xml2 version])
-dnl XML_VERSION=`xml2-config --version`
-dnl XML_VERSION_major=`echo $XML_VERSION | ${PERL} -pe 's|^(\d+).*|\1|'`
-dnl XML_VERSION_minor=`echo $XML_VERSION | ${PERL} -pe 's|^(\d+)\.(\d+).*|\2|'`
-dnl dnl First test the minimum version of 2.6
-dnl if test $XML_VERSION_major -lt 2 || ( test $XML_VERSION_major -eq 2 && test $XML_VERSION_minor -lt 6 )
-dnl then
-dnl     AC_MSG_ERROR([requires libxml2 2.6.0 or greater, found $XML_VERSION.  Install newer version or use --with-xml2-config to specify another location.])
-dnl else
-dnl     AC_MSG_RESULT([$XML_VERSION... yes])
-dnl fi
-dnl
-dnl AC_MSG_CHECKING([xml2 cflags])
-dnl XML_CFLAGS="`${XML_CONFIG} --cflags`"
-dnl AC_MSG_RESULT([${XML_CFLAGS}])
-dnl
-dnl AC_MSG_CHECKING([xml2 ldflags])
-dnl XML_LDFLAGS="`${XML_CONFIG} --libs`"
-dnl AC_MSG_RESULT([${XML_LDFLAGS}])
-dnl
-dnl PSLIB_CFLAGS="${PSLIB_CFLAGS=} ${XML_CFLAGS}"
-dnl PSLIB_LIBS="${PSLIB_LIBS=} ${XML_LDFLAGS}"
-dnl
-dnl AC_SUBST([XML_CFLAGS])
 dnl ------------------- SWIG options ---------------------
 …
   utils/Makefile
 ])
-dnl src/xml/Makefile
-dnl test/xml/Makefile
 #if test "$SWIG_REQ" == "yes"

branches/cnb_branches/cnb_branch_20090301/psLib/src/fits/psFitsHeader.c

-              r22432
+              r23352
 bool psFitsCheckSingleCompressedImagePHU(const psFits *fits, psMetadata *header)
+bool psFitsCheckCompressedImagePHU(const psFits *fits, psMetadata *header)
+{
     PS_ASSERT_FITS_NON_NULL(fits, false);
 …
+    }
     if (psFitsGetSize(fits) != 2) {
         // No second extension, or multiple extensions
+    if (psFitsGetSize(fits) == 1) {
+        // No extension present
         return false;
+    }
 …
     // Explore the potential case that this is an empty PHU, and the first extension contains the sole image,
     // which is compressed.
     if (psFitsCheckSingleCompressedImagePHU(fits, header)) {
+    if (psFitsCheckCompressedImagePHU(fits, header)) {
         // This is really what we want, not the empty PHU
         psTrace("psLib.fits", 1,

branches/cnb_branches/cnb_branch_20090301/psLib/src/fits/psFitsHeader.h

-              r15179
+              r23352
 /// Determine whether the current HDU is an empty PHU with a single compressed image following.
+/// Determine whether the current HDU is an empty PHU with a compressed image following.
 ///
 /// In that case, what should be treated as an image PHU is technically an empty PHU with a binary table
 …
 /// following compressed image to determine if this is the case.  If so, the FITS file pointer is left
 /// pointing at the compressed image.
 bool psFitsCheckSingleCompressedImagePHU(const psFits *fits, ///< FITS file pointer
                                          const psMetadata *header ///< Header, or NULL
+bool psFitsCheckCompressedImagePHU(const psFits *fits, ///< FITS file pointer
+                                   const psMetadata *header ///< Header, or NULL
     );

branches/cnb_branches/cnb_branch_20090301/psLib/src/fits/psFitsImage.c

-              r21535
+              r23352
     PS_ASSERT_FITS_NON_NULL(fits, NULL);
     if (psFitsCheckSingleCompressedImagePHU(fits, NULL)) {
+    if (psFitsCheckCompressedImagePHU(fits, NULL)) {
         // This is really what we want, not the empty PHU
         psTrace("psLib.fits", 1,
 …
     PS_ASSERT_INT_NONNEGATIVE(z, NULL);
     if (psFitsCheckSingleCompressedImagePHU(fits, NULL)) {
+    if (psFitsCheckCompressedImagePHU(fits, NULL)) {
         // This is really what we want, not the empty PHU
         psTrace("psLib.fits", 1,
 …
+    }
     if (psFitsCheckSingleCompressedImagePHU(fits, NULL)) {
+    if (psFitsCheckCompressedImagePHU(fits, NULL)) {
         // This is really what we want, not the empty PHU
         psTrace("psLib.fits", 1,
 …
     // code replication, and should be sufficient for our needs.
     if (psFitsCheckSingleCompressedImagePHU(fits, NULL)) {
+    if (psFitsCheckCompressedImagePHU(fits, NULL)) {
         // This is really what we want, not the empty PHU
         psTrace("psLib.fits", 1,

branches/cnb_branches/cnb_branch_20090301/psLib/src/fits/psFitsScale.c

-              r21183
+              r23352
     // psImageBackground automatically excludes pixels that are non-finite, so we don't need to bother about a
     // mask.
+    psU64 seed = p_psRandomGetSystemSeed(false);
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, seed);
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
     psStats *stats = psStatsAlloc(MEAN_STAT | STDEV_STAT); // Statistics object
     if (!psImageBackground(stats, NULL, image, mask, maskVal, rng)) {
 …
     if (!psMemIncrRefCounter(rng) && options->fuzz) {
         // Don't blab about which seed we're going to get --- it's not necessary for this purpose
+        psU64 seed = p_psRandomGetSystemSeed(false);
+        rng = psRandomAlloc(PS_RANDOM_TAUS, seed);
+        rng = psRandomAlloc(PS_RANDOM_TAUS);
+    }

branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageCovariance.c

-              r21467
+              r23352
     } else {
         covar = psImageCovarianceNone();
+    }
+    // Check for non-finite elements
+    for (int y = kernel->yMin; y <= kernel->yMax; y++) {
+        for (int x = kernel->xMin; x <= kernel->xMax; x++) {
+            if (!isfinite(kernel->kernel[y][x])) {
+                psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                        "Non-finite covariance matrix element in kernel at %d,%d", x, y);
+                psFree(covar);
+                return NULL;
+            }
+        }
+    }
+    for (int y = covar->yMin; y <= covar->yMax; y++) {
+        for (int x = covar->xMin; x <= covar->xMax; x++) {
+            if (!isfinite(covar->kernel[y][x])) {
+                psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                        "Non-finite covariance matrix element in covariance matrix at %d,%d", x, y);
+                psFree(covar);
+                return NULL;
+            }
+        }
+    }
 …
 float psImageCovarianceFactor(const psKernel *covariance)
+{
     return covariance ? covariance->kernel[0][0] : NAN;
+    return covariance ? covariance->kernel[0][0] : 1.0;
+}
 …
         for (int y = covar->yMin; y <= covar->yMax; y++) {
             for (int x = covar->xMin; x <= covar->xMax; x++) {
+                if (!isfinite(covar->kernel[y][x])) {
+                    psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                            "Non-finite covariance matrix element at %d,%d for input %d",
+                            x, y, i);
+                    psFree(sum);
+                    return NULL;
+                }
                 sum->kernel[y][x] += covar->kernel[y][x];
+            }
 …
             int radius = PS_MAX(abs(x), abs(y)); // Squarish radius
             psAssert(radius <= maxRadius, "Radius doesn't fit");
+            if (!isfinite(covar->kernel[y][x])) {
+                psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Non-finite covariance matrix element at %d,%d",
+                        x, y);
+                return NULL;
+            }
             radiusSum->data.F64[radius] += fabsf(covar->kernel[y][x]);
             sum += fabsf(covar->kernel[y][x]);

branches/cnb_branches/cnb_branch_20090301/psLib/src/imageops/psImageInterpolate.c

-              r21290
+              r23352
+    }
     float norm1 = 2.0 / PS_SQR(M_PI); // Normalisation for laczos
     float norm2 = M_PI * 4.0 / (float)size; // Normalisation for sinc function 1
     float norm3 = M_PI_2 * 4.0 / (float)size; // Normalisation for sinc function 2
+    float norm1 = size / 2.0 / PS_SQR(M_PI); // Normalisation for laczos
+    float norm2 = M_PI;                 // Normalisation for sinc function 1
+    float norm3 = M_PI * 2.0 / (float)size; // Normalisation for sinc function 2
     float pos = - (size - 1)/2 - frac;  // Position of interest
     for (int i = 0; i < size; i++, pos += 1.0) {

branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psRandom.c

-              r20547
+              r23352
 #include <inttypes.h>
+#include "psAbort.h"
 #include "psMemory.h"
 #include "psRandom.h"
 …
+unsigned long seed = 0;                 // Seed for RNG
 psU64 p_psRandomGetSystemSeed(bool log)
+{
+    FILE*  fd;
+    psU64  seedVal = 0;
+    time_t timeVal;
+    psU64 seedVal = 0;                  // Seed value to return
+    fd = fopen("/dev/urandom","r");
+    if(fd == NULL) {
+        // Read system clock to get seed
+        seedVal = (psU64)time(&timeVal);
+    } else {
+        // Read urandom to get seed
+        if (fread(&seedVal, sizeof(psU64),1,fd)) {;} // ignore return value
+        // Close file
+        fclose(fd);
+    // Since zero is a special value in our context, don't allow the final value chosen to be zero
+    while (seedVal == 0) {
+        FILE *fd = fopen("/dev/urandom", "r");
+        if (fd) {
+            // Read urandom to get seed
+            if (fread(&seedVal, sizeof(psU64), 1, fd)) {;} // ignore return value
+            // Close file
+            fclose(fd);
+        } else {
+            // Read system clock to get seed
+            time_t timeVal;                 // Time value
+            seedVal = (psU64)time(&timeVal);
+        }
+    }
 …
+}
+psRandom *psRandomAlloc(psRandomType type,
+                        unsigned long seed)
+psU64 psRandomSeed(psU64 value)
+{
+    gsl_rng   *r      = NULL;
+    psRandom  *myRNG  = NULL;
+    while (value == 0) {
+        value = p_psRandomGetSystemSeed(false);
+    }
+    seed = value;
+    return seed;
+}
+// Destructor for psRandom
+static void randomFree(psRandom *rng)
+{
+    if (rng->gsl) {
+        gsl_rng_free(rng->gsl);
+    }
+    return;
+}
+// Constructor for psRandom
+static psRandom *randomAlloc(psRandomType type)
+{
+    psRandom *rng = psAlloc(sizeof(psRandom)); // Random number generator to return
+    psMemSetDeallocator(rng, (psFreeFunc)randomFree);
+    rng->type = type;
+    const gsl_rng_type *gslType;        // Type of RNG according to GSL
     switch (type) {
+    case PS_RANDOM_TAUS:
+        myRNG = (psRandom*)psAlloc(sizeof(psRandom));
+        r = gsl_rng_alloc(gsl_rng_taus);
+        myRNG->gsl = r;
+        if(seed == 0) {
+            gsl_rng_set(myRNG->gsl, p_psRandomGetSystemSeed(true));
+        } else {
+            gsl_rng_set(myRNG->gsl, seed);
+        }
+        myRNG->type = type;
+      case PS_RANDOM_TAUS:
+        gslType = gsl_rng_taus;
         break;
+    default:
+        psError(PS_ERR_UNEXPECTED_NULL, true, _("Unknown Random Number Generator Type"));
+      default:
+        psAbort("Unknown Random Number Generator Type: %x", type);
         break;
+    }
+    return(myRNG);
+    rng->gsl = gsl_rng_alloc(gslType);
+    return rng;
+}
+void psRandomReset(psRandom *rand,
+                   unsigned long seed)
+psRandom *psRandomAlloc(psRandomType type)
+{
+    // Check null psRandom
+    if(rand==NULL) {
+        psError(PS_ERR_UNEXPECTED_NULL,
+                true,
+                _("Random variable is NULL."));
+    } else {
+        // Check seed value to see if system seed should be used
+        if(seed == 0) {
+            gsl_rng_set(rand->gsl,p_psRandomGetSystemSeed(true));
+        } else {
+            gsl_rng_set(rand->gsl, seed);
+        }
+    psRandom *rng = randomAlloc(type);
+    psRandomReset(rng);
+    return rng;
+}
+psRandom *psRandomAllocSpecific(psRandomType type, psU64 specificSeed)
+{
+    psRandom *rng = randomAlloc(type);
+    if (specificSeed == 0) {
+        specificSeed = p_psRandomGetSystemSeed(true);
+    }
+    gsl_rng_set(rng->gsl, specificSeed);
+    return rng;
+}
+bool psRandomReset(psRandom *rand)
+{
+    PS_ASSERT_RANDOM_NON_NULL(rand, false);
+    if (seed == 0) {
+        seed = p_psRandomGetSystemSeed(true);
+    }
+    gsl_rng_set(rand->gsl, seed);
+    return true;
+}
 double psRandomUniform(const psRandom *r)
+{
+    // Check null psRandom variable
+    if(r == NULL) {
+        psError(PS_ERR_UNEXPECTED_NULL,
+                true,
+                _("Random variable is NULL."));
+        return(0);
+    } else {
+        return(gsl_rng_uniform(r->gsl));
+    }
+    PS_ASSERT_RANDOM_NON_NULL(r, NAN);
+    return gsl_rng_uniform(r->gsl);
+}
 double psRandomGaussian(const psRandom *r)
+{
+    // Check null psRandom variable
+    if(r == NULL) {
+        psError(PS_ERR_UNEXPECTED_NULL,
+                true,
+                _("Random variable is NULL."));
+        return(0);
+    } else {
+        // XXX: What should sigma be?
+        return(gsl_ran_gaussian(r->gsl, 1.0));
+    }
+    PS_ASSERT_RANDOM_NON_NULL(r, NAN);
+    return gsl_ran_gaussian(r->gsl, 1.0);
+}
 double p_psRandomGaussian(const psRandom *r, double sigma)
+{
+    // Check null psRandom variable
+    if(r == NULL) {
+        psError(PS_ERR_UNEXPECTED_NULL,
+                true,
+                _("Random variable is NULL."));
+        return(0);
+    } else {
+        return(gsl_ran_gaussian(r->gsl, sigma));
+    }
+    PS_ASSERT_RANDOM_NON_NULL(r, NAN);
+    return gsl_ran_gaussian(r->gsl, sigma);
+}
 double psRandomPoisson(const psRandom *r, double mean)
+{
+    // Check null psRandom variable
+    if(r == NULL) {
+        psError(PS_ERR_UNEXPECTED_NULL,
+                true,
+                _("Random variable is NULL."));
+        return(0);
+    } else {
+        return((psF64) gsl_ran_poisson(r->gsl, mean));
+    }
+    PS_ASSERT_RANDOM_NON_NULL(r, NAN);
+    return gsl_ran_poisson(r->gsl, mean);
+}

branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psRandom.h

-              r15627
+              r23352
 #include <gsl/gsl_randist.h>
 /** Enumeration containing a flag for psRandom types.  */
+/// Random number generator types
 typedef enum {
     PS_RANDOM_TAUS                     ///< A maximally equidistributed combined Tausworthe generator.
 } psRandomType;
+/** Data structure for psRandom.
+ *  Contains information on the psRandom type and GNU Scientific Library random number generator.
+ */
+/// Random number generator
 typedef struct {
     psRandomType type;                 ///< The type of RNG
 …
     );
+/** Allocates a psRandom struct.
+ *
+ *  @return psRandom*:    A new psRandom structure.
+ */
+/// Set the seed to use for random number generators.
+///
+/// A seed value of zero indicates that the seed is to be generated from the system.
+/// The new seed value is returned
+psU64 psRandomSeed(psU64 seed           ///< Seed for RNG
+                   );
+/// Allocate a random number generator
+///
+/// The currently defined seed (via psRandomSeed) is used
 psRandom *psRandomAlloc(
+    psRandomType type,                 ///< The type of RNG
+    unsigned long seed                 ///< Known value with which to seed the RNG
+    psRandomType type                   ///< The type of RNG
 ) PS_ATTR_MALLOC;
+/** Resets an existing psRandom struct.
+ *
+ *  @return void
+ */
+void psRandomReset(
+    psRandom *rand,                    ///< Existing psRandom struct to reset
+    unsigned long seed                 ///< Known value with which to seed the RNG
+/// Allocate a random number generator with a specific seed
+///
+/// A seed value of zero indicates that the seed is to be generated from the system.
+psRandom *psRandomAllocSpecific(psRandomType type, ///< The type of RNG
+                                psU64 specificSeed ///< The specific seed to use
+    );
+/// Resets an existing random number generator
+bool psRandomReset(
+    psRandom *rand                      ///< Random number generator to reset
 );
 …
  */
 double psRandomUniform(
     const psRandom *r                  ///< psRandom struct for RNG
+    const psRandom *r                  ///< Random number generator
 );
 …
  */
 double psRandomGaussian(
     const psRandom *r                  ///< psRandom struct for RNG
+    const psRandom *r                  ///< Random number generator
 );
 /** Random number generator based on a Gaussian deviate, N(0,1).
+/** Random number generator based on a Gaussian deviate with specified standard deviation.
  *  Uses gsl_ran_gaussian.
+ *
- *  XXX: I created this since the above psLib spec for p_psRandomGaussian
- *  had no argument for sigma.  Verify that with IfA.
+ *
  *  @return double:     Random number.
  */
 double p_psRandomGaussian(
     const psRandom *r,                  ///< psRandom struct for RNG
+    const psRandom *r,                  ///< Random number generator
     double sigma
 );
 …
  */
 double psRandomPoisson(
     const psRandom *r,                 ///< psRandom struct for RNG
+    const psRandom *r,                  ///< Random number generator
     double mean                         ///< Mean value
 );

branches/cnb_branches/cnb_branch_20090301/psLib/src/math/psStats.c

-              r21183
+              r23352
     for (long i = 0; i < numData; i++) {
         // Check if the data is with the specified range
+        if (!isfinite(data[i]))
+            continue;
         if (useRange && (data[i] < stats->min))
             continue;
 …
     // into the temporary vectors.
     for (long i = 0; i < inVector->n; i++) {
+        if (!isfinite(input[i]))
+            continue;
         if (useRange && (input[i] < stats->min))
             continue;
 …
     for (long i = 0; i < myVector->n; i++) {
         // Check if the data is with the specified range
+        if (!isfinite(data[i]))
+            continue;
         if (useRange && (data[i] < stats->min)) {
             continue;
 …
     for (long i = 0; i < myVector->n; i++) {
         // Check if the data is with the specified range
+        if (!isfinite(data[i]))
+            continue;
         if (useRange && (data[i] < stats->min)) {
             continue;

branches/cnb_branches/cnb_branch_20090301/psLib/src/pslib_strict.h

r21283	r23352
110	110	#include "psMetadataItemParse.h"
111	111	#include "psMetadataItemCompare.h"
	112	#include "psMetadataHeader.h"
112	113	#include "psPixels.h"
113	114	#include "psArguments.h"

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/Makefile.am

-              r19056
+              r23352
 noinst_LTLIBRARIES = libpslibsys.la
+libpslibsys_la_CPPFLAGS = $(SRCINC) $(PSLIB_CFLAGS) $(CFITSIO_CFLAGS)
+# PSLIB_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+# PSLIB_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+# PSLIB_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+# Force recompilation of psConfigure.c, since it gets the version information
+psConfigure.c: FORCE
+        touch psConfigure.c
+FORCE: ;
+libpslibsys_la_CPPFLAGS = $(SRCINC) $(PSLIB_CFLAGS) $(CFITSIO_CFLAGS) -DPSLIB_VERSION=$(SVN_VERSION) -DPSLIB_BRANCH=$(SVN_BRANCH) -DPSLIB_SOURCE=$(SVN_SOURCE)
 libpslibsys_la_SOURCES = \
         psAbort.c \

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psConfigure.c

-              r18953
+              r23352
  *  @author Ross Harman, MHPCC
  *  @author Robert DeSonia, MHPCC
+ *  @author Paul Price, IfA
+ *
+ *  @version $Revision: 1.28 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2008-08-08 18:05:08 $
+ *
+ *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
+ *  Copyright 2004-2009 Institute for Astronomy, University of Hawaii
  */
 …
 #include <stdlib.h>
 #include <string.h>
+#include <fitsio.h>
+#include <longnam.h>
+#include <fftw3.h>
+#include <gsl/gsl_version.h>
+#ifdef HAVE_PSDB
+#include <mysql.h>
+#endif
 #include "psAbort.h"
 …
 static FILE *memCheckFile = NULL;       // File to which to write results of mem check
+static const char *cvsTag = "$Name: not supported by cvs2svn $"; // CVS tag name
+#ifndef PSLIB_VERSION
+#error "PSLIB_VERSION is not set"
+#endif
+#ifndef PSLIB_BRANCH
+#error "PSLIB_BRANCH is not set"
+#endif
+#ifndef PSLIB_SOURCE
+#error "PSLIB_SOURCE is not set"
+#endif
+#define xstr(s) str(s)
+#define str(s) #s
 psString psLibVersion(void)
+{
+    psString version = NULL;            // Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", xstr(PSLIB_BRANCH), xstr(PSLIB_VERSION));
+    return value;
+}
+psString psLibSource(void)
+{
+    return psStringCopy(xstr(PSLIB_SOURCE));
+}
+psString psLibDependencies(void)
+{
+    psString deps = NULL;               // Dependencies, to return
+    float cfitsioVersion;               // CFITSIO version number
+    psStringAppend(&deps, "cfitsio-%.3f gsl-%s %s",
+                   fits_get_version(&cfitsioVersion), gsl_version, fftwf_version);
+#ifdef HAVE_PSDB
+    psStringAppend(&deps, " mysql-%s", mysql_get_client_info());
+#endif
+    return deps;
+}
 psString psLibVersionLong(void)
+{
+    psString version = psLibVersion();    // Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag
+    psString version = psLibVersion();  // Version, to return
+    psString source = psLibSource();    // Source
+    psString deps = psLibDependencies();// Dependencies
+#ifdef HAVE_PSDB
+    psStringAppend(&version, " (cvs tag %s), %s, %s with psDB", tag, __DATE__, __TIME__);
+    psStringPrepend(&version, "psLib ");
+    psStringAppend(&version, " from %s, built %s, %s with %s",
+                   source, __DATE__, __TIME__, deps);
+    psFree(source);
+    psFree(deps);
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
 #else
     psStringAppend(&version, " (cvs tag %s), %s, %s without psDB", tag, __DATE__, __TIME__);
+    psStringAppend(&version, " unoptimised");
 #endif
+    psFree(tag);
+#ifdef PS_NO_TRACE
+    psStringAppend(&version, " without trace");
+#else
+    psStringAppend(&version, " with trace");
+#endif
     return version;
+}
+};
 // Check the memory; intended for use on exit, but might be used elsewhere

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psConfigure.h

-              r13950
+              r23352
 #define PS_CONFIGURE_H
+#include <psString.h>
 /// @addtogroup SysUtils System Utilities
 /// @{
 …
  *  @return psString: String with version name.
  */
+psString psLibVersion(
+    void
+);
+psString psLibVersion(void);
+/** Get current psLib source
+ *
+ * Returns the current psLib source name as a string.
+ *
+ * @return psString: String with source name.
+ */
+psString psLibSource(void);
+/** Get psLib dependencies' versions
+ *
+ * Returns the psLib dependency versions as a string.
+ *
+ * @return psString: String with dependencies.
+ */
+psString psLibDependencies(void);
 /** Get current psLib version (full identification)

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psLogMsg.c

-              r20546
+              r23352
+    }
     int fileD = creat(dest, 0666);
+    int fileD = open(dest, O_WRONLY | O_CREAT, 0666);
     if (fileD == 0) {
         psError(PS_ERR_IO, true, _("Could not open file '%s' for output."), dest);
 …
     if (write(logFD, head, strlen(head))) {;} // ignore return value
     if (logMsg) {
         psString msg = NULL;            // Message to print

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psMemory.c

-              r21346
+              r23352
                    memBlock->func, memBlock->file, memBlock->lineno, (unsigned long)memBlock->tid);
+}
+bool psMemTypeEqual (void *ptr1, void *ptr2) {
+    // if ptr is a psAlloc()'d memory, find the actual address of the memBlock
+    psMemBlock *memBlock1 = ((psMemBlock *)ptr1) - 1;
+    HANDLE_BAD_BLOCK(memBlock1, __FILE__, __LINE__, __func__);
+    if (!memBlock1->freeFunc) return false;
+    // if ptr is a psAlloc()'d memory, find the actual address of the memBlock
+    psMemBlock *memBlock2 = ((psMemBlock *)ptr2) - 1;
+    HANDLE_BAD_BLOCK(memBlock2, __FILE__, __LINE__, __func__);
+    if (!memBlock2->freeFunc) return false;
+    return (memBlock1->freeFunc == memBlock2->freeFunc);
+}

branches/cnb_branches/cnb_branch_20090301/psLib/src/sys/psMemory.h

-              r15047
+              r23352
 );
+/** test for matching types (equal free functions)
+ *
+ * This function returns true if the two pointers have matching, non-NULL free functions.
+ * Supplied pointers must have been allocated within the psLib memory system (ie, with a psAlloc) or the function will abort. (XXX just return false?)
+ * Supplied pointers must have been provided with free function or the function returns false.
+ */
+bool psMemTypeEqual (void *ptr1, ///< pointer to first psMemory object
+                     void *ptr2 ///< pointer to second psMemory object
+  );
 // Ensure this is a psLib pointer
 #define PS_ASSERT_PTR_HEAVY(PTR, RVAL) \

branches/cnb_branches/cnb_branch_20090301/psLib/src/types/Makefile.am

-              r18145
+              r23352
         psMetadataItemParse.c \
         psMetadataItemCompare.c \
+        psMetadataHeader.c \
         psPixels.c \
         psArguments.c   \
 …
         psMetadataItemParse.h \
         psMetadataItemCompare.h \
+        psMetadataHeader.h \
         psPixels.h \
         psArguments.h   \

branches/cnb_branches/cnb_branch_20090301/psLib/src/types/psMetadata.h

r21278	r23352
30	30	#include "psLookupTable.h"
31	31	#include "psMutex.h"
	32	#include "psError.h"
32	33
33	34	#define PS_DATA_IS_PRIMITIVE(TYPE) \

branches/cnb_branches/cnb_branch_20090301/psLib/src/types/psMetadataConfig.c

-              r22707
+              r23352
+            }
+            psString newStr = psMetadataConfigFormat(item->data.md);
+            if (newStr) {
+            psMetadata *md = item->data.md; // Metadata at new level
+            if (md) {
+                psString newStr = psMetadataConfigFormat(item->data.md);
+                if (!newStr) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to format metadata %s", item->name);
+                    psFree(content);
+                    return NULL;
+                }
                 // add 3 extra spaces to each metadata folder item
                 char *buf = strtok(newStr, "\n");
 …
                 psFree(newStr);
+            }
             psStringAppend(&content, "\nEND\n");
             break;

branches/cnb_branches/cnb_branch_20090301/psLib/test/fits/tap_psFitsImage.c

r19382	r23352
12	12	static psImage *generateImage(void)
13	13	{
14		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 12345); // Random number generator
	14	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 12345); // Random number generator
15	15	psImage *image = psImageAlloc(NUMCOLS, NUMROWS, PS_TYPE_F32); // Generated image
16	16	for (int y = 0; y < NUMROWS; y++) {

branches/cnb_branches/cnb_branch_20090301/psLib/test/imageops/convolutionBench.c

r17320	r23352
68	68	int main(int argc, char *argv[])
69	69	{
70		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator
	70	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 0); // Random number generator
71	71
72	72	printf("#%14s%16s %8s %8s\n", "Image", "Kernel", "Direct", "FFT");

branches/cnb_branches/cnb_branch_20090301/psLib/test/imageops/tap_psImageInterpolate2.c

r21465	r23352
77	77	psImage *variance = NULL; // generateVariance(xSize, ySize, type);
78	78	psImage *mask = NULL; // generateMask(xSize, ySize);
79		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 12345);
	79	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 12345);
80	80
81	81	psImageInterpolation *interp = psImageInterpolationAlloc(mode, image, variance, mask, 0, NAN, NAN,

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psMinimizeLMM.c

-              r13124
+              r23352
+    {
         psMemId id = psMemGetId();
         psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
+        psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
         psMinimization *min = psMinimizationAlloc(NUM_ITERATIONS, ERR_TOL);
         psArray *ordinates = psArrayAlloc(NUM_DATA_POINTS);
 …
+    {
         psMemId id = psMemGetId();
         psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
+        psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
         psMinimization *min = psMinimizationAlloc(NUM_ITERATIONS, ERR_TOL);
         psArray *ordinates = psArrayAlloc(NUM_DATA_POINTS);

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psPolyFit1D.c

r13337	r23352
109	109	psVector *xTruth = psVectorAlloc(numData, PS_TYPE_F64);
110	110	psVector *fTruth = psVectorAlloc(numData, PS_TYPE_F64);
111		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Using a known seed
	111	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Using a known seed
112	112	for (int i = 0; i < numData; i++) {
113	113	xTruth->data.F64[i] = (flags & TS00_X_NULL) ? i : 2.0*psRandomUniform(rng) - 1.0;

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psPolyFit2D.c

r17567	r23352
108	108	yTruth->n = numData;
109	109	fTruth->n = numData;
110		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Using an RNG with a known seed
	110	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Using an RNG with a known seed
111	111	for (int i = 0; i < numData; i++) {
112	112	xTruth->data.F64[i] = 2.0*psRandomUniform(rng) - 1.0;

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psPolyFit3D.c

r17567	r23352
108	108	zTruth->n = numData;
109	109	fTruth->n = numData;
110		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Using known seed
	110	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Using known seed
111	111	for (int i = 0; i < numData; i++) {
112	112	xTruth->data.F64[i] = 2.0*psRandomUniform(rng) - 1.0;

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psPolyFit4D.c

r17567	r23352
132	132	tTruth->n = numData;
133	133	fTruth->n = numData;
134		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Using known seed
	134	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Using known seed
135	135	for (int i = 0; i < numData; i++) {
136	136	xTruth->data.F64[i] = 2.0*psRandomUniform(rng) - 1.0;

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psRandom.c

-              r13123
+              r23352
 work properly.
     ensure that psRandom structs are properly allocated by psRandomAlloc().
+    ensure that psRandom structs are properly allocated by psRandomAllocSpecific().
     ensure that psRandomUniform() produces a sequence of numbers with
         proper mean and stdev.
 …
     plan_tests(34);
     // ensure that psRandom structs are properly allocated by psRandomAlloc()
+    // ensure that psRandom structs are properly allocated by psRandomAllocSpecific()
+    {
         psMemId id = psMemGetId();
         // Valid type allocation
         psRandom *myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
         ok(myRNG != NULL, "psRandom struct was allocated properly");
         skip_start(myRNG == NULL, 1, "Skipping tests because psRandomAlloc() failed");
         ok(myRNG->type == PS_RANDOM_TAUS, "psRandomAlloc() set type properly");
+        psRandom *myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
+        ok(myRNG != NULL, "psRandom struct was allocated properly");
+        skip_start(myRNG == NULL, 1, "Skipping tests because psRandomAllocSpecific() failed");
+        ok(myRNG->type == PS_RANDOM_TAUS, "psRandomAllocSpecific() set type properly");
         psFree(myRNG);
         skip_end();
 …
+    {
         psMemId id = psMemGetId();
         psRandom *myRNG = psRandomAlloc(100,SEED);
         ok(myRNG == NULL, "psRandomAlloc() refused to generate psRandom with unallowed type");
+        psRandom *myRNG = psRandomAllocSpecific(100,SEED);
+        ok(myRNG == NULL, "psRandomAllocSpecific() refused to generate psRandom with unallowed type");
         psFree(myRNG);
         ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
 …
+    {
         psMemId id = psMemGetId();
         psRandom *myRNG = psRandomAlloc(PS_RANDOM_TAUS,-5);
         ok(myRNG != NULL, "psRandomAlloc() allows negative seed");
+        psRandom *myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS,-5);
+        ok(myRNG != NULL, "psRandomAllocSpecific() allows negative seed");
         psFree(myRNG);
         ok(!psMemCheckLeaks (id, NULL, NULL, false), "no memory leaks");
 …
         rans->n = rans->nalloc;
         psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN);
         psRandom *myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
         ok(myRNG != NULL, "psRandom struct was allocated properly");
         skip_start(myRNG == NULL, 2, "Skipping tests because psRandomAlloc() failed");
+        psRandom *myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
+        ok(myRNG != NULL, "psRandom struct was allocated properly");
+        skip_start(myRNG == NULL, 2, "Skipping tests because psRandomAllocSpecific() failed");
         // Initialize vector data with random number
 …
         rans->n = rans->nalloc;
         psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN);
         psRandom *myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
         ok(myRNG != NULL, "psRandom struct was allocated properly");
         skip_start(myRNG == NULL, 2, "Skipping tests because psRandomAlloc() failed");
+        psRandom *myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
+        ok(myRNG != NULL, "psRandom struct was allocated properly");
+        skip_start(myRNG == NULL, 2, "Skipping tests because psRandomAllocSpecific() failed");
         // Initialize vector with random data
 …
         rans->n = rans->nalloc;
         psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN);
         psRandom *myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
         ok(myRNG != NULL, "psRandom struct was allocated properly");
         skip_start(myRNG == NULL, 2, "Skipping tests because psRandomAlloc() failed");
+        psRandom *myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
+        ok(myRNG != NULL, "psRandom struct was allocated properly");
+        skip_start(myRNG == NULL, 2, "Skipping tests because psRandomAllocSpecific() failed");
         // Initialize vector with random data
 …
         rans02->n = rans02->nalloc;
         psRandom *myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
         ok(myRNG != NULL, "psRandom struct was allocated properly");
         skip_start(myRNG == NULL, 1, "Skipping tests because psRandomAlloc() failed");
+        psRandom *myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
+        ok(myRNG != NULL, "psRandom struct was allocated properly");
+        skip_start(myRNG == NULL, 1, "Skipping tests because psRandomAllocSpecific() failed");
 …
     if (0) {
         psRandom *myRNG1 = NULL;
         myRNG1 = psRandomAlloc(PS_RANDOM_TAUS, SEED);
+        myRNG1 = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
         //    psLogSetDestination("dest:stderr");
         psLogSetDestination(0);
 …
         rans02->n = rans02->nalloc;
         psRandom *myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
         ok(myRNG != NULL, "psRandom struct was allocated properly");
         skip_start(myRNG == NULL, 1, "Skipping tests because psRandomAlloc() failed");
+        psRandom *myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
+        ok(myRNG != NULL, "psRandom struct was allocated properly");
+        skip_start(myRNG == NULL, 1, "Skipping tests because psRandomAllocSpecific() failed");
 …
         rans02->n = rans02->nalloc;
         psRandom *myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
         ok(myRNG != NULL, "psRandom struct was allocated properly");
         skip_start(myRNG == NULL, 1, "Skipping tests because psRandomAlloc() failed");
+        psRandom *myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
+        ok(myRNG != NULL, "psRandom struct was allocated properly");
+        skip_start(myRNG == NULL, 1, "Skipping tests because psRandomAllocSpecific() failed");
         // Initialize vectors with random data

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psStats07.c

-              r12247
+              r23352
     PS_ASSERT_INT_NONNEGATIVE(Npts, NULL);
     //    psRandom *r = psRandomAlloc(PS_RANDOM_TAUS, p_psRandomGetSystemSeed());
     psRandom *r = psRandomAlloc(PS_RANDOM_TAUS, PS_XXX_GAUSSIAN_SEED);
+    //    psRandom *r = psRandomAllocSpecific(PS_RANDOM_TAUS, p_psRandomGetSystemSeed());
+    psRandom *r = psRandomAllocSpecific(PS_RANDOM_TAUS, PS_XXX_GAUSSIAN_SEED);
     psVector* gauss = psVectorAlloc(Npts, PS_TYPE_F32);
     for (unsigned int i = 0; i < Npts; i++) {

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psStats09.c

r12783	r23352
52	52	srand(SEED);
53	53
54		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
	54	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
55	55	psVector *truth = psVectorAlloc(numData, PS_TYPE_F64);
56	56	for (long i = 0; i < numData; i++) {

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tap_psStatsTiming.c

r12513	r23352
20	20
21	21	// build a gauss-deviate vector (mean = 0.0, sigma = 1.0) for tests
22		psRandom *seed = psRandomAlloc (PS_RANDOM_TAUS, 0);
	22	psRandom *seed = psRandomAllocSpecific (PS_RANDOM_TAUS, 0);
23	23	psVector *rnd = psVectorAlloc (1000, PS_TYPE_F32);
24	24	for (int i = 0; i < rnd->n; i++) {

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tst_psMinimizeLMM.c

r11686	r23352
66	66	psBool testStatus = true;
67	67
68		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
	68	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
69	69	psMinimization *min = psMinimizationAlloc(NUM_ITERATIONS, ERR_TOL);
70	70	psArray *ordinates = psArrayAlloc(NUM_DATA_POINTS);

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tst_psPolyFit1D.c

r11686	r23352
101	101	xTruth->n = numData;
102	102	fTruth->n = numData;
103		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Using a known seed
	103	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Using a known seed
104	104	for (int i = 0; i < numData; i++) {
105	105	xTruth->data.F64[i] = (flags & TS00_X_NULL) ? i : 2.0*psRandomUniform(rng) - 1.0;

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tst_psPolyFit2D.c

r11686	r23352
100	100	yTruth->n = numData;
101	101	fTruth->n = numData;
102		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Using an RNG with a known seed
	102	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Using an RNG with a known seed
103	103	for (int i = 0; i < numData; i++) {
104	104	xTruth->data.F64[i] = 2.0*psRandomUniform(rng) - 1.0;

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tst_psPolyFit3D.c

r11686	r23352
103	103	zTruth->n = numData;
104	104	fTruth->n = numData;
105		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Using known seed
	105	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Using known seed
106	106	for (int i = 0; i < numData; i++) {
107	107	xTruth->data.F64[i] = 2.0*psRandomUniform(rng) - 1.0;

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tst_psPolyFit4D.c

r11686	r23352
124	124	tTruth->n = numData;
125	125	fTruth->n = numData;
126		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Using known seed
	126	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Using known seed
127	127	for (int i = 0; i < numData; i++) {
128	128	xTruth->data.F64[i] = 2.0*psRandomUniform(rng) - 1.0;

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tst_psRandom.c

-              r6484
+              r23352
 work properly.
     t00(): ensure that psRandom structs are properly allocated by psRandomAlloc().
+    t00(): ensure that psRandom structs are properly allocated by psRandomAllocSpecific().
     t01(): ensure that psRandomUniform() produces a sequence of numbers with
     proper mean and stdev.
 …
 testDescription tests[] = {
                               {testRandomAlloc,000,"psRandomAlloc",0,false},
+                              {testRandomAlloc,000,"psRandomAllocSpecific",0,false},
                               {testRandomUniform,000,"psRandomUniform",0,false},
                               {testRandomGaussian,000,"psRandomGaussian",0,false},
 …
     // Valid type allocation
     myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
+    myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
     if (myRNG == NULL) {
         psError(PS_ERR_UNKNOWN,true,"Could not allocate psRandom structure");
 …
     //    psLogSetDestination("file:seed_msglog1.txt");
     psLogSetDestination(fd1);
     myRNG = psRandomAlloc(PS_RANDOM_TAUS, 0);
+    myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, 0);
     //    psLogSetDestination("dest:stderr");
     psLogSetDestination(2);
 …
     // Invalid type allocation
     psLogMsg(__func__,PS_LOG_INFO,"Invalid type, should generate error message");
     myRNG = psRandomAlloc(100,SEED);
+    myRNG = psRandomAllocSpecific(100,SEED);
     if (myRNG != NULL) {
         psError(PS_ERR_UNKNOWN,true,"Did not return NULL for invalid type");
 …
     // Negative seed value
     myRNG = psRandomAlloc(PS_RANDOM_TAUS,-5);
+    myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS,-5);
     if(myRNG == NULL) {
         psError(PS_ERR_UNKNOWN,true,"Did not return allocated psRandom");
 …
     psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN);
     myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
+    myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
     if (myRNG == NULL) {
         psError(PS_ERR_UNKNOWN,true,"Could not allocate psRandom structure");
 …
     psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN);
     myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
+    myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
     if (myRNG == NULL) {
         psError(PS_ERR_UNKNOWN,true,"Could not allocate psRandom structure");
 …
     psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN);
     myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
+    myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
     if (myRNG == NULL) {
         psError(PS_ERR_UNKNOWN,true,"Could not allocate psRandom structure");
 …
     rans02->n = rans02->nalloc;
     myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
+    myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
     if (myRNG == NULL) {
         psError(PS_ERR_UNKNOWN,true,"Could not allocate psRandom structure");
 …
     psRandom *myRNG1 = NULL;
     myRNG1 = psRandomAlloc(PS_RANDOM_TAUS, SEED);
+    myRNG1 = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
     //    psLogSetDestination("dest:stderr");
     psLogSetDestination(0);
 …
     rans02->n = rans02->nalloc;
     myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
+    myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
     if (myRNG == NULL) {
         psError(PS_ERR_UNKNOWN,true,"Could not allocate psRandom structure");
 …
     rans02->n = rans02->nalloc;
     myRNG = psRandomAlloc(PS_RANDOM_TAUS, SEED);
+    myRNG = psRandomAllocSpecific(PS_RANDOM_TAUS, SEED);
     if (myRNG == NULL) {
         psError(PS_ERR_UNKNOWN,true,"Could not allocate psRandom structure");

branches/cnb_branches/cnb_branch_20090301/psLib/test/math/tst_psStats09.c

r11686	r23352
52	52	printPositiveTestHeader(stdout, "psMathUtils functions", "psVectorStats Clipped Stats Routine");
53	53
54		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
	54	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 1); // Random number generator; using known seed
55	55	psVector *truth = psVectorAlloc(numData, PS_TYPE_F64);
56	56	truth->n = numData;

branches/cnb_branches/cnb_branch_20090301/psLib/test/mathtypes/tap_psVectorSelect.c

-              r18333
+              r23352
     psVector *vector = psVectorAlloc(size, PS_TYPE_F32);
     psVectorInit(vector, 0.0);
     psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 12345);
+    psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 12345);
     for (long i = 0; i < numNonzero; i++) {
         long index = psRandomUniform(rng) * size;
 …
     psVector *vector = psVectorAlloc(size, PS_TYPE_F32);
     psVectorInit(vector, 0.0);
     psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 12345);
+    psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 12345);
     for (long i = 0; i < numNonzero; i++) {
         long index = psRandomUniform(rng) * size;

branches/cnb_branches/cnb_branch_20090301/psLib/test/types/tap_psTree.c

r18145	r23352
16	16	psVector *y = psVectorAlloc(NUM, PS_TYPE_F64);
17	17
18		psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0);
	18	psRandom *rng = psRandomAllocSpecific(PS_RANDOM_TAUS, 0);
19	19	for (int i = 0; i < NUM; i++) {
20	20	x->data.F64[i] = 2.0 * psRandomUniform(rng) - 1.0;

branches/cnb_branches/cnb_branch_20090301/psastro

Property svn:mergeinfo set to (toggle deleted branches)

/branches/cnb_branches/cnb_branch_20090215/psastro	merged	eligible
/trunk/psastro	merged	eligible
/branches/cnb_branch_20090215/psastro	21495-22685
/branches/eam_branches/eam_branch_20090303/psastro	23158-23228

branches/cnb_branches/cnb_branch_20090301/psastro/src/Makefile.am

-              r21422
+              r23352
+lib_LTLIBRARIES = libpsastro.la
+lib_LTLIBRARIES = libpsastro.la
+libpsastro_la_CFLAGS = $(PSASTRO_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
+libpsastro_la_LDFLAGS = $(PSASTRO_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
+libpsastro_la_CFLAGS = $(PSASTRO_CFLAGS) $(PPSTATS_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPSASTRO_VERSION=$(SVN_VERSION) -DPSASTRO_BRANCH=$(SVN_BRANCH) -DPSASTRO_SOURCE=$(SVN_SOURCE)
+libpsastro_la_LDFLAGS = $(PSASTRO_LIBS) $(PPSTATS_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)
+# Force recompilation of psastroVersion.c, since it gets the version information
+# can we do this with dependency info?
+# psastroVersion.c: FORCE
+# touch psastroVersion.c
+# FORCE: ;
 bin_PROGRAMS = psastro psastroModel psastroModelFit gpcModel

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastro.c

-              r21409
+              r23352
     if (!config) usage ();
+    psastroVersionPrint();
     // load identify the data sources
     if (!psastroParseCamera (config)) {
         psErrorStackPrint(stderr, "error setting up the camera\n");
         exit (1);
+        psErrorStackPrint(stderr, "error setting up the camera\n");
+        exit (1);
+    }
 …
     // select subset of stars for astrometry
     if (!psastroDataLoad (config)) {
         psErrorStackPrint(stderr, "error loading input data\n");
         exit (1);
+        psErrorStackPrint(stderr, "error loading input data\n");
+        exit (1);
+    }
     // run the full astrometry analysis (chip and/or mosaic)
     if (!psastroAnalysis (config)) {
         psErrorStackPrint(stderr, "failure in psastro analysis\n");
         exit (1);
+        psErrorStackPrint(stderr, "failure in psastro analysis\n");
+        exit (1);
+    }
     // write out the results
     if (!psastroDataSave (config)) {
         psErrorStackPrint(stderr, "failed to write out data\n");
         exit (1);
+        psErrorStackPrint(stderr, "failed to write out data\n");
+        exit (1);
+    }

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastro.h

-              r21422
+              r23352
 // Return version strings.
 psString          psastroVersion(void);
+psString          psastroSource(void);
 psString          psastroVersionLong(void);
+bool              psastroVersionHeader(psMetadata *header);
+bool              psastroVersionHeaderFull(psMetadata *header);
+void              psastroVersionPrint(void);
 // demo plots

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroArguments.c

-              r21422
+              r23352
+ *
  *  @author IfA
  *  @version $Revision: 1.34 $ $Name: not supported by cvs2svn $
  *  @date $Date: 2009-02-09 21:25:34 $
+ *  @version $Revision: 1.34.2.1 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-19 17:59:50 $
  *  Copyright 2009 Institute for Astronomy, University of Hawaii
  */
 …
     if ((N = psArgumentGet (argc, argv, "-visual"))) {
         psArgumentRemove (N, &argc, argv);
         pmAstromSetVisual (true);
+        pmVisualSetVisual (true);
+    }

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroCleanup.c

-              r21422
+              r23352
+ *
  *  @author IfA
  *  @version $Revision: 1.8 $ $Name: not supported by cvs2svn $
  *  @date $Date: 2009-02-09 21:25:34 $
+ *  @version $Revision: 1.8.2.1 $ $Name: not supported by cvs2svn $
+ *  @date $Date: 2009-02-19 17:59:50 $
  *  Copyright 2009 Institute for Astronomy, University of Hawaii
  */
 …
     psFree (config);
     pmAstromVisualClose ();
+    pmVisualClose ();
     psTimerStop ();

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroConvert.c

-              r21409
+              r23352
         int n = index->data.S32[i];
         pmSource *source = sources->data[n];
+        // XXX this needs to be flexible
+        psTrace ("psastro", 6, "mag: %f +/- %f, mode: %x, skip: %x\n", source->psfMag, source->errMag, source->mode, skip);
         if (source->mode & skip) {
           nModeSkip ++;
           continue;
+        }
         if ((iMagMax != 0.0) && (source->psfMag > iMagMax)) {
           nFaintSkip ++;

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroDataSave.c

-              r21409
+              r23352
   return false; \
+}
 /**
  * this loop saves the photometry/astrometry data files
 …
     psMetadata *recipe  = psMetadataLookupPtr (NULL, config->recipes, PSASTRO_RECIPE);
     if (!recipe) {
         psError(PSASTRO_ERR_CONFIG, true, "Can't find PSASTRO recipe!\n");
         return false;
+        psError(PSASTRO_ERR_CONFIG, true, "Can't find PSASTRO recipe!\n");
+        return false;
+    }
 …
     pmFPAfile *output = psMetadataLookupPtr (NULL, config->files, "PSASTRO.OUTPUT");
     if (!output) {
         psError(PSASTRO_ERR_CONFIG, true, "Can't find or interpret output file rule PSASTRO.OUTPUT!\n");
         return false;
+        psError(PSASTRO_ERR_CONFIG, true, "Can't find or interpret output file rule PSASTRO.OUTPUT!\n");
+        return false;
+    }
 …
     pmFPAview *view = pmFPAviewAlloc (0);
+    pmHDU *lastHDU = NULL;              // Last HDU updated
     // open/load files as needed
 …
         psTrace ("psastro", 4, "Chip %d: %x %x\n", view->chip, chip->file_exists, chip->process);
         if (!chip->process || !chip->file_exists) { continue; }
         if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
+        if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
         while ((cell = pmFPAviewNextCell (view, output->fpa, 1)) != NULL) {
+        while ((cell = pmFPAviewNextCell (view, output->fpa, 1)) != NULL) {
             psTrace ("psastro", 4, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
             if (!cell->process || !cell->file_exists) { continue; }
             if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
+            if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
             // process each of the readouts
             while ((readout = pmFPAviewNextReadout (view, output->fpa, 1)) != NULL) {
                 if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
                 if (!readout->data_exists) { continue; }
+            // process each of the readouts
+            while ((readout = pmFPAviewNextReadout (view, output->fpa, 1)) != NULL) {
+                if (!pmFPAfileIOChecks (config, view, PM_FPA_BEFORE)) ESCAPE;
+                if (!readout->data_exists) { continue; }
+                if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+            }
+            if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+        }
+        if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+                // Put version information into the header
+                pmHDU *hdu = pmHDUGetHighest(output->fpa, chip, cell);
+                if (hdu && hdu != lastHDU) {
+                    psastroVersionHeaderFull(hdu->header);
+                    lastHDU = hdu;
+                }
+                if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+            }
+            if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+        }
+        if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;
+    }
     if (!pmFPAfileIOChecks (config, view, PM_FPA_AFTER)) ESCAPE;

branches/cnb_branches/cnb_branch_20090301/psastro/src/psastroVersion.c

-              r21409
+              r23352
  *  @ingroup libpsastro
+ *
- *  @author IfA
- *  @version $Revision: 1.3 $ $Name: not supported by cvs2svn $
- *  @date $Date: 2009-02-07 02:03:34 $
  *  Copyright 2009 Institute for Astronomy, University of Hawaii
  */
 …
 #include "psastroInternal.h"
+static const char *cvsTag = "$Name: not supported by cvs2svn $";///< CVS tag name
+#ifndef PSASTRO_VERSION
+#error "PSASTRO_VERSION is not set"
+#endif
+#ifndef PSASTRO_BRANCH
+#error "PSASTRO_BRANCH is not set"
+#endif
+#ifndef PSASTRO_SOURCE
+#error "PSASTRO_SOURCE is not set"
+#endif
+#define xstr(s) str(s)
+#define str(s) #s
 psString psastroVersion(void)
+{
+    psString version = NULL;            // Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", xstr(PSASTRO_BRANCH), xstr(PSASTRO_VERSION));
+    return value;
+}
+psString psastroSource(void)
+{
+  return psStringCopy(xstr(PSASTRO_SOURCE));
+}
 psString psastroVersionLong(void)
+{
+    psString version = psastroVersion(); // Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag
+    psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__);
+    psFree(tag);
+    psString version = psastroVersion();  // Version, to return
+    psString source = psastroSource();    // Source
+    psStringPrepend(&version, "psastro ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
     return version;
+};
+bool psastroVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+    psString version = psastroVersion(); // Software version
+    psString source = psastroSource();   // Software source
+    psStringPrepend(&version, "psastro version: ");
+    psStringPrepend(&source, "psastro source: ");
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+    psFree(version);
+    psFree(source);
+    return true;
+}
+bool psastroVersionHeaderFull(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "psastro at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+    ppStatsVersionHeader(header);
+    psastroVersionHeader(header);
+    return true;
+}
+void psastroVersionPrint(void)
+{
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("psastro", PS_LOG_INFO, "psastro at %s", timeString);
+    psFree(timeString);
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString ppStats = ppStatsVersionLong(); // ppStats version
+    psString psastro = psastroVersionLong(); // psastro version
+    psLogMsg("psastro", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("psastro", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("psastro", PS_LOG_INFO, "%s", ppStats);
+    psLogMsg("psastro", PS_LOG_INFO, "%s", psastro);
+    psFree(pslib);
+    psFree(psmodules);
+    psFree(ppStats);
+    psFree(psastro);
+    return;
+}

branches/cnb_branches/cnb_branch_20090301/psconfig/psbuild

-              r22697
+              r23352
 $stop = "";
 $verbose = 0;
+$use_svn = 1;
 $extlibs = "none";
 …
     if ($ARGV[0] eq "-bootstrap") {
         &bootstrap ();
+    }
+    if ($ARGV[0] eq "-skip-svn") {
+        $use_svn = 0;
+        shift; next;
+    }
     if ($ARGV[0] eq "-env") {
 …
 sub build_distribution {
+    # set environment variables used to supply SVN info to the compilation
+    if ($use_svn) {
+        # example dump from svn info:
+        # pikake: svn info
+        # Path: .
+        # URL: https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp/branches/eam_branches/eam_branch_20090303/ppImage
+        # Repository Root: https://svn.pan-starrs.ifa.hawaii.edu/repo/ipp
+        # Repository UUID: 60eb6cdc-a59c-4636-a4e0-dba66a9721fd
+        # Revision: 23158
+        # Node Kind: directory
+        # Schedule: normal
+        # Last Changed Author: price
+        # Last Changed Rev: 23125
+        # Last Changed Date: 2009-03-03 15:41:16 -1000 (Tue, 03 Mar 2009)
+        $svn_version = `svnversion`; chomp $svn_version;
+        @svn_info = `svn info`;
+        # get the svn_root first:
+        foreach $line (@svn_info) {
+            if ($line =~ m|^Repository Root:|) {
+                ($svn_root) = $line =~ m|^Repository Root:\s*(\S*)|;
+                last;
+            }
+        }
+        # now get the branch and UUID values
+        foreach $line (@svn_info) {
+            if ($line =~ m|^URL:|) {
+                ($svn_branch) = $line =~ m|^URL:\s*$svn_root/*(\S*)|;
+            }
+            if ($line =~ m|^Repository UUID:|) {
+                ($svn_source) = $line =~ m|^Repository UUID:\s*(\S*)|;
+            }
+        }
+        $ENV{SVN_VERSION} = $svn_version;
+        $ENV{SVN_BRANCH}  = $svn_branch;
+        $ENV{SVN_SOURCE}  = $svn_source;
+    } else {
+        # alternatively, grab these from the following files:
+        if (! -e "SVNINFO") {
+            print "missing SVNINFO file for repository info, skipping\n";
+        } else {
+            @svn_info = `cat SVNINFO`;
+            foreach $line (@svn_info) {
+                ($name, $value) = split (" ", $line);
+                $ENV{$name} = $value;
+            }
+        }
+    }
+    print "SVN_VERSION $ENV{SVN_VERSION}\n";
+    print "SVN_BRANCH  $ENV{SVN_BRANCH}\n";
+    print "SVN_SOURCE  $ENV{SVN_SOURCE}\n";
     # use psconfig.csh to set needed build aliases
     if ($extlibs eq "check") {
         $status = vsystem ("pschecklibs");
 …
         if (-e "Build.PL") {
             vsystem ("$psperlbuild");
             if ($?) { &failure($module[$i], "failure in perl Build.PL"); }
             vsystem ("./Build");
             if ($?) { &failure($module[$i], "failure in Build"); }
             vsystem ("./Build install");
             if ($?) { &failure($module[$i], "failure in Build install"); }
+            $status = vsystem ("$psperlbuild");
+            if ($status) { &failure($module[$i], "failure in perl Build.PL"); }
+            $status = vsystem ("./Build");
+            if ($status) { &failure($module[$i], "failure in Build"); }
+            $status = vsystem ("./Build install");
+            if ($status) { &failure($module[$i], "failure in Build install"); }
             next;
 …
         if ($developer && $rebuild_this && ! -e "configure" && -e "autogen.sh") {
             $skip_configure = 1;
             vsystem ("$psautogen $psopts");
             if ($?) { &failure($module[$i], "failure in psautogen"); }
+            $status = vsystem ("$psautogen $psopts");
+            if ($status) { &failure($module[$i], "failure in psautogen"); }
+        }
         if ($rebuild_this && -e "configure" && !$skip_configure) {
             vsystem ("$psconfigure $psopts");
             if ($?) { &failure($module[$i], "failure in psconfigure"); }
+            $status = vsystem ("$psconfigure $psopts");
+            if ($status) { &failure($module[$i], "failure in psconfigure"); }
+        }
 …
         if ($clean) {
             vsystem ("$make clean");
             if ($?) { &failure($module[$i], "failure in make clean"); }
+            $status = vsystem ("$make clean");
+            if ($status) { &failure($module[$i], "failure in make clean"); }
+        }
         vsystem ("$make");
         if ($?) { &failure($module[$i], "failure in make"); }
         vsystem ("$make install");
         if ($?) { &failure($module[$i], "failure in make install"); }
+        $status = vsystem ("$make");
+        if ($status) { &failure($module[$i], "failure in make"); }
+        $status = vsystem ("$make install");
+        if ($status) { &failure($module[$i], "failure in make install"); }
         print "*** done with $module[$i] ***\n";

branches/cnb_branches/cnb_branch_20090301/psconfig/pschecklibs

r22721	r23352
487	487
488	488	sub usage {
489		print STDERR "USAGE: pscheck~~perl~~ [-version] [-build]\n";
	489	print STDERR "USAGE: pschecklibs [-version] [-build]\n";
490	490	exit 2;
491	491	}

branches/cnb_branches/cnb_branch_20090301/psphot

Property svn:mergeinfo set to (toggle deleted branches)
/branches/cnb_branches/cnb_branch_20090215/psphot merged eligible
/trunk/psphot merged eligible
/branches/cnb_branch_20090215/psphot 21495-22685
/branches/eam_branches/eam_branch_20090303/psphot 23158-23228

branches/cnb_branches/cnb_branch_20090301/psphot/src/Makefile.am

-              r21392
+              r23352
 lib_LTLIBRARIES = libpsphot.la
+libpsphot_la_CFLAGS = $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS)
+# PSPHOT_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+# PSPHOT_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+# PSPHOT_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+#
+# # Force recompilation of psphotVersion.c, since it gets the version information
+# psphotVersion.c: FORCE
+#       touch psphotVersion.c
+# FORCE: ;
+libpsphot_la_CFLAGS = $(PSPHOT_CFLAGS) $(PSMODULE_CFLAGS) $(PSLIB_CFLAGS) -DPSPHOT_VERSION=$(PSPHOT_VERSION) -DPSPHOT_BRANCH=$(PSPHOT_BRANCH) -DPSPHOT_SOURCE=$(SVN_SOURCE)
 libpsphot_la_LDFLAGS = $(PSPHOT_LIBS) $(PSMODULE_LIBS) $(PSLIB_LIBS)

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphot.c

-              r20411
+              r23352
     psTimerStart ("complete");
     pmErrorRegister();                  // register psModule's error codes/messages
+    pmErrorRegister();                  // register psModule's error codes/messages
     psphotInit();
 …
     pmConfig *config = psphotArguments (argc, argv);
     if (!config) {
         psErrorStackPrint(stderr, "Error reading arguments\n");
         usage ();
+        psErrorStackPrint(stderr, "Error reading arguments\n");
+        usage ();
+    }
+    psphotVersionPrint();
     // load input data (config and images (signal, noise, mask)

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphot.h

-              r21392
+              r23352
 const char     *psphotCVSName(void);
 psString        psphotVersion(void);
+psString        psphotSource(void);
 psString        psphotVersionLong(void);
+bool            psphotVersionHeader(psMetadata *header);
+bool            psphotVersionHeaderFull(psMetadata *header);
+void            psphotVersionPrint(void);
 bool            psphotModelTest (pmConfig *config, const pmFPAview *view, psMetadata *recipe);
 …
 // psphotVisual functions
-bool psphotSetVisual (bool mode);
 bool psphotVisualShowImage (pmReadout *readout);
 bool psphotVisualShowBackground (pmConfig *config, const pmFPAview *view, pmReadout *readout);

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotArguments.c

-              r21458
+              r23352
     if ((N = psArgumentGet (argc, argv, "-visual"))) {
         psArgumentRemove (N, &argc, argv);
+        psphotSetVisual (true);
+        // pmSourceSetVisual (true);
+        pmVisualSetVisual(true);
+    }

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotImageLoop.c

-              r21458
+              r23352
     pmFPAview *view = pmFPAviewAlloc (0);
+    pmHDU *lastHDU = NULL;              // Last HDU updated
     // files associated with the science image
 …
                 psLogMsg ("psphot", 6, "Cell %d: %x %x\n", view->cell, cell->file_exists, cell->process);
                 if (! readout->data_exists) { continue; }
+                // Update the header
+                {
+                    pmHDU *hdu = pmHDUGetHighest(input->fpa, chip, cell);
+                    if (hdu && hdu != lastHDU) {
+                        psphotVersionHeaderFull(hdu->header);
+                        lastHDU = hdu;
+                    }
+                }
                 // run the actual photometry analysis on this chip/cell/readout

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotMagnitudes.c

-              r21519
+              r23352
     int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads
     if (!status) {
         nThreads = 0;
+        nThreads = 0;
+    }
 …
     // the binning details are saved on the analysis metadata
+    psImageBinning *binning = psMetadataLookupPtr(&status, recipe, "PSPHOT.BACKGROUND.BINNING");
+    psImageBinning *binning = NULL;
+    if (backModel) {
+        binning = psMetadataLookupPtr(&status, backModel->analysis, "PSPHOT.BACKGROUND.BINNING");
+    }
     bool IGNORE_GROWTH = psMetadataLookupBool (&status, recipe, "IGNORE_GROWTH");
 …
     for (int i = 0; i < cellGroups->n; i++) {
         psArray *cells = cellGroups->data[i];
         for (int j = 0; j < cells->n; j++) {
             // allocate a job -- if threads are not defined, this just runs the job
             psThreadJob *job = psThreadJobAlloc ("PSPHOT_MAGNITUDES");
             psArrayAdd(job->args, 1, cells->data[j]); // sources
             psArrayAdd(job->args, 1, psf);
             psArrayAdd(job->args, 1, binning);
             psArrayAdd(job->args, 1, backModel);
             psArrayAdd(job->args, 1, backStdev);
             PS_ARRAY_ADD_SCALAR(job->args, photMode, PS_TYPE_S32);
             PS_ARRAY_ADD_SCALAR(job->args, maskVal,  PS_TYPE_IMAGE_MASK);
             PS_ARRAY_ADD_SCALAR(job->args, 0,        PS_TYPE_S32); // this is used as a return value for nAp
             if (!psThreadJobAddPending(job)) {
                 psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
                 psFree (job);
                 return false;
+            }
             psFree(job);
+        psArray *cells = cellGroups->data[i];
+        for (int j = 0; j < cells->n; j++) {
+            // allocate a job -- if threads are not defined, this just runs the job
+            psThreadJob *job = psThreadJobAlloc ("PSPHOT_MAGNITUDES");
+            psArrayAdd(job->args, 1, cells->data[j]); // sources
+            psArrayAdd(job->args, 1, psf);
+            psArrayAdd(job->args, 1, binning);
+            psArrayAdd(job->args, 1, backModel);
+            psArrayAdd(job->args, 1, backStdev);
+            PS_ARRAY_ADD_SCALAR(job->args, photMode, PS_TYPE_S32);
+            PS_ARRAY_ADD_SCALAR(job->args, maskVal,  PS_TYPE_IMAGE_MASK);
+            PS_ARRAY_ADD_SCALAR(job->args, 0,        PS_TYPE_S32); // this is used as a return value for nAp
+            if (!psThreadJobAddPending(job)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+                psFree (job);
+                return false;
+            }
+            psFree(job);
 # if (0)
                 int nap = 0;
                 if (!psphotMagnitudes_Unthreaded (&nap, cells->data[j], psf, binning, backModel, backStdev, photMode, maskVal)) {
                     psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
                     return false;
+                }
                 Nap += nap;
+                int nap = 0;
+                if (!psphotMagnitudes_Unthreaded (&nap, cells->data[j], psf, binning, backModel, backStdev, photMode, maskVal)) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+                    return false;
+                }
+                Nap += nap;
 # endif
+        }
         // wait for the threads to finish and manage results
         if (!psThreadPoolWait (false)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             return false;
+        }
         // we have only supplied one type of job, so we can assume the types here
         psThreadJob *job = NULL;
         while ((job = psThreadJobGetDone()) != NULL) {
             if (job->args->n < 1) {
                 fprintf (stderr, "error with job\n");
             } else {
                 psScalar *scalar = job->args->data[7];
                 Nap += scalar->data.S32;
+            }
             psFree(job);
+        }
+        }
+        // wait for the threads to finish and manage results
+        if (!psThreadPoolWait (false)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+            return false;
+        }
+        // we have only supplied one type of job, so we can assume the types here
+        psThreadJob *job = NULL;
+        while ((job = psThreadJobGetDone()) != NULL) {
+            if (job->args->n < 1) {
+                fprintf (stderr, "error with job\n");
+            } else {
+                psScalar *scalar = job->args->data[7];
+                Nap += scalar->data.S32;
+            }
+            psFree(job);
+        }
+    }
 …
         if (status && isfinite(source->apMag)) Nap ++;
         if (backModel) {
             psAssert (binning, "if backModel is defined, so should binning be");
             source->sky = psImageUnbinPixel(source->peak->x, source->peak->y, backModel->image, binning);
             if (isnan(source->sky) && false) {
                 psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.sky");
+            }
         } else {
             source->sky = NAN;
+        }
         if (backStdev) {
             psAssert (binning, "if backStdev is defined, so should binning be");
             source->skyErr = psImageUnbinPixel(source->peak->x, source->peak->y, backStdev->image, binning);
             if (isnan(source->skyErr) && false) {
                 psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.skyErr");
+            }
         } else {
             source->skyErr = NAN;
+        }
+        if (backModel) {
+            psAssert (binning, "if backModel is defined, so should binning be");
+            source->sky = psImageUnbinPixel(source->peak->x, source->peak->y, backModel->image, binning);
+            if (isnan(source->sky) && false) {
+                psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.sky");
+            }
+        } else {
+            source->sky = NAN;
+        }
+        if (backStdev) {
+            psAssert (binning, "if backStdev is defined, so should binning be");
+            source->skyErr = psImageUnbinPixel(source->peak->x, source->peak->y, backStdev->image, binning);
+            if (isnan(source->skyErr) && false) {
+                psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.skyErr");
+            }
+        } else {
+            source->skyErr = NAN;
+        }
+    }
 …
         if (status && isfinite(source->apMag)) Nap ++;
         if (backModel) {
             psAssert (binning, "if backModel is defined, so should binning be");
             source->sky = psImageUnbinPixel(source->peak->x, source->peak->y, backModel->image, binning);
             if (isnan(source->sky) && false) {
                 psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.sky");
+            }
         } else {
             source->sky = NAN;
+        }
         if (backStdev) {
             psAssert (binning, "if backStdev is defined, so should binning be");
             source->skyErr = psImageUnbinPixel(source->peak->x, source->peak->y, backStdev->image, binning);
             if (isnan(source->skyErr) && false) {
                 psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.skyErr");
+            }
         } else {
             source->skyErr = NAN;
+        }
+        if (backModel) {
+            psAssert (binning, "if backModel is defined, so should binning be");
+            source->sky = psImageUnbinPixel(source->peak->x, source->peak->y, backModel->image, binning);
+            if (isnan(source->sky) && false) {
+                psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.sky");
+            }
+        } else {
+            source->sky = NAN;
+        }
+        if (backStdev) {
+            psAssert (binning, "if backStdev is defined, so should binning be");
+            source->skyErr = psImageUnbinPixel(source->peak->x, source->peak->y, backStdev->image, binning);
+            if (isnan(source->skyErr) && false) {
+                psLogMsg ("psphot.magnitudes", PS_LOG_DETAIL, "error setting pmSource.skyErr");
+            }
+        } else {
+            source->skyErr = NAN;
+        }
+    }
 …
     int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads
     if (!status) {
         nThreads = 0;
+        nThreads = 0;
+    }
     nThreads = 0; // XXX until testing is complete, do not thread this function
 …
     for (int i = 0; i < cellGroups->n; i++) {
         psArray *cells = cellGroups->data[i];
         for (int j = 0; j < cells->n; j++) {
             // allocate a job -- if threads are not defined, this just runs the job
             psThreadJob *job = psThreadJobAlloc ("PSPHOT_PSF_WEIGHTS");
             psArrayAdd(job->args, 1, cells->data[j]); // sources
             PS_ARRAY_ADD_SCALAR(job->args, maskVal,  PS_TYPE_IMAGE_MASK);
             if (!psThreadJobAddPending(job)) {
                 psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
                 psFree (job);
                 return false;
+            }
             psFree(job);
+        }
         // wait for the threads to finish and manage results
         if (!psThreadPoolWait (false)) {
             psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
             return false;
+        }
         // we have only supplied one type of job, so we can assume the types here
         psThreadJob *job = NULL;
         while ((job = psThreadJobGetDone()) != NULL) {
             if (job->args->n < 1) {
                 fprintf (stderr, "error with job\n");
+            }
             psFree(job);
+        }
+        psArray *cells = cellGroups->data[i];
+        for (int j = 0; j < cells->n; j++) {
+            // allocate a job -- if threads are not defined, this just runs the job
+            psThreadJob *job = psThreadJobAlloc ("PSPHOT_PSF_WEIGHTS");
+            psArrayAdd(job->args, 1, cells->data[j]); // sources
+            PS_ARRAY_ADD_SCALAR(job->args, maskVal,  PS_TYPE_IMAGE_MASK);
+            if (!psThreadJobAddPending(job)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+                psFree (job);
+                return false;
+            }
+            psFree(job);
+        }
+        // wait for the threads to finish and manage results
+        if (!psThreadPoolWait (false)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+            return false;
+        }
+        // we have only supplied one type of job, so we can assume the types here
+        psThreadJob *job = NULL;
+        while ((job = psThreadJobGetDone()) != NULL) {
+            if (job->args->n < 1) {
+                fprintf (stderr, "error with job\n");
+            }
+            psFree(job);
+        }
+    }
 …
         pmSource *source = (pmSource *) sources->data[i];
         // we must have a valid model
         pmModel *model = pmSourceGetModel (&isPSF, source);
         if (model == NULL) {
           psTrace ("psphot", 3, "fail mag : no valid model");
           source->pixWeight = NAN;
           continue;
+        }
+        // we must have a valid model
+        pmModel *model = pmSourceGetModel (&isPSF, source);
+        if (model == NULL) {
+          psTrace ("psphot", 3, "fail mag : no valid model");
+          source->pixWeight = NAN;
+          continue;
+        }
         status = pmSourcePixelWeight (&source->pixWeight, model, source->maskObj, maskVal);
         if (!status) {
           psTrace ("psphot", 3, "fail to measure pixel weight");
           source->pixWeight = NAN;
           continue;
+        }
+    }
     return true;
+}
+        if (!status) {
+          psTrace ("psphot", 3, "fail to measure pixel weight");
+          source->pixWeight = NAN;
+          continue;
+        }
+    }
+    return true;
+}

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotMaskReadout.c

-              r21366
+              r23352
+    }
+    // make this an option via the recipe
+    if (0) {
+      psImage *im = readout->image;
+      psImage *wt = readout->variance;
+      psImage *mk = readout->mask;
+      for (int j = 0; j < im->numRows; j++) {
+        for (int i = 0; i < im->numCols; i++) {
+          if (isfinite(im->data.F32[j][i]) && isfinite(wt->data.F32[j][i])) continue;
+          mk->data.PS_TYPE_IMAGE_MASK_DATA[j][i] |= maskBad;
+        }
+      }
+    }
     // mask the excluded outer pixels
     // these coordinates refer to the parent image

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotModelBackground.c

-              r21183
+              r23352
 static bool backgroundModel(psImage *model,  // Model image
                             psImage *modelStdev, // Model stdev image
                             psImage *image, // Image for which to generate a background model
                             psImage *mask, // Mask for image
+                            psMetadata *analysis, // Analysis metadata for outputs
+                            pmReadout *readout, // Readout for which to generate a background model
                             psImageBinning *binning, // Binning parameters
                             const pmConfig *config // Configuration
 …
     bool status = true;
+    psImage *image = readout->image, *mask = readout->mask; // Image and mask for readout
     // select the appropriate recipe information
     psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, PSPHOT_RECIPE);
 …
     assert (maskVal);
+    // user supplied seed, if available
+    unsigned long seed = psMetadataLookupS32 (&status, recipe, "IMSTATS_SEED");
+    if (!status) {
+        seed = 0;
+    }
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, seed);
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
     // subtract this amount extra from the sky
 …
     // we save the binning structure for use in psphotMagnitudes
+    status = psMetadataAddPtr(recipe, PS_LIST_TAIL, "PSPHOT.BACKGROUND.BINNING", PS_DATA_UNKNOWN | PS_META_REPLACE, "Background binning", binning);
+    PS_ASSERT (status, false);
+    psMetadataAddPtr(analysis, PS_LIST_TAIL, "PSPHOT.BACKGROUND.BINNING",
+                     PS_DATA_UNKNOWN | PS_META_REPLACE, "Background binning", binning);
     psF32 **modelData = model->data.F32;
 …
     psImage *modelStdev = psImageAlloc(binning->nXruff, binning->nYruff, PS_TYPE_F32); // Standard deviation
     if (!backgroundModel(model, modelStdev, ro->image, ro->mask, binning, config)) {
+    if (!backgroundModel(model, modelStdev, ro->analysis, ro, binning, config)) {
         psFree(model);
         psFree(modelStdev);
 …
     pmFPAfile *file = psMetadataLookupPtr (&status, config->files, filename);
     pmFPA *inFPA = file->fpa;
+    pmReadout *readout = pmFPAviewThisReadout (view, inFPA);
+    psImage *image = readout->image;
+    psImage *mask  = readout->mask;
+    psImageBinning *binning = backgroundBinning(image, config); // Image binning parameters
+    pmReadout *readout = pmFPAviewThisReadout(view, inFPA);
+    psImageBinning *binning = backgroundBinning(readout->image, config); // Image binning parameters
     pmReadout *model = pmFPAGenerateReadout(config, view, "PSPHOT.BACKMDL", inFPA, binning);
     pmReadout *modelStdev = pmFPAGenerateReadout(config, view, "PSPHOT.BACKMDL.STDEV", inFPA, binning);
     if (!backgroundModel(model->image, modelStdev->image, image, mask, binning, config)) {
+    if (!backgroundModel(model->image, modelStdev->image, model->analysis, readout, binning, config)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to generate background model");
         return false;

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotSetThreads.c

r21392	r23352
25	25	psFree(task);
26	26
27		task = psThreadTaskAlloc("PSPHOT_SOURCE_STATS", 4);
	27	task = psThreadTaskAlloc("PSPHOT_SOURCE_STATS", 5);
28	28	task->function = &psphotSourceStats_Threaded;
29	29	psThreadTaskAdd(task);

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotSourceStats.c

-              r21519
+              r23352
     int nThreads = psMetadataLookupS32(&status, config->arguments, "NTHREADS"); // Number of threads
     if (!status) {
         nThreads = 0;
+        nThreads = 0;
+    }
 …
         pmSource *source = pmSourceAlloc();
         // add the peak
+        // add the peak
         source->peak = psMemIncrRefCounter(peak);
         // allocate space for moments
+        // allocate space for moments
         source->moments = pmMomentsAlloc();
 …
         pmSourceDefinePixels (source, readout, source->peak->x, source->peak->y, OUTER);
         peak->assigned = true;
         psArrayAdd (sources, 100, source);
         psFree (source);
+        peak->assigned = true;
+        psArrayAdd (sources, 100, source);
+        psFree (source);
+    }
     if (!strcasecmp (breakPt, "PEAKS")) {
         psLogMsg ("psphot", PS_LOG_INFO, "%ld sources : %f sec\n", sources->n, psTimerMark ("psphot.stats"));
         psLogMsg ("psphot", PS_LOG_INFO, "break point PEAKS, skipping MOMENTS\n");
         psphotVisualShowMoments (sources);
         return sources;
+        psLogMsg ("psphot", PS_LOG_INFO, "%ld sources : %f sec\n", sources->n, psTimerMark ("psphot.stats"));
+        psLogMsg ("psphot", PS_LOG_INFO, "break point PEAKS, skipping MOMENTS\n");
+        psphotVisualShowMoments (sources);
+        return sources;
+    }
 …
     int Nfail = 0;
     int Nmoments = 0;
+    int Nfaint = 0;
     // choose Cx, Cy (see psphotThreadTools.c for overview of the concepts)
 …
     for (int i = 0; i < cellGroups->n; i++) {
+        psArray *cells = cellGroups->data[i];
+        for (int j = 0; j < cells->n; j++) {
+            // allocate a job -- if threads are not defined, this just runs the job
+            psThreadJob *job = psThreadJobAlloc ("PSPHOT_SOURCE_STATS");
+            psArrayAdd(job->args, 1, cells->data[j]); // sources
+            psArrayAdd(job->args, 1, recipe);
+            PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nmoments
+            PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfail
+            if (!psThreadJobAddPending(job)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+                psFree (job);
+                return NULL;
+            }
+            psFree(job);
+        psArray *cells = cellGroups->data[i];
+        for (int j = 0; j < cells->n; j++) {
+            // allocate a job -- if threads are not defined, this just runs the job
+            psThreadJob *job = psThreadJobAlloc ("PSPHOT_SOURCE_STATS");
+            psArrayAdd(job->args, 1, cells->data[j]); // sources
+            psArrayAdd(job->args, 1, recipe);
+            PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nmoments
+            PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfail
+            PS_ARRAY_ADD_SCALAR(job->args, 0, PS_TYPE_S32); // this is used as a return value for Nfaint
+            if (!psThreadJobAddPending(job)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+                psFree (job);
+                return NULL;
+            }
+            psFree(job);
 # if (0)
                 int nfail = 0;
                 int nmoments = 0;
                 if (!psphotSourceStats_Unthreaded (&nfail, &nmoments, cells->data[j], recipe)) {
                     psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
                     return NULL;
+                }
                 Nfail += nfail;
                 Nmoments += nmoments;
+                int nfail = 0;
+                int nmoments = 0;
+                if (!psphotSourceStats_Unthreaded (&nfail, &nmoments, cells->data[j], recipe)) {
+                    psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+                    return NULL;
+                }
+                Nfail += nfail;
+                Nmoments += nmoments;
 # endif
+        }
+        // wait for the threads to finish and manage results
+        if (!psThreadPoolWait (false)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+            return NULL;
+        }
+        // we have only supplied one type of job, so we can assume the types here
+        psThreadJob *job = NULL;
+        while ((job = psThreadJobGetDone()) != NULL) {
+            if (job->args->n < 1) {
+                fprintf (stderr, "error with job\n");
+            } else {
+                psScalar *scalar = NULL;
+                scalar = job->args->data[2];
+                Nmoments += scalar->data.S32;
+                scalar = job->args->data[3];
+                Nfail += scalar->data.S32;
+            }
+            psFree(job);
+        }
+        }
+        // wait for the threads to finish and manage results
+        if (!psThreadPoolWait (false)) {
+            psError(PS_ERR_UNKNOWN, false, "Unable to guess model.");
+            return NULL;
+        }
+        // we have only supplied one type of job, so we can assume the types here
+        psThreadJob *job = NULL;
+        while ((job = psThreadJobGetDone()) != NULL) {
+            if (job->args->n < 1) {
+                fprintf (stderr, "error with job\n");
+            } else {
+                psScalar *scalar = NULL;
+                scalar = job->args->data[2];
+                Nmoments += scalar->data.S32;
+                scalar = job->args->data[3];
+                Nfail += scalar->data.S32;
+                scalar = job->args->data[4];
+                Nfaint += scalar->data.S32;
+            }
+            psFree(job);
+        }
+    }
     psFree (cellGroups);
     psLogMsg ("psphot", PS_LOG_INFO, "%ld sources, %d moments, %d failed: %f sec\n", sources->n, Nmoments, Nfail, psTimerMark ("psphot.stats"));
+    psLogMsg ("psphot", PS_LOG_INFO, "%ld sources, %d moments, %d faint, %d failed: %f sec\n", sources->n, Nmoments, Nfaint, Nfail, psTimerMark ("psphot.stats"));
     psphotVisualShowMoments (sources);
 …
     int Nfail = 0;
     int Nmoments = 0;
+    int Nfaint = 0;
     for (int i = 0; i < sources->n; i++) {
         pmSource *source = sources->data[i];
 …
         // skip faint sources for moments measurement
         if (source->peak->SN < MIN_SN) {
+            source->mode |= PM_SOURCE_MODE_BELOW_MOMENTS_SN;
+            source->mode |= PM_SOURCE_MODE_BELOW_MOMENTS_SN;
+            Nfaint++;
             continue;
+        }
 …
         status = pmSourceLocalSky (source, PS_STAT_SAMPLE_MEDIAN, INNER, maskVal, markVal);
         if (!status) {
             source->mode |= PM_SOURCE_MODE_SKY_FAILURE;
             psErrorClear(); // XXX re-consider the errors raised here
             Nfail ++;
             continue;
+            source->mode |= PM_SOURCE_MODE_SKY_FAILURE;
+            psErrorClear(); // XXX re-consider the errors raised here
+            Nfail ++;
+            continue;
+        }
 …
         status = pmSourceLocalSkyVariance (source, PS_STAT_SAMPLE_MEDIAN, INNER, maskVal, markVal);
         if (!status) {
             source->mode |= PM_SOURCE_MODE_SKYVAR_FAILURE;
             Nfail ++;
             psErrorClear();
             continue;
+            source->mode |= PM_SOURCE_MODE_SKYVAR_FAILURE;
+            Nfail ++;
+            psErrorClear();
+            continue;
+        }
 …
         status = pmSourceMoments (source, BIG_RADIUS);
         if (status) {
             source->mode |= PM_SOURCE_MODE_BIG_RADIUS;
+            source->mode |= PM_SOURCE_MODE_BIG_RADIUS;
             Nmoments ++;
             continue;
+        }
         source->mode |= PM_SOURCE_MODE_MOMENTS_FAILURE;
+        source->mode |= PM_SOURCE_MODE_MOMENTS_FAILURE;
         Nfail ++;
         psErrorClear();
 …
     scalar = job->args->data[3];
     scalar->data.S32 = Nfail;
+    scalar = job->args->data[4];
+    scalar->data.S32 = Nfaint;
     return true;
+}
 …
         status = pmSourceLocalSky (source, PS_STAT_SAMPLE_MEDIAN, INNER, maskVal, markVal);
         if (!status) {
             psErrorClear(); // XXX re-consider the errors raised here
             Nfail ++;
             continue;
+            psErrorClear(); // XXX re-consider the errors raised here
+            Nfail ++;
+            continue;
+        }
 …
         status = pmSourceLocalSkyVariance (source, PS_STAT_SAMPLE_MEDIAN, INNER, maskVal, markVal);
         if (!status) {
             Nfail ++;
             psErrorClear();
             continue;
+            Nfail ++;
+            psErrorClear();
+            continue;
+        }
 …
     *nmoments = Nmoments;
     *nfail = Nfail;
     return true;
+}
 # endif
+# endif

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotSubtractBackground.c

-              r21183
+              r23352
 // generate the median in NxN boxes, clipping heavily
 // linear interpolation to generate full-scale model
 bool psphotSubtractBackground (pmConfig *config, const pmFPAview *view, const char *filename)
+bool psphotSubtractBackground (pmConfig *config, const pmFPAview *view, const char *filename)
+{
     bool status = true;
 …
     assert (maskVal);
     psImageBinning *binning = psMetadataLookupPtr(&status, recipe, "PSPHOT.BACKGROUND.BINNING");
+    psImageBinning *binning = psMetadataLookupPtr(&status, model->analysis, "PSPHOT.BACKGROUND.BINNING");
     assert (binning);

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotVersion.c

-              r12805
+              r23352
 # include "psphotInternal.h"
+#include "psphotInternal.h"
 # if (HAVE_KAPA)
 # include <kapa.h>
 # endif
+#ifdef HAVE_KAPA
+#include <kapa.h>
+#endif
+static const char *cvsTag = "$Name: not supported by cvs2svn $";// CVS tag name
+#ifndef PSPHOT_VERSION
+#error "PSPHOT_VERSION is not set"
+#endif
+#ifndef PSPHOT_BRANCH
+#error "PSPHOT_BRANCH is not set"
+#endif
+#ifndef PSPHOT_SOURCE
+#error "PSPHOT_SOURCE is not set"
+#endif
+#define xstr(s) str(s)
+#define str(s) #s
 psString psphotVersion(void)
+{
+    psString version = NULL;            // Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+    char *value = NULL;
+    psStringAppend(&value, "%s@%s", xstr(PSPHOT_BRANCH), xstr(PSPHOT_VERSION));
+    return value;
+}
+psString psphotSource(void)
+{
+    return psStringCopy(xstr(PSPHOT_SOURCE));
+}
 psString psphotVersionLong(void)
+{
+    psString version = psphotVersion(); // Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag
+    psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__);
+    psString version = psLibVersion();  // Version, to return
+    psString source = psLibSource();    // Source
+# if (HAVE_KAPA)
     psString ohanaVersion = psStringStripCVS (ohana_version(), "Name");
     psString libdvoVersion = psStringStripCVS (libdvo_version(), "Name");
+    psStringPrepend(&version, "psphot ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+    psStringAppend (&version, " with libkapa (ohana %s, libdvo: %s)", ohanaVersion, libdvoVersion);
+    psFree (ohanaVersion);
+    psFree (libdvoVersion);
+# else
+    psStringAppend (&version, " WITHOUT libkapa");
+# endif
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
+    psFree(tag);
+#ifdef HAVE_KAPA
+#if 0
+    // XXX Need to get ohana and libdvo versions
+    psString ohanaVersion = psStringStripCVS(ohana_version(), "Name");
+    psString libdvoVersion = psStringStripCVS(libdvo_version(), "Name");
+    psStringAppend(&version, " with libkapa (ohana %s, libdvo: %s)", ohanaVersion, libdvoVersion);
+    psFree(ohanaVersion);
+    psFree(libdvoVersion);
+#else
+    psStringAppend(&version, " with libkapa");
+#endif
+#else
+    psStringAppend (&version, " without libkapa");
+#endif
     return version;
+}
+// Defined by RHL; leaving for backwards compatibility.
+const char *psphotCVSName(void) {
+   return cvsTag;
+bool psphotVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+    psString version = psphotVersion(); // Software version
+    psString source = psphotSource();   // Software source
+    psStringPrepend(&version, "psphot version: ");
+    psStringPrepend(&source, "psphot source: ");
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+    psFree(version);
+    psFree(source);
+    return true;
+}
+bool psphotVersionHeaderFull(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "psphot at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+    psphotVersionHeader(header);
+    return true;
+}
+void psphotVersionPrint(void)
+{
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("psphot", PS_LOG_INFO, "psphot at %s", timeString);
+    psFree(timeString);
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString psphot = psphotVersionLong(); // psphot version
+    psLogMsg("psphot", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("psphot", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("psphot", PS_LOG_INFO, "%s", psphot);
+    psFree(pslib);
+    psFree(psmodules);
+    psFree(psphot);
+    return;
+}

branches/cnb_branches/cnb_branch_20090301/psphot/src/psphotVisual.c

-              r21519
+              r23352
 // these are invoked by the -visual options
-static bool isVisual = false;
 static int kapa = -1;
 static int kapa2 = -1;
 static int kapa3 = -1;
-bool psphotSetVisual (bool mode) {
-    isVisual = mode;
-    return true;
+}
 bool psphotVisualShowMask (int kapaFD, psImage *inImage, const char *name, int channel) {
 …
     psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV);
     psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0);
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
     if (!psImageBackground(stats, NULL, inImage, NULL, 0, rng)) {
         fprintf (stderr, "failed to get background values\n");
 …
     psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV);
     psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0);
+    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
     if (!psImageBackground(stats, NULL, inImage, NULL, 0, rng)) {
         fprintf (stderr, "failed to get background values\n");
 …
 bool psphotVisualShowImage (pmReadout *readout) {
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
         if (kapa == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
     pmReadout *backgnd;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
         if (kapa == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
 bool psphotVisualShowSignificance (psImage *image) {
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
         if (kapa == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
     KiiOverlay *overlay;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
     KiiOverlay *overlay;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
     psEllipseAxes axes;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
     Graphdata graphdata;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa3 == -1) {
 …
         if (kapa3 == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
 bool psphotVisualShowRoughClass (psArray *sources) {
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
 bool psphotVisualShowPSFModel (pmReadout *readout, pmPSF *psf) {
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa2 == -1) {
 …
         if (kapa2 == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
     bool status;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa2 == -1) {
 …
         if (kapa2 == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
     bool status;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa2 == -1) {
 …
         if (kapa2 == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
     KapaSection section;  // put the positive profile in one and the residuals in another?
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa3 == -1) {
 …
         if (kapa3 == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
     psEllipseAxes axes;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
     KiiOverlay *overlay;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
     KapaSection section;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa3 == -1) {
 …
         if (kapa3 == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
 bool psphotVisualShowResidualImage (pmReadout *readout) {
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa == -1) {
 …
         if (kapa == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
     Graphdata graphdata;
     if (!isVisual) return true;
+    if (!pmVisualIsVisual()) return true;
     if (kapa3 == -1) {
 …
         if (kapa3 == -1) {
             fprintf (stderr, "failure to open kapa; visual mode disabled\n");
             isVisual = false;
+            pmVisualSetVisual(false);
             return false;
+        }
 …
 # else
-bool psphotSetVisual (bool mode){}
 bool psphotVisualShowImage (pmConfig *config, pmReadout *readout) { return true; }
 bool psphotVisualShowBackground (pmConfig *config, const pmFPAview *view, pmReadout *readout) { return true; }

branches/cnb_branches/cnb_branch_20090301/pstamp/scripts/pstamp_finish.pl

-              r21410
+              r23352
 use PS::IPP::Config qw( :standard );
 use PStamp::RequestFile qw( :standard );
 use PStamp::Job qw( :standard );
+use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $verbose, $save_temps, $redirect_output);
 …
+    }
     # this function is PStamp::RequestFile::read_request_file
+    # this function is PS::IPP::PStamp::RequestFile::read_request_file
     my ($header, $rows) = read_request_file($req_file);

branches/cnb_branches/cnb_branch_20090301/pstamp/scripts/pstamp_job_run.pl

r21410	r23352
13	13	use File::Basename;
14	14	use Digest::MD5::File qw( file_md5_hex );
15		use PStamp::RequestFile qw( :standard );
	15	use PS::IPP::PStamp::RequestFile qw( :standard );
16	16
17	17	my $verbose;

branches/cnb_branches/cnb_branch_20090301/pstamp/scripts/pstampparse.pl

-              r21410
+              r23352
 use Sys::Hostname;
 use Getopt::Long qw( GetOptions );
 use PStamp::RequestFile qw( :standard );
 use PStamp::Job qw( :standard );
+use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 my $verbose;
 …
+    }
     # Call PStamp::Job's locate_images routine to get the parameters for this request specification
+    # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this request specification
     my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id,
             $x, $y, $mjd_min, $mjd_max, $filter, $verbose);

branches/cnb_branches/cnb_branch_20090301/pstamp/test/test_lookup.pl

r20113	r23352
9	9
10	10	use Getopt::Long qw( GetOptions );
11		use PStamp::Job qw( :standard );
	11	use PS::IPP::PStamp::Job qw( :standard );
12	12
13	13	use PS::IPP::Config qw( :standard );

branches/cnb_branches/cnb_branch_20090301/pstest/src/tst_psPolynomial.c

-              r5873
+              r23352
             fillVectors(x, f, NORD, NPTS, type[k]);
             psTime *now = psTimeGetNow(PS_TIME_UTC);
             const psRandom *r = psRandomAlloc(PS_RANDOM_TAUS, now->sec);
+            const psRandom *r = psRandomAllocSpecific(PS_RANDOM_TAUS, now->sec);
             for (int i = 0; i < NPTS; i++) {
                 double errgen = psRandomGaussian(r);
 …
             // add random errors to the vector
             psTime *now = psTimeGetNow(PS_TIME_UTC);
             const psRandom *r = psRandomAlloc(PS_RANDOM_TAUS, now->sec);
+            const psRandom *r = psRandomAllocSpecific(PS_RANDOM_TAUS, now->sec);
             for (int i = 0; i < NPTS; i++) {
                 double errgen = psRandomGaussian(r);
 …
             fillVectors(x, f, NORD, NPTS, type[k]);
             psTime *now = psTimeGetNow(PS_TIME_UTC);
             const psRandom *r = psRandomAlloc(PS_RANDOM_TAUS, now->sec);
+            const psRandom *r = psRandomAllocSpecific(PS_RANDOM_TAUS, now->sec);
             for (int i = 0; i < NPTS; i++) {
                 double errgen = psRandomGaussian(r);

branches/cnb_branches/cnb_branch_20090301/pswarp/src/Makefile.am

-              r18839
+              r23352
 bin_PROGRAMS = pswarp
+pswarp_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PSWARP_CFLAGS)
+PSWARP_VERSION=`if [ -e ../../VERSION ]; then cat ../../VERSION; else svnversion; fi`
+PSWARP_BRANCH=`if [ -e ../../BRANCH ]; then cat ../../BRANCH; else svn info | sed -n -e '/URL:/ h' -e '/Repository Root:/ { x; H; x; s|Repository Root: \(.*\)\nURL: \1\(.*\)|\2| ; s|^/|| ; s|/[a-zA-Z]*/src.*|| ; p }'; fi`
+PSWARP_SOURCE=`if [ -e ../../SOURCE ]; then cat ../../SOURCE; else svn info | sed -n -e 's/Repository UUID: // p'; fi`
+# Force recompilation of pswarpVersion.c, since it gets the version information
+pswarpVersion.c: FORCE
+        touch pswarpVersion.c
+FORCE: ;
+pswarp_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PSWARP_CFLAGS) -DPSWARP_VERSION=\"$(PSWARP_VERSION)\" -DPSWARP_BRANCH=\"$(PSWARP_BRANCH)\" -DPSWARP_SOURCE=\"$(PSWARP_SOURCE)\"
 pswarp_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PPSTATS_LIBS) $(PSPHOT_LIBS) $(PSWARP_LIBS)
 …
         pswarpSetThreads.c              \
         pswarpTransformReadout.c        \
         pswarpTransformSources.c \
+        pswarpTransformSources.c        \
         pswarpTransformTile.c           \
         pswarpVersion.c

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarp.c

r21442	r23352
39	39	if (!config) usage();
40	40
	41	pswarpVersionPrint();
	42
41	43	// load identify the data sources
42	44	if (!pswarpParseCamera(config)) {

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarp.h

-              r21368
+              r23352
  */
 bool pswarpSetThreads ();
+/// Return software version
+psString pswarpVersion(void);
+/// Return software souce
+psString pswarpSource(void);
+/// Return long software version information
+psString pswarpVersionLong(void);
+/// Populate header with version information
+bool pswarpVersionHeader(
+    psMetadata *header                  ///< Header to populate
+    );
+/// Print version information
+void pswarpVersionPrint(void);

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpArguments.c

-              r21368
+              r23352
         if ((arg = psArgumentGet(argc, argv, "-psphot-visual"))) {
             psArgumentRemove(arg, &argc, argv);
             psphotSetVisual(true);
+            pmVisualSetVisual(true);
+        }
+    }
 …
+    if (!pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list")) {
+        psError(PSWARP_ERR_ARGUMENTS, true, "Missing -file (input) or -list (input)");
+        return NULL;
+    }
+    // the mask and variance entries are optional (build from gain?)
+    pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK",   "-mask",   "-masklist");
+    pmConfigFileSetsMD (config->arguments, &argc, argv, "INPUT", "-file", "-list");
+    pmConfigFileSetsMD (config->arguments, &argc, argv, "MASK", "-mask", "-masklist");
     pmConfigFileSetsMD (config->arguments, &argc, argv, "VARIANCE", "-variance", "-variancelist");
 …
     // output position is fixed
     psMetadataAddStr (config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "", argv[1]);
+    psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "", argv[1]);
     // skycell position is fixed
     array = psArrayAlloc(1);
     array->data[0] = psStringCopy (argv[2]);
     status = psMetadataAddPtr (config->arguments, PS_LIST_TAIL, "SKYCELL", PS_DATA_ARRAY, "", array);
     psFree (array);
+    array->data[0] = psStringCopy(argv[2]);
+    status = psMetadataAddPtr(config->arguments, PS_LIST_TAIL, "SKYCELL", PS_DATA_ARRAY, "", array);
+    psFree(array);
     psTrace("pswarp", 1, "Done with pswarpArguments...\n");
     return (config);
+    return config;
+}
 …
     psTrace("pswarp", 1, "Done with pswarpArguments...\n");
-    // Dump configuration, now that's it's settled
-    psString dump_file =  psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
-    if (dump_file) {
-        const char *skyCamera = psMetadataLookupStr(NULL, config->arguments,
-                                                    "SKYCELL.CAMERA");  ///< Name of camera for skycell
-        pmConfigCamerasCull(config, skyCamera);
-        pmConfigRecipesCull(config, "PSWARP,PPSTATS,PSPHOT,MASKS");
-        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PSWARP.INPUT"); // Input file
-        pmConfigDump(config, input->fpa, dump_file);
+    }
     return (config);
+}

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpLoop.c

-              r21468
+              r23352
     bool status;
+    const char *skyCamera = psMetadataLookupStr(NULL, config->arguments,
+                                                "SKYCELL.CAMERA");  ///< Name of camera for skycell
+    pmConfigCamerasCull(config, skyCamera);
+    pmConfigRecipesCull(config, "PSWARP,PPSTATS,PSPHOT,MASKS,JPEG");
     // load the recipe
     psMetadata *recipe = psMetadataLookupPtr (&status, config->recipes, PSWARP_RECIPE);
 …
         psError(PSPHOT_ERR_CONFIG, false, "missing recipe %s", PSWARP_RECIPE);
         return false;
+    }
+    if (!pswarpSetMaskBits(config)) {
+        psError(PS_ERR_IO, false, "failed to set mask bits");
+        return NULL;
+    }
 …
+    }
+    pswarpVersionHeader(hdu->header);
     if (!pmAstromWriteWCS(hdu->header, outFPA, outChip, WCS_NONLIN_TOL)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to generate WCS header.");
 …
+    }
+    // Dump configuration
+    psString dump_file = psMetadataLookupStr(&status, config->arguments, "DUMP_CONFIG");
+    if (dump_file) {
+        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PSWARP.INPUT"); // Input file
+        pmConfigDump(config, input->fpa, dump_file);
+    }
     return true;
+}

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpParseCamera.c

-              r21368
+              r23352
  */
+# include "pswarp.h"
+#include "pswarp.h"
+// Define an input file
+static pmFPAfile *defineInputFile(pmConfig *config,// Configuration
+                                  pmFPAfile *bind,    // File to which to bind, or NULL
+                                  char *filerule,     // Name of file rule
+                                  char *argname,      // Argument name
+                                  pmFPAfileType fileType // Type of file
+    )
+{
+    bool status;
+    // look for the file on the RUN metadata
+    pmFPAfile *file = pmFPAfileDefineFromRun(&status, config, filerule); // File to return
+    if (!status) {
+        psError(PSWARP_ERR_CONFIG, false, "Failed to load file definition for %s", filerule);
+        return NULL;
+    }
+    if (!file) {
+        // look for the file on the argument list
+        if (bind) {
+            file = pmFPAfileBindFromArgs(&status, bind, config, filerule, argname);
+        } else {
+            file = pmFPAfileDefineFromArgs(&status, config, filerule, argname);
+        }
+        if (!status) {
+            psError(PSWARP_ERR_CONFIG, false, "Failed to load file definition for %s", filerule);
+            return false;
+        }
+    }
+    if (!file) {
+        return NULL;
+    }
+    if (file->type != fileType) {
+        psError(PSWARP_ERR_CONFIG, true, "%s is not of type %s", filerule, pmFPAfileStringFromType(fileType));
+        return NULL;
+    }
+    return file;
+}
 bool pswarpParseCamera(pmConfig *config)
+{
+    bool status;
+    bool mdok;                          ///< Status of MD lookup
+    pmFPAfile *skycell = NULL;
+    pmConfig *skyConfig = NULL;
+    psAssert(config, "Require configuration");
+    // the input image(s) are required arguments; they define the camera
+    status = false;
+    pmFPAfile *input = pmFPAfileDefineFromArgs(&status, config, "PSWARP.INPUT", "INPUT");
+    if (!input || !status) {
+    // The input image(s) is required: it defines the camera
+    pmFPAfile *input = defineInputFile(config, NULL, "PSWARP.INPUT", "INPUT", PM_FPA_FILE_IMAGE);
+    if (!input) {
         psError(PSWARP_ERR_CONFIG, false, "Failed to build FPA from PSWARP.INPUT");
         return false;
+    }
+    // the input image(s) are required arguments; they define the camera
+    status = false;
+    pmFPAfile *astrom = pmFPAfileDefineFromArgs(&status, config, "PSWARP.ASTROM", "ASTROM");
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "failed to load find definition");
+        return NULL;
+    }
+    if (astrom) {
+        psLogMsg ("pswarp", 3, "using supplied astrometry\n");
+    } else {
+        psLogMsg ("pswarp", 3, "using header astrometry\n");
+    pmFPAfile *astrom = defineInputFile(config, NULL, "PSWARP.ASTROM", "ASTROM", PM_FPA_FILE_CMF);
+    psLogMsg("pswarp", PS_LOG_INFO, "Astrometry source: %s", astrom ? "supplied" : "header");
+    pmFPAfile *inMask = defineInputFile(config, input, "PSWARP.MASK", "MASK", PM_FPA_FILE_MASK);
+    if (!inMask) {
+        psLogMsg("pswarp", PS_LOG_INFO, "No mask supplied");
+    }
+    // the mask is not required - but must conform to input camera
+    pmFPAfile *inMask = pmFPAfileBindFromArgs(&status, input, config, "PSWARP.MASK", "MASK");
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "failed to load find definition");
+        return NULL;
+    }
+    if (!inMask) {
+        psLogMsg ("pswarp", 3, "no mask supplied\n");
+    pmFPAfile *inVariance = defineInputFile(config, input, "PSWARP.VARIANCE", "VARIANCE",
+                                            PM_FPA_FILE_VARIANCE);
+    if (!inVariance) {
+        psLogMsg("pswarp", PS_LOG_INFO, "No variance supplied");
+    }
+    // loading the mask here should have invoked pmConfigMaskReadHeader()
+    if (!pswarpSetMaskBits (config)) {
+        psError(PS_ERR_IO, false, "failed to set mask bits");
+        return NULL;
+    }
+    pmFPAfile *inVariance = pmFPAfileBindFromArgs(&status, input, config, "PSWARP.VARIANCE", "VARIANCE");
+    if (!status) {
+        psError (PS_ERR_UNKNOWN, false, "failed to load find definition");
+        return NULL;
+    }
+    if (!inVariance) {
+        psLogMsg ("pswarp", 3, "no variance supplied\n");
+    }
+    // the input skycell is a required argument: it defines the output image
+    // The input skycell is a required argument: it defines the output image
     // XXX we may need a different skycell structure here
+    status = pswarpDefineSkycell(&skycell, &skyConfig, config, "PSWARP.SKYCELL", "SKYCELL");
+    pmFPAfile *skycell = NULL;
+    pmConfig *skyConfig = NULL;
+    bool status = pswarpDefineSkycell(&skycell, &skyConfig, config, "PSWARP.SKYCELL", "SKYCELL");
     if (!status) {
         psError(PSWARP_ERR_CONFIG, false, "Failed to build FPA from PSWARP.SKYCELL");
 …
         return false;
+    }
+    output->save = true;
     pmFPAfile *outMask = pmFPAfileDefineSkycell(config, output->fpa, "PSWARP.OUTPUT.MASK");
     if (!outMask) {
 …
         return false;
+    }
-    output->save = true;
     outMask->save = true;
 …
+    }
+    bool mdok;                          // Status of MD lookup
     if (psMetadataLookupBool(&mdok, config->arguments, "PSF")) {
         // This file, PSPHOT.INPUT, is just used as a carrier; output files (eg, PSPHOT.RESID) are defined by
 …
+            }
+        }
         psFree (chips);
+        psFree(chips);
+    }

branches/cnb_branches/cnb_branch_20090301/pswarp/src/pswarpVersion.c

-              r21323
+              r23352
 #include <pslib.h>
 #include <psmodules.h>
+#include <psphot.h>
+#include <ppStats.h>
 #include "pswarp.h"
-static const char *cvsTag = "$Name: not supported by cvs2svn $";///< CVS tag name
 psString pswarpVersion(void)
+{
+    psString version = NULL;            ///< Version, to return
+    psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION);
+    return version;
+#ifndef PSWARP_VERSION
+#error "PSWARP_VERSION is not set"
+#endif
+#ifndef PSWARP_BRANCH
+#error "PSWARP_BRANCH is not set"
+#endif
+    return psStringCopy(PSWARP_BRANCH "@" PSWARP_VERSION);
+}
+psString pswarpSource(void)
+{
+#ifndef PSWARP_SOURCE
+#error "PSWARP_SOURCE is not set"
+#endif
+    return psStringCopy(PSWARP_SOURCE);
+}
 psString pswarpVersionLong(void)
+{
+    psString version = pswarpVersion(); ///< Version, to return
+    psString tag = psStringStripCVS(cvsTag, "Name"); ///< CVS tag
+    psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__);
+    psFree(tag);
+    psString version = pswarpVersion();  // Version, to return
+    psString source = pswarpSource();    // Source
+    psStringPrepend(&version, "pswarp ");
+    psStringAppend(&version, " from %s, built %s, %s", source, __DATE__, __TIME__);
+    psFree(source);
+#ifdef __OPTIMIZE__
+    psStringAppend(&version, " optimised");
+#else
+    psStringAppend(&version, " unoptimised");
+#endif
     return version;
+};
+bool pswarpVersionHeader(psMetadata *header)
+{
+    PS_ASSERT_METADATA_NON_NULL(header, false);
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psString history = NULL;               // History string
+    psStringAppend(&history, "pswarp at %s", timeString);
+    psFree(timeString);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, history);
+    psFree(history);
+    psLibVersionHeader(header);
+    psModulesVersionHeader(header);
+    psphotVersionHeader(header);
+    ppStatsVersionHeader(header);
+    psString version = pswarpVersion(); // Software version
+    psString source  = pswarpSource();  // Software source
+    psStringPrepend(&version, "pswarp version: ");
+    psStringPrepend(&source, "pswarp source: ");
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, version);
+    psMetadataAddStr(header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, NULL, source);
+    psFree(version);
+    psFree(source);
+    return true;
+}
+void pswarpVersionPrint(void)
+{
+    psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now
+    psString timeString = psTimeToISO(time); // The time in an ISO string
+    psFree(time);
+    psLogMsg("pswarp", PS_LOG_INFO, "pswarp at %s", timeString);
+    psFree(timeString);
+    psString pslib = psLibVersionLong();// psLib version
+    psString psmodules = psModulesVersionLong(); // psModules version
+    psString psphot = psphotVersionLong(); // psphot version
+    psString ppStats = ppStatsVersionLong(); // ppStats version
+    psString pswarp = pswarpVersionLong(); // pswarp version
+    psLogMsg("pswarp", PS_LOG_INFO, "%s", pslib);
+    psLogMsg("pswarp", PS_LOG_INFO, "%s", psmodules);
+    psLogMsg("pswarp", PS_LOG_INFO, "%s", psphot);
+    psLogMsg("pswarp", PS_LOG_INFO, "%s", ppStats);
+    psLogMsg("pswarp", PS_LOG_INFO, "%s", pswarp);
+    psFree(pslib);
+    psFree(psmodules);
+    psFree(psphot);
+    psFree(ppStats);
+    psFree(pswarp);
+    return;
+}

/trunk	merged	eligible
/branches/eam_branches/eam_branch_20090303	23158-23228

Context Navigation

Legend: