Changeset 27751

Timestamp:

Apr 23, 2010, 11:33:14 AM (16 years ago)

Author:

bills

Message:

Enable the postage stamp server to generate stamps from cleaned up data.
"Works" for chip and warp stages. diff and maybe stack to do.

Location:

trunk

Files:

• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (43 diffs)
• ippTasks/pstamp.pro (modified) (6 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (11 diffs)
• pstamp/scripts/pstamp_finish.pl (modified) (4 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_request_file (modified) (2 diffs)
• pstamp/scripts/pstampparse.pl (modified) (25 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -27 +27 @@
 use PS::IPP::Config qw( :standard );
 use Carp;
+use DateTime;
+use File::Temp qw(tempfile);
+use File::Basename;
+use POSIX;
+
+my $save_temps = 0;
 
 # caches of camProcessedExp objects. 
@@ -34 +40 @@
 my %camRunByCamIDCache;
 
-### my @images = locate_images($image_db, $req_type, $img_type, $id, $component,
-###            $mjd_min, $mjd_max, $filter);
+# cache of last project looked up
+my $last_project = "";
+
+my ($regtool, $chiptool, $camtool, $warptool, $stacktool, $difftool);
+my ($pstamptool, $dvoImagesAtCoords, $whichimage, $ppConfigDump);
 
 sub locate_images {
     my $ipprc    = shift;   # required
-    my $image_db = shift;   # required
+    my $imagedb = shift;   # required
+    my $rowList  = shift;   # required
     my $req_type = shift;   # required
     my $img_type = shift;   # required
@@ -47 +57 @@
     my $option_mask  = shift;
     my $need_magic = shift;
-    my $x        = shift;
-    my $y        = shift;
     my $mjd_min  = shift;
     my $mjd_max  = shift;
@@ -56 +64 @@
 
     # we die in response to bad data in request files
-    # The wrapper script is responsible for updating the database
-    die "Unknown req_type: $req_type" 
+    # The caller is responsible for updating the database
+    # pstampparse.pl error checks now so this shouldn't happen
+    &my_die("Unknown req_type: $req_type", $PS_EXIT_PROG_ERROR) 
         if ($req_type ne "byid") and
            ($req_type ne "byexp") and 
@@ -84 +93 @@
 
     if ($req_type eq "bycoord") {
-        my $results = lookup_bycoord($ipprc, $image_db, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+        my $num_rows = scalar @$rowList;
+        die "Unexpected number of rows found in rowList: $num_rows" if $num_rows != 1;
+        my $row = $rowList->[0];
+        my $x = $row->{CENTER_X};
+        my $y = $row->{CENTER_Y};
+        my $results = lookup_bycoord($ipprc, $rowList, $imagedb, $img_type, $tess_id, $component, $need_magic, $x, $y, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
         return $results;
     }
@@ -93 +107 @@
         # in one place
         $req_type = "bydiff";
-        my $results = lookup_diff($ipprc, $image_db, $id, $component, 1, $option_mask, $img_type, $verbose);
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 1, $option_mask, $img_type, $verbose);
         return $results;
     }
 
     if ($req_type eq "bydiff") {
-        my $results = lookup_diff($ipprc, $image_db, $id, $component, 0, $option_mask, $img_type, $verbose);
+        # for bydiff reuqests we go look up the diffRun to obtain exp_id, chip_id, warp_id, stack_id, or 
+        # the image from the diffRun
+        my $results = lookup_diff($ipprc, $rowList, $imagedb, $id, $component, 0, $option_mask, $img_type, $verbose);
         if (!$results) {
             return undef;
@@ -130 +146 @@
             # fall though and lookup by stack_id
         } else {
-            # shouldn't I check this elsewhere?
+            # This is checked this elsewhere?
             print STDERR "Error: $img_type is an unknown image type\n";
             return undef;
@@ -145 +161 @@
     }
 
-    my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic,
-        $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
+    my $results = lookup($ipprc, $rowList, $imagedb, $req_type, $img_type, $id, $tess_id, $component,
+        $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
 
     return $results;
 }
+
+# The subroutine lookup handles byexp, byid, and byskycell lookups including lookups that are
+# triggered by a bydiff request
 
 sub lookup
 {
     my $ipprc    = shift;
-    my $image_db = shift;
+    my $rowList    = shift;
+    my $imagedb = shift;
     my $req_type = shift;
-    my $img_type = shift;
+    my $stage = shift;
     my $id       = shift;
     my $tess_id  = shift;
@@ -172 +192 @@
     my $use_imfile_id = ($req_type eq "byid") && ($option_mask & $PSTAMP_USE_IMFILE_ID);
 
-    my $missing_tools;
-    my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
-    my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
-    my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
-    my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
-    my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
     my $command;
     my $id_opt;     # option for the lookup
@@ -203 +213 @@
     my $magic_arg = $need_magic ? " -destreaked" : "";
 
+    # all rows have the same center type
+    my $skycenter = ! ($rowList->[0]->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+
     my $component_args;
-    if ($img_type eq "raw") {
+    my $choose_components = 0;
+    if ($stage eq "raw") {
         $class_id = $component;
-        $command = "$regtool -processedimfile -dbname $image_db";
+        $command = "$regtool -processedimfile -dbname $imagedb";
         # XXX: for now restrict lookups to type object 
         # are stamps of detrend exposures interesting?
@@ -214 +228 @@
         $want_astrom = 1;
         $set_class_id = 1;
-    } elsif ($img_type eq "chip") {
+    } elsif ($stage eq "chip") {
         $class_id = $component;
-        $command = "$chiptool -processedimfile -dbname $image_db";
-        $component_args = " -class_id $class_id" if $class_id;
+        # if the request is such that it will yield a single image per "run" or the
+        # center is specified in pixel coordinates 
+        # use chiptool -processsedimfile. Otherwise use chiptool -listrun and then
+        # choose the chips containing the center pixel by calling selectComponets() below
+        if ($class_id or $use_imfile_id or !$skycenter) {
+            $command = "$chiptool -processedimfile -dbname $imagedb";
+            $component_args = " -class_id $class_id" if $class_id;
+        } else {
+            $command = "$chiptool -listrun -pstamp_order -dbname $imagedb";
+            $choose_components = 1;
+        }
         $id_opt = $use_imfile_id ? "-chip_imfile_id" : "-chip_id";
+        # XXX: perhaps we should stop resolving images to this level here and do it at job run time.
+        # With the mdc file it has all of the information that it needs.
         $image_name   = "PPIMAGE.CHIP";
         $mask_name    = "PPIMAGE.CHIP.MASK";
@@ -228 +253 @@
         $want_astrom  = 1;
         $set_class_id = 1;
-    } elsif ($img_type eq "warp") {
+    } elsif ($stage eq "warp") {
         $skycell_id = $component;
-        $command = "$warptool -warped -dbname $image_db";
-        $component_args = " -skycell_id $skycell_id" if $skycell_id;
+        if ($skycell_id or $use_imfile_id or !$skycenter) {
+            $command = "$warptool -warped -dbname $imagedb";
+            $component_args = " -skycell_id $skycell_id" if $skycell_id;
+        } else {
+            $command = "$warptool -listrun -pstamp_order -dbname $imagedb";
+            $choose_components = 1;
+        }
         $id_opt = $use_imfile_id ? "-warp_skyfile_id" : "-warp_id";
         $image_name   = "PSWARP.OUTPUT";
@@ -239 +269 @@
         $psf_name     = "PSPHOT.PSF.SKY.SAVE";
         $base_name    = "path_base"; # name of the field for the warptool output
-    } elsif ($img_type eq "diff") {
+    } elsif ($stage eq "diff") {
         $skycell_id = $component;
-        $command = "$difftool -diffskyfile -dbname $image_db";
-        $component_args = " -skycell_id $skycell_id" if $skycell_id;
+        if ($skycell_id or $use_imfile_id or !$skycenter) {
+            $command = "$difftool -diffskyfile -dbname $imagedb";
+            $component_args = " -skycell_id $skycell_id" if $skycell_id;
+        } else {
+            $command = "$difftool -listrun -pstamp_order -dbname $imagedb";
+            $choose_components = 1;
+        }
         $id_opt = $use_imfile_id ? "-diff_skyfile_id" : "-diff_id";
         $image_name  = "PPSUB.OUTPUT";
@@ -250 +285 @@
         $psf_name     = "PSPHOT.PSF.SKY.SAVE";
         $base_name   = "path_base";
-    } elsif ($img_type eq "stack") {
+    } elsif ($stage eq "stack") {
         $skycell_id = $component;
-        $command = "$stacktool -sumskyfile -dbname $image_db";
+        $command = "$stacktool -sumskyfile -dbname $imagedb";
         $id_opt = "-stack_id";
         $component_args = " -skycell_id $skycell_id" if $skycell_id;
@@ -272 +307 @@
         $base_name   = "path_base";
     } else {
-        die "Unknown img_type supplied: $img_type";
-    }
-
+        die "Unknown IMG_TYPe supplied: $stage";
+    }
+
+    my $filter_runs = 0;
     if ($req_type eq "byid") {
         $command .= " $id_opt $id";
@@ -288 +324 @@
         # the reason as 'not destreaked'
         $magic_arg = "";
+        # remove duplicate runs for the same exposure.
+        $filter_runs = 1;
     } elsif ($req_type eq "byskycell") {
         die "tess_id and component are required for byskycell" if !$tess_id or ! $skycell_id;
         $command .= " -tess_id $tess_id -skycell_id $skycell_id";
     } else {
-        die "Unknown req_type supplied: $req_type";
-    }
-
-    if ($img_type ne "stack") {
+        # this should be caught by caller
+        &my_die("Unexpected req_type supplied: $req_type", $PS_EXIT_PROG_ERROR);
+    }
+
+    if ($stage ne "stack") {
         $command .= $magic_arg;
         $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
@@ -304 +343 @@
     $command .= " -data_group $data_group" if !isnull($data_group);
 
-    # run the tool and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        # not sure if we should die here
-        print STDERR @$stderr_buf;
+    my $images = runToolAndParse($command, $verbose);
+
+    if (!$images or scalar @$images == 0) {
         return undef;
     }
 
-    my $buf = join "", @$stdout_buf;
-    if (!$buf) {
-        return;
-    }
-
-    my $mdcParser = PS::IPP::Metadata::Config->new;
-    my $images = parse_md_fast($mdcParser, $buf)
-        or die ("Unable to parse metadata config doc");
-
+    if ($filter_runs) {
+        # The image selectors are such that multiple runs my have be returned for the same exposure.
+        # Return only the latest one.
+        $images = filterRuns($stage, $need_magic, $images, $verbose);
+    }
+    if ($choose_components) {
+        # the list of "images" is actually a list of "Runs"
+        # match the coords in the rows to the components in the runs
+        # returns an actual list of images
+        $images = selectComponents($ipprc, $imagedb, $req_type, $stage, $rowList, $images, $verbose);
+    } else {
+        # put each of the rows into all of the the images
+        setRowRefs($rowList, $images);
+    }
+
+    my $camera;
     my $output = [];
-
-    my $camera;
     foreach my $image (@$images) {
         my $base;
 
         next if $image->{fault};
-        next if ($img_type ne "raw") and $image->{quality};
+        next if ($stage ne "raw") and $image->{quality};
 
         if ($base_name) {
@@ -361 +402 @@
         }
         my $stage_id;
-        if ($img_type eq "raw") {
+        if ($stage eq "raw") {
             $stage_id = $image->{exp_id};
-        } elsif ($img_type eq "chip") {
+        } elsif ($stage eq "chip") {
             $stage_id = $image->{chip_id};
-        } elsif ($img_type eq "warp") {
+        } elsif ($stage eq "warp") {
             $stage_id = $image->{warp_id};
-        } elsif ($img_type eq "diff") {
+        } elsif ($stage eq "diff") {
             $stage_id = $image->{diff_id};
             if ($inverse && $image->{bothways}) {
@@ -375 +416 @@
                 $cmf_name    = "PPSUB.INVERSE.SOURCES";
             }
-        } elsif ($img_type eq "stack") {
+        } elsif ($stage eq "stack") {
             $stage_id = $image->{stack_id};
         }
         $out->{stage_id} = $stage_id;
-        $out->{stage}    = $img_type;
-        $out->{image_db} = $image_db;
+        $out->{stage}    = $stage;
+        $out->{imagedb} = $imagedb;
 
         if ($base) {
@@ -390 +431 @@
             $out->{backmdl}= $ipprc->filename($backmdl_name, $base, $class_id) if $backmdl_name;
         }
-        $out->{astrom} = find_astrometry($ipprc, $image_db, $image, $verbose) if $want_astrom;
+        if ($want_astrom and ! defined $out->{astrom}) {
+            $out->{astrom} = find_astrometry($ipprc, $imagedb, $image, $verbose);
+        }
 
         push @$output, $out;
@@ -397 +440 @@
     return $output;
 }
+
 sub lookup_diff {
     my $ipprc    = shift;
-    my $image_db = shift;
+    my $rowList  = shift;
+    my $imagedb = shift;
     my $id       = shift;
     my $skycell_id = shift;
@@ -409 +454 @@
     my $inverse = $option_mask & $PSTAMP_SELECT_INVERSE;
 
-    my $missing_tools;
-    my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
-
-    my $command = "$difftool -diffskyfile -dbname $image_db";
+    my $command = "$difftool -diffskyfile -dbname $imagedb";
     
     if ($byid) {
@@ -425 +463 @@
     $command .= " -skycell_id $skycell_id" if $skycell_id;
 
-    # run the tool and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        # not sure if we should die here
-        print STDERR @$stderr_buf;
-        return undef;
-    }
-
-    my $buf = join "", @$stdout_buf;
-    if (!$buf) {
-        return undef;
-    }
-
-    my $mdcParser = PS::IPP::Metadata::Config->new;
-    my $images = parse_md_fast($mdcParser, $buf)
-        or die ("Unable to parse metadata config doc");
+    my $images = runToolAndParse($command, $verbose);
 
     my $n = @$images;
@@ -527 +549 @@
 
         if ($img_type eq "diff") {
+            my @imageList = ($image);
+
+            setRowRefs($rowList, \@imageList);
             # the $image is going to be returned directly in this case so we need to duplicate
             # some of processing that lookup does for other img_types
@@ -539 +564 @@
                 $image->{stage_id} = $image->{diff_id};
                 $image->{stage}    = "diff";
-                $image->{image_db} = $image_db;
+                $image->{imagedb} = $imagedb;
                 $image->{component} = $image->{skycell_id};
             } else {
@@ -554 +579 @@
 sub lookup_bycoord {
     my $ipprc      = shift;
-    my $image_db   = shift;
+    my $rowList    = shift;
+    my $imagedb    = shift;
     my $img_type   = shift;
     my $tess_id    = shift;
@@ -568 +594 @@
     my $verbose    = shift;
 
-    my $missing_tools;
-    my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
-    my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
-    my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
-    my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
-
     my $results = ();
     if (($img_type eq "raw") or ($img_type eq "chip")) {
 
-        my $runs = lookup_runs_by_camid_and_coords($ipprc, $image_db, $img_type,
+        my $runs = lookup_runs_by_cam_id_and_coords($ipprc, $imagedb, $img_type,
             $ra, $dec, $need_magic, $dateobs_begin, $dateobs_end, $filter, $data_group, $verbose);
 
+        # lookup is going to filter these we don't need to do it here
+#        $runs = filterRuns($img_type, $need_magic, $runs, $verbose);
         foreach my $run (@$runs) {
             next if $component and ($run->{component} ne $component);
-            my $these_results = lookup($ipprc, $image_db, "byid", $img_type, $run->{id},
+            my $these_results = lookup($ipprc, $rowList, $imagedb, "byid", $img_type, $run->{id},
                 $tess_id, $run->{component}, $need_magic, 
                 $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
@@ -602 +620 @@
 
         foreach my $skycell (@$skycells) {
-            my $these_results = lookup($ipprc, $image_db, "byskycell", $img_type, undef,
+            my $these_results = lookup($ipprc, $rowList, $imagedb, "byskycell", $img_type, undef,
                 $skycell->{tess_id}, $skycell->{component}, $need_magic, 
                 $dateobs_begin, $dateobs_end, $filter, $data_group, $option_mask, $verbose);
@@ -614 +632 @@
 }
 
-# lookup_runs_by_camid_and_coords
+# lookup_runs_by_cam_id_and_coords
 # given an ra, dec, and optionally other paramters, find camera runs for exposures
 # that are within some distance of the coordinates
-sub lookup_runs_by_camid_and_coords {
+sub lookup_runs_by_cam_id_and_coords {
     my $ipprc      = shift;
-    my $image_db   = shift;
+    my $imagedb   = shift;
     my $img_type   = shift;
     my $ra         = shift;
@@ -631 +649 @@
 
     my $camruns;
-    my $missing_tools;
-    my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
-    my $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
 
     {
-        my $command = "$camtool -dbname $image_db -processedexp";
+        my $command = "$camtool -dbname $imagedb -processedexp -pstamp_order";
            $command .= " -ra $ra -decl $dec -radius 1.6";
            $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
@@ -652 +663 @@
            $command .= " -data_group $data_group" if $data_group;
            $command .= " -destreaked" if $need_magic;
+
         # run the tool and parse the output
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-        unless ($success) {
-            print STDERR @$stderr_buf;
-            return undef;
-        }
-        my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-        my $output = join "", @$stdout_buf;
-        if (!$output) {
-            print STDERR "no output returned from $command\n" if $verbose;
-            return undef;
-        }
-        $camruns = parse_md_fast($mdcParser, $output);
+        $camruns = runToolAndParse($command, $verbose);
     }
     if (!$camruns) {
@@ -672 +671 @@
     }
     my $runs;
+    my $last_exp_id = 0;
     foreach my $camRun (@$camruns) {
         my $cam_id = $camRun->{cam_id};
-
-        updateCamRunCache($camRun);
+        my $exp_id = $camRun->{exp_id};
+
+        next if $exp_id eq $last_exp_id;
 
         next if $camRun->{quality};
         next if $camRun->{fault};
+
+        updateCamRunCache($camRun);
+
+        $last_exp_id = $exp_id;
+
         # XXX Use file rule
         my $astrom = $camRun->{path_base} . ".smf";
@@ -684 +690 @@
         next if !$astrom_resolved;
 
+        my $start_dvo = DateTime->now->mjd;
         my $command = "$dvoImagesAtCoords -astrom $astrom_resolved $ra $dec";
         # run the tool and parse the output
+        my $dvo_verbose = 0;
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
+                    run(command => $command, verbose => $dvo_verbose);
+        my $dtime_dvo = (DateTime->now->mjd - $start_dvo)* 86400.;
+#        print "Time to run dvoImagesAtCoords: $dtime_dvo\n";
         unless ($success) {
             my $result_code = $error_code >> 8;
             if ($result_code == $PSTAMP_NO_OVERLAP) {
-                print STDERR "no overlap for $astrom\n" if $verbose;
+                print "no overlap for $astrom\n" if $verbose;
                 next;
             }
@@ -721 +731 @@
             die "unexpected output from dvoImagesAtCoords: $lines[0]";
         }
+
+        print "cam_id $cam_id exp_id $camRun->{exp_id} chip_id $camRun->{chip_id} $class_id\n";
+
         # build the hash to return
         my $run = {
@@ -732 +745 @@
         };
         if ($img_type eq "chip") {
-            $run->{id} = $run->{chip_id};
+            $run->{id} = $camRun->{chip_id};
+            $run->{state} = $camRun->{chip_state};
+            $run->{magicked} = $camRun->{chip_magicked};
         } else {
-            $run->{id} = $run->{exp_id};
+            $run->{id} = $camRun->{exp_id};
+            $run->{state} = 'full';
+            $run->{magicked} = $camRun->{raw_magicked};
         }
         push @$runs, $run;
@@ -749 +766 @@
     my $dec        = shift;
     my $verbose    = shift;
-
-    my $missing_tools;
-    my $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
 
     $requested_tess_id = "" if isnull($requested_tess_id);
@@ -817 +827 @@
 sub find_astrometry {
     my $ipprc = shift;
-    my $image_db = shift;
+    my $imagedb = shift;
     my $image = shift;      # hashref to output of the lookup tool
     my $verbose = shift;
@@ -825 +835 @@
     my $exp_id = $image->{exp_id};
     if (($exp_id eq $last_exp_id) and $lastAstromFile) {
-        # running camtool 60 times is really expensive when the answer is the same
+        # running camtool 60 times is really expensive when the answer is the same every time
         return $lastAstromFile;
     }
@@ -831 +841 @@
     $lastAstromFile = undef;
 
-    my $missing_tools;
-    my $camtool = can_run("camtool") or (warn "Can't find camtool" and $missing_tools = 1);
-    my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
-
     my $camRun = getCamRunByExpID($exp_id);
     if (!$camRun) {
-        my $command = "$camtool -dbname $image_db -processedexp -exp_id $exp_id";
+        my $command = "$camtool -dbname $imagedb -processedexp -exp_id $exp_id -pstamp_order";
         # run the tool and parse the output
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -864 +866 @@
         # If there are multiple cam runs for this exposure, take the last completed one with good quality
         # on the assumption that it has the best astrometry.
-        while ($camRun = pop @$camruns) {
-            last if (($camRun->{quality} eq 0) and ($camRun->{fault} eq 0));
+        foreach $camRun (@$camruns) {
+            last if (($camRun->{state} eq 'full') and ($camRun->{quality} eq 0) and ($camRun->{fault} eq 0));
         }
         # XXX: this looks like a bug at least if ASTROM.SOURCE eq PSASTRO.OUTPUT
@@ -959 +961 @@
 # resolve_project()
 # get project specific information
-my $last_project = "";
 sub resolve_project {
     my $ipprc = shift;
     my $project_name = shift;
 
+    findTools();
+
     if (!$project_name) {
         carp ("project is not defined");
@@ -978 +981 @@
     my $verbose = 0;
 
-    my $missing_tools;
-    my $pstamptool = can_run("pstamptool") or (warn "Can't find pstamptool" and $missing_tools = 1);
-    if ($missing_tools) {
-        warn("Can't find required tools.");
-        exit ($PS_EXIT_CONFIG_ERROR);
-    }
-
     my $command = "$pstamptool -project -name $project_name";
     $command .= " -dbname $dbname" if defined $dbname;
     $command .= " -dbserver $dbserver" if defined $dbserver;
-    # run the tool and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        print STDERR @$stderr_buf;
+
+    my $proj_hash = runToolAndParse($command, $verbose);
+    if (!$proj_hash or scalar @$proj_hash == 0) {
+        print STDERR "Project $project_name not found\n";
         return undef;
     }
-
-    my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-    my $output = join "", @$stdout_buf;
-    if (!$output) {
-        print STDERR "no output returned from $command\n" if $verbose;
-        return undef;
-    }
-    my $proj_hash = parse_md_fast($mdcParser, $output);
 
     $last_project = $proj_hash->[0];
@@ -1035 +1021 @@
 sub getCamRunByExpID {
     my $exp_id = shift;
-    die "getCamRun: exp_id is nil" if !$exp_id;
+    &my_die ("getCamRun: exp_id is nil", $PS_EXIT_PROG_ERROR) if !$exp_id;
 
     return $camRunByExpIDCache{$exp_id};
 }
+
 sub getCamRunByCamID {
     my $cam_id = shift;
-    die "getCamRun: cam_id is nil" if !$cam_id;
+    &my_die ("getCamRun: cam_id is nil", $PS_EXIT_PROG_ERROR) if !$cam_id;
 
     return $camRunByCamIDCache{$cam_id};
 }
+
+sub selectComponents {
+    my $ipprc = shift;
+    my $imagedb = shift;
+    my $req_type = shift;
+    my $stage  = shift;
+    my $rowList = shift;
+    my $runList = shift;
+    my $verbose = shift;
+    my $results = [];
+    
+    my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
+    my $npoints = 0;
+    foreach my $row (@$rowList) {
+        print $pointsList "$npoints $row->{CENTER_X} $row->{CENTER_Y}\n";
+        $npoints++;
+    }
+    close $pointsList;
+
+    # XXX: need to loop over these
+    # my $run = $runList->[0];
+    if (($req_type eq "byid") or ($req_type eq "byexp")) {
+        my ($last_tess_id, $tess_dir_abs, $astrom_file) = ("", "", "");
+        foreach my $run (@$runList) {
+            my $command = "$dvoImagesAtCoords -coords $pointsListName";
+            if (($stage eq "chip") or ($stage eq "raw")) {
+                # XXX: use file rule and handle cameras where the astrometry is solved at
+                # the chip stage.
+                $astrom_file = $run->{cam_path_base} . ".smf";
+                my $astrom_file_resolved = $ipprc->file_resolve($astrom_file);
+                $command .= " -astrom $astrom_file_resolved";
+            } else {
+                my $tess_id = $run->{tess_id};
+                if ($tess_id ne $last_tess_id) {
+                    $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id );
+                    $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs );
+                    $last_tess_id = $tess_id;
+                }
+                $command .= " -D CATDIR $tess_dir_abs"
+            }
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                 run(command => $command, verbose => $verbose);
+            unless ($success) {
+                # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP
+                #                     # That just means that the coordinate didn't match any image/skycell
+                if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) {
+                    print STDERR @$stderr_buf;
+                    my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR;
+                    &my_die( "dvoImagesAtCoords failed: $rc", $rc);
+                }
+            }
+            my %components;
+            my @lines = split "\n", join "", @$stdout_buf;
+            foreach my $line (@lines) {
+                my ($ptnum, undef, undef, $component) = split " ", $line;
+                my $ref = $components{$component};
+                if (!$ref) {
+                    $ref = $components{$component} = [];
+                }
+                push @$ref, $ptnum;
+            }
+            if ($verbose) {
+                foreach my $c (keys %components) {
+                    my $ref = $components{$c};
+                    print "component $c contains: ";
+                    foreach my $i (@$ref) {
+                        print "$i ";
+                    }
+                    print "\n";
+                }
+            }
+            # now find the images for this run
+            foreach my $c (keys %components) {
+                my $command;
+                my $stage_id;
+                if ($stage eq 'raw') {
+                    $stage_id = $run->{exp_id};
+                    $command = "$regtool -processedimfile -exp_id $stage_id -class_id $c";
+                } elsif ($stage eq 'chip') {
+                    $stage_id = $run->{chip_id};
+                    $command = "$chiptool -processedimfile -chip_id $stage_id -class_id $c";
+                } elsif ($stage eq 'warp') {
+                    $stage_id = $run->{warp_id};
+                    $command = "$warptool -warped -warp_id $stage_id -skycell_id $c";
+                } elsif ($stage eq 'diff') {
+                    $stage_id = $run->{diff_id};
+                    $command = "$difftool -diffskyfile -diff_id $stage_id -skycell_id $c";
+                }
+                $command .= " -dbname $imagedb";
+                my $images = runToolAndParse($command, $verbose);
+                if (!defined $images) {
+                    # XXX: need to get the set error_code to $PSTAMP_NO_OVERLAP
+                    print "No components containing coordinates found for ${stage}_id $stage_id\n";
+                    foreach my $row (@$rowList) {
+                        $row->{error_code} = $PSTAMP_NO_OVERLAP;
+                    }
+                    next;
+                }
+                if (scalar @$images != 1) {
+                    my $num_images = scalar @$images;
+                    &my_die ("unexpected number of images returned: $num_images\n", $PS_EXIT_PROG_ERROR);
+                }
+                my $image = $images->[0];
+                    
+                my $ref = $components{$c};
+                $image->{row_index} = $ref;
+                if (($stage eq "raw") or ($stage eq 'chip')) {
+                    $image->{astrom} = $astrom_file;
+                    $image->{cam_id} = $run->{cam_id};
+                    $image->{cam_path_base} = $run->{cam_path_base};
+                }
+                push @$results, $image;
+            }
+        }
+    } else {
+        &my_die ("sorry not done with $req_type\n", $PS_EXIT_PROG_ERROR);
+    }
+    return $results;
+}
+
+sub filterRuns {
+    my $stage      = shift;
+    my $need_magic = shift;
+    my $inputs     = shift;
+    my $verbose    = shift;
+
+    if ($inputs and (scalar @$inputs) == 1)  {
+        # one run nothing to do
+        return $inputs;
+    }
+
+    my $output = [];
+
+    return $output if (!$inputs or scalar @$inputs == 0);
+
+    my $id_name = $stage . "_id";
+
+    # input list is "order by exp_id, run_id DESC"   run_id is one of (chip_id, warp_id, diff_id)
+    print "Starting filterRuns\n";
+    my $last_exp_id = 0;
+    my $last_run_id = 0;
+    my $printed = 0;
+    foreach my $input (@$inputs) {
+        my $exp_id = $input->{exp_id};
+        my $run_id = $input->{$id_name};
+        my $magicked = $input->{magicked};  # this will be either stageRun.magicked or stage%file.magicked
+        my $state = $input->{state};
+
+        # can't process run in these states
+        if (($state eq 'new') or ($state eq 'purged') or ($state eq 'scrubbed')) {
+            print "skipping ${stage}Run $run_id for exp_id $exp_id in state $state\n";
+            next;
+        }
+
+        $printed = 1 if (($exp_id == $last_exp_id) and ($run_id == $last_run_id));
+
+        # skip if we need magicked run and this one has never been magicked
+        if ($need_magic and !$magicked) {
+            print "skipping ${stage}Run $run_id for exp_id $exp_id not magicked\n" if !$printed;
+            next;
+        }
+
+        if (($exp_id == $last_exp_id) and ($run_id != $last_run_id)) {
+            print "Skipping duplicate ${stage}Run $run_id for $exp_id\n" if !$printed;
+            next;
+        }
+        print "Keeping ${stage}Run $run_id for $exp_id\n";
+        push @$output, $input;
+        $last_exp_id = $exp_id;
+        $last_run_id = $run_id;
+        $printed = 0;
+    }
+
+    my $num_runs = scalar @$output;
+    print "filterRuns returning $num_runs ${stage}Run\n";
+
+    return $output;
+}
+
+# run a command that produces metadata output and parse the results into an array of objects
+sub runToolAndParse {
+    my $command = shift;
+    my $verbose = shift;
+
+    my ($program) = split " ", $command;
+    $program = basename($program);
+
+    print "Running $command\n" if !$verbose;
+    my $start_tool = DateTime->now->mjd;
+    # run the command and parse the output
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+    unless ($success) {
+        # not sure if we should die here
+        print STDERR @$stderr_buf;
+        return undef;
+    }
+
+    my $now = DateTime->now->mjd;
+    my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.;
+    print "Time to run $program: $dtime_tool\n";
+
+    my $buf = join "", @$stdout_buf;
+    if (!$buf) {
+        return undef;
+    }
+
+    my $start_parse = DateTime->now->mjd;
+
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $results = parse_md_fast($mdcParser, $buf)
+        or die ("Unable to parse metadata config doc");
+
+    my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.;
+    print "Time to parse results from $program: $dtime_parse\n";
+
+    return $results;
+}
+
+sub findTools {
+    return if ($regtool);
+
+    my $missing_tools;
+    $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+    $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1);
+    $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
+    $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
+    $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+    $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
+    $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
+    $dvoImagesAtCoords = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" 
+                                                                and $missing_tools = 1);
+    $whichimage = can_run('whichimage') or (warn "Can't find whichimage" and $missing_tools = 1);
+    $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
+    if ($missing_tools) {
+        warn("Can't find required tools.");
+        exit ($PS_EXIT_CONFIG_ERROR);
+    }
+}
+
+# add a row_index array to a set of images with entries for each of the rows in a list
+sub setRowRefs {
+    my $rowList = shift;
+    my $images  = shift;
+
+    my $row_index = [];
+    for (my $i = 0; $i < scalar @$rowList; $i++) {
+        push @$row_index, $i;
+    }
+    foreach my $image (@$images) {
+        $image->{row_index} = $row_index;
+    }
+}
+
+sub my_die
+{
+    my $msg = shift;
+    my $fault = shift;
+
+    carp $msg;
+
+    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
+
+    return $fault;
+}
 1;

trunk/ippTasks/pstamp.pro

@@ -13 +13 @@
 $pstampFin_DB = 0
 $pstampRev_DB = 0
+$pstampDep_DB = 0
 
 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config
@@ -28 +29 @@
     book init pstampJob
     book init pstampFinish
+    book init pstampDependent
 end
 
 pstamp.reset
-
 
 macro pstamp.on
@@ -60 +61 @@
         active true
     end
+    task pstamp.dependent.load
+        active true
+    end
 end
 
@@ -88 +92 @@
     task pstamp.job.revert
         active false
+    end
+    task pstamp.dependent.load
+        active true
     end
 end
@@ -208 +215 @@
         book getword pstampRequest $pageName uri -var URI
         book getword pstampRequest $pageName ds_outProduct -var PRODUCT
+        book getword pstampRequest $pageName label -var LABEL
 
         host anyhost
 
-        $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --redirect-output
+        $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --label $LABEL --redirect-output
 
         add_standard_args run
@@ -512 +520 @@
 end
 
+task pstamp.dependent.load
+    host        local
+
+    periods     -poll $LOADPOLL
+    # XXX: create a macro for this
+    periods     -exec 30
+#    periods     -exec $LOADEXEC
+    periods     -timeout 30
+    npending    1
+
+    task.exec
+        stdout $LOGSUBDIR/pstamp.dependent.load.log
+        stderr $LOGSUBDIR/pstamp.dependent.load.log
+        $run = pstamptool -pendingdependent
+        if ($DB:n == 0)
+            option DEFAULT
+        else
+            option $DB:$pstampDep_DB
+            $run = $run -dbname $DB:$pstampDep_DB $PS_DBSERVER
+            $pstampDep_DB ++
+            if ($pstampDep_DB >= $DB:n) set pstampDep_DB = 0
+        end
+        add_poll_args run
+        add_poll_labels run 
+        command $run
+    end
+
+    task.exit $EXIT_SUCCESS
+        ipptool2book stdout pstampDependent -key dep_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+
+        book npages pstampDependent -var N
+        if ($VERBOSE > 2)
+            book listbook pstampDependent
+        end
+
+        # delete existing entries in the appropriate pantaskStates
+        process_cleanup pstampDependent
+    end
+
+    task.exit   crash
+        showcommand crash
+    end
+
+    task.exit   timeout
+        showcommand timeout
+    end
+
+    task.exit   default
+        showcommand failure
+    end
+
+end
+
+task pstamp.dependent.run
+    periods     -poll $RUNPOLL
+    periods     -exec $RUNEXEC
+    periods     -timeout 300
+
+    task.exec
+#        stdout $LOGSUBDIR/pstamp.dependent.run.log
+#        stderr $LOGSUBDIR/pstamp.dependent.run.log
+        book npages pstampDependent -var N
+        if ($N == 0) 
+            periods -exec $RUNEXEC
+            break
+        end
+        periods -exec 0.05
+        
+        book getpage pstampDependent 0 -var pageName -key pantaskState INIT
+        if ("$pageName" == "NULL") break
+
+        #echo pageName: $pageName
+
+        book setword pstampDependent $pageName pantaskState RUN
+        book getword pstampDependent $pageName dep_id     -var DEP_ID
+        book getword pstampDependent $pageName state      -var STATE
+        book getword pstampDependent $pageName stage      -var STAGE
+        book getword pstampDependent $pageName stage_id   -var STAGE_ID
+        book getword pstampDependent $pageName component  -var COMPONENT
+        book getword pstampDependent $pageName imagedb    -var IMAGEDB
+        book getword pstampDependent $pageName rlabel     -var RLABEL
+#        book getword pstampDependent $pageName output_base -var OUTPUT_BASE
+        book getword pstampDependent $pageName need_magic -var NEED_MAGIC
+        book getword pstampDependent $pageName dbname     -var DBNAME
+
+        if ($VERBOSE > 1) 
+            book listpage pstampDependent $pageName
+        end
+
+        host anyhost
+
+        if ("$NEED_MAGIC" == "T")
+            $NEED_MAGIC="--need_magic"
+        else
+            $NEED_MAGIC=""
+        end
+
+        # XXX: use OUTPUT_BASE for this
+        $LOGFILE = /data/ipp053.0/home/bills/pstamp/logs/pstampdep.$DEP_ID.log
+#        echo logfile is $LOGFILE
+#        stdout $LOGFILE
+#        stderr $LOGFILE
+
+        $run = pstamp_checkdependent.pl --dep_id $DEP_ID --stage_id $STAGE_ID --stage $STAGE --component $COMPONENT --imagedb $IMAGEDB --rlabel $RLABEL $NEED_MAGIC
+        add_standard_args run
+
+        options $pageName
+
+        if ($VERBOSE > 1) 
+            echo command $run
+        end
+        command $run
+    end
+
+
+    task.exit $EXIT_SUCCESS
+        if ($VERBOSE > 1)
+            echo pstamp.dependent.run task.exit $DEP_ID status: $JOB_STATUS
+        end
+        process_exit pstampDependent $options:0 $JOB_STATUS
+    end
+    task.exit default
+        if ($VERBOSE > 1)
+            echo pstamp.job.run task.exit $DEP_ID status: $JOB_STATUS
+        end
+        showcommand failure
+        process_exit pstampDependent $options:0 $JOB_STATUS
+    end
+
+    task.exit crash
+        echo pstamp.job.run task.crash $DEP_ID status: $JOB_STATUS
+        process_exit pstampDependent $options:0 $JOB_STATUS
+        showcommand crash
+        book setword pstampDependent $options:0 pantaskState CRASH
+    end
+
+    task.exit timeout
+        echo pstamp.job.run task.timeout $DEP_ID status: $JOB_STATUS
+        process_exit pstampDependent $options:0 $JOB_STATUS
+        showcommand timeout
+        book setword pstampDependent $options:0 pantaskState TIMEOUT
+    end
+end
+

trunk/pstamp/scripts/pstamp_checkdependent.pl

@@ -48 +48 @@
 my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1);
 my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $stacktool = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $magicdstool = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
 my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools = 1);
@@ -86 +87 @@
 } else {
     if ($stage eq "chip") {
-        $cmd = "$chiptool -processedimfile -chip_id $stage_id -class_id $component";
+        $cmd = "$chiptool -processedimfile -allfiles -chip_id $stage_id -class_id $component";
     } elsif ($stage eq "warp") {
         $cmd = "$warptool -warped -warp_id $stage_id -skycell_id $component";
@@ -96 +97 @@
 }
 
-my $components = runToolAndParse($cmd, $verbose);
-
-my $n = scalar @$components;
-my_die("Unexpected number of components found $n", $PS_EXIT_PROG_ERROR) if ($n != 1);
+my $it = runToolAndParseExpectOne($cmd, $verbose);
+
+my_die("no components found", $PS_EXIT_PROG_ERROR) if ( !$it);
 
 # Got "it"
-my $it = $components->[0];
 
 my $status = 0;
@@ -130 +129 @@
     # need to queue the updates
     if ($stage eq 'chip') {
-        # queue_update_chip takes an array so that queue_update_warp can pass a set of
-        # chips to it
+        # queue_update_chip takes an array so that queue_update_warp can pass it set of chips
         my $chips = [$it];
         $status = queue_update_chip($it->{chip_id}, $whole_run, $chips, $rlabel, $need_magic);
@@ -145 +143 @@
 
     # detect states that cannot be updated and update the job state
-    # XXX: we should be more assertive here and check for the specific states where we can continue
+    # XXX: Perhaps I should be more assertive here and check for the specific states that we know
+    # that we can continue.
+
     my $state = $it->{state};
     my $job_fault = 0;
-    if (($state =~ /scrub/) or ($state =~ /purge/)) {
+
+    # temporary hack to deal with data with old burntool tables. This check is now done by pstampparse.pl
+    # so runs in this state will not get queued
+    if ($stage eq 'chip') {
+        my $burntool_state = $it->{burntool_state};
+        if ($burntool_state and (abs($burntool_state) < 14)) {
+            print STDERR "chip $it->{chip_id} $it->{class_id} has burntool_state $burntool_state. Not avaiable.\n";
+            $job_fault = $PSTAMP_NOT_AVAILABLE;
+        }
+    }
+    if ($state eq 'error_cleaned') {
+        $job_fault = $PSTAMP_NOT_AVAILABLE;
+    } elsif (($state =~ /scrub/) or ($state =~ /purge/)) {
         # jobs must have changed state since depenency was made
         print STDERR "Dependency cannot be satisfied\n";
@@ -155 +167 @@
         # Dependency never should have been inserted
         my_die ("Unexpected state for ${stage}Run $stage_id $state", $PS_EXIT_PROG_ERROR);
+    }
+    if (!$job_fault and ($stage eq 'chip')) {
+        $job_fault = queue_update_magicDSRun($stage, $stage_id, $rlabel, $it->{raw_magicked}, $it->{dsRun_state});
     }
     if ($job_fault) {
@@ -213 +228 @@
     }
 
+    my $status = queue_update_magicDSRun('chip', $chip_id, $rlabel, $raw_all_magicked, $dsRun_state);
+
+    return $status;
+}
+
+sub queue_update_warp {
+    # check status of input chips. If state is not updatable set error code for job
+
+    # if chipProcessedImfile.state is cleaned call queue_update_chip
+
+    # need to code warptool -setskyfiletoupdate
+    my $metadata = shift;
+    my $whole_run = shift;  # if true queue entire run for update
+    my $rlabel = shift;     # if defined a new label for the chipRun
+    my $need_magic = shift; 
+
+    my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked
+
+    my $warp_id = $metadata->{warp_id};
+    if (!$whole_run) {
+        my $skycell = $metadata;
+        my $skycell_id = $skycell->{skycell_id};
+
+        my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
+        my $data = runToolAndParse($command, $verbose);
+        my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR)
+            if !$data or scalar @$data == 0;
+
+        my $good_to_go = 1;
+        my @chipsToUpdate;
+        my $chip_id;
+        foreach my $chip (@$data) {
+            $chip_id = $chip->{chip_id};
+            if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
+                $good_to_go = 0;
+                push @chipsToUpdate, $chip;
+            } else {
+                # this chip is good to go
+            }
+        }
+
+        if ($good_to_go) {
+            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
+            $command .= " -set_label $rlabel" if $rlabel;
+
+            if (!$no_update) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                            run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+                }
+            } else {
+                print "skipping $command\n";
+            }
+        } elsif (scalar @chipsToUpdate > 0) {
+            return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic);
+        }
+    } else {
+        my $warpRun = $metadata;
+        my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}";
+        my $data = runToolAndParse($command, $verbose);
+        my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR)
+            if !$data or scalar @$data != 1;
+
+        my $chipRun = $data->[0];
+
+        my $chipRunState = $chipRun->{state};
+        if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) {
+            print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n";
+            faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE);
+            return 0;
+        }
+        my $warpRunState = $warpRun->{state};
+        if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) {
+            # The inputs and outputs are ready. Queue the warpRun for update.
+
+            # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update'
+            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id";
+            $command .= " -set_label $rlabel" if $rlabel;
+
+            if (!$no_update) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                            run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+                }
+            } else {
+                print "skipping $command\n";
+            }
+        } elsif ($chipRunState eq 'cleaned' or 
+            (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) {
+            my $data = [$chipRun];
+            return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic);
+        }
+    }
+
+    # return value may be the return status of script so zero is good
+    return 0;
+}
+sub queue_update_diff {
+    my $metadata = shift;
+    my $whole_run = shift;  # if true queue entire run for update
+    my $rlabel = shift;     # if defined a new label for the chipRun
+    my $need_magic = shift; 
+
+# XXXX This is not ready to go yet XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXxx
+
+    my $diff_id   = $metadata->{diff_id};
+    my $diff_mode = $metadata->{diff_mode};
+    if (!$whole_run) {
+        my $skycell = $metadata;
+        my $skycell_id = $skycell->{skycell_id};
+
+        # put this in a module somewhere
+        my $IPP_DIFF_MODE_WARP_WARP   = 1;
+        my $IPP_DIFF_MODE_WARP_STACK  = 2;
+        my $IPP_DIFF_MODE_STACK_WARP  = 2;
+        my $IPP_DIFF_MODE_STACK_STACK = 4;
+        if ($diff_mode == $IPP_DIFF_MODE_WARP_STACK ) {
+            # check the state of the stack
+            my $command = "$stacktool -sumkskyfile -stack_id $skycell->{stack2}";
+            my $stack = runToolAndParseExpectOne($command, $verbose);
+            my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack;
+
+            if ($stack->{state} ne 'full') {
+                print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
+                # this faults all jobs depending on this dep_id
+                faultJobs('stop', 'diff', $diff_id, $PSTAMP_GONE);
+                return $PSTAMP_GONE;
+            }
+
+            # now check the warp
+            $command = "$warptool -warped -warp_id $skycell->{warp1} -skycell_id $skycell_id";
+            my $warp = runToolAndExpectOne($command, $verbose);
+            my_die("failed to find warpSkyfile for warpRun $skycell->{warp_id} skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR)
+                if !$warp;
+        }
+
+        my $good_to_go = 1;
+        my @chipsToUpdate;
+        my $chip_id;
+        my $warp_id;
+        my $data ;  #XXXX
+        foreach my $chip (@$data) {
+            $chip_id = $chip->{chip_id};
+            if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
+                $good_to_go = 0;
+                push @chipsToUpdate, $chip;
+            } else {
+                # this chip is good to go
+            }
+        }
+
+        if ($good_to_go) {
+            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
+            $command .= " -set_label $rlabel" if $rlabel;
+
+            if (!$no_update) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                            run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+                }
+            } else {
+                print "skipping $command\n";
+            }
+        } elsif (scalar @chipsToUpdate > 0) {
+            return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic);
+        }
+    } else {
+        my $warpRun = $metadata;
+        my $warp_id = $warpRun->{warp_id};
+        my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}";
+        my $data = runToolAndParse($command, $verbose);
+        my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR)
+            if !$data or scalar @$data != 1;
+
+        my $chipRun = $data->[0];
+
+        my $chipRunState = $chipRun->{state};
+        if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) {
+            print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n";
+            faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE);
+            return 0;
+        }
+        my $warpRunState = $warpRun->{state};
+        if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) {
+            # The inputs and outputs are ready. Queue the warpRun for update.
+
+            # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update'
+            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id";
+            $command .= " -set_label $rlabel" if $rlabel;
+
+            if (!$no_update) {
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                            run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
+                }
+            } else {
+                print "skipping $command\n";
+            }
+        } elsif ($chipRunState eq 'cleaned' or 
+            (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) {
+            my $data = [$chipRun];
+            return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic);
+        }
+    }
+
+    # return value may be the return status of script so zero is good
+    return 0;
+}
+
+
+# run a command that produces metadata output and parse the results into an array of objects
+sub runToolAndParse {
+    my $command = shift;
+    my $verbose = shift;
+
+    my ($program) = split " ", $command;
+    $program = basename($program);
+
+    print "Running $command\n" if !$verbose;
+    my $start_tool = DateTime->now->mjd;
+    # run the command and parse the output
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+    unless ($success) {
+        print STDERR @$stderr_buf if !$verbose;
+        return undef;
+    }
+
+    my $now = DateTime->now->mjd;
+    my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.;
+    print "Time to run $program: $dtime_tool\n";
+
+    my $buf = join "", @$stdout_buf;
+    if (!$buf) {
+        return undef;
+    }
+
+    my $start_parse = DateTime->now->mjd;
+
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $results = parse_md_fast($mdcParser, $buf)
+        or my_die ("Unable to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
+
+    my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.;
+    print "Time to parse results from $program: $dtime_parse\n";
+
+    return $results;
+}
+
+# run an command returning metadata where we expect 1 entry
+sub runToolAndParseExpectOne {
+    my $command = shift;
+    my $verbose = shift;
+
+    my $data = runToolAndParse($command, $verbose);
+
+    if (!$data) {
+        return undef;
+    }
+
+    my $n = scalar @$data;
+    if ($n > 1) {
+        my_die("Unexpected number of components $n returned by $command", $PS_EXIT_PROG_ERROR);
+    }
+
+    return $data->[0];
+}
+
+# splits meta data config input stream into single units to work around the pathalogically
+# slow parser. This is similar to and adapted from code in various ippScripts.
+sub parse_md_fast {
+    my $mdcParser = shift;
+    my $input = shift;
+    my $output = ();
+
+    my @whole = split /\n/, $input;
+    my @single = ();
+
+    my $n;
+    while ( ($n = @whole) > 0) {
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or
+                print STDERR "Unable to parse metdata config doc" and return undef;
+            push @$output, $list->[0];
+
+            @single = ();
+        }
+    }
+    return $output;
+}
+
+sub faultJobs {
+    my ($state, $stage, $stage_id, $job_fault) = @_;
+
+    my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id";
+    $command .= " -dbname $dbname" if $dbname;
+    $command .= " -dbserver $dbserver" if $dbserver;
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("failed to set pstampJob.fault for dep_id: $dep_id",
+                $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping $command\n";
+    }
+}
+
+sub queue_update_magicDSRun {
+    my $stage = shift;
+    my $stage_id = shift;
+    my $rlabel  = shift;
+    my $input_magicked = shift;
+    my $dsRun_state = shift;
+
+    # XXX: this code is only fully coded and tested for chip stage
+    my_die ("queue_update_magicDSRun only works for stage chip", $PS_EXIT_PROG_ERROR) if $stage ne 'chip';
+
     # if called from queue_update_warp dsRun_state is unknown. Go find it.
     if (!$dsRun_state) {
-        my $command = "$chiptool -listrun -chip_id $chip_id";
+        my $command = "$chiptool -listrun -chip_id $stage_id";
         my $data = runToolAndParse($command, $verbose);
         my $chipRun = $data->[0];
@@ -222 +564 @@
 
     # if the input file is already magicked no need to queue destreaking for this chipRun
-    if ($need_magic and !$raw_all_magicked) {
+    if ($need_magic and !$input_magicked) {
         if ($dsRun_state eq 'cleaned') {
             my $command = "$magicdstool -updaterun -set_state new -stage $stage -stage_id $stage_id";
@@ -242 +584 @@
         }
     }
-
-    # return value may be the return status of script so zero is good
     return 0;
 }
 
-sub queue_update_warp {
-    # check status of input chips. If state is not updatable set error code for job
-
-    # if chipProcessedImfile.state is cleaned call queue_update_chip
-
-    # need to code warptool -setskyfiletoupdate
-    my $metadata = shift;
-    my $whole_run = shift;  # if true queue entire run for update
-    my $rlabel = shift;     # if defined a new label for the chipRun
-    my $need_magic = shift; 
-
-    my $raw_all_magicked = 1; # this gets cleared if any of the inputs aren't destreaked
-
-    my $warp_id = $metadata->{warp_id};
-    if (!$whole_run) {
-        my $skycell = $metadata;
-        my $skycell_id = $skycell->{skycell_id};
-
-        my $command = "$warptool -scmap -warp_id $warp_id -skycell_id $skycell_id";
-        my $data = runToolAndParse($command, $verbose);
-        my_die("failed to find warpSkyCelllMap for warpRun $warp_id skycell_id $skycell_id", $PS_EXIT_UNKNOWN_ERROR)
-            if !$data or scalar @$data == 0;
-
-        my $good_to_go = 1;
-        my @chipsToUpdate;
-        my $chip_id;
-        foreach my $chip (@$data) {
-            $chip_id = $chip->{chip_id};
-            if (($chip->{data_state} ne 'full') or ($need_magic and ($chip->{magicked} <= 0))) {
-                $good_to_go = 0;
-                push @chipsToUpdate, $chip;
-            } else {
-                # this chip is good to go
-            }
-        }
-
-        if ($good_to_go) {
-            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id -skycell_id $skycell->{skycell_id}";
-            $command .= " -set_label $rlabel" if $rlabel;
-
-            if (!$no_update) {
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                            run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
-                }
-            } else {
-                print "skipping $command\n";
-            }
-        } elsif (scalar @chipsToUpdate > 0) {
-            return queue_update_chip($chip_id, 0, \@chipsToUpdate, $rlabel, $need_magic);
-        }
-    } else {
-        my $warpRun = $metadata;
-        my $command = "$chiptool -listrun -chip_id $warpRun->{chip_id}";
-        my $data = runToolAndParse($command, $verbose);
-        my_die("failed to find chipRun $warpRun->{chip_id} for warpRun $warp_id", $PS_EXIT_UNKNOWN_ERROR)
-            if !$data or scalar @$data != 1;
-
-        my $chipRun = $data->[0];
-
-        my $chipRunState = $chipRun->{state};
-        if (($chipRunState =~ /purge/) or ($chipRunState =~ /scrub/)) {
-            print STDERR "warpRun $warp_id depends on chipRun $chipRun->{chip_id} which is in state $chipRunState\n";
-            faultJobs('stop', 'warp', $warp_id, $PSTAMP_GONE);
-            return 0;
-        }
-        my $warpRunState = $warpRun->{state};
-        if (($chipRunState eq 'full') and (! $need_magic or ($chipRun->{magicked} > 0)) and ($warpRunState eq 'cleaned')) {
-            # The inputs and outputs are ready. Queue the warpRun for update.
-
-            # providing no -skycell_id arguments changes all skyfiles with data_state = 'cleaned' to 'update'
-            my $command = "$warptool -setskyfiletoupdate -warp_id $warp_id";
-            $command .= " -set_label $rlabel" if $rlabel;
-
-            if (!$no_update) {
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                            run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    my_die("failed to queue ${stage}Run $stage_id $component for update", $PS_EXIT_UNKNOWN_ERROR);
-                }
-            } else {
-                print "skipping $command\n";
-            }
-        } elsif ($chipRunState eq 'cleaned' or 
-            (($chipRun->{state} eq 'full') and ($need_magic and ($chipRun->{magicked} < 0) and ($chipRun->{dsRun_state} ne 'new')))) {
-            my $data = [$chipRun];
-            return queue_update_chip($chipRun->{chip_id}, 1, $data, $rlabel, $need_magic);
-        }
-    }
-
-    # return value may be the return status of script so zero is good
-    return 0;
-}
-sub queue_update_diff {
-    return 0;
-}
-
-
-# run a command that produces metadata output and parse the results into an array of objects
-sub runToolAndParse {
-    my $command = shift;
-    my $verbose = shift;
-
-    my ($program) = split " ", $command;
-    $program = basename($program);
-
-    print "Running $command\n" if !$verbose;
-    my $start_tool = DateTime->now->mjd;
-    # run the command and parse the output
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-    unless ($success) {
-        print STDERR @$stderr_buf if !$verbose;
-        return undef;
-    }
-
-    my $now = DateTime->now->mjd;
-    my $dtime_tool = (DateTime->now->mjd - $start_tool) * 86400.;
-    print "Time to run $program: $dtime_tool\n";
-
-    my $buf = join "", @$stdout_buf;
-    if (!$buf) {
-        return undef;
-    }
-
-    my $start_parse = DateTime->now->mjd;
-
-    my $mdcParser = PS::IPP::Metadata::Config->new;
-    my $results = parse_md_fast($mdcParser, $buf)
-        or my_die ("Unable to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
-
-    my $dtime_parse = (DateTime->now->mjd - $start_parse) * 86400.;
-    print "Time to parse results from $program: $dtime_parse\n";
-
-    return $results;
-}
-
-# splits meta data config input stream into single units to work around the pathalogically
-# slow parser. This is similar to and adapted from code in various ippScripts.
-sub parse_md_fast {
-    my $mdcParser = shift;
-    my $input = shift;
-    my $output = ();
-
-    my @whole = split /\n/, $input;
-    my @single = ();
-
-    my $n;
-    while ( ($n = @whole) > 0) {
-        my $value = shift @whole;
-        push @single, $value;
-        if ($value =~ /^\s*END\s*$/) {
-            push @single, "\n";
-
-            my $list = parse_md_list( $mdcParser->parse( join("\n", @single ) ) ) or
-                print STDERR "Unable to parse metdata config doc" and return undef;
-            push @$output, $list->[0];
-
-            @single = ();
-        }
-    }
-    return $output;
-}
-sub faultJobs {
-    my ($state, $stage, $stage_id, $job_fault) = @_;
-
-    my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id";
+sub my_die
+{
+    my $msg = shift;
+    my $fault = shift;
+    carp $msg;
+
+    my $command = "$pstamptool -updatedepedent -fault $fault -dep_id $dep_id";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -421 +600 @@
                     run(command => $command, verbose => $verbose);
         unless ($success) {
-            my_die("failed to set pstamJob.fault for dep_id: $dep_id",
+            my_die("failed to set pstampDependent.fault for dep_id: $dep_id",
                 $PS_EXIT_UNKNOWN_ERROR);
         }
@@ -427 +606 @@
         print "skipping $command\n";
     }
-}
-
-sub my_die
-{
-    my $msg = shift;
-    my $fault = shift;
-
-    carp $msg;
-
-    return $fault;
-}
+
+    exit $fault;
+}

trunk/pstamp/scripts/pstamp_finish.pl

@@ -124 +124 @@
     my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps);
     print $rlf "results.fits|||table|\n";
-    print $rlf "results.mdc|||txt|\n";
+    print $rlf "results.mdc|||text|\n";
 
     my $err_file = "parse_error.txt";
@@ -171 +171 @@
         my $exp_id = $job->{exp_id};
 
+
         if (($fault eq $PSTAMP_DUP_REQUEST) and ($req_name eq "NULL")) {
             # this request had a duplicate request name. We can't put the results
@@ -188 +189 @@
         }
 
-        if ($exp_id ne $last_exp_id) {
+        my $job_params = get_job_parameters($job);
+        my $stage = "";
+        if ($job_params) {
+            $stage = $job_params->{stage};
+        }
+
+        if ($stage ne 'stack') {
             # get the metadata for the exposure (if any i.e. stack) 
             # returns an appropriate string if !$exp_id
             $exp_info = get_exposure_info($image_db, $exp_id);
             $last_exp_id = $exp_id;
+        } else {
+            my $filter = $job_params->{filter};
+            $filter = "0" if !$filter;
+            $exp_info = "0|0|0|$filter|0|0";
+            $last_exp_id = -1;
         }
 
@@ -344 +356 @@
 }
 
+sub get_job_parameters {
+    my $job = shift;
+    if (!$job->{outputBase}) {
+        print "get_job_parameters: $job->{job_id} has no outputBase\n";
+        return undef;
+    }
+    my $params_file = $job->{outputBase} . '.mdc';
+    if (! -e $params_file ) {
+        print "get_job_parameters: $job->{job_id} has no parameters file\n";
+        return undef;
+    }
+    open IN, "<$params_file" or die "unable to open $params_file";
+    my $data = join "", (<IN>);
+    close IN;
+    if (! $data ) {
+        print "get_job_parameters: parameters file is empty\n";
+        return undef;
+    }
+    my $metadata = $mdcParser->parse($data) or die("Unable to parse metdata config doc");
+
+    my $results = parse_md_list($metadata);
+    if (scalar @$results != 1) {
+        print STDERR "get_job_params: failed to parse_md_list\n";
+        return undef;
+    }
+    return $results->[0];
+}
+
 sub get_exposure_info {
     my $image_db= shift;

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -250 +250 @@
             copy_and_register_file($f, $backmdl_file, $outdir, $prefix);
         }
-        if (0) {
+       if (0) {
         # don't enable this yet
         if ($pattern_file) {
@@ -256 +256 @@
             copy_and_register_file($f, $pattern_file, $outdir, $prefix);
         }
-        }
+       }
     }
 }

trunk/pstamp/scripts/pstamp_parser_run.pl

@@ -28 +28 @@
 my $redirect_output;
 my $product;
+my $label;
 my $verbose;
 my $dbname;
@@ -36 +37 @@
     'uri=s'             =>  \$uri,
     'product=s'         =>  \$product,
+    'label=s'           =>  \$label,
     'redirect-output'   =>  \$redirect_output,
     'verbose'           =>  \$verbose,
@@ -146 +148 @@
         if ($request_type eq "PS1_PS_REQUEST") {
             $reqType = 'pstamp';
-            $parse_cmd = $pstampparse;
+            $parse_cmd = "$pstampparse";
+            $parse_cmd .= " --label $label" if $label;
         } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {
             $reqType = 'dquery';

trunk/pstamp/scripts/pstamp_request_file

@@ -187 +187 @@
         for (my $i = 0; $i < scalar @colNames; $i++) {
             my $writeType = $colWriteType[$i];
-            $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $colData->[$i], $status );
+            my $data = $colData->[$i];
+            if ($writeType == TULONG) {
+                die "invalid integer data found in column $i\n" unless validIntegers($data);
+            } elsif ($writeType == TDOUBLE) {
+                die "invalid numeric data found in column $i\n" unless validNumbers($data);
+            }
+            $outFits->write_col( $writeType, $i + 1, 1, 1, $numRows, $data, $status );
             check_fitsio( $status );
         }
@@ -312 +318 @@
 }
 
+sub checkValid {
+    my $aref = shift;
+    my $float = shift;
+
+    return 0 if !defined $aref;
+
+    my $valid = 0;
+    my $row = 0;
+    foreach my $val (@$aref) {
+        $row++;
+        if ($float) {
+            if (!($val =~  /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/)) {
+                $valid = 0;
+                print STDERR "Error on row $row: '$val' is not a valid number\n";
+                last;
+            }
+        } else {
+            if ($val =~ /\D/) {
+                $valid = 0;
+                print STDERR "Error on row $row: '$val' is not a valid integer\n";
+                last;
+            }
+        }
+        $valid = 1;
+    }
+
+    return $valid;
+}
+sub validIntegers {
+    return checkValid(@_, 0);
+}
+
+sub validNumbers
+{
+    return checkValid(@_, 1);
+}
+
+
 sub printhelp
 {

trunk/pstamp/scripts/pstampparse.pl

@@ -17 +17 @@
 use Carp;
 use POSIX;
+use DateTime;
 
 my $verbose;
@@ -26 +27 @@
 my $out_dir;
 my $product;
+my $label;
 my $save_temps;
 my $no_update;
@@ -34 +36 @@
     'out_dir=s' =>  \$out_dir,
     'product=s' =>  \$product,
+    'label=s'   =>  \$label,
     'mode=s'    =>  \$mode,
     'dbname=s'  =>  \$dbname,
@@ -42 +45 @@
 );
 
-die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job");
+die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
 die "--file is required"     if !defined($request_file_name);
 
@@ -64 +67 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
-my $dvoImagesAtCoords  = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
 my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
@@ -159 +161 @@
 }
 
+#
+# Loop over rows in the request file collecting consecutive rows that have the "same images of interest"
+# in the sense that their selection parameters will yield the same "Runs".
+# Process the groups of rows together to reduce lookup time and to potentially make multiple
+# stamps from the same ppstamp process.
+#
 my @rowList;
 my $num_jobs = 0;
@@ -165 +173 @@
 my $need_magic;
 foreach my $row (@$rows) {
-    # XXX: TODO: sanity check all parameters
-
-    # If we encounter an error for a particular row add a job with the proper fault code.
-    # If we encounter an error in this loop we shouldn't really just die.
-    # We only do that now in the case of I/O or DB errors or the like.
-
-    my $rownum   = $row->{ROWNUM};
-    my $job_type = $row->{JOB_TYPE};
-    
-    # parameters that select the images of interest
-    my $project  = $row->{PROJECT};
-
-    # note: resolve_project avoids running pstamptool every time by remembering the
-    # last project resolved
-    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
-    if (!$proj_hash) {
-        print STDERR "project $project not found\n"  if $verbose;
-        insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
+    # santiy check the paramaters
+    if (!checkRow($row)) {
+        # when it enconters an error checkRow adds a fake job with an appropriate error code to the database
         $num_jobs++;
         next;
     }
+    # initialize counter for "job number"
+    $row->{job_num} = 0;
+    $row->{error_code} = 0;
+
+    if (scalar @rowList == 0) {
+        push @rowList, $row;
+        next;
+    }
+
+    my $firstRow = $rowList[0];
+    if (same_images_of_interest($firstRow, $row)) {
+        # add this row to the list and move on
+        push @rowList, $row;
+        next;
+    }
+
+    # the images of interest for this new row doesn't match the list. 
+    # process the list ...
+    $num_jobs += processRows(\@rowList);
+
+    # and reset the list to contain just the new row
+    @rowList = ($row);
+}
+
+# out of rows process the list
+if (scalar @rowList > 0) {
+    $num_jobs += processRows(\@rowList);
+}
+
+if (($mode eq "queue_job") and ($num_jobs eq 0)) {
+    print STDERR "no jobs created for $req_name\n" if $verbose;
+    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
+}
+
+exit 0;
+
+sub checkRow {
+        
+    my $row = shift;
+
+    # If we encounter an error for a particular row add a job with the proper fault code.
+
+    my $rownum   = $row->{ROWNUM};
+    if (!validID($rownum)) {
+        print STDERR "$rownum is not a valid ROWNUM\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+    my $job_type = $row->{JOB_TYPE};
+    if (($job_type ne "stamp") and ($job_type ne "get_image")) {
+        print STDERR "$job_type is not a valid JOB_TYPE\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+    
     my $req_type = $row->{REQ_TYPE};
-    $stage       = $row->{IMG_TYPE};
-    my $id      = $row->{ID};
+    if (($req_type ne "byid") and ($req_type ne "bycoord") and ($req_type ne "byexp") and
+        ($req_type ne "byskycell") and ($req_type ne "bydiff")) {
+        print STDERR "$req_type is not a valid REQ_TYPE\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+
+
     my $component = $row->{COMPONENT};
-    my $tess_id = $row->{TESS_ID};
+    if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
+        $row->{COMPONENT} = $component = "";
+    }
+    $row->{TESS_ID} = "" if !defined $row->{TESS_ID};
 
     my $filter  = $row->{REQFILT};
@@ -196 +253 @@
         if (length($filter) == 1) {
             # allow single character filter cuts to work
-            $filter .= '%';
+            $row->{REQFILT} .= '%';
         }
     }
     my $mjd_min = $row->{MJD_MIN};
+    if (defined($mjd_min) and !validNumber($mjd_min)) {
+        print STDERR "$mjd_min is not a valid MJD_MIN\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
     my $mjd_max = $row->{MJD_MAX};
+    if (defined($mjd_max) and !validNumber($mjd_max)) {
+        print STDERR "$mjd_max is not a valid MJD_MAX\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
     my $data_group = $row->{DATA_GROUP};
     if (!defined $data_group) {
@@ -209 +276 @@
     }
         
-    my $x       = $row->{CENTER_X};
-    my $y       = $row->{CENTER_Y};
-
-    # XXX things don't work if bit zero of option mask is not set;
+    # req_finish doesn't work if bit zero of option mask is not set;
     $row->{OPTION_MASK} |= 1;
+
     my $option_mask= $row->{OPTION_MASK};
     my $inverse = ($option_mask & $PSTAMP_SELECT_INVERSE) ? 1 : 0;
@@ -222 +287 @@
     my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
 
-    my $search_component = (!defined($component) or ($component eq "null")) ? "" : $component;
-    
-    if (!$skycenter and !$search_component) {
+    if (!$skycenter and !$component) {
         print STDERR "COMPONENT must be specified for pixel coordinate ROI center\n" if $verbose;
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
-        $num_jobs++;
-        next;
-    }
-
-    # user requested us to search all components. Set to ""
-    $search_component = "" if $search_component eq "all";
-
+        return 0;
+    }
+
+    my $stage = $row->{IMG_TYPE};
     if (!check_image_type($stage)) {
         print STDERR "invalid IMG_TYPE for row $rownum\n" if $verbose;
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
-        $num_jobs++;
-        next;
+        return 0;
     }
 
@@ -244 +303 @@
         print STDERR "invalid ROI for row $rownum\n" if $verbose;
         insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
-        $num_jobs++;
-        next;
+        return 0;
     }
 
@@ -251 +309 @@
         print STDERR "byexp not implemented for stack stage. row: $rownum\n" if $verbose;
         insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
-        $num_jobs++;
-        next;
-    }
-
-    
+        return 0;
+    }
+
     # $mode list_uri is a debugging mode (it may used by the http interface)
     # if this happens just croak
-    my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
-
-    my $image_db   = $proj_hash->{dbname};
-    my $camera     = $proj_hash->{camera};
-    $need_magic = $proj_hash->{need_magic};
-
-    # Temporary hack so that MOPS can get at non-magicked data
-#    if ($product and ($product eq "mops-pstamp-results")) {
-#        $need_magic = 0;
-#    }
-
-    # For "stamp" and "list_uri" jobs collect rows with the same images of interest in a list so that they
-    # can be looked up together.
-    if (@rowList) {
-        my $firstRow = $rowList[0];
-        if (($firstRow->{JOB_TYPE} ne "get_image") and same_images_of_interest($firstRow, $row)) {
-
-            # add this row to the list and move on
-            push @rowList, $row;
-
-            next;
-
-        } else {
-            # this row has different selectors
-            # queue the jobs for the ones we've collected
-            $num_jobs += queueJobs($mode, \@rowList, $imageList);
-            @rowList = ();
-        }
-    }
-
-    # look up images for the current row
+   # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
+
+
     if ($req_type eq "bycoord") {
         if (!$skycenter) {
             print STDERR "center must be specified in sky coordintes for bycoord" if $verbose;
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
-            $num_jobs++;
-            next;
-        }
-    }
+            return 0;
+        }
+    }
+
     if (($req_type eq "byid") or ($req_type eq "bydiff")) {
-        if (!validID($id)) {
+        if (!validID($row->{ID})) {
             print STDERR "ID must be a positive integer for req_type $req_type\n" if $verbose;
             insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+            return 0
+        }
+    }
+
+    return 1;
+}
+
+sub processRows {
+    my $rowList = shift;
+    my $num_jobs = 0;
+
+    # all rows in the list are compatible
+    my $row = $rowList->[0];
+
+    my $project  = $row->{PROJECT};
+
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+    if (!$proj_hash) {
+        print STDERR "project $project not found\n"  if $verbose;
+        foreach $row (@$rowList) {
+            insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
             $num_jobs++;
-            next;
-        }
-    }
+        }
+        return $num_jobs;
+    }
+    my $req_type  = $row->{REQ_TYPE};
+    $stage        = $row->{IMG_TYPE};
+    my $id        = $row->{ID};
+    my $component = $row->{COMPONENT};
+    my $tess_id   = $row->{TESS_ID};
+
+    my $filter    = $row->{REQFILT};
+    my $mjd_min   = $row->{MJD_MIN};
+    my $mjd_max   = $row->{MJD_MAX};
+    my $data_group = $row->{DATA_GROUP};
+
+    my $rownum     = $row->{ROWNUM};
+    my $job_type   = $row->{JOB_TYPE};
+    my $option_mask= $row->{OPTION_MASK};
+    
+    my $image_db   = $proj_hash->{dbname};
+    my $camera     = $proj_hash->{camera};
+    $need_magic    = $proj_hash->{need_magic};
+
+    # Temporary hack so that MOPS can get at non-magicked data
+    my $allow_mops_unmagicked = 1;
+    if ($allow_mops_unmagicked) {
+        if ($product and (($product eq "mops-pstamp-results") or
+                          ($product eq "mops-pstamp-results2"))) {
+            $need_magic = 0;
+        }
+    }
+    
+    my $numRows = scalar @$rowList;
+
+#    $tess_id = "" if !defined $tess_id;
+#    $component = "" if !defined $component;
+
+    print "Collected $numRows rows beginning with row $rownum. $req_type $stage $id $tess_id $component\n";
+    
     # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
     # request specification. An array reference is returned.
-    $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $tess_id, $search_component,
-                $option_mask, $need_magic, $x, $y, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
+    my $start_locate = DateTime->now->mjd;
+
+    # XXX: perhaps we should get rid of most of this argument list.
+    # Now that we are passing down compatible rows all of the
+    # information required is contained there
+
+    $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component,
+                $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
+
+    # XXX: why use mjd? It doesn't have great precision.
+    my $dtime_locate = (DateTime->now->mjd - $start_locate) * 86400.;
+    print "Time to locate_images for row $rownum $dtime_locate\n";
 
     if (!$imageList or !@$imageList) {
@@ -314 +410 @@
         # note in this case queueJobs inserts the fake job for these rows
     }
+    # handle this
     $row->{need_magic} = $need_magic;
-    push @rowList, $row;
-}
-
-if (@rowList) {
+
     $num_jobs += queueJobs($mode, \@rowList, $imageList);
-}
-
-if (($mode eq "queue_job") and ($num_jobs eq 0)) {
-    print STDERR "no jobs created for $req_name\n" if $verbose;
-    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
-}
-
-# PAU
-
-exit 0;
-
-
-sub queueJobsForRow
+
+    # if a row slipped through with no jobs add one
+    foreach my $row (@rowList) {
+        if ($row->{job_num} == 0) {
+            print "row $row->{ROWNUM} produced no jobs\n";
+            print STDERR "row $row->{ROWNUM} produced no jobs\n";
+            my $error_code = $row->{error_code};
+            $error_code =  $PSTAMP_NO_IMAGE_MATCH if !$error_code;
+            insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
+        }
+    }
+
+    return $num_jobs;
+}
+
+sub queueJobForImage
 {
     my $row = shift;
     my $stage = shift;
-    my $imageList = shift;
-    my $have_skycells = shift;
+    my $image = shift;
     my $need_magic = shift;
     my $mode = shift;
 
-    my $num_jobs = 0;
     my $rownum = $row->{ROWNUM};
     my $option_mask = $row->{OPTION_MASK};
@@ -365 +460 @@
     }
 
+    my $component = $image->{component};
+
+    my $job_num = ++($row->{job_num});
+
+    my $imagefile = $image->{image};
+    if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
+        # XXX: should we add a faulted job so the client can know what happened if no images come back?
+        # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback
+        # to users, but speeds up processing significantly
+        print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
+
+        # for now assume yes.
+
+        insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
+        return 1;
+    } elsif ($stage eq "stack") {
+        # unconvolved stack images weren't available prior to some point in time.
+        # XXX: handle this more correctly by examining the stack run's config dump file.
+        # It looks like # the feature was turned on sometime around November 11, 2009. stackRun 30067 is the lowest
+        # one that I found with an unconvolved image.
+        my $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES = 30067;
+        if ($row->{unconvolved} and ($row->{PROJECT} eq 'gpc1') and 
+            ($image->{stack_id} < $MIN_GPC1_STACK_ID_WITH_UNCONVOLVED_IMAGES)) {
+            print STDERR "Unconvolved stack image is not available for stackRun.stack_id: $image->{stack_id}\n";
+            insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_AVAILABLE);
+            return 1;
+        }
+    }
+    my $exp_id = $image->{exp_id};
+            
+    my $args = $roi_string ? $roi_string : "";
+    if ($stage eq "raw" or $stage eq "chip") {
+        $args .= " -class_id $component" if $component;
+    }
+
+    # add astrometry file for raw and chip images if one is available
+    if (($stage eq "chip") || ($stage eq "raw")) {
+        $args .= " -astrom $image->{astrom}" if $image->{astrom};
+    }
+
+    $image->{job_args} = $args;
+
+    # XXX: we can get rid of the following everything that we need is
+    # in the params file
+
+    $args .= " -file $imagefile";
+
+    if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
+        $args .= " -mask $image->{mask}";
+    }
+    if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
+        $args .= " -variance $image->{weight}";
+    }
+
+    my $base = basename($image->{image});
+    if (! $base =~ /.fits$/ ) {
+        my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR);
+    }
+    $base =~ s/.fits$//;
+            
+    my $output_base = "$out_dir/${rownum}_${job_num}_${base}";
+    my $argslist = "${output_base}.args";
+
+    # copy the argument list to a file
+    open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR);
+    print ARGSLIST "$args\n";
+    close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR);
+
+    write_params($output_base, $image);
+
+    my $newState = "run";
+    my $fault = 0;
+    my $dep_id;
+
+    # XXX: this code is repeated in queueGetImageJobs we should encapsulate it in a subroutine and share it
+    if ($stage ne 'raw') {
+        # not ready to handle diff update yet. May never support stack
+        my $allow_wait_for_update = (($stage ne 'stack') and ($stage ne 'diff'));
+        my $run_state = $image->{state};
+        my $data_state = $image->{data_state};
+        $data_state = $run_state if $stage eq 'stack';
+        if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
+            ($run_state eq 'drop') or 
+            ($run_state eq 'error_cleaned') or ($data_state eq 'error_cleaned') or
+            ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
+            # image is gone and it's not coming back
+            $newState = 'stop';
+            $fault = $PSTAMP_GONE;
+        } elsif ($allow_wait_for_update and 
+                (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0)))) {
+            if ($stage eq 'chip') {
+                my $burntool_state = $image->{burntool_state};
+                if ($burntool_state and (abs($burntool_state) < 14)) {
+                    $newState = 'stop';
+                    $fault = $PSTAMP_NOT_AVAILABLE;
+                }
+            }
+            if (!$fault) {
+                # don't wait for update unless the caller asks us to
+                if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) {
+                    $newState = 'stop';
+                    $fault = $PSTAMP_NOT_AVAILABLE;
+                } elsif (!$image->{magicked}) {
+                    $newState = 'stop';
+                    $fault = $PSTAMP_NOT_DESTREAKED;
+                } else {
+                    # cause the image to be re-made
+                    # set up to queue an update run
+                    queue_update_run(\$newState, \$fault, \$dep_id, $image->{imagedb}, 
+                        $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic);
+                }
+            }
+        }
+    }
+
+    my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
+                    . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask";
+    $command .= " -fault $fault" if $fault;
+    $command .= " -exp_id $exp_id" if $exp_id;
+    $command .= " -dep_id $dep_id" if $dep_id;
+
+    if (!$no_update) {
+        # mode eq "queue_job"
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            print STDERR @$stderr_buf;
+            # XXX TODO: now what? Should we mark the error state for the request?
+            # should we keep going for other uris? If so how do we report that some
+            # of the work that the request wanted isn't going to get done
+            my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping command: $command\n";
+    }
+
+    return 1;
+}
+
+# queue jobs for a collection of request specifications that have the same Images of Interest
+sub queueJobs
+{
+    my $mode = shift;
+    my $rowList = shift;
+    my $imageList = shift;
+
+    my $firstRow = $rowList[0];
+    my $stage    = $firstRow->{IMG_TYPE};
+    my $job_type = $firstRow->{JOB_TYPE};
+    my $need_magic = $firstRow->{need_magic};
+
+    my $num_jobs = 0;
+
+    if ($mode eq "list_uri") {
+        foreach my $image (@$imageList) {
+            print "$image->{image}\n";
+        }
+    } elsif ($job_type eq "get_image") {
+        my $n = scalar @$rowList;
+
+        my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1;
+
+        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+
+    } else {
+        if (!$imageList or (scalar @$imageList eq 0)) {
+            # We didn't find any images for this set of rows. Insert a fake job to carry
+            # the status back to the requestor.
+            foreach my $row (@$rowList) {
+                my $error_code = $row->{error_code};
+                $error_code = $PSTAMP_NO_IMAGE_MATCH if !$error_code;
+                insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
+                $num_jobs++;
+            }
+            return $num_jobs;
+        }
+
+        foreach my $image (@$imageList) {
+            # get the array of row indices that touch this image
+            my $row_index = $image->{row_index};
+            if (!$row_index or scalar @$row_index == 0) {
+                # XXX should this happen? Why did something get returned.
+                print "image ${stage}_id: $image->{stage_id} component: $image->{component} matched no rows\n";
+                next;
+            }
+            # XXX: TODO: eventually we may change ppstamp to be able to make multiple stamps per invocation
+
+            foreach my $i (@$row_index) {
+                my $row = $rowList->[$i];
+
+                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode);
+            }
+        }
+    }
+
+    return $num_jobs;
+}
+
+#        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+sub queueGetImageJobs
+{
+    my $row = shift;
+    my $imageList = shift;
+    my $stage = shift;
+    my $need_magic = shift;
+    my $mode = shift;
+
+    my $num_jobs = 0;
+    my $rownum = $row->{ROWNUM};
+    my $option_mask = $row->{OPTION_MASK};
+
+    # For dist_bundle we need
+    #  --camera from $image
+    #  --stage 
+    #  --stage_id from $image
+    #  --component from $image
+    #  --path_base 
+    #  --outdir global to this script
+
     # loop over images
-    my $job_num = 0;
     foreach my $image (@$imageList) {
-        my $component;
-        if ($have_skycells) {
-            $component = $image->{skycell_id};
-        } else {
-            $component = $image->{class_id};
-        }
-
-        # skip this component if it is not in the list for this row
-        next if ! $components->{$component};
-
-        $job_num++;
+        my $stage_id = $image->{stage_id};
+        my $component = $image->{component};
+
+        # skip faulted components for now. Should we even be here?
+        if ($image->{fault} > 0) {
+            printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose;
+            next;
+        }
+
+        my $job_num = ++($row->{job_num});
 
         my $imagefile = $image->{image};
         if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
-            # XXX: should we add a faulted job so the client can know what happened if no images come back?
-            # The test for destreaked is made in locate_images now so this code never runs. This leads to no feedback
-            # to users, but speeds up processing significantly
+            # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed
+            # in locate_images because it's much more efficient to do the test in the database.
+            # For these two modes we fall through to here in order to give feedback to the requestor as
+            # to why the request failed to queue jobs.
             print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
-
-            # for now assume yes.
-
             insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
             $num_jobs++;
@@ -396 +706 @@
         my $exp_id = $image->{exp_id};
             
-        my $args = $roi_string ? $roi_string : "";
-        if ($stage eq "raw" or $stage eq "chip") {
-            $args .= " -class_id $component" if $component;
-        }
-
-        # add astrometry file for raw and chip images if one is available
-        if (($stage eq "chip") || ($stage eq "raw")) {
-            $args .= " -astrom $image->{astrom}" if $image->{astrom};
-        }
-
-        $image->{job_args} = $args;
-
-        # XXX: we can get rid of the following everything that we need is
-        # in the params file
-
-        $args .= " -file $imagefile";
-
-        if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
-            $args .= " -mask $image->{mask}";
-        }
-        if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
-            $args .= " -variance $image->{weight}";
-        }
-
-        my $base = basename($image->{image});
-        if (! $base =~ /.fits$/ ) {
-            my_die("unexpected image file name found $image->{image}", $PS_EXIT_PROG_ERROR);
-        }
-        $base =~ s/.fits$//;
-            
-        # XXX: TODO use filerule for this. I don't have a camera defined here
-        if (($stage eq 'chip') and ($image->{camera} eq 'GPC1')) {
-            $base = "${base}.${component}";
-        }
-
-        my $output_base = "$out_dir/${rownum}_${job_num}_${base}";
-        my $argslist = "${output_base}.args";
-
-        # copy the argument list to a file
-        open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR);
-        print ARGSLIST "$args\n";
-        close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR);
+        my $output_base = "$out_dir/${rownum}_${job_num}";
 
         write_params($output_base, $image);
@@ -454 +723 @@
                 $newState = 'stop';
                 $fault = $PSTAMP_GONE;
-            } elsif (($data_state ne 'full') or ($run_state ne 'full' )) {
+            } elsif (($data_state ne 'full') or ($need_magic and ($image->{magicked} < 0))) {
                 # don't wait for update unless the caller asks us to
                 if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) {
@@ -463 +732 @@
                     # set up to queue an update run
                     queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, 
-                        $run_state, $stage, $image->{stage_id}, $need_magic);
+                        $run_state, $stage, $image->{stage_id}, $image->{component}, $need_magic);
                 }
             }
@@ -475 +744 @@
         $command .= " -dep_id $dep_id" if $dep_id;
 
-        if ($mode eq "list_job") { 
-            # this is a debugging mode, just print the pstamptool that would have run
-            # this is sort of like the mode -noupdate that some other tools support
-            print "$command\n";
-        } elsif (!$no_update) {
-            # mode eq "queue_job"
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                print STDERR @$stderr_buf;
-                # XXX TODO: now what? Should we mark the error state for the request?
-                # should we keep going for other uris? If so how do we report that some
-                # of the work that the request wanted isn't going to get done
-                my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
-            }
-        } else {
-            print "skipping command: $command\n";
-        }
-    }
-    if ( $num_jobs == 0 ) {
-        print STDERR "no jobs for row $rownum\n" if $verbose;
-        insertFakeJobForRow($row, 1, $PSTAMP_NO_OVERLAP);
-        $num_jobs = 1;
-    }
-    return $num_jobs;
-}
-
-# queue jobs for a collection of request specifications that have the same Images of Interest
-sub queueJobs
-{
-    my $mode = shift;
-    my $rowList = shift;
-    my $imageList = shift;
-
-    my $firstRow = $rowList[0];
-    my $stage    = $firstRow->{IMG_TYPE};
-    my $job_type = $firstRow->{JOB_TYPE};
-    my $need_magic = $firstRow->{need_magic};
-
-    my $num_jobs = 0;
-
-    if ($mode eq "list_uri") {
-        foreach my $image (@$imageList) {
-            print "$image->{image}\n";
-        }
-    } elsif ($job_type eq "get_image") {
-        my $n = scalar @$rowList;
-
-        my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1;
-
-        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
-
-    } else {
-        if (!$imageList or (scalar @$imageList eq 0)) {
-            # we didn't find any images for this set of rows. Insert a fake job to carry
-            # the status back to the requestor
-            foreach my $row (@$rowList) {
-                insertFakeJobForRow($row, 1, $PSTAMP_NO_IMAGE_MATCH);
-                $num_jobs++;
-            }
-            return $num_jobs;
-        }
-
-        my $have_skycells;
-        if (($stage eq "raw") or ($stage eq "chip")) {
-            $have_skycells = 0;
-        } else {
-            $have_skycells = 1;
-        }
-        
-        my $thisRun;
-
-        my $npoints = 0;
-        my ($pointsList, $pointsListName);
-        if (scalar @$imageList > 1) {
-            ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
-            foreach my $row (@$rowList) {
-                $row->{components} = {};
-                if ($row->{skycenter}) {
-                    print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
-                    $npoints++;
-                } else {
-                    # this row's center is in pixel coordinates add all images to the component list for this row
-                    foreach my $i (@$imageList) {
-                        my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id};
-                        my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR)
-                                if !$component;
-                        $row->{components}->{$component} = 1;
-                    }
-                }
-            }
-            close $pointsList;
-        } else {
-            # only one image. Avoid the expense of dvoImagesAtCoords. 
-            # queue the job and let ppstamp figure out if the center is valid for the image
-            # the resulting result code will be the same.
-            foreach my $row (@$rowList) {
-                $row->{components} = {};
-                my $i = $imageList->[0];
-                my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id};
-                my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) if !$component;
-                $row->{components}->{$component} = 1;
-            }
-        }
-
-        my $tess_dir_abs;
-        my $last_tess_id = "";
-        while ($thisRun = getOneRun($stage, $imageList)) {
-            if ($npoints) {
-                my_die( "pointsListName is not defined", $PS_EXIT_PROG_ERROR) if !$pointsListName;
-                # we collected a set of sky coordinates above.
-                # filter the images so that only those that contain the centers of the ROIs are processed
-                my $command = "$dvoImagesAtCoords -coords $pointsListName";
-                if ($have_skycells) {
-                    my $tess_id = $thisRun->[0]->{tess_id};
-                    if ($tess_id ne $last_tess_id) {
-                        $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id );
-                        $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs );
-                        $last_tess_id = $tess_id;
-                    }
-                    $command .= " -D CATDIR $tess_dir_abs";
-                } else {
-                    my $astrom = $thisRun->[0]->{astrom};
-                    my_die( "no astrometry file found", $PS_EXIT_UNKNOWN_ERROR) if !$astrom;
-                    my $astrom_resolved = $ipprc->file_resolve($astrom);
-                    $command .= " -astrom $astrom_resolved";
-                }
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP
-                    # That just means that the coordinate didn't match any image/skycell
-                    if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) {
-                        print STDERR @$stderr_buf;
-                        my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR;
-                        my_die( "dvoImagesAtCoords failed: $rc", $rc);
-                    }
-                }
-                # now we have a list of row numbers and components
-                # eventually we might want to multiple stamp requests for the same image
-                # into the same ppstamp job but not yet. For now we will queue a new
-                my @lines = split "\n", join "", @$stdout_buf;
-                foreach my $line (@lines) {
-                    # parse the line, ignoring the ra and dec
-                    my ($rownum, undef, undef, $component) = split " ", $line;
-
-                    # I guess since we need this function we should be using a hash for rowList 
-                    my $row = findRow($rownum, $rowList);
-                    $row->{components}->{$component} = 1;
-                }
-            }
-            
-            foreach my $row (@$rowList) {
-                $num_jobs += queueJobsForRow($row, $stage, $thisRun, $have_skycells, $need_magic, $mode);
-            }
-        }
-    }
-    return $num_jobs;
-}
-
-#        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
-sub queueGetImageJobs
-{
-    my $row = shift;
-    my $imageList = shift;
-    my $stage = shift;
-    my $need_magic = shift;
-    my $mode = shift;
-
-    my $num_jobs = 0;
-    my $rownum = $row->{ROWNUM};
-    my $option_mask = $row->{OPTION_MASK};
-
-    # For dist_bundle we need
-    #  --camera from $image
-    #  --stage 
-    #  --stage_id from $image
-    #  --component from $image
-    #  --path_base 
-    #  --outdir global to this script
-
-    # loop over images
-    my $job_num = 0;
-    foreach my $image (@$imageList) {
-        my $stage_id = $image->{stage_id};
-        my $component = $image->{component};
-
-        # skip faulted components for now. Should we even be here?
-        if ($image->{fault} > 0) {
-            printf STDERR "skipping faulted component for $stage $stage_id $component\n" if $verbose;
-            next;
-        }
-
-        $job_num++;
-
-        my $imagefile = $image->{image};
-        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
-            # we only get here if req_type is (byid or byexp). For other types the test for magicked is performed
-            # in locate_images because it's much more efficient to do the test in the database.
-            # For these two modes we fall through to here in order to give feedback to the requestor as
-            # to why the request failed to queue jobs.
-            print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
-            insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
-            $num_jobs++;
-
-            next;
-        }
-        my $exp_id = $image->{exp_id};
-            
-        my $output_base = "$out_dir/${rownum}_${job_num}";
-
-        write_params($output_base, $image);
-
-        my $newState = "run";
-        my $fault = 0;
-        my $dep_id;
-
-        if ($stage ne 'raw') {
-            my $run_state = $image->{state};
-            my $data_state = $image->{data_state};
-            $data_state = $run_state if $stage eq "stack";
-            if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
-                ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
-                # image is gone and it's not coming back
-                $newState = 'stop';
-                $fault = $PSTAMP_GONE;
-            } elsif (($data_state ne 'full') or ($run_state ne 'full' )) {
-                # don't wait for update unless the caller asks us to
-                if (!($option_mask & $PSTAMP_WAIT_FOR_UPDATE)) {
-                    $newState = 'stop';
-                    $fault = $PSTAMP_NOT_AVAILABLE;
-                } else {
-                    # cause the image to be re-made
-                    # set up to queue an update run
-                    queue_update_run(\$newState, \$fault, \$dep_id, $image->{image_db}, 
-                        $run_state, $stage, $image->{stage_id}, $need_magic);
-                }
-            }
-        }
-
-        $num_jobs++;
-        my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
-                        . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask";
-        $command .= " -fault $fault" if $fault;
-        $command .= " -exp_id $exp_id" if $exp_id;
-        $command .= " -dep_id $dep_id" if $dep_id;
-
-        if ($mode eq "list_job") { 
-            # this is a debugging mode, just print the pstamptool that would have run
-            # this is sort of like the mode -noupdate that some other tools support
-            print "$command\n";
-        } elsif (!$no_update) {
+        if (!$no_update) {
             # mode eq "queue_job"
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -776 +794 @@
                         . " -rownum $rownum -state stop -fault $fault";
 
-    if ($mode eq "list_job") { 
-        # this is a debugging mode, just print the pstamptool that would have run
-        # this is sort of like the mode -noupdate that some other tools support
-        print "$command\n";
-    } elsif (!$no_update) {
+    if (!$no_update) {
         # mode eq "queue_job"
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -796 +810 @@
 }
 
-sub get_run_id
-{
-    my $stage = shift;
-    my $image = shift;
-
-    if ($stage eq "raw") {
-        return $image->{exp_id};
-    } elsif ($stage eq "chip") {
-        return $image->{chip_id};
-    } elsif ($stage eq "warp") {
-        return $image->{warp_id};
-    } elsif ($stage eq "stack") {
-        return $image->{stack_id};
-    } elsif ($stage eq "diff") {
-        return $image->{diff_id};
-    } else {
-        my_die("unenexpected stage: $stage found", $PS_EXIT_PROG_ERROR);
-    }
-}
-
-# extract components from the imageList that have the same run id and return the list
-sub getOneRun {
-    my $stage = shift;
-    my $imageList = shift;
-
-    # return if array is empty
-    return undef if ! @$imageList;
-
-    my $last_run_id = 0;
-    my @runList;
-    while ($imageList->[0]) {
-        my $run_id = get_run_id($stage, $imageList->[0]);
-
-        last if ($last_run_id and ($run_id ne $last_run_id));
-
-        my $image = shift @$imageList;
-        $image->{stage} = $stage;
-        push @runList, $image;
-        $last_run_id = $run_id;
-    }
-    return \@runList;
-}
-
 sub same_images_of_interest {
     my $r1 = shift;
     my $r2 = shift;
 
-    return 0 if (($r1->{REQ_TYPE} eq "bycoord") or ($r2->{REQ_TYPE} eq "bycoord"));
-    return 0 if ($r1->{PROJECT}  ne $r2->{PROJECT});
-    return 0 if ($r1->{JOB_TYPE} ne $r2->{JOB_TYPE});
+    return 0 if (($r1->{REQ_TYPE} eq "bycoord")   or ($r2->{REQ_TYPE} eq "bycoord"));
+    return 0 if (($r1->{JOB_TYPE} eq "get_image") or ($r2->{JOB_TYPE} eq "get_image"));
     return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE});
     return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
     return 0 if ($r1->{ID}       ne $r2->{ID});
     return 0 if ($r1->{TESS_ID}  ne $r2->{TESS_ID});
+    return 0 if ($r1->{COMPONENT}  ne $r2->{COMPONENT});
     return 0 if ($r1->{REQFILT}  ne $r2->{REQFILT});
     return 0 if ($r1->{DATA_GROUP}    ne $r2->{DATA_GROUP});
     return 0 if ($r1->{MJD_MIN}  ne $r2->{MJD_MAX});
     return 0 if ($r1->{MJD_MAX}  ne $r2->{MJD_MAX});
+    return 0 if ($r1->{OPTION_MASK}  ne $r2->{OPTION_MASK});
+    return 0 if ($r1->{PROJECT}  ne $r2->{PROJECT});
     return 0 if ($r1->{inverse}  ne $r2->{inverse});
     return 0 if ($r1->{unconvolved}  ne $r2->{unconvolved});
-
-    if (defined($r1->{COMPONENT})) {
-        return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT});
-    } elsif (defined($r2->{COMPONENT})) {
-        # if first row has no component all of the images will be retrieved, so
-        # the fact that this row has a component is ok. Fall through to return 1
-        # XXX Nope this doesn't work. It is consistent with the other logic in some way
-        # that i haven't fully thought through
-        return 0;
-    }
+    # don't combine requests in pixel coordinates
+    return 0 if (($r1->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS) || ($r2->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS));
 
     return 1;
@@ -911 +876 @@
 sub queue_update_run 
 {
-    my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $need_magic) = @_;
+    my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
 
     if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) {
@@ -918 +883 @@
 
     my $dep_id;
-    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb";
-    $command .= " -no_magic" if !$need_magic;
+    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component";
+    $command .= " -need_magic" if $need_magic;
+
+    # compute rlabel for the run.
+    # XXX: This bit of policy shouldn't be buried so deeply in the code
+    # For now use one that implies 'postage stamp server' 'update' 'request_label"
+    my $rlabel = "ps_ud_" . $label if $label;
+    $command .= " -rlabel $rlabel" if $rlabel;
+
     if (!$no_update) {
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -929 +901 @@
         chomp $output;
         $dep_id = $output;
+        #
+        # XXX: need to fault the request or something
         my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id;
     } else {
@@ -936 +910 @@
     
     $$r_dep_id = $dep_id;
-    $r_fault = 0;
-    $r_jobState = 'blocked';
+    $$r_fault = 0;
+    $$r_jobState = 'run';
 }
 
@@ -990 +964 @@
     # we don't fault the request here pstamp_parser_run.pl handles that if necessary
 
-    return $fault;
-}
+    exit $fault;
+}