Changeset 25143

Timestamp:

Aug 20, 2009, 11:12:36 AM (17 years ago)

Author:

watersc1

Message:

 

Location:

branches/czw_branch/cleanup/pstamp

Files:

• scripts/pstamp_finish.pl (modified) (12 diffs)
• scripts/pstamp_job_run.pl (modified) (1 diff)
• scripts/pstamp_listjobs.pl (modified) (1 diff)
• scripts/pstampparse.pl (modified) (23 diffs)
• src/ppstampMakeStamp.c (modified) (4 diffs)

branches/czw_branch/cleanup/pstamp/scripts/pstamp_finish.pl

@@ -8 +8 @@
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
+use Carp;
 
 use Time::Local;
@@ -47 +48 @@
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
-
 if ($redirect_output) {
     my $logDest = "$out_dir/psfinish.$req_id.log";
@@ -94 +94 @@
         if (!mkdir $out_dir) {
             print STDERR "cannot create output directory $out_dir";
-            stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR);
+            stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
         }
 
@@ -102 +102 @@
         # request
         print STDERR "output directory $out_dir exists but is not a directory";
-        stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR);
+        stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
     }
 
     if (! -e $req_file ) {
         print STDERR "request file $req_file is missing\n";
-        stop_request($req_id, $PS_EXIT_CONFIG_ERROR);
+        stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR);
     }
 
@@ -117 +117 @@
         # point 
         print STDERR "failed to read request_file $req_file" ;
-        stop_request($req_id, $PS_EXIT_CONFIG_ERROR);
+        stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR);
     }
 
@@ -154 +154 @@
                 print STDERR "Request $req_id produced no jobs.\n"
             }
-            # No Jobs. 
-            # XXXX Ouch. We need results for each rownum (each request specification) in the request file 
-            # including those that produced no jobs.
-            # for now add an entry for rownum 1 and a phony error code.
-            # we've included parse_error.txt to the fileset if it exists
-            #
-            my $rownum = 0;
-            my $fault = 42; # get a real error code
-            print $tdf "$rownum|$fault|0|0|";
-            print $tdf "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|\n";
+            # This should not happen. A fake job should be queued
+            stop_request_and_exit($req_id, $PS_EXIT_PROG_ERROR);
         } else {
             my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
@@ -182 +174 @@
         my $exp_id = $job->{exp_id};
 
+        if ($fault eq $PSTAMP_DUP_REQUEST) {
+            # this request had a duplicate request name. We can't put the results
+            # on the data store since the product name is already used
+            # maybe we could be more clever about this and choose a fileset name
+            stop_request_and_exit($req_id, $fault);
+        }
+
         my ($row, $req_info, $project) = get_request_info($rows, $rownum);
 
@@ -188 +187 @@
         if (!$image_db) {
             carp("failed to find imagedb for project: $project");
-            stop_request($req_id, $PS_EXIT_CONFIG_ERROR);
+            if (!$fault) {
+                stop_request_and_exit($req_id, $PS_EXIT_CONFIG_ERROR);
+            }
         }
 
@@ -227 +228 @@
                 print $tdf "$exp_info|";
                 print $tdf "$req_info|";
+                print $tdf "\n";
             }
         } else {
@@ -277 +279 @@
 }
 
-sub stop_request {
+sub stop_request_and_exit {
     my $req_id = shift;
     my $fault  = shift;
@@ -299 +301 @@
     my $row = $rows->{$rownum};
 
+    # these may be set to null during processing
+    my $component = $row->{COMPONENT};
+    $component = "null" if !$component;
+    my $tess_id = $row->{TESS_ID};
+    $tess_id = "null" if !$tess_id;
+
     # This is ugly, error prone and hard to change.
     # Create a results file module and provide a list of the names (we have the data in the columns)
     my $rowinfo = "$row->{PROJECT}|$row->{JOB_TYPE}|$row->{REQ_TYPE}|$row->{IMG_TYPE}|";
-    $rowinfo   .= "$row->{ID}|$row->{TESS_ID}|$row->{COMPONENT}|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|";
+    $rowinfo   .= "$row->{ID}|$tess_id|$component|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|";
     $rowinfo   .= "$row->{REQFILT}|$row->{COORD_MASK}|$row->{CENTER_X}|$row->{CENTER_Y}|";
     $rowinfo   .= "$row->{WIDTH}|$row->{HEIGHT}";
@@ -313 +321 @@
     my $exp_id = shift;
 
-    if (!$exp_id) {
+    if (!$exp_id or !$image_db) {
         # no exposure id just return zeros
         # XXX: we could put a value in for filter, but we don't have a good place to find it

branches/czw_branch/cleanup/pstamp/scripts/pstamp_job_run.pl

@@ -208 +208 @@
         $command .= " -job_id $job_id";
         $command .= " -fault $exit_code";
+        # XXX: fix pstamptool to not require -state when -fault with nonzero value is provided
+        $command .= " -state run";
         $command .= " -dbname $dbname" if defined $dbname;
         $command .= " -dbserver $dbserver" if defined $dbserver;

branches/czw_branch/cleanup/pstamp/scripts/pstamp_listjobs.pl

@@ -12 +12 @@
 my $verbose;
 my $dbname;
-my $dserver;
+my $dbserver;
 
 GetOptions(

branches/czw_branch/cleanup/pstamp/scripts/pstampparse.pl

@@ -14 +14 @@
 use PS::IPP::PStamp::Job qw( :standard );
 use File::Temp qw(tempfile);
+use Carp;
 
 my $verbose;
@@ -91 +92 @@
 # make sure the file contains what we are expecting
 
-die "$request_file_name is not a PS1_PS_REQEST" if $extname ne "PS1_PS_REQUEST";
-die "REQ_NAME not found in $request_file_name"  if (!$req_name);
-die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
+# we shouldn't get here if this is the case so just die. No need to notify the client
+my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
+my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
+my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1");
+
+# check for duplicate request name
+if (!$no_update) {
+    my $command = "$pstamptool -listreq  -name $req_name";
+    $command .= " -dbname $dbname" if $dbname;
+    $command .= " -dbserver $dbserver" if $dbserver;
+    # no verbose so that error message about request not found doesn't appear in parse_error.txt
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 0);
+    my $exitStatus = $error_code >> 8;
+    if ($success) {
+        # -listreq succeeded duplicate request name
+        insertFakeJobForRow(undef, 0, $PSTAMP_DUP_REQUEST);
+        exit 0;
+    } elsif ($exitStatus ne $PS_EXIT_DATA_ERROR) {
+        # wrong error code something else is wrong
+        my_die("$command failed", $exitStatus);
+    }
+}
 
 if ($req_id and !$no_update) {
+    # update the database with the request name. This will be used as the
+    # the output data store's product name
     my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
     $command .= " -outProduct $product";
@@ -103 +126 @@
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        die "$command failed";
+        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
     }
 }
@@ -120 +143 @@
     }
     my $table =  $mdcParser->parse(join "", @$stdout_buf) or
-        die("Unable to parse metdata config doc");
+        my_die("Unable to parse metdata config doc", $PS_EXIT_UNKNOWN_ERROR);
 
     $rows = parse_md_list($table);
@@ -143 +166 @@
     # parameters that select the images of interest
     my $project  = $row->{PROJECT};
+
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+    if (!$proj_hash) {
+        print STDERR "project $project not found\n"  if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PRODUCT);
+        $num_jobs++;
+        next;
+    }
     my $req_type = $row->{REQ_TYPE};
     $stage = $row->{IMG_TYPE};
@@ -153 +186 @@
 
     my $option_mask= $row->{OPTION_MASK};
-
-    die "valid region of interest is required to make postage stamps"
-        if (($job_type eq "stamp") and ! validROI($row));
+    my $inverse = ($option_mask & $PSTAMP_SELECT_INVERSE) ? 1 : 0;
+    $row->{inverse} = $inverse;
 
     my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
-    $component = "" if (defined($component) and ($component eq "null" or $component eq "all"));
+
+    my $search_component = (!defined($component) or ($component eq "null")) ? "" : $component;
     
-    die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");
-
-
-    # note: resolve_project avoids running pstamptool every time by remembering the
-    # last project resolved
-    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
-    die "project $project not found\n" unless $proj_hash;
+    if (!$skycenter and !$search_component) {
+        print STDERR "COMPONENT must be specified for pixel coordinate ROI\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        $num_jobs++;
+        next;
+    }
+
+    $search_component = "" if $search_component eq "all";
+
+    if (($job_type eq "stamp") and ! validROI($row)) {
+        print STDERR "invalid ROI for row $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        $num_jobs++;
+        next;
+    }
+
+    
+    # $mode list_uri is a debugging mode (it may used by the http interface)
+    # if this happens just croak
+    my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
+
 
     my $image_db   = $proj_hash->{dbname};
@@ -173 +220 @@
 
     # collect rows with the same images of interest in a list so that they
-    # can be processed optimially
+    # can be looked up together
     if (@rowList) {
         my $firstRow = $rowList[0];
-        # XXX: the collecting might work with !$skycenter but I need to think about it
-        if ($skycenter and same_images_of_interest($row, $firstRow)) {
+        # note order of these parameters matters
+        if (same_images_of_interest($firstRow, $row)) {
+
+            # add this row to the list and move on
             push @rowList, $row;
-            # On to the next row
+
             next;
+
         } else {
-            # queue the jobs for this set of rows
+            # this row has different selectors
+            # queue the jobs for the ones we've collected
             $num_jobs += queueJobs($mode, \@rowList, $imageList);
             @rowList = ();
@@ -188 +239 @@
     }
 
+    # look up images for the current row
     if ($req_type eq "bycoord") {
-        die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter);
-        die "lookup bycoord is not yet implemented";
+        if (!$skycenter) {
+            print STDERR "center must be specified in sky coordintes for bycoord" if $verbose;
+            insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+            $num_jobs++;
+            next;
+        }
+        print STDERR "lookup bycoord is not yet implemented" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
+        $num_jobs++;
+        next;
     } else {
-
-        # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this
-        # request specification. An array reference is returned
+        # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
+        # request specification. An array reference is returned.
         my ($x, $y);
-        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $component, $skycenter,
-                $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
-    }
-
-    if (!$imageList or !@$imageList) {
-        print STDERR "no matching images found for row $rownum\n";
-        next;
-    }
-    $row->{need_magic} = $need_magic;
-    push @rowList, $row;
+
+        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $search_component,
+                $inverse, $skycenter, $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
+
+        if (!$imageList or !@$imageList) {
+            print STDERR "no matching images found for row $rownum\n" if $verbose;
+            # note in this case queueJobs inserts the fake job for these rows
+        }
+        $row->{need_magic} = $need_magic;
+        push @rowList, $row;
+    }
 }
 
@@ -212 +272 @@
 }
 
-if ($mode eq "queue_jobs") {
-    if ($num_jobs) {
-        exit 0;
-    } else {
-        # no jobs queued return error to get the request stopped
-        # TODO: need to define meaningful error codes
-        exit $PS_EXIT_UNKNOWN_ERROR;
-    }
-} else {
-    exit 0;
-}
+if (($mode eq "queue_jobs") and ($num_jobs eq 0)) {
+    print STDERR "no jobs created for $req_name\n" if $verbose;
+    insertFakeJobForRow(undef, 0, $PSTAMP_UNKNOWN_ERROR);
+}
+
+# PAU
+
+exit 0;
 
 
@@ -232 +289 @@
     my $have_skycells = shift;
     my $need_magic = shift;
+    my $num_jobs = 0;
+
+    my $rownum = $row->{ROWNUM};
+
+    my $components = $row->{components};
+
+    my $roi_string;
+
+    # note values were insured to be numbers in validROI()
+    my $x = $row->{CENTER_X};
+    my $y = $row->{CENTER_Y};
+    my $w = $row->{WIDTH};
+    my $h = $row->{HEIGHT};
+    my $coord_mask = $row->{COORD_MASK};
+    if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
+        if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
+            $roi_string = "-pixcenter $x $y";
+        } else {
+            $roi_string = "-skycenter $x $y";
+        }
+        if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
+            $roi_string .= " -pixrange $w $h";
+        } else {
+            $roi_string .= " -arcrange $w $h";
+        }
+    }
 
     # loop over images
@@ -244 +327 @@
 
         # skip this component if it is not in the list for this row
-        next if !$row->{components}->{$component};
-
-        my $rownum = $row->{ROWNUM};
-
-        my $roi_string;
-
-        # note values were insured to be numbers in validROI()
-        my $x = $row->{CENTER_X};
-        my $y = $row->{CENTER_Y};
-        my $w = $row->{WIDTH};
-        my $h = $row->{HEIGHT};
-        my $coord_mask = $row->{COORD_MASK};
-        if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
-            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
-                $roi_string = "-pixcenter $x $y";
-            } else {
-                $roi_string = "-skycenter $x $y";
-            }
-            if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
-                $roi_string .= " -pixrange $w $h";
-            } else {
-                $roi_string .= " -arcrange $w $h";
-            }
-        }
+        next if ! $components->{$component};
+
+        $job_num++;
 
         my $imagefile = $image->{image};
         if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
-            # XXX: should we add a faulted job so the client can know what happened?
+            # XXX: should we add a faulted job so the client can know what happened if no images come back?
             print STDERR "skippping non-magicked image $imagefile\n" if $verbose;
+
+            # for now assume yes.
+
+            insertFakeJobForRow($row, $job_num, $PSTAMP_NOT_DESTREAKED);
+
             next;
         }
@@ -282 +349 @@
         }
 
-        $job_num++;
-
         # add astrometry file for raw and chip images if one is available
         if (($stage eq "chip") || ($stage eq "raw")) {
@@ -302 +367 @@
 
         # copy the argument list to a file
-        open ARGSLIST, ">$argslist" or die "failed to open $argslist";
+        open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR);
         print ARGSLIST "$args\n";
-        close ARGSLIST or die "failed to close $argslist";
-
-        # XXX: TODO: here is where we need to check whether or not the source inputs still exist
-        # and if not, queue an update job and set the job state appropriately.
+        close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR);
 
         my $newState = "run";
+        my $fault = 0;
+
+        if (($stage ne 'stack') and ($stage ne 'raw')) {
+            if (($image->{state} eq 'goto_purged') or ($image->{data_state} eq 'purged')) {
+                $newState = 'stop';
+                $fault = $PSTAMP_GONE;
+            } elsif (($image->{data_state} ne 'full') or ($image->{state} ne 'full' )) {
+                # XXX here is where we need to queue an update job
+                # for now just say that the image is not available
+                $newState = 'stop';
+                $fault = $PSTAMP_NOT_AVAILABLE;
+            }
+        }
 
         $num_jobs++;
         my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
                         . " -outputBase $output_base -rownum $rownum -state $newState";
+        $command .= " -fault $fault" if $fault;
         $command .= " -exp_id $exp_id" if $exp_id;
         $command .= " -dbname $dbname" if $dbname;
@@ -331 +407 @@
                 # should we keep going for other uris? If so how do we report that some
                 # of the work that the request wanted isn't going to get done
-                die "failed to queue job for request $req_id";
+                my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
             }
         } else {
@@ -337 +413 @@
         }
     }
+    if ( $num_jobs == 0 ) {
+        print STDERR "no jobs for row $rownum\n" if $verbose;
+        insertFakeJobForRow($row, 1, $PSTAMP_NO_OVERLAP);
+        $num_jobs = 1;
+    }
     return $num_jobs;
 }
 
+# queue jobs for a collection of request specifications that have the same Images of Interest
 sub queueJobs
 {
@@ -358 +440 @@
         }
     } elsif ($job_type eq "get_image") {
-        die "get_image jobs not implemented yet";
+        print STDERR "get_image jobs not implemented yet" if $verbose;
+        foreach my $row (@$rowList) {
+            insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
+            $num_jobs++;
+        }
     } else {
-        my $thisRun;
-
-        my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
-        foreach my $row (@$rowList) {
-            print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
-            $row->{components} = {};
-        }
-        close $pointsList;
+        if (!$imageList or (scalar @$imageList eq 0)) {
+            # we didn't find any images for this set of rows. Insert a fake job to carry
+            # the status back to the requestor
+            foreach my $row (@$rowList) {
+                insertFakeJobForRow($row, 1, $PSTAMP_NO_IMAGE_MATCH);
+                $num_jobs++;
+            }
+            return $num_jobs;
+        }
 
         my $have_skycells;
@@ -376 +463 @@
         }
         
+        my $thisRun;
+
+        my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
+        my $npoints = 0;
+        foreach my $row (@$rowList) {
+            $row->{components} = {};
+            if ($row->{skycenter}) {
+                print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
+                $npoints++;
+            } else {
+                # this row's center is in pixel coordinates add all images to the component list for this row
+                foreach my $i (@$imageList) {
+                    my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id};
+                    my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) if !$component;
+                    $row->{components}->{$component} = 1;
+                }
+            }
+        }
+        close $pointsList;
+
         my $tess_dir_abs;
         my $last_tess_id = "";
         while ($thisRun = getOneRun($stage, $imageList)) {
-            {
+            if ($npoints) {
+                # we collected a set of sky coordintates above filter the images so that only
+                # those tat contain the centers are processed
                 my $command = "$dvoImagesAtCoords $pointsListName";
                 if ($have_skycells) {
@@ -391 +500 @@
                 } else {
                     my $astrom = $thisRun->[0]->{astrom};
-                    die "no astrometry file found" if !$astrom;
+                    my_die( "no astrometry file found", $PS_EXIT_UNKNOWN_ERROR) if !$astrom;
                     my $astrom_resolved = $ipprc->file_resolve($astrom);
                     $command .= " -astrom $astrom_resolved";
@@ -400 +509 @@
                     print STDERR @$stderr_buf;
                     my $rc = $error_code >> 8;
-                    die "dvoImagesAtCoords failed: $rc";
+                    my_die( "dvoImagesAtCoords failed: $rc", $PS_EXIT_UNKNOWN_ERROR);
                 }
                 # now we have a list of row numbers and components
@@ -422 +531 @@
     }
     return $num_jobs;
+}
+sub insertFakeJobForRow
+{
+    my $row = shift;
+    my $job_num = shift;
+    my $fault = shift;
+
+    my ($job_type, $rownum);
+    if ($row) {
+        $job_type = $row->{JOB_TYPE};
+        $rownum = $row->{ROWNUM};
+    } else {
+        $job_type = "none";
+        $rownum = 0;
+    }
+
+    my $command = "$pstamptool -addjob  -req_id $req_id -job_type $job_type"
+                        . " -rownum $rownum -state stop -fault $fault";
+    $command .= " -dbname $dbname" if $dbname;
+    $command .= " -dbserver $dbserver" if $dbserver;
+
+    if ($mode eq "list_job") { 
+        # this is a debugging mode, just print the pstamptool that would have run
+        # this is sort of like the mode -noupdate that some other tools support
+        print "$command\n";
+    } elsif (!$no_update) {
+        # mode eq "queue_job"
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            print STDERR @$stderr_buf;
+            # XXX TODO: now what? Should we mark the error state for the request?
+            # should we keep going for other uris? If so how do we report that some
+            # of the work that the request wanted isn't going to get done
+            my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    } else {
+        print "skipping command: $command\n";
+    }
 }
 
@@ -440 +588 @@
         return $image->{diff_id};
     } else {
-        die "unenexpected stage: $stage found";
+        my_die("unenexpected stage: $stage found", $PS_EXIT_PROG_ERROR);
     }
 }
@@ -476 +624 @@
     return 0 if ($r1->{IMG_TYPE} ne $r2->{IMG_TYPE});
     return 0 if ($r1->{ID} ne $r2->{ID});
+    return 0 if ($r1->{inverse} ne $r2->{inverse});
 
     if (defined($r1->{COMPONENT})) {
         return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT});
     } elsif (defined($r2->{COMPONENT})) {
+        # if first row has no component all of the images will be retrieved, so
+        # the fact that this row has a component is ok. Fall through to return 1
+        # XXX Nope that doesn't work (yet)
         return 0;
     }
@@ -514 +666 @@
     return undef;
 }
+
+sub my_die
+{
+    my $msg = shift;
+    my $fault = shift;
+
+    carp $msg;
+
+    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
+
+    return $fault;
+}

branches/czw_branch/cleanup/pstamp/src/ppstampMakeStamp.c

@@ -17 +17 @@
 
 static void skyToChip(pmAstromObj *pt, pmFPA *fpa, pmChip *chip);
+static bool setMaskedToNAN(pmConfig *config, psImage *image, psImage *mask, psImage *variance);
 
 // convert the input chip's transforms to the output
@@ -325 +326 @@
             break;
         }
+        if (readout->variance) {
+            outReadout->variance = extractStamp(readout->variance, extractRegion,  0);
+            if (!outReadout->variance) {
+                psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp weight image\n");
+                status = false;
+                break;
+            }
+        }
         if (readout->mask) {
             outReadout->mask = extractStamp(readout->mask, extractRegion,  0);
@@ -332 +341 @@
                 break;
             }
-        }
-        if (readout->variance) {
-            outReadout->variance = extractStamp(readout->variance, extractRegion,  0);
-            if (!outReadout->variance) {
-                psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp weight image\n");
-                status = false;
-                break;
+
+            if (!setMaskedToNAN(config, outReadout->image, outReadout->mask, outReadout->variance)) {
+                 psError(PS_ERR_UNKNOWN, false, "failed to create postage stamp mask image\n");
+                 status = false;
+                 break;
             }
         }
@@ -668 +675 @@
 
 
+static bool setMaskedToNAN(pmConfig *config, psImage *image, psImage *mask, psImage *variance)
+{
+    bool status;
+    psMetadata *masks = psMetadataLookupMetadata(&status, config->recipes, "MASKS");
+    if (!status) {
+        psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n");
+        return false;
+    }
+    // we set anything masked to NAN except if CONV.POOR is the only bit set
+    // First check the old value
+    psU32 convPoor = psMetadataLookupU32(&status, masks, "POOR.WARP");
+    if (!status) {
+        convPoor = psMetadataLookupU32(&status, masks, "CONV.POOR");
+        if (!status) {
+            psError(PM_ERR_CONFIG, false, "failed to lookup mask value for CONV.POOR in recipes\n");
+            return false;
+        }
+    }
+    psU32 maskMask = ~convPoor;
+
+    double exciseValue;
+    if (image->type.type == PS_TYPE_U16) {
+        exciseValue = 0xffff;
+    } else if (image->type.type == PS_TYPE_F32) {
+        exciseValue = NAN;
+    } else {
+         psError(PS_ERR_PROGRAMMING, true, "unexpected image type: %d\n", image->type.type);
+        return false;
+    }
+    long numExcised = 0;
+    for (int y=0; y<image->numRows; y++) {
+        for (int x=0; x<image->numCols; x++) {
+            psU16 maskVal = psImageGet(mask, x, y);
+            if (maskVal & maskMask) {
+                numExcised++;
+                psImageSet(image, x, y, exciseValue);
+                if (variance) {
+                    psImageSet(variance, x, y, exciseValue);
+                }
+            }
+        }
+    }
+    fprintf(stderr, "excised %ld masked pixels\n", numExcised);
+
+    return true;
+}