Changeset 24921

Timestamp:

Jul 24, 2009, 2:45:59 PM (17 years ago)

Author:

bills

Message:

when parsing postage stamp request files run dvoImagesAtCoords to
narrow down the images that contain the center pixel. This prevents
queuing many many failing jobs

Location:

trunk

Files:

• Ohana/src/getstar/src/dvoImagesAtCoords.c (modified) (1 diff)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (10 diffs)
• PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm (modified) (1 diff)
• pstamp/scripts/pstampparse.pl (modified) (9 diffs)

trunk/Ohana/src/getstar/src/dvoImagesAtCoords.c

@@ -104 +104 @@
   for (i = 0; i < Nmatches; i++) {
     int N = matches[i];
-    fprintf (stdout, "%d (%lf, %lf)  :  %s\n", id, ra, dec, dbImages[N].name);
+
+    char *name;
+    // output of lookup is filename[class_id.hdr] for astrometry files
+    char *left_bracket = rindex(dbImages[N].name, '[');
+    if (left_bracket) {
+        name = strdup(left_bracket + 1);
+        // zap the .hdr]
+        char *dot = index(name, '.');
+        if (dot) {
+            *dot = 0;
+        }
+    } else {
+        name = dbImages[N].name;
+    }
+    fprintf (stdout, "%d %lf %lf %s\n", id, ra, dec, name);
   }
   

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -24 +24 @@
 use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Config qw( :standard );
+use Carp;
 
 ### my @images = locate_images($image_db, $req_type, $img_type, $id, $component,
@@ -121 +122 @@
     my $command;
     my $id_opt;     # option for the lookup
+    my $image_name;
     my $mask_name;
     my $weight_name;
@@ -130 +132 @@
 
     # special class_id value "null" means ignore
-    if ($component eq "null") {
+    if ($component and ($component eq "null")) {
         $component = undef;
     }
@@ -146 +148 @@
         $command .= " -class_id $class_id" if $class_id;
         $id_opt = "-chip_id";
+        $image_name   = "PPIMAGE.CHIP";
         $mask_name    = "PPIMAGE.CHIP.MASK";
         $weight_name  = "PPIMAGE.CHIP.VARIANCE";
@@ -156 +159 @@
         $command .= " -skycell_id $skycell_id" if $skycell_id;
         $id_opt = "-warp_id";
+        $image_name   = "PSWARP.OUTPUT";
         $mask_name    = "PSWARP.OUTPUT.MASK";
         $weight_name  = "PSWARP.OUTPUT.VARIANCE";
@@ -164 +168 @@
         $command .= " -skycell_id $skycell_id" if $skycell_id;
         $id_opt = "-diff_id";
+        $image_name  = "PPSUB.OUTPUT";
         $mask_name   = "PPSUB.OUTPUT.MASK";
         $weight_name = "PPSUB.OUTPUT.VARIANCE";
@@ -171 +176 @@
         $id_opt = "-stack_id";
 
+        $image_name  = "PPSTACK.OUTPUT";
         $mask_name   = "PPSTACK.OUTPUT.MASK";
         $weight_name = "PPSTACK.OUTPUT.VARIANCE";
@@ -225 +231 @@
             $ipprc->define_camera($camera);
         }
-        my $out = {};
-        $out->{exp_id} = $image->{exp_id};
-        $out->{exp_name} = $image->{exp_name};
+        my $out = $image;
+        # if uri is nil this will get overridded below
         $out->{image}  = $image->{uri};
-        $out->{state}  = $image->{state}; # state is undef for rawExp, but that's ok
-        if ($set_class_id) {
-            $class_id = $image->{class_id};
-            $out->{class_id} = $class_id;
-        }
+#        $out->{exp_id} = $image->{exp_id};
+#        $out->{exp_name} = $image->{exp_name};
+#        $out->{state}  = $image->{state}; # state is undef for rawExp, but that's ok
+#        if ($set_class_id) {
+#            $class_id = $image->{class_id};
+#            $out->{class_id} = $class_id;
+#        }
 
         # find the mask and weight images
         if ($base) {
+            $out->{image}  = $ipprc->filename($image_name,   $base, $class_id) if $image_name;
             $out->{mask}   = $ipprc->filename($mask_name,   $base, $class_id) if $mask_name;
             $out->{weight} = $ipprc->filename($weight_name, $base, $class_id) if $weight_name;
         }
-        $out->{camera} = $camera;
+#        $out->{camera} = $camera;
 
         $out->{astrom} = find_astrometry($ipprc, $image_db, $image, $verbose) if $want_astrom;
@@ -539 +547 @@
 # resolve_project()
 # get project specific information
+my $last_project = "";
 sub resolve_project {
     my $ipprc = shift;
     my $project_name = shift;
+
+    if (!$project_name) {
+        carp ("project is not defined");
+        return undef;
+    }
+
+    if ($project_name eq $last_project) {
+        return $last_project;
+    }
+
     my $dbname = shift;
     my $dbserver = shift;
-    die "project is not defined" if !$project_name;
 
     my $verbose = 0;
@@ -575 +593 @@
     my $proj_hash = parse_md_fast($mdcParser, $output);
 
-    return $proj_hash->[0];
+    $last_project = $proj_hash->[0];
+
+    return $last_project;
 }
 1;

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/RequestFile.pm

@@ -26 +26 @@
 
 
+# these values need to match the ones in pstamp/src/pstamp.h
 our $PSTAMP_CENTER_IN_PIXELS = 1;
 our $PSTAMP_RANGE_IN_PIXELS  = 2;

trunk/pstamp/scripts/pstampparse.pl

@@ -13 +13 @@
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use PS::IPP::PStamp::Job qw( :standard );
+use File::Temp qw(tempfile);
 
 my $verbose;
@@ -22 +23 @@
 my $out_dir;
 my $product;
+my $save_temps;
+my $no_update;
 
 GetOptions(
@@ -32 +35 @@
     'dbserver=s'=>  \$dbserver,
     'verbose'   =>  \$verbose,
+    'save-temps'=>  \$save_temps,
+    'no-update' =>  \$no_update,
 );
 
@@ -56 +61 @@
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
+my $dvoImagesAtCoords  = can_run('dvoImagesAtCoords') or (warn "Can't find dvoImagesAtCoords" and $missing_tools = 1);
 my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
@@ -62 +68 @@
     exit ($PS_EXIT_CONFIG_ERROR);
 }
+
+$no_update = 1 if $mode eq "list_job";
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -87 +95 @@
 die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
 
-if ($req_id) {
+if ($req_id and !$no_update) {
     my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
     $command .= " -outProduct $product";
@@ -117 +125 @@
 }
 
+my @rowList;
 my $num_jobs = 0;
+my $imageList;
+my $stage;
+my $need_magic;
 foreach my $row (@$rows) {
-    my $rownum   = $row->{ROWNUM};
-    my $job_type = $row->{JOB_TYPE};
-    my $project  = $row->{PROJECT};
-    my $req_type = $row->{REQ_TYPE};
-    my $img_type = $row->{IMG_TYPE};
-    my $id       = $row->{ID};
-    my $class_id = $row->{CLASS_ID};
-    my $filter   = $row->{REQFILT};
-    my $mjd_min = $row->{MJD_MIN};
-    my $mjd_max = $row->{MJD_MAX};
-    my $x = $row->{CENTER_X};
-    my $y = $row->{CENTER_Y};
-    my $w = $row->{WIDTH};
-    my $h = $row->{HEIGHT};
-    my $coord_mask = $row->{COORD_MASK};
-    my $option_mask= $row->{OPTION_MASK};
-
-    $class_id = "" if ($class_id eq "null" or $class_id eq "all");
-    
     # XXX: TODO: sanity check all parameters
 
@@ -144 +137 @@
     # add a job with the proper fault code. If there is a db or config error we should probably just
     # trash the request.
+
+    my $rownum   = $row->{ROWNUM};
+    my $job_type = $row->{JOB_TYPE};
+    
+    # parameters that select the images of interest
+    my $project  = $row->{PROJECT};
+    my $req_type = $row->{REQ_TYPE};
+    $stage = $row->{IMG_TYPE};
+    my $id       = $row->{ID};
+    my $class_id = $row->{CLASS_ID};
+    my $component = $row->{COMPONENT};
+    my $filter   = $row->{REQFILT};
+    my $mjd_min = $row->{MJD_MIN};
+    my $mjd_max = $row->{MJD_MAX};
+    my $option_mask= $row->{OPTION_MASK};
+
+    die "region of interest is required to make postage stamps"
+        if (($job_type eq "stamp") and ! validROI($row));
+
+    my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
+    $class_id = "" if (defined($class_id) and ($class_id eq "null" or $class_id eq "all"));
+    $component = "" if (defined($component) and ($component eq "null" or $component eq "all"));
+    
     die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");
 
+
+    # note: resolve_project remembers the last project returned so avoids running
+    # pstamptool every time
     my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
     die "project $project not found\n" unless $proj_hash;
 
-    my $image_db = $proj_hash->{dbname};
-    my $camera = $proj_hash->{camera};
-    my $need_magic = $proj_hash->{need_magic};
-
-    my $roi_string;
-    # XXX we're depending on other code to insure valid values for roi components
-    # this is checked in and ppstamp but I should check here so that we don't get that far,
-    # but not today
-    if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
-        if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
-            $roi_string = "-pixcenter $x $y";
+    my $image_db   = $proj_hash->{dbname};
+    my $camera     = $proj_hash->{camera};
+    $need_magic = $proj_hash->{need_magic};
+
+    # collect rows with the same images of interest in a list so that they
+    # can be processed optimially
+    if (@rowList) {
+        my $firstRow = $rowList[0];
+        if ($skycenter and same_images_of_interest($row, $firstRow)) {
+            push @rowList, $row;
+            # On to the next row
+            next;
         } else {
-            $roi_string = "-skycenter $x $y";
-        }
-        if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
-            $roi_string .= " -pixrange $w $h";
-        } else {
-            $roi_string .= " -arcrange $w $h";
-        }
-    }
-
-    die "region of interest is required to make postage stamps"
-        if (($job_type eq "stamp") && !defined($roi_string));
-
-    # find the uris for this request
+            # Process the jobs for this set of rows
+            $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList);
+            @rowList = ();
+        }
+    }
 
     if ($req_type eq "bycoord") {
-        die "region of interest is required for request type bycoord" if !defined($roi_string) ;
-        die "center must be specified in sky coordintes for bycoord" 
-            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS);
-    }
-
-    # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this request specification
-    my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id,
-            $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
-
-    if (!$images) {
+        die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter);
+        die "lookup bycoord is not yet implemented";
+    } else {
+
+        # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this
+        # request specification. An array reference is returned
+        my ($x, $y);
+        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $class_id, $skycenter,
+                $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
+    }
+
+    if (!$imageList or !@$imageList) {
         print STDERR "no matching images found for row $rownum\n";
         next;
     }
-    if ($mode eq "list_uri") {
-        foreach my $image (@$images) {
-            print "$image->{image}\n";
-        }
-    } elsif ($job_type eq "get_image") {
-        # XXX TODO: Get rid of this block and use the same code as the stamp jobs
-        # XXX This doesn't work to get the mask and weight images
-        my $listfile = "$out_dir/filelist";
-
-
-        # map our img_type to the Data Store file types.
-        my %filelist_img_types = ( "raw" => "chip", 
-                                   "chip" => "chipproc",
-                                   "warp" => "warp", 
-                                   "stack"=>"stack",
-                                   "diff" => "diff");
-
-        my $filelist_img_type = $filelist_img_types{$img_type};
-
-        open LISTFILE, ">$listfile"
-            or die "failed to open file list: $listfile while parsing get_image request $req_id";
-
-        foreach my $image (@$images) {
-            my $class_id = $image->{class_id} ? $image->{class_id} : "";
-            print LISTFILE "$image->{image}|$filelist_img_type|$class_id|\n";
-        }
-        close LISTFILE;
-        $num_jobs++;
-        my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image"
-                    . " -uri $listfile -outputBase $out_dir -rownum $rownum";
-        $command .= " -dbname $dbname" if $dbname;
-        $command .= " -dbserver $dbserver" if $dbserver;
-        if ($mode eq "list_job") {
-            print "$command\n";
-        } else {
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                run(command => $command, verbose => $verbose);
-            unless ($success) {
-                print STDERR @$stderr_buf;
-                # XXX TODO: now what? Should we mark the error state for the request?
-                die "failed to queue job for request $req_id: rownum: $rownum";
-            }
-        }
-    } else {
-        # sequence number for the the job for a request spec. Not to be confused with job_id
-        my $job_num = 0;
-
-        # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images
-        # use -list instead of -file
-        foreach my $image (@$images) {
-            my $uri = $image->{image};
-            if (($img_type ne "stack") and ($need_magic and !$image->{magicked})) {
-                print STDERR "skippping non-magicked image $uri\n" if $verbose;
-                next;
-            }
-            my $exp_id = $image->{exp_id};
-            
-            my $args = $roi_string ? $roi_string : "";
-            $args .= " -class_id $class_id" if $class_id;
-
-            $job_num++;
-
-            my $output_base = "$out_dir/${rownum}_${job_num}";
-
-            # add astrometry file for raw and chip images if one is available
-            if (($img_type eq "chip") || ($img_type eq "raw")) {
-                $args .= " -astrom $image->{astrom}" if $image->{astrom};
-            }
-
-            if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
-                $args .= " -mask $image->{mask}";
-            }
-            if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
-                $args .= " -weight $image->{weight}";
-            }
-
-            # XXX: TODO: here is where we need to check whether or not the source inputs still exist
-            # and if not, queue an update job and set the job state appropriately.
-
-            my $newState = "run";
-
-            $num_jobs++;
-            my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type"
-                . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum"
-                . " -state $newState";
-            $command .= " -exp_id $exp_id" if $exp_id;
-            $command .= " -dbname $dbname" if $dbname;
-            $command .= " -dbserver $dbserver" if $dbserver;
-
-            if ($mode eq "list_job") { 
-                # this is a debugging mode, just print the pstamptool that would have run
-                # this is sort of like the mode -noupdate that some other tools support
-                print "$command\n";
-            } else {
-                # mode eq "queue_job"
-                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                    run(command => $command, verbose => $verbose);
-                unless ($success) {
-                    print STDERR @$stderr_buf;
-                    # XXX TODO: now what? Should we mark the error state for the request?
-                    # should we keep going for other uris? If so how do we report that some
-                    # of the work that the request wanted isn't going to get done
-                    die "failed to queue job for request $req_id";
-                }
-            }
-        }
-    }
+    push @rowList, $row;
+}
+
+if (@rowList) {
+    $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList);
 }
 
@@ -309 +221 @@
     exit 0;
 }
+
+
+sub makeJobsForRow
+{
+    my $row = shift;
+    my $imageList = shift;
+    my $have_skycells = shift;
+    my $need_magic = shift;
+
+    # loop over images
+    my $job_num = 0;
+    foreach my $image (@$imageList) {
+        my $component;
+        my $class_id;   # get rid of this use componet
+        if ($have_skycells) {
+            $component = $image->{skycell_id};
+        } else {
+            $component = $image->{class_id};
+            $class_id = $component;
+        }
+
+        # skip this component if it is not in the list for this row
+        next if !$row->{components}->{$component};
+
+        my $rownum = $row->{ROWNUM};
+
+        my $roi_string;
+
+        # XXX we're depending on other code to insure valid values for roi components
+        # this is checked in and ppstamp but I should check here so that we don't get that far,
+        # but not today
+        my $x = $row->{CENTER_X};
+        my $y = $row->{CENTER_Y};
+        my $w = $row->{WIDTH};
+        my $h = $row->{HEIGHT};
+        my $coord_mask = $row->{COORD_MASK};
+        if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
+            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
+                $roi_string = "-pixcenter $x $y";
+            } else {
+                $roi_string = "-skycenter $x $y";
+            }
+            if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
+                $roi_string .= " -pixrange $w $h";
+            } else {
+                $roi_string .= " -arcrange $w $h";
+            }
+        }
+
+        my $uri = $image->{image};
+        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
+            print STDERR "skippping non-magicked image $uri\n" if $verbose;
+            next;
+        }
+        my $exp_id = $image->{exp_id};
+            
+        my $args = $roi_string ? $roi_string : "";
+        $args .= " -class_id $class_id" if $class_id;
+
+        $job_num++;
+
+        my $output_base = "$out_dir/${rownum}_${job_num}";
+
+        # add astrometry file for raw and chip images if one is available
+        if (($stage eq "chip") || ($stage eq "raw")) {
+            $args .= " -astrom $image->{astrom}" if $image->{astrom};
+        }
+
+        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
+            $args .= " -mask $image->{mask}";
+        }
+        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
+            $args .= " -weight $image->{weight}";
+        }
+
+        # XXX: TODO: here is where we need to check whether or not the source inputs still exist
+        # and if not, queue an update job and set the job state appropriately.
+
+        my $newState = "run";
+
+        $num_jobs++;
+        my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}"
+            . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum"
+            . " -state $newState";
+        $command .= " -exp_id $exp_id" if $exp_id;
+        $command .= " -dbname $dbname" if $dbname;
+        $command .= " -dbserver $dbserver" if $dbserver;
+
+        if ($mode eq "list_job") { 
+            # this is a debugging mode, just print the pstamptool that would have run
+            # this is sort of like the mode -noupdate that some other tools support
+            print "$command\n";
+        } else {
+            # mode eq "queue_job"
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR @$stderr_buf;
+                # XXX TODO: now what? Should we mark the error state for the request?
+                # should we keep going for other uris? If so how do we report that some
+                # of the work that the request wanted isn't going to get done
+                die "failed to queue job for request $req_id";
+            }
+        }
+    }
+    return $num_jobs;
+}
+
+sub makeJobs
+{
+    my $mode = shift;
+    my $stage = shift;
+    my $need_magic = shift;
+    my $rowList = shift;
+    my $imageList = shift;
+
+    my $firstRow = $rowList[0];
+    my $job_type = $firstRow->{JOB_TYPE};
+
+    my $num_jobs = 0;
+
+    if ($mode eq "list_uri") {
+        foreach my $image (@$imageList) {
+            print "$image->{image}\n";
+        }
+    } elsif ($job_type eq "get_image") {
+        die "get_image jobs not implemented yet";
+    } else {
+        my $thisRun;
+
+        my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
+        foreach my $row (@$rowList) {
+            print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
+            $row->{components} = {};
+        }
+        close $pointsList;
+
+        my $have_skycells;
+        if (($stage eq "raw") or ($stage eq "chip")) {
+            $have_skycells = 0;
+        } else {
+            $have_skycells = 1;
+        }
+        
+        my $tess_dir_abs;
+        my $last_tess_id = "";
+        while ($thisRun = getOneRun($stage, $imageList)) {
+            {
+                my $command = "$dvoImagesAtCoords $pointsListName";
+                if ($have_skycells) {
+                    my $tess_id = $thisRun->[0]->{tess_id};
+                    if ($tess_id ne $last_tess_id) {
+                        $tess_dir_abs = $ipprc->tessellation_catdir( $tess_id );
+                        $tess_dir_abs = $ipprc->convert_filename_absolute( $tess_dir_abs );
+                        $last_tess_id = $tess_id;
+                    }
+                    $command .= " -D CATDIR $tess_dir_abs";
+                } else {
+                    my $astrom = $thisRun->[0]->{astrom};
+                    die "no astrometry file found" if !$astrom;
+                    $command .= " -astrom $astrom";
+                }
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    print STDERR @$stderr_buf;
+                    my $rc = $error_code >> 8;
+                    die "dvoImagesAtCoords failed: $rc";
+                }
+                # now we have a list of row numbers and components
+                # eventually we might want to multiple stamp requests for the same image
+                # into the same ppstamp job but not yet. For now we will queue a new
+                my @lines = split "\n", join "", @$stdout_buf;
+                foreach my $line (@lines) {
+                    # parse the line, ignoring the ra and dec
+                    my ($rownum, undef, undef, $component) = split " ", $line;
+
+                    # I guess since we need this function we should be useing a hash for rowList 
+                    my $row = findRow($rownum, $rowList);
+                    $row->{components}->{$component} = 1;
+                }
+            }
+            
+
+                # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images
+                # use -list instead of -file
+            foreach my $row (@$rowList) {
+                $num_jobs += makeJobsForRow($row, $thisRun, $have_skycells, $need_magic);
+            }
+        }
+    }
+    return $num_jobs;
+}
+
+sub get_run_id
+{
+    my $stage = shift;
+    my $image = shift;
+
+    if ($stage eq "raw") {
+        return $image->{exp_id};
+    } elsif ($stage eq "chip") {
+        return $image->{chip_id};
+    } elsif ($stage eq "warp") {
+        return $image->{warp_id};
+    } elsif ($stage eq "stack") {
+        return $image->{stack_id};
+    } elsif ($stage eq "diff") {
+        return $image->{diff_id};
+    } else {
+        die "unenexpected stage: $stage found";
+    }
+}
+
+# extract components from the imageList that have the same run id and return the list
+sub getOneRun {
+    my $stage = shift;
+    my $imageList = shift;
+
+    # return if array is empty
+    return undef if ! @$imageList;
+
+    my $last_run_id = 0;
+    my @runList;
+    while ($imageList->[0]) {
+        my $run_id = get_run_id($stage, $imageList->[0]);
+
+        last if ($last_run_id and ($run_id ne $last_run_id));
+
+        my $image = shift @$imageList;
+        push @runList, $image;
+        $last_run_id = $run_id;
+    }
+    return \@runList;
+}
+
+sub same_images_of_interest {
+    my $r1 = shift;
+    my $r2 = shift;
+
+    return 0 if ($r1->{project} ne $r2->{project});
+    return 0 if ($r1->{job_type} ne $r2->{job_type});
+    return 0 if ($r1->{req_type} ne $r2->{req_type});
+    return 0 if ($r1->{img_type} ne $r2->{img_type});
+    return 0 if ($r1->{id} ne $r2->{id});
+#    XXX: turn this on when we change the request file format to change class_id into component
+#    return false if ($r1->{component} ne $r2->{component});
+
+    return 1;
+}
+
+sub validROI
+{
+    my $row = shift;
+    return 0 if !defined $row->{CENTER_X};
+    return 0 if !defined $row->{CENTER_Y};
+    return 0 if !defined $row->{WIDTH};
+    return 0 if !defined $row->{HEIGHT};
+
+    return 1;
+}
+
+sub findRow
+{
+    my $rownum = shift;
+    my $rowList = shift;
+
+    foreach my $row (@$rowList) {
+        return $row if $row->{ROWNUM} eq $rownum;
+    }
+    return undef;
+}