Changeset 24944

Timestamp:

Jul 30, 2009, 10:19:40 AM (17 years ago)

Author:

bills

Message:

use a file for the list of arguments for a postage stamp job instead of the database

Location:

trunk/pstamp/scripts

Files:

• pstamp_job_run.pl (modified) (1 diff)
• pstampparse.pl (modified) (16 diffs)

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -118 +118 @@
 my $uri = $psjob->{uri};
 my $outputBase = $psjob->{outputBase};
-my $argString = $psjob->{args};
 my $jobType = $psjob->{jobType};
+#my $argString = $psjob->{args};
 
 my $jobStatus;
 if ($jobType eq "stamp") {
-    my $command = "$ppstamp -file $uri $outputBase $argString";
+    open ARGSLIST, "<$uri" or die "failed to open argslist file $uri";
+    my $argString = <ARGSLIST>;
+    close ARGSLIST;
+    chomp $argString;
+
+    # XXX: should we do any other sanity checking?
+    die "arglist file $uri is empty" if !$argString;
+
+    my $command = "$ppstamp $outputBase $argString";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;

trunk/pstamp/scripts/pstampparse.pl

@@ -133 +133 @@
     # XXX: TODO: sanity check all parameters
 
-    # XXX: TODO: We shouldn't just die in this loop.
+    # XXX: TODO: We shouldn't really just die in this loop.
     # If we encounter an error for a particular row
     # add a job with the proper fault code. If there is a db or config error we should probably just
@@ -148 +148 @@
     my $class_id = $row->{CLASS_ID};
     my $component = $row->{COMPONENT};
+
     my $filter   = $row->{REQFILT};
     my $mjd_min = $row->{MJD_MIN};
     my $mjd_max = $row->{MJD_MAX};
+
     my $option_mask= $row->{OPTION_MASK};
 
-    die "region of interest is required to make postage stamps"
+    die "valid region of interest is required to make postage stamps"
         if (($job_type eq "stamp") and ! validROI($row));
 
@@ -163 +165 @@
 
 
-    # note: resolve_project remembers the last project returned so avoids running
-    # pstamptool every time
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
     my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
     die "project $project not found\n" unless $proj_hash;
@@ -176 +178 @@
     if (@rowList) {
         my $firstRow = $rowList[0];
+        # the collecting might work with !$skycenter but I need to think about it
         if ($skycenter and same_images_of_interest($row, $firstRow)) {
             push @rowList, $row;
@@ -181 +184 @@
             next;
         } else {
-            # Process the jobs for this set of rows
-            $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList);
+            # queue the jobs for this set of rows
+            $num_jobs += queueJobs($mode, $stage, $need_magic, \@rowList, $imageList);
             @rowList = ();
         }
@@ -207 +210 @@
 
 if (@rowList) {
-    $num_jobs += makeJobs($mode, $stage, $need_magic, \@rowList, $imageList);
+    $num_jobs += queueJobs($mode, $stage, $need_magic, \@rowList, $imageList);
 }
 
@@ -223 +226 @@
 
 
-sub makeJobsForRow
+sub queueJobsForRow
 {
     my $row = shift;
@@ -270 +273 @@
         }
 
-        my $uri = $image->{image};
+        my $imagefile = $image->{image};
         if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
-            print STDERR "skippping non-magicked image $uri\n" if $verbose;
+            # XXX: should we add a faulted job so the client can know what happened?
+            print STDERR "skippping non-magicked image $imagefile\n" if $verbose;
             next;
         }
@@ -283 +287 @@
 
         my $output_base = "$out_dir/${rownum}_${job_num}";
+        my $argslist = "${output_base}.argslist";
 
         # add astrometry file for raw and chip images if one is available
@@ -289 +294 @@
         }
 
+        $args .= " -file $imagefile";
+
         if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
             $args .= " -mask $image->{mask}";
@@ -296 +303 @@
         }
 
+        # copy the argument list to a file
+        open ARGSLIST, ">$argslist" or die "failed to open $argslist";
+        print ARGSLIST "$args\n";
+        close ARGSLIST or die "failed to close $argslist";
+
         # XXX: TODO: here is where we need to check whether or not the source inputs still exist
         # and if not, queue an update job and set the job state appropriately.
@@ -303 +315 @@
         $num_jobs++;
         my $command = "$pstamptool -addjob -req_id $req_id -job_type $row->{JOB_TYPE}"
-            . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum"
+            . " -uri $argslist -outputBase $output_base -rownum $rownum"
             . " -state $newState";
+#        $command .= " -args '$args'" if $args;
         $command .= " -exp_id $exp_id" if $exp_id;
         $command .= " -dbname $dbname" if $dbname;
@@ -329 +342 @@
 }
 
-sub makeJobs
+sub queueJobs
 {
     my $mode = shift;
@@ -404 +417 @@
             }
             
-
-                # XXX TODO: for raw and chip level images and class_id "null" if there are multiple images
-                # use -list instead of -file
             foreach my $row (@$rowList) {
-                $num_jobs += makeJobsForRow($row, $thisRun, $have_skycells, $need_magic);
+                $num_jobs += queueJobsForRow($row, $thisRun, $have_skycells, $need_magic);
             }
         }
@@ -461 +471 @@
     my $r2 = shift;
 
-    return 0 if ($r1->{PROJECT} ne $r2->{PROJECT});
+    return 0 if ($r1->{PROJECT}  ne $r2->{PROJECT});
     return 0 if ($r1->{JOB_TYPE} ne $r2->{JOB_TYPE});
     return 0 if ($r1->{REQ_TYPE} ne $r2->{REQ_TYPE});
@@ -467 +477 @@
     return 0 if ($r1->{ID} ne $r2->{ID});
 
-#    XXX: turn this on when we change the request file format to change class_id into component
-#    return false if ($r1->{component} ne $r2->{component});
+    if (defined($r1->{COMPONENT})) {
+        return 0 if !defined $r2->{COMPONENT} or ($r1->{COMPONENT} ne $r2->{COMPONENT});
+    } elsif (defined($r2->{COMPONENT})) {
+        return 0;
+    }
 
     return 1;
 }
 
+sub validNumber
+{
+    my $val = shift;
+
+    return $val =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/;
+}
+
 sub validROI
 {
     my $row = shift;
-    return 0 if !defined $row->{CENTER_X};
-    return 0 if !defined $row->{CENTER_Y};
-    return 0 if !defined $row->{WIDTH};
-    return 0 if !defined $row->{HEIGHT};
+    return 0 if !validNumber($row->{CENTER_X});
+    return 0 if !validNumber($row->{CENTER_Y});
+    return 0 if !validNumber($row->{WIDTH});
+    return 0 if !validNumber($row->{HEIGHT});
 
     return 1;