Changeset 25712

Timestamp:

Oct 1, 2009, 10:07:51 AM (17 years ago)

Author:

bills

Message:

implement REQ_TYPE byskycell and various other features

Location:

trunk

Files:

• PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm (modified) (10 diffs)
• pstamp/scripts/pstamp_finish.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (4 diffs)
• pstamp/scripts/pstampparse.pl (modified) (11 diffs)

trunk/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

@@ -34 +34 @@
     my $img_type = shift;   # required
     my $id       = shift;   # required unless req_type eq bycoord or byskycell
+    my $tess_id  = shift;
     my $component= shift;   # class_id or skycell_id
     my $inverse  = shift;
+    my $need_magic = shift;
     my $x        = shift;
     my $y        = shift;
@@ -41 +43 @@
     my $mjd_max  = shift;
     my $filter   = shift;
+    my $label    = shift;
     my $verbose  = shift;
 
 
     # we die in response to bad data in request files
-    die "Unknown req_type: $req_type" if ($req_type ne "byid") and ($req_type ne "byexp")
-                                        and ($req_type ne "bycoord") and ($req_type ne "bydiff");
+    die "Unknown req_type: $req_type" 
+        if ($req_type ne "byid") and
+           ($req_type ne "byexp") and 
+           ($req_type ne "bycoord") and 
+           ($req_type ne "bydiff") and 
+           ($req_type ne "byskycell");
+
+    my $dateobs_begin;
+    my $dateobs_end;
+    if (!iszero($mjd_min)) {
+        $dateobs_begin = mjd_to_dateobs($mjd_min);
+    }
+    if (!iszero($mjd_max)) {
+        $dateobs_end = mjd_to_dateobs($mjd_max);
+    }
 
     if (($req_type eq "byid") and ($img_type eq "diff")) {
@@ -95 +111 @@
         # regtool -dbname $image_db -processedimfile -time_begin $mjd_min -time_end = $mjd_max -filter $filter
         #
-        my $results = lookup_bycoord($ipprc, $image_db, $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
+        my $results = lookup_bycoord($ipprc, $image_db, $x, $y, $dateobs_begin, $dateobs_end, $filter, $verbose);
 
         # now take the results and lookup byexp
@@ -102 +118 @@
         $req_type = "byexp";
         $id = $results->{exp_name};
-    }
-
-    my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $component, $verbose);
+    } elsif ($req_type eq "byskycell") {
+        if (($img_type eq "raw") or ($img_type eq "chip")) {
+            print STDERR "REQ_TYPE byskycell not supported for IMG_TYPE raw or chip\n";
+            return undef;
+        }
+        if (!$tess_id or !$component) {
+            print STDERR "component and tess_id are required for REQ_TYPE byskycell\n";
+            return undef;
+        }
+    }
+
+    my $results = lookup($ipprc, $image_db, $req_type, $img_type, $id, $tess_id, $component, $need_magic, $dateobs_begin, $dateobs_end, $filter, $label, $verbose);
 
     return $results;
@@ -115 +140 @@
     my $img_type = shift;
     my $id       = shift;
+    my $tess_id  = shift;
     my $component= shift;
+    my $need_magic = shift;
+    my $dateobs_begin = shift;
+    my $dateobs_end   = shift;
+    my $filter = shift;
+    my $label = shift;
     my $verbose  = shift;
 
@@ -139 +170 @@
     my $skycell_id;
 
-    # special class_id value "null" means ignore
-    if ($component and ($component eq "null")) {
+    if (isnull($component)) {
         $component = undef;
     }
 
+    my $magic_arg = $need_magic ? " -destreaked" : "";
     if ($img_type eq "raw") {
         $class_id = $component;
@@ -196 +227 @@
     } elsif ($req_type eq "byexp") {
         $command .= " -exp_name $id";
+    } elsif ($req_type eq "byskycell") {
+        $command .= " -tess_id $tess_id -skycell_id $skycell_id";
     } else {
         die "Unknown req_type supplied: $req_type";
     }
+
+    if ($img_type ne "stack") {
+        $command .= $magic_arg;
+        $command .= " -dateobs_begin $dateobs_begin" if $dateobs_begin;
+        $command .= " -dateobs_end $dateobs_end" if $dateobs_end;
+    }
+
+    $command .= " -filter $filter" if !isnull($filter);
+    $command .= " -label $label" if !isnull($label);
 
     # run the tool and parse the output
@@ -386 +428 @@
     my $x        = shift;
     my $y        = shift;
-    my $mjd_min  = shift;
-    my $mjd_max  = shift;
+    my $dateobs_begin  = shift;
+    my $dateobs_end  = shift;
     my $filter   = shift;
     my $verbose  = shift;
@@ -403 +445 @@
 
     my $args;
-    if ($mjd_min) {
-        my $dateobs_min = mjd_to_dateobs($mjd_min);
-        $args .= " -dateobs_begin $dateobs_min";
-    }
-    if ($mjd_max) {
-        my $dateobs_max = mjd_to_dateobs($mjd_max);
-        $args .= " -dateobs_end $dateobs_max";
-    }
-    if ($filter) {
+    if (!isnull($dateobs_begin)) {
+        $args .= " -dateobs_begin $dateobs_begin";
+    }
+    if (!isnull($dateobs_end)) {
+        $args .= " -dateobs_end $dateobs_end";
+    }
+    if (!isnull($filter)) {
         $args .= " -filter $filter";
     }
@@ -567 +607 @@
     my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks);
 
-    return sprintf "'%4d-%02d-%02d %02d:%02d:%02d'", $year+1900, $mon+1, $day, $hr, $min, $sec;
+    return sprintf "'%4d-%02d-%02dT%02d:%02d:%02dZ'", $year+1900, $mon+1, $day, $hr, $min, $sec;
+}
+
+sub isnull {
+    my $val = shift;
+
+    return (!defined($val) or (lc($val) eq "null"));
+}
+
+sub iszero {
+    my $val = shift;
+    return (!defined($val) or ($val == 0));
 }
 

trunk/pstamp/scripts/pstamp_finish.pl

@@ -81 +81 @@
     my $fileset = $req_name;
 
-
-    # Here we invoke the assumption that the output for the request is placed in the
-    # fileset directory directly
-#    my $out_dir = "$outputDataStoreRoot/$product/$fileset";
-
-    # now we are assuming that the output directory is the dirname of the request file
-    # XXX: put this in the database
-
     print STDERR "product: $product  REQ_NAME: $req_name $out_dir\n" if $verbose;
 
@@ -216 +208 @@
 
                     # ra_deg and dec_deg are the coordinates of center of the stamp
-                    # XXX: parse the stamp header to find it
-                    print $tdf "0.0|0.0|";
+                    # XXX do this more cleanly
+                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields RA_DEG DEC_DEG`;
+                    $ra_deg = 0.0 if (!$ra_deg);
+                    $dec_deg = 0.0 if (!$dec_deg);
+                    print $tdf "$ra_deg|$dec_deg|";
 
                     print $tdf "$exp_info|";

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -16 +16 @@
 use PS::IPP::PStamp::RequestFile qw( :standard );
 use IPC::Cmd 0.36 qw( can_run run );
+use POSIX;
 
 use PS::IPP::Metadata::Config;
@@ -93 +94 @@
         run(command => $command, verbose => $verbose);
 
-    if ($success) {
+    my $exitStatus;
+    if (WIFEXITED($error_code)) {
+        $exitStatus = WEXITSTATUS($error_code);
+    } else {
+        print STDERR "ppstamp failed error_code: $error_code\n";
+        $exitStatus = $PS_EXIT_SYS_ERROR;
+    }
+
+    if ($exitStatus == 0) {
         my $dir = dirname($outputBase);
 
@@ -133 +142 @@
         close F;
         $jobStatus = $PS_EXIT_SUCCESS;
-    } else {
-        $jobStatus = $error_code >> 8;
-        my_die( "ppstamp failed with error code: $jobStatus", $job_id, $jobStatus);
+    } elsif ($exitStatus == $PSTAMP_NO_OVERLAP) {
+        $jobStatus = $PSTAMP_NO_OVERLAP
+    } else {
+        my_die( "ppstamp failed with error code: $exitStatus", $job_id, $exitStatus);
     }
 } elsif ($jobType eq "get_image") {
@@ -162 +172 @@
 {
     my $command = "$pstamptool -updatejob -job_id $job_id -state stop"; 
+    $command .= " -fault $jobStatus" if $jobStatus;
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;

trunk/pstamp/scripts/pstampparse.pl

@@ -16 +16 @@
 use File::Basename qw(basename);
 use Carp;
-use POSIX qw( strftime );
+use POSIX;
 
 my $verbose;
@@ -72 +72 @@
 }
 
+# list_job is a deugging mode
 $no_update = 1 if $mode eq "list_job";
 
@@ -185 +186 @@
     }
     my $req_type = $row->{REQ_TYPE};
-    $stage = $row->{IMG_TYPE};
-    my $id       = $row->{ID};
+    $stage       = $row->{IMG_TYPE};
+    my $id      = $row->{ID};
     my $component = $row->{COMPONENT};
-
-    my $filter   = $row->{REQFILT};
+    my $tess_id = $row->{TESS_ID};
+
+    my $filter  = $row->{REQFILT};
     my $mjd_min = $row->{MJD_MIN};
     my $mjd_max = $row->{MJD_MAX};
+    my $label   = $row->{LABEL};
 
     my $option_mask= $row->{OPTION_MASK};
@@ -222 +225 @@
     my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
 
-
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
@@ -228 +230 @@
 
     # Temporary hack so that MOPS can get at non-magicked data
-    if ($product eq "mops-pstamp-results") {
+    if ($product and ($product eq "mops-pstamp-results")) {
         $need_magic = 0;
     }
@@ -269 +271 @@
         my ($x, $y);
 
-        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $search_component,
-                $inverse, $skycenter, $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
+        $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $tess_id, $search_component,
+                $inverse, $need_magic, $x, $y, $mjd_min, $mjd_max, $filter, $label, $verbose);
 
         if (!$imageList or !@$imageList) {
@@ -305 +307 @@
     my $num_jobs = 0;
     my $rownum = $row->{ROWNUM};
+    my $option_mask = $row->{OPTION_MASK};
     my $components = $row->{components};
 
@@ -346 +349 @@
         if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
             # XXX: should we add a faulted job so the client can know what happened if no images come back?
+            # This test is made in locate_images now so this code never runs. This leads to no feedback
+            # to users, but speeds up processing significatnly
             print STDERR "skipping non-magicked image $imagefile\n" if $verbose;
 
@@ -369 +374 @@
         $args .= " -file $imagefile";
 
-        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
+        if (($option_mask & $PSTAMP_SELECT_MASK) &&  $image->{mask} ) {
             $args .= " -mask $image->{mask}";
         }
-        if (($row->{OPTION_MASK} & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
+        if (($option_mask & $PSTAMP_SELECT_WEIGHT) and $image->{weight} ) {
             $args .= " -variance $image->{weight}";
         }
@@ -407 +412 @@
         $num_jobs++;
         my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
-                        . " -outputBase $output_base -rownum $rownum -state $newState";
+                        . " -outputBase $output_base -rownum $rownum -state $newState -options $option_mask";
         $command .= " -fault $fault" if $fault;
         $command .= " -exp_id $exp_id" if $exp_id;
@@ -542 +547 @@
                     run(command => $command, verbose => $verbose);
                 unless ($success) {
-                    print STDERR @$stderr_buf;
-                    my $rc = $error_code >> 8;
-                    my_die( "dvoImagesAtCoords failed: $rc", $PS_EXIT_UNKNOWN_ERROR);
+                    # don't fail if the program exited normally and exit status was PSTAMP_NO_OVERLAP
+                    # That just means that the coordinate didn't match any image/skycell
+                    if (!WIFEXITED($error_code) || (WEXITSTATUS($error_code) ne $PSTAMP_NO_OVERLAP)) {
+                        print STDERR @$stderr_buf;
+                        my $rc = WIFEXITED($error_code) ? WEXITSTATUS($error_code) : $PS_EXIT_SYS_ERROR;
+                        my_die( "dvoImagesAtCoords failed: $rc", $rc);
+                    }
                 }
                 # now we have a list of row numbers and components