Changeset 18985

Timestamp:

Aug 8, 2008, 3:47:34 PM (18 years ago)

Author:

bills

Message:

Major re-work to get image data from the database. Supports
looking up mask, and weight files and getting req_id for all
requests (except for stack imgages) not done yet

File:

• trunk/pstamp/scripts/pstampparse.pl (modified) (12 diffs)

trunk/pstamp/scripts/pstampparse.pl

@@ -11 +11 @@
 use Sys::Hostname;
 use Getopt::Long qw( GetOptions );
+use PStamp::RequestFile qw( :standard );
+use PStamp::Job qw( :standard );
 
 my $verbose;
@@ -19 +21 @@
 my $out_dir;
 my $product;
-
-# values for COORD_MASK
-use constant PSTAMP_CENTER_IN_PIXELS => 1;
-use constant PSTAMP_RANGE_IN_PIXELS  => 2;
-
-# values for OPTION_MASK
-use constant PSTAMP_SELECT_IMAGE  => 1;
-use constant PSTAMP_SELECT_MASK   => 2;
-use constant PSTAMP_SELECT_WEIGHT => 4;
 
 GetOptions(
@@ -40 +33 @@
 
 die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job");
-die "req_id is required"   if (!defined($req_id) && ($mode ne "list_uri"));
 die "file is required"     if !defined($request_file_name);
-die "out_dir is required"  if !defined($out_dir);
-die "product is required"  if !defined($product);
-
+
+if ($mode ne "list_uri") {
+    die "req_id is required"   if !$req_id;
+    die "out_dir is required"  if !$out_dir;
+    die "product is required"  if !$product;
+}
 
 use IPC::Cmd 0.36 qw( can_run run );
@@ -52 +47 @@
 use PS::IPP::Metadata::List qw( parse_md_list );
 
-use PS::IPP::Config qw($PS_EXIT_SUCCESS
-                       $PS_EXIT_UNKNOWN_ERROR
-                       $PS_EXIT_SYS_ERROR
-                       $PS_EXIT_CONFIG_ERROR
-                       $PS_EXIT_PROG_ERROR
-                       $PS_EXIT_DATA_ERROR
-                       $PS_EXIT_TIMEOUT_ERROR
-                       metadataLookupStr
-                       metadataLookupBool
-                       caturi
-                       );
-
+use PS::IPP::Config qw( :standard );
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
 
@@ -102 +86 @@
 die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
 
-$out_dir .= "/$req_name";
-if (! -e $out_dir ) {
-    mkdir $out_dir or die "cannot create output directory $out_dir";
-} elsif (! -d $out_dir ) {
-    die "output fileset directory $out_dir exists but is not a directory";
-}
-
-{
-    my $command = "$pstamptool -processedreq -req_id $req_id -name $req_name -outProduct $product";
-    $command .= " -dbname $dbname";
+if ($out_dir) {
+    $out_dir .= "/$req_name";
+    if (! -e $out_dir ) {
+        mkdir $out_dir or die "cannot create output directory $out_dir";
+    } elsif (! -d $out_dir ) {
+        die "output fileset directory $out_dir exists but is not a directory";
+    }
+}
+
+if ($req_id) {
+    my $command = "$pstamptool -processedreq -req_id $req_id  -name $req_name";
+    $command .= " -outProduct $product";
+    $command .= " -dbname $dbname" if $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -158 +145 @@
     $stamp_name = "" if $stamp_name eq "null";
     
-
-    # XXX: TODO: don't die in this loop. If we encounter an error add a job with the proper fault
-    # code
-    # todo this needs some implementation work
+    # XXX: TODO: sanity check all parameters
+
+    # XXX: TODO: We shouldn't just die in this loop.
+    # If we encounter an error for a particular row
+    # add a job with the proper fault code. If there is a db or config error we should probably just
+    # trash the request.
     die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");
 
-    # TODO: sanity check all arguments
-
-    my ($image_db, $lookup_class_id ) = resolve_project($project, $img_type, $class_id);
+
+    my ($image_db, $lookup_class_id, $camera, $magic_required ) = resolve_project($project, $img_type, $class_id);
 
     my $roi_string;
     # XXX we're depending on other code to insure valid values for roi components
-    # this is checked in pstamprequest and ppstamp
-    # I should check here to but not today
+    # this is checked in and ppstamp but I should check here so that we don't get that far,
+    # but not today
     if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
-        if ($coord_mask & PSTAMP_CENTER_IN_PIXELS) {
+        if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
             $roi_string = "-pixcenter $x $y";
         } else {
             $roi_string = "-skycenter $x $y";
         }
-        if ($coord_mask & PSTAMP_RANGE_IN_PIXELS) {
+        if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
             $roi_string .= " -pixrange $w $h";
         } else {
@@ -193 +181 @@
         die "region of interest is required for request type bycoord" if !defined($roi_string) ;
         die "center must be specified in sky coordintes for bycoord" 
-            if ($coord_mask & PSTAMP_CENTER_IN_PIXELS);
-        # replace the id with center coordinates
-        $id = "$x $y";
-    }
-
-    # XXX: replace the invokcation of the hacked locateimages with a new ippTool that finds
-    # images/runs and prints out the uri and other metadata as a metadata config doc
-
-    my $command = "$locateimages -dbname $image_db -$req_type $img_type $id $lookup_class_id";
-    $command .= " -mjd_min $mjd_min" if ($mjd_min);
-    $command .= " -mjd_max $mjd_max" if ($mjd_max);
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-    unless ($success) {
-        print STDERR @$stderr_buf;
-        exit 1;
-    }
-    my @uris = split "^", join("", @$stdout_buf);
+            if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS);
+    }
+
+    # Call PStamp::Job's locate_images routinte to get the parameters for this request specification
+    my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $lookup_class_id,
+            $x, $y, $mjd_min, $mjd_max, $filter);
 
     if ($mode eq "list_uri") {
-        foreach my $uri (@uris) {
-            print $uri;
+        foreach my $image (@$images) {
+            print "$image->{image}\n";
         }
     } elsif ($job_type eq "get_image") {
+        # XXX TODO: Get rid of this block and use the same code as the stamp jobs
+        # XXX This doesn't work to get the mask and weight images
         my $listfile = "$out_dir/filelist";
 
@@ -223 +200 @@
             or die "failed to open file list: $listfile while parsing get_image request $req_id";
 
-        # XXX TODO: how do we expand URIS to include wt and mk images
-        # I've been assuming that ppstamp would just know what to do based on flags and file rules.
-        # Actually if we change the code that lists images to list outputBase's instead we can
-        # translate the rules here
-        foreach my $uri (@uris) {
-            chomp $uri;
-            print LISTFILE "$uri|$img_type\n";
+        foreach my $image (@$images) {
+            print LISTFILE "$image->{image}|$img_type\n";
         }
         close LISTFILE;
-        $command = "$pstamptool -dbname $dbname -addjob -req_id $req_id -job_type get_image"
+        my $command = "$pstamptool -addjob -req_id $req_id -job_type get_image"
                     . " -uri $listfile -outputBase $out_dir -rownum $rownum";
+        $command .= " -dbname $dbname" if $dbname;
         if ($mode eq "list_job") {
             print "$command\n";
@@ -249 +222 @@
         $args .= " -class_id $class_id" if ($class_id and ($class_id ne "null"));
 
-#        my $numJobs = @uris;
-        my $num = 0;
-        foreach my $uri (@uris) {
-            chomp $uri;
-
-            $num++;
-            my $output_base = "$out_dir/${rownum}_${num}";
-            $output_base .= "_${stamp_name}" if $stamp_name;
+        # sequence number for the the job for a request spec. Not to be confused with job_id
+        my $job_num = 0;
+
+        # XXX for raw and chip level images and class_id "null" if there are multiple images
+        # use -list instead of -file
+        foreach my $image (@$images) {
+            my $uri = $image->{image};
+
+            $job_num++;
+
+            my $output_base = "$out_dir/${rownum}_${job_num}";
+
+            # By commenting out the next line, I hereby drop the use of STAMP_NAME
+            # $output_base .= "_${stamp_name}" if $stamp_name;
+
+            # add astrometry file for raw and chip images if one is available
+            if (($img_type eq "chip") || ($img_type eq "raw")) {
+                $args .= " -astrom $image->{astrom}" if $image->{astrom};
+            }
+
+            if ($option_mask & $PSTAMP_SELECT_IMAGE) {
+                $args .= " -mask $image->{mask}";
+            }
+            if ($option_mask & $PSTAMP_SELECT_WEIGHT) {
+                $args .= " -weight $image->{weight}";
+            }
+
+            # XXX sounds like magic will be handled outside of the postage stamp server
+            #if ($magic_required) {
+            #    die "no magic mask available" if ! $image->{magic_mask};
+            #    $args .= " -magic_mask $image->{magic_mask}" 
+            #}
+
 
             # XXX: TODO: here is where we need to check whether or not the source inputs still exist
             # and if not, queue a regeneration job and set the job state appropriately.
-            # a simpler but slower option would be to set state to 'staged' and have a new task do
-            # the checking. This would add latency to the processing when the images are available.
-            # just share code.
-
-            $command = "$pstamptool -dbname $dbname -addjob -req_id $req_id -job_type $job_type"
+
+
+#            # XXX TEMPORARY HACK won't work in general don't commit this
+#            if (($img_type eq "chip") && ($image_db eq "gpc1")) {
+#                my $astrom = $uri;
+#                $astrom =~ s/.ch.fits/.smf/;
+#                if ( -e $astrom ) {
+#                    $astrom_args .= " -astrom $astrom";
+#                }
+#            }
+
+            my $command = "$pstamptool -addjob -req_id $req_id -job_type $job_type"
                 . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum";
+            $command .= " -dbname $dbname" if $dbname;
+
             if ($mode eq "list_job") { 
                 # this is a debugging mode, just print the pstamptool that would have run
@@ -288 +295 @@
 exit 0;
 
+
 # resolve_project()
 # get project specific information
@@ -308 +316 @@
     my $ret_dbname;
     my $ret_class_id;
+    my $ret_inst;
+    my $ret_magic;
+    my $ret_camera;
 
     if ($project_name eq "megacam-mops") {
         $ret_dbname = "mops";
         $ret_class_id = "null";
+        $ret_camera = "MEGACAM";
+        $ret_magic = 0;
     } elsif ($project_name eq "simtest") {
         $ret_dbname = "simtest";
         $ret_class_id = "null";
+        $ret_camera = "SIMTEST";
+        $ret_magic = 0;
     } elsif ($project_name eq "gpc1") {
         $ret_dbname = "gpc1";
         $ret_class_id = $class_id;
+        $ret_camera = "GPC1";
+        $ret_magic = 0; # switch this to 1 when we're ready
     } else {
         die "unknown project $project_name";
@@ -327 +344 @@
     }
 
-    return ($ret_dbname, $ret_class_id);
-}
+    return ($ret_dbname, $ret_class_id, $ret_camera, $ret_magic);
+}