Changeset 18541

Timestamp:

Jul 15, 2008, 10:06:26 AM (18 years ago)

Author:

bills

Message:

Significant changes to how requests flow through the system. Improved
fault handling and reporting.

Resolve a request's project into database name. (Hardcoded list in this
program for now. Will move this to a new database table at some point).

File:

• trunk/pstamp/scripts/pstampparse.pl (modified) (13 diffs)

trunk/pstamp/scripts/pstampparse.pl

@@ -17 +17 @@
 my $request_file_name;
 my $mode = "list_uri";
-my $out_dir = ".";
+my $out_dir;
+my $product;
 
 # values for COORD_MASK
@@ -24 +25 @@
 
 # values for OPTION_MASK
-use constant PSTAMP_SELECT_IMAGE => 1;
-use constant PSTAMP_SELECT_MASK  => 2;
+use constant PSTAMP_SELECT_IMAGE  => 1;
+use constant PSTAMP_SELECT_MASK   => 2;
 use constant PSTAMP_SELECT_WEIGHT => 4;
 
@@ -32 +33 @@
     'req_id=s'  =>  \$req_id,
     'out_dir=s' =>  \$out_dir,
+    'product=s' =>  \$product,
     'mode=s'    =>  \$mode,
     'dbname=s'  =>  \$dbname,
@@ -37 +39 @@
 );
 
-die "req_id is required"   if !defined($req_id);
+die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job");
+die "req_id is required"   if (!defined($req_id) && ($mode ne "list_uri"));
 die "file is required"     if !defined($request_file_name);
 die "out_dir is required"  if !defined($out_dir);
-
-die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "list_job") or ($mode eq "queue_job");
+die "product is required"  if !defined($product);
+
 
 use IPC::Cmd 0.36 qw( can_run run );
@@ -66 +69 @@
 
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
-my $pstampparse = can_run('pstampparse') or (warn "Can't find pstampparse" and $missing_tools = 1);
+my $lookup_images = can_run('pstampparse') or (warn "Can't find pstampparse" and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
+my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -77 +81 @@
 
 #
-# convert the request table to an array of metadata config docs
-#
-# TODO: check the header for the correct EXTNAME
+# get the data from the extension header
+#
+my $fields_output;
+{
+    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        print STDERR @$stderr_buf;
+    }
+    $fields_output = join "", @$stdout_buf;
+}
+my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
+
+# make sure the file contains what we are expecting
+
+die "$request_file_name is not a PS1_PS_REQEST" if $extname ne "PS1_PS_REQUEST";
+die "REQ_NAME not found in $request_file_name"  if (!$req_name);
+die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
+
+$out_dir .= "/$req_name";
+if (! -e $out_dir ) {
+    mkdir $out_dir or die "cannot create output directory $out_dir";
+} elsif (! -d $out_dir ) {
+    die "output fileset directory $out_dir exists but is not a directory";
+}
+
+{
+    my $command = "$pstamptool -processedreq -req_id $req_id -name $req_name -outProduct $product";
+    $command .= " -dbname $dbname";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die "$command failed";
+    }
+}
+
+#
+# now convert the request table to an array of metadata config docs
 #
 
@@ -96 +136 @@
 }
 
-# TODO: Make sure that the output directory exists 
-
 foreach my $row (@$rows) {
+    my $rownum   = $row->{ROWNUM};
     my $job_type = $row->{JOB_TYPE};
     my $project  = $row->{PROJECT};
@@ -105 +144 @@
     my $id       = $row->{ID};
     my $class_id = $row->{CLASS_ID};
-    my $stamp_select = $row->{STAMP_SELECT};
     my $stamp_name   = $row->{STAMP_NAME};
     my $filter   = $row->{FILTER};
@@ -117 +155 @@
     my $option_mask= $row->{OPTION_MASK};
 
+    # XXX: TODO: don't die in this loop. If we encounter an error add a job with the proper fault
+    # code
+    # todo this needs some implementation work
+    die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");
+
+    # TODO: sanity check all arguments
+
+    my ($image_db, $lookup_class_id ) = resolve_project($project, $img_type, $class_id);
+
     my $roi_string;
     # XXX we're depending on other code to insure valid values for roi components
     # this is checked in pstamprequest and ppstamp
-    # arguably I should check here to but not today
+    # I should check here to but not today
     if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
         if ($coord_mask & PSTAMP_CENTER_IN_PIXELS) {
@@ -140 +187 @@
         if (($job_type eq "stamp") && !defined($roi_string));
 
-    my $args = $roi_string ? $roi_string : "";
-
     # find the uris for this request
 
@@ -147 +192 @@
     # images/runs and prints out the uri and other metadata as a metadata config doc
 
-    my $command = "$pstampparse -dbname $project -$req_type $img_type $id $class_id";
+    my $command = "$lookup_images -dbname $image_db -$req_type $img_type $id $lookup_class_id";
+
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
     unless ($success) {
         print STDERR @$stderr_buf;
+        exit 1;
     }
     my @uris = split "^", join("", @$stdout_buf);
 
-    foreach my $uri (@uris) {
-        chomp $uri;
-        if ($mode eq "list_uri") {
-            print "$uri\n";
+    if ($mode eq "list_uri") {
+        foreach my $uri (@uris) {
+            print $uri;
+        }
+    } elsif ($job_type eq "get_image") {
+        my $listfile = "$out_dir/filelist";
+
+        open LISTFILE, ">$listfile"
+            or die "failed to open file list: $listfile while parsing get_image request $req_id";
+
+        # XXX TODO: how do we expand URIS to include wt and mk images
+        # I've been assuming that ppstamp would just know what to do based on flags and file rules.
+        # Actually if we change the code that lists images to list outputBase's instead we can
+        # translate the rules here
+        foreach my $uri (@uris) {
+            chomp $uri;
+            print LISTFILE "$uri|$img_type\n";
+        }
+        close LISTFILE;
+        $command = "$pstamptool -dbname $dbname -addjob -req_id $req_id -job_type get_image"
+                    . " -uri $listfile -outputBase $out_dir -rownum $rownum";
+        if ($mode eq "list_job") {
+            print "$command\n";
         } else {
-
-            # TODO: here is where we need to check whether or not the source image still exists
-            # and queue a regeneration job. Also remove streaks here ???
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+            unless ($success) {
+                print STDERR @$stderr_buf;
+                # XXX TODO: now what? Should we mark the error state for the request?
+                die "failed to queue job for request $req_id: rownum: $rownum";
+            }
+        }
+    } else {
+        my $args = $roi_string ? $roi_string : "";
+        $args .= " -class_id $class_id" if ($class_id and ($class_id ne "null"));
+
+#        my $numJobs = @uris;
+        my $num = 0;
+        foreach my $uri (@uris) {
+            chomp $uri;
+
+            $num++;
+            my $output_base = "$out_dir/${rownum}_${num}_${stamp_name}";
+
+            # XXX: TODO: here is where we need to check whether or not the source inputs still exist
+            # and if not, queue a regeneration job and set the job state appropriately.
+            # a simpler but slower option would be to set state to 'staged' and have a new task do
+            # the checking. This would add latency to the processing when the images are available.
+            # just share code.
 
             $command = "$pstamptool -dbname $dbname -addjob -req_id $req_id -job_type $job_type"
-                . " -uri $uri -outputBase $out_dir/$stamp_name -args '$args'";
+                . " -uri $uri -outputBase $output_base -args '$args' -rownum $rownum";
             if ($mode eq "list_job") { 
                 # this is a debugging mode, just print the pstamptool that would have run
@@ -179 +267 @@
                     # should we keep going for other uris? If so how do we report that some
                     # of the work that the request wanted isn't going to get done
-                    die "failed to queue job";
+                    die "failed to queue job for request $req_id";
                 }
             }
@@ -188 +276 @@
 exit 0;
 
+# resolve_project()
+# get project specific information
+# This is a placeholder final implementation will use configuration file and database to
+# map from "project" to image database, dvo database, telescope, camera, etc
+#
+# For now just get the database and the class_id that will be used for image lookup
+
+sub resolve_project {
+    my $project_name = shift;
+    my $img_type     = shift;
+    my $class_id     = shift;
+    die "project is not defined" if !$project_name;
+
+    # for megacam and simtest since images are at the fpa level, leave out the
+    # class_id at the lookup stage
+    #
+    # TODO: get this information from a database or a configuration file
+
+    my $ret_dbname;
+    my $ret_class_id;
+
+    if ($project_name eq "megacam-mops") {
+        $ret_dbname = "mops";
+        $ret_class_id = "null";
+    } elsif ($project_name eq "simtest") {
+        $ret_dbname = "simtest";
+        $ret_class_id = "null";
+    } elsif ($project_name eq "gpc1") {
+        $ret_dbname = "gpc1";
+        $ret_class_id = $class_id;
+    } else {
+        die "unknown project $project_name";
+    }
+
+    if (($img_type ne "raw") and ($img_type ne "chip")) {
+        # don't need class_id
+        $ret_class_id = "";
+    }
+
+    return ($ret_dbname, $ret_class_id);
+}