Changeset 35412 for branches/eam_branches/ipp-20130307/pstamp/scripts/pstampparse.pl

Timestamp:

Apr 19, 2013, 3:39:39 PM (13 years ago)

Author:

eugene

Message:

merged from trunk

Location:

branches/eam_branches/ipp-20130307/pstamp/scripts

Files:

• . (modified) (1 prop)
• pstampparse.pl (modified) (34 diffs)

branches/eam_branches/ipp-20130307/pstamp/scripts/pstampparse.pl

@@ -27 +27 @@
 my $outdir;
 my $product;
-my $label;
+my $label = "";
 my $save_temps;
 my $no_update;
@@ -57 +57 @@
     die "outdir is required"  if !$outdir;
     die "product is required"  if !$product;
+} else {
+    $req_id = 0;
+    $outdir = "nowhere";
+    $product = "dummy";
 }
 
@@ -83 +87 @@
 $pstamptool .= " -dbserver $dbserver" if $dbserver;
 
-# list_job is a deugging mode
-$no_update = 1 if $mode eq "list_job";
+# If $mode is not queue_job we are using a debugging mode
+# do not update the database
+$no_update = 1 if $mode ne "queue_job";
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 #
-# get the data from the extension header
+# Read the keywords from the extension header
 #
 my $fields_output;
 {
-    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION USER EMAIL";
+    my $command = "echo $request_file_name | $fields -x 0 EXTNAME EXTVER REQ_NAME ACTION EMAIL";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-    # fields doesn't return zero when it succeeds
-    #unless ($success) {
-    #    print STDERR @$stderr_buf;
-    #}
+
+    # note fields doesn't return zero when it succeeds.
     $fields_output = join "", @$stdout_buf;
 }
-my (undef, $extname, $extver, $req_name, $action, $username, $email) = split " ", $fields_output;
+my (undef, $extname, $extver, $req_name, $action, $email) = split " ", $fields_output;
 
 # make sure the file contains what we are expecting
-# This program shouldn't have been run if the request file is bogus.
-# No need to notify the client
-my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
-my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
-my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
-
+# pstamp_parser_run.pl would not have run this program unless the request file was ok
+my_die("$request_file_name does not contain EXTNAME\n", $PS_EXIT_PROG_ERROR) if !$extname;
+my_die("$request_file_name is not a PS1_PS_REQUEST\n", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
+my_die("REQ_NAME not found in $request_file_name\n", $PS_EXIT_PROG_ERROR)  if (!$req_name);
+my_die("wrong EXTVER $extver found in $request_file_name\n", $PS_EXIT_PROG_ERROR) if ($extver ne "1" and $extver ne "2");
+
+if ($extver >= 2) {
+    # We have a version 2 file. Require that the new keywords be supplied. 
+    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
+
+    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
+
+    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
+    # XXX check for "valid" $email
+} else {
+    # for version 1 file the action is process and email is not used
+    $action = 'PROCESS';
+    $email = 'null';
+}
+
+print "Request Header Keywords EXTVER: $extver REQ_NAME: $req_name ACTION: $action EMAIL: $email\n";
 
 # check for duplicate request name
@@ -161 +179 @@
     # the output data store's product name
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
+    $command .= " -set_username $email" if $email ne 'null';
     $command .= " -set_outProduct $product";
     $command .= " -set_label $label" if $label_changed;
@@ -174 +193 @@
 
 #
-# now convert the request table to an array of metadata config docs
+# now convert the request table to an array of metadatas
 #
 
@@ -191 +210 @@
     my $dtime_request_file = gettimeofday() - $start_request_file;
     print "Time to read and parse request file: $dtime_request_file\n";
-
-}
-
-#
-# Loop over rows in the request file collecting consecutive rows that have the "same images of interest"
-# in the sense that their selection parameters will yield the same "Runs".
-# Process the groups of rows together to reduce lookup time and to potentially make multiple
-# stamps from the same ppstamp process.
-#
-my @rowList;
+}
+
+my $nRows = scalar @$rows;
+print "\n$nRows rows read from request file\n";
+
+
 my $num_jobs = 0;
 my $imageList;
 my $stage;
-my $need_magic;
 foreach my $row (@$rows) {
-    # santiy check the paramaters
+
+    if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) {
+        # this is a big request and it came from the upload page and doesn't have a specific label assigned
+        # change it to the generic one that has lower with lower priority
+        $label = 'WEB.BIG';
+        print "\nChanging label for big WEB.UP request to $label\n";
+
+        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+    }
+
+    # validate the paramaters
     if (!checkRow($row)) {
         # when it enconters an error checkRow adds a fake job with an appropriate error code to the database
@@ -216 +245 @@
     $row->{error_code} = 0;
 
-    if (scalar @rowList == 0) {
-        push @rowList, $row;
-        next;
-    }
-
-    my $firstRow = $rowList[0];
-    if (same_images_of_interest($firstRow, $row)) {
-        # add this row to the list and move on
-        push @rowList, $row;
-        next;
-    }
-
-    # the images of interest for this new row doesn't match the list. 
-    # process the list ...
-    $num_jobs += processRows($action, \@rowList);
-
-    # and reset the list to contain just the new row
-    @rowList = ($row);
-}
-
-# out of rows process the list
-if (scalar @rowList > 0) {
-    $num_jobs += processRows($action, \@rowList);
+    $num_jobs += processRow($action, $row);
 }
 
@@ -252 +259 @@
     my $row = shift;
 
+    # check validity of the paramters in a request specification
     # If we encounter an error for a particular row add a job with the proper fault code.
+    # also adjust some paramterers like filter and set defaults for some others
 
     my $stage = $row->{IMG_TYPE};
@@ -284 +293 @@
     }
 
-
     my $component = $row->{COMPONENT};
     if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
@@ -314 +322 @@
         return 0;
     }
+
+    my $fwhm_min = $row->{FWHM_MIN};
+    if (!defined $fwhm_min) {
+        $row->{FWHM_MIN} = 0;
+    } elsif (!validNumber($fwhm_min)) {
+        print STDERR "$fwhm_min is not a valid FWHM_MIN\n";
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+    my $fwhm_max = $row->{FWHM_MAX};
+    if (!defined $fwhm_max) {
+        $row->{FWHM_MAX} = 0;
+    } elsif (!validNumber($fwhm_max)) {
+        print STDERR "$fwhm_max is not a valid FWHM_MAX\n";
+        insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
+        return 0;
+    }
+
     my $data_group = $row->{DATA_GROUP};
     if (!defined $data_group) {
@@ -363 +389 @@
     }
 
-    # $mode list_uri is a debugging mode (it may used by the http interface)
-    # if this happens just croak
-   # my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
-
-
     if ($req_type eq "bycoord") {
         if (!$skycenter) {
@@ -384 +405 @@
     }
 
+
     return 1;
 }
@@ -392 +414 @@
 
     $num_jobs = 1;
+    return $num_jobs;
+}
+
+sub processRow {
+    my $action  = shift;
+    my $row     = shift;
+
+    my $num_jobs = 0;
+
+    my $project  = $row->{PROJECT};
+
+    # note: resolve_project avoids running pstamptool every time by remembering the
+    # last project resolved
+    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
+    if (!$proj_hash) {
+        insertFakeJobForRow($row, 1, $PSTAMP_UNKNOWN_PROJECT);
+        $num_jobs++;
+        return $num_jobs;
+    }
+    my $image_db   = $proj_hash->{dbname};
+    my $camera     = $proj_hash->{camera};
+
+    my $req_type    = $row->{REQ_TYPE};
+    my $rownum      = $row->{ROWNUM};
+
+    # Since user can get unmagicked data "by coordinate" requests can go back in time
+    # to dredge unusable data from the "dark days"...
+    if ($req_type eq 'bycoord' and $row->{IMG_TYPE} ne 'stack' and $row->{MJD_MIN} == 0) {
+        # ... so unless the user sets mjd_min clamp it to 2009-04-01
+        # XXX: This value should live in the pstampProject table not be hardcoded here
+        $row->{MJD_MIN} = 54922;
+    }
+
+    if (($req_type eq 'byskycell') or ($req_type eq 'bycoord')) {
+        # avoid error from print below if $id isn't needed
+        $row->{ID} = "" if !$row->{ID};
+    }
+    
+    # Call PS::IPP::PStamp::Job locate_images subroutine to get the images for this
+    # request specification. An array reference is returned.
+    my $start_locate = gettimeofday();
+
+    print "\nCalling new_locate_images for row: $rownum\n";
+
+    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
+
+    my $dtime_locate = gettimeofday() - $start_locate;
+    print "Time to locate_images for row $rownum $dtime_locate\n";
+
+    my @rowList = ($row);
+    $num_jobs += queueJobs($action, \@rowList, $imageList);
+
+    # if this row slipped through without a job being added add one. 
+    if ($row->{job_num} == 0) {
+        print "row $row->{ROWNUM} produced no jobs\n";
+        print STDERR "row $row->{ROWNUM} produced no jobs\n";
+        my $error_code = $row->{error_code};
+        $error_code =  $PSTAMP_NO_IMAGE_MATCH if !$error_code;
+        insertFakeJobForRow($row, ++$row->{job_num}, $error_code);
+    }
+
     return $num_jobs;
 }
@@ -401 +484 @@
 
     if ($action eq 'LIST') {
+        # LIST is not allowed by caller. Can't get here.
         return list_targets($rowList);
     }
@@ -420 +504 @@
     }
     my $req_type  = $row->{REQ_TYPE};
-    $stage        = $row->{IMG_TYPE};
+    my $stage     = $row->{IMG_TYPE};
     my $id        = $row->{ID};
     my $component = $row->{COMPONENT};
@@ -436 +520 @@
     my $image_db   = $proj_hash->{dbname};
     my $camera     = $proj_hash->{camera};
-    $need_magic    = $proj_hash->{need_magic};
+    my $need_magic    = $proj_hash->{need_magic};
     # Since user can get unmagicked data "by coordinate" requests can go back in time
     # to dredge unusable data from the "dark days"...
@@ -496 +580 @@
     # information required is contained there
 
-    $imageList = locate_images($ipprc, $image_db, \@rowList, $req_type, $stage, $id, $tess_id, $component,
+    $imageList = locate_images($ipprc, $image_db, $rowList, $req_type, $stage, $id, $tess_id, $component,
                 $option_mask, $need_magic, $mjd_min, $mjd_max, $filter, $data_group, $verbose);
 
@@ -506 +590 @@
     $row->{need_magic} = $need_magic;
 
-    $num_jobs += queueJobs($mode, \@rowList, $imageList);
+    $num_jobs += queueJobs($action, $rowList, $imageList);
 
     # if a row slipped through with no jobs add a faulted one
-    foreach my $row (@rowList) {
+    foreach my $row (@$rowList) {
         if ($row->{job_num} == 0) {
             print "row $row->{ROWNUM} produced no jobs\n";
@@ -528 +612 @@
     my $image = shift;
     my $need_magic = shift;
-    my $mode = shift;
+    my $action = shift;
 
     my $rownum = $row->{ROWNUM};
@@ -611 +695 @@
     write_params($output_base, $image);
 
-    my $newState = "run";
+    my $newState = $action eq 'PROCESS' ? "run" : "parsed";
     my $fault = 0;
     my $dep_id;
 
-    queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
+    queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
 
     my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
@@ -644 +728 @@
 sub queueJobs
 {
-    my $mode = shift;
+    my $action = shift;
     my $rowList = shift;
     my $imageList = shift;
 
-    my $firstRow = $rowList[0];
+    my $firstRow = $rowList->[0];
     my $stage    = $firstRow->{IMG_TYPE};
     my $job_type = $firstRow->{JOB_TYPE};
@@ -656 +740 @@
 
     if ($mode eq "list_uri") {
+        $num_jobs = $imageList ? scalar @$imageList : 0;
+        print "List of $num_jobs Images selected for row: $firstRow->{ROWNUM}\n";
         foreach my $image (@$imageList) {
             print "$image->{image}\n";
+            ++$firstRow->{job_num};
         }
     } elsif ($job_type eq "get_image") {
@@ -664 +751 @@
         my_die( "error: unexpected number of rows for get_image request: $n", $PS_EXIT_PROG_ERROR) if $n != 1;
 
-        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $mode);
+        $num_jobs = queueGetImageJobs($firstRow, $imageList, $stage, $need_magic, $action);
 
     } else {
@@ -692 +779 @@
                 my $row = $rowList->[$i];
 
-                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $mode);
+                $num_jobs += queueJobForImage($row, $stage, $image, $need_magic, $action);
             }
         }
@@ -706 +793 @@
     my $stage = shift;
     my $need_magic = shift;
-    my $mode = shift;
+    my $action = shift;
 
     my $num_jobs = 0;
@@ -753 +840 @@
         write_params($output_base, $image);
 
-        my $newState = "run";
+        my $newState = $action eq 'PROCESS' ? "run" : "parsed";
         my $fault = 0;
         my $dep_id;
 
-        queueUpdatesIfNeeded($stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
+        queueUpdatesIfNeeded($action, $stage, $image, $option_mask, \$newState, \$fault, \$dep_id);
 
         $num_jobs++;
@@ -899 +986 @@
 sub get_dependent 
 {
-    my ($r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
+    my ($action, $r_jobState, $r_fault, $r_dep_id, $imagedb, $state, $stage, $stage_id, $component, $need_magic) = @_;
 
     # chipRun's can be in full state if destreaking is necessary
@@ -918 +1005 @@
     my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $outdir";
     $command .= " -need_magic" if $need_magic;
-
-    # compute rlabel for the run.
-    # XXX: This bit of policy shouldn't be buried so deeply in the code
-    # For now use one that implies 'postage stamp server' 'update' 'request_label"
-    my $rlabel = "ps_ud_" . $label if $label;
-    $command .= " -rlabel $rlabel" if $rlabel;
+    $command .= ' -hold' if $action eq 'PREVIEW';
+
+    if ($label) {
+        # compute rlabel for the run.
+        # XXX: This bit of policy shouldn't be buried so deeply in the code
+        # For now use one that implies 'postage stamp server' 'update' 'request_label"
+        my $rlabel = "ps_ud_" . $label;
+        $command .= " -rlabel $rlabel";
+    }
 
     if (!$no_update) {
@@ -931 +1021 @@
             my $fault = $error_code >> 8;
             print STDERR "$command failed with fault $fault\n";
-            if ($fault < 10) {
+            if ($fault < $PSTAMP_FIRST_ERROR_CODE) {
                 # pstamptool returns an error if an existing depenent is faulted
-                # Set the object to not available even if the fault < 10
+                # Set the object to not available even if the fault < $PSTAMP_FIRST_ERROR_CODE
                 # which is nominally a recoverable error in order to keep
                 # the request from faulting (which can be very expensive if
@@ -939 +1029 @@
                 $fault = $PSTAMP_NOT_AVAILABLE
             }
-            if ($fault >= 10) {
+            if ($fault >= $PSTAMP_FIRST_ERROR_CODE) {
                 $$r_dep_id = 0;
                 $$r_fault = $fault;
@@ -967 +1057 @@
 
 sub queueUpdatesIfNeeded {
+    my $action = shift;
     my $stage = shift;
     my $image = shift;
@@ -973 +1064 @@
     my $r_fault = shift;
     my $r_dep_id = shift;
+
+    my $need_magic = 0;
 
     if ($stage ne 'raw') {
@@ -1016 +1109 @@
                     # set up to queue an update run
                     my $require_magic = ($need_magic or $image->{magicked});
-                    get_dependent(\$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, 
+                    get_dependent($action, \$$r_newState, \$$r_fault, $r_dep_id, $image->{imagedb}, 
                         $run_state, $stage, $image->{stage_id}, $image->{component}, $require_magic );
                 }
@@ -1070 +1163 @@
     my $fault = shift;
 
-    carp $msg;
+    print STDERR $msg;
 
     # we don't fault the request here pstamp_parser_run.pl handles that if necessary