Changeset 33126

Timestamp:

Jan 20, 2012, 5:42:08 PM (14 years ago)

Author:

bills

Message:

In detectability requests use child jobs to manage the updates. The parent job runs
when all of the children are finished and does the actual work. This is a work in progress.
(It turns out detectability query requests have been unable to run for months due to
a bug in pstamp_parser_run.

Location:

trunk/pstamp/scripts

Files:

• detectability_respond.pl (modified) (1 diff)
• dqueryparse.pl (modified) (2 diffs)
• pstamp_job_run.pl (modified) (3 diffs)
• pstamp_parser_run.pl (modified) (1 diff)

trunk/pstamp/scripts/detectability_respond.pl

@@ -332 +332 @@
         }
         print WISDOM "\n";
+        my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
         if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
-            @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 
-                ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
-                 $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
+            if ($data_state ne 'full') {
+                
+                @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } = 
+                    ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
+                     $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
+            }
             push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
         }

trunk/pstamp/scripts/dqueryparse.pl

@@ -115 +115 @@
 my $response_file = "$outdir/${req_name}.dresponse.${req_id}.fits";
 my $fault;
-my $data_to_update = '';
+# my $data_to_update = '';
 {
     my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir";
     $command .= " --save-temps" if $save_temps;
     $command .= " --verbose" if $verbose;
+    $command .= " --ignore-wisdom" if !$job_id;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -137 +138 @@
 
 my $result;
-# If we returned correctly with a valid response file, get a job ID
-# for the completed work, and move the response to a standardized name.
-if ($fault == 0) {
-    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
-    $command .= " -job_type detect_query -state stop -fault 0";
-    $command .= " -rownum 1";
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    if ($success) {
-        $job_id = join "", @$stdout_buf;
-        chomp $job_id;
-        if ($job_id && -e $response_file) {
-            rename $response_file, "$outdir/response${job_id}.fits";
+unless ($job_id) {
+    # If we returned correctly with a valid response file, get a job ID
+    # for the completed work, and move the response to a standardized name.
+    if ($fault == 0) {
+        my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
+        $command .= " -job_type detect_query -state stop -fault 0";
+        $command .= " -rownum 1"; 
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        if ($success) {
+            $job_id = join "", @$stdout_buf;
+            chomp $job_id;
+            if ($job_id && -e $response_file) {
+                rename $response_file, "$outdir/response${job_id}.fits";
+            }
+            $result = 0;
+        } else {
+            warn("Unable to perform $command error code: $error_code");
+            $result = $error_code >> 8;
         }
-        $result = 0;
-    } else {
-        warn("Unable to perform $command error code: $error_code");
-        $result = $error_code >> 8;
-    }
-}
-elsif ($fault == $PSTAMP_NOT_AVAILABLE) {
-    # Failed to run correctly, which means that we need to queue a job and flag data for updating.
-    # Get the dependency id for the data we're requesting be updated.
-    open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
-    while (<UPDATE_REQUEST>) {
-        my $data_to_update = $_;
-        chomp($data_to_update);
-        my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
-
-        # Link this request to a job and link that job to any dependency
-        my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
-        $command .= " -job_type detect_query -state run -fault 0";
-        $command .= " -rownum 1";
-        $command .= " -dep_id $dep_id" if $dep_id;
-
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-
-        if ($success) {
-            $job_id = join "", @$stdout_buf;
-            chomp $job_id;
-            if ($job_id && -e $response_file) {
-                # We shouldn't have a response file at this stage.
-                rename $response_file, "$outdir/response${job_id}.fits";
-            }
-            $result = 0;
-        } else {
-            warn("Unable to perform $command error code: $error_code");
-            $result = $error_code >> 8;
-        }
-    }
-    close(UPDATE_REQUEST);
-}
-else {
-    my_die ("Parse fault!!", $PS_EXIT_UNKNOWN_ERROR);
+    }
+    elsif ($fault == $PSTAMP_NOT_AVAILABLE) {
+        # Failed to run correctly, which means that we need to queue a job and flag data for updating.
+        # first create a parent job for the actual detectabilty query
+        my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/1_"; 
+        $command .= " -job_type detect_query -state run -fault 0 -is_parent";
+        $command .= " -rownum 1";
+
+        my $parent_job_id;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        if ($success) {
+            # pstamptool returns the job_id of the new job
+            $parent_job_id = join "", @$stdout_buf;
+            chomp $parent_job_id;
+            if ($parent_job_id) {
+                $result = 0;
+            } else {
+                print STDERR "pstamptool did not return a valid job_id for parent job\n";
+                $result = $PS_EXIT_UNKNOWN_ERROR;
+            }
+        } else {
+            warn("Unable to perform $command error code: $error_code");
+            $result = $error_code >> 8;
+        }
+        # now create child jobs for each dependent. All these jobs do is finish once the dependency
+        # is satisfied
+        if (!$result) {
+            # Get the dependency id for the data we need to have updated.
+            my $job_num = 2;
+            open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
+            while (<UPDATE_REQUEST>) {
+                my $data_to_update = $_;
+                chomp($data_to_update);
+                my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
+
+                # Link this request to a job and link that job to the dependent
+                my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/${job_num}_"; 
+                $command .= " -job_type child -state run -fault 0 -parent_id $parent_job_id";
+                $command .= " -rownum 1"; # XXX: we should choose a correct rownum
+                $command .= " -dep_id $dep_id" if $dep_id;
+
+                $job_num++;
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+
+                if ($success) {
+                    $job_id = join "", @$stdout_buf;
+                    chomp $job_id;
+                    $result = 0;
+                } else {
+                    warn("Unable to perform $command error code: $error_code");
+                    $result = $error_code >> 8;
+                    last;
+                }
+            }
+            close(UPDATE_REQUEST);
+        }
+    }
+    else {
+        my_die ("Parse fault!!", $fault);
+    }
+} else {
+        # in some cases we will need to finish off the request
+        my_die ("Run fault!!", $fault);
 }
 

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -90 +90 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my $params = read_params_file($outputBase);
 
 my $jobStatus;
 if ($jobType eq "stamp") {
+    my $params = read_params_file($outputBase);
 
     my $argString;
@@ -283 +283 @@
 
     my $pstamp_bundle_root = metadataLookupStr($ipprc->{_siteConfig}, "PSTAMP_BUNDLE_ROOT");
+
+    my $params = read_params_file($outputBase);
     my $imagedb = $params->{imagedb};
 
@@ -327 +329 @@
         my_die("dqueryparse.pl failed with error code: $jobStatus", $job_id, $jobStatus);
     }
+} elsif ($jobType eq "child") {
+    # the only thing jobs of jobType child is to finish
+    $jobStatus = 0;
 } else {
     my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR);

trunk/pstamp/scripts/pstamp_parser_run.pl

@@ -294 +294 @@
     carp($msg);
 
+    if (!$req_id) {
+        exit $PS_EXIT_CONFIG_ERROR;
+    }
+
     my $command = "$pstamptool -updatereq -req_id $req_id  -set_fault $fault";
     $command   .= " -dbname $dbname" if $dbname;