Changeset 18619

Timestamp:

Jul 18, 2008, 12:56:11 PM (18 years ago)

Author:

bills

Message:

Add the job id to the response file to support a dquery request generating
multiple jobs

Location:

trunk/pstamp/scripts

Files:

• dquery_finish.pl (modified) (3 diffs)
• dqueryparse.pl (modified) (3 diffs)

trunk/pstamp/scripts/dquery_finish.pl

@@ -67 +67 @@
 exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot; # lookup failure outputs a message
 
-my $response_file = "$outputDataStoreRoot/$product/$req_name/response.fits";
+my $out_dir = "$outputDataStoreRoot/$product/$req_name";
+if (! -e $out_dir ) {
+    # something must have gone wrong at the parse stage
+    print STDERR "output fileset directory $out_dir does not exist\n" if $verbose;
+    if (! mkdir $out_dir ) {
+        stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
+        die "cannot create output directory $out_dir";
+    }
+} elsif (! -d $out_dir) {
+    stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
+    die "output fileset directory $out_dir exists but is not a directory";
+}
+
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 my $request_fault = 0;
-if (-e $response_file) {
+my @jobs;
+{
+    my $command = "$pstamptool -listjob -req_id $req_id";
+    $command .= " -dbname $dbname" if $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die("Unable to perform $command error code: $error_code");
+    }
+    my $output = join "", @$stdout_buf;
+    if (!$output) {
+        if ($verbose) {
+            print STDERR "Request $req_id produced no jobs.\n";
+        }
+        # assume that the parser set the fault
+    } else {
+        my $metadata =  $mdcParser->parse($output) or die("Unable to parse metdata config doc for jobs list");
+        my $jobs = parse_md_list($metadata);
+        @jobs = @$jobs;
+    }
+}
 
-    # the job generated a response file put it into the Data Store
-    # XXX: for now we have one response file. In the real system we may have multiple jobs with
-    # multiple response files
+my ($REGLIST, $reg_list) = tempfile("$out_dir/reqlist.XXXX", UNLINK => !$save_temps);
+foreach my $job (@jobs) {
+    my $job_id = $job->{job_id};
+    my $response_file = "response${job_id}.fits";
+    my $response_path = "$out_dir/$response_file";
 
-    my $command = "echo 'response.fits|||table|' | $dsreg --add $req_name --product $product --list -";
+    if (-e $response_path) {
+        # the job generated a response file put it into the Data Store
+        print $REGLIST "$response_file|||table|\n";
+    } else  {
+        print STDERR "detect_query response file for job $job_id not found\n";
+        $request_fault = $PS_EXIT_UNKNOWN_ERROR;
+    }
+}
+close $REGLIST;
+
+if (-s $reg_list) {
+    my $command = "$dsreg --add $req_name --product $product --list $reg_list";
     $command .= " --type MOPS_DETECTABILITY_RESPONSE";
     $command .= " --dbname $dbname" if $dbname;
@@ -85 +132 @@
         $request_fault = $error_code >> 8;
         print STDERR "Unable to perform $command return code: $request_fault";
+        # fall through to stop request
     }
-
-} else  {
-    print STDERR "detect_query response file not found\n";
-    $request_fault = $PS_EXIT_UNKNOWN_ERROR;
 }
 
-# set the request's state to stop
-{
-    my $command = "$pstamptool -processedreq -req_id $req_id -state stop -fault $request_fault";
+stop_request($req_id, $request_fault, $verbose);
+
+exit 0;
+
+sub stop_request {
+    my $req_id = shift;
+    my $fault = shift;
+    my $verbose = shift;
+    
+    my $command = "$pstamptool -processedreq -req_id $req_id -state stop";
+    $command   .= " -fault $fault" if $fault;
     $command   .= " -dbname $dbname" if $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -102 +154 @@
     }
 }
-

trunk/pstamp/scripts/dqueryparse.pl

@@ -86 +86 @@
 my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
 
+die "$req_file is missing one of EXTNAME EXTVER or QUERY_ID" 
+    if !(defined($extname) and defined($extver) and defined($req_name));
+
 die "$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table"
                 if $extname ne         "MOPS_DETECTABILITY_QUERY";
-die "$req_file does not have a QUERY_ID" if ! $req_name;
 die "$req_file is version $extver expecting 1" if $extver ne 1;
 
@@ -130 +132 @@
         $job_id = join "", @$stdout_buf;
         chomp $job_id;
+        if ($job_id && -e $response_file) {
+            rename $response_file, "$out_dir/response${job_id}.fits";
+        }
         $result = 0;
     } else {
@@ -136 +141 @@
     }
 }
+
 
 {