Changeset 18539

Timestamp:

Jul 15, 2008, 10:02:24 AM (18 years ago)

Author:

bills

Message:

Significant changes to how requests flow through the system. Improved
fault handling and reporting.

File:

• trunk/pstamp/scripts/pstamp_parser_run.pl (modified) (13 diffs)

trunk/pstamp/scripts/pstamp_parser_run.pl

@@ -2 +2 @@
 ###
 ### pstampparser_run.pl
-###     Run the postage stamp request parser for a given request id
+###     Run the request parser for a given request id
 ###
 
@@ -15 +15 @@
 my $verbose;
 my $dbname;
-my $request_id;
+my $req_id;
 
 GetOptions(
     'verbose'   =>  \$verbose,
-    'dbname=s'    =>  \$dbname,
-    'req_id=s'  =>  \$request_id,
+    'dbname=s'  =>  \$dbname,
+    'req_id=s'  =>  \$req_id,
 );
 
@@ -29 +29 @@
 }
 
-die "req_id is required" if !defined($request_id);
+die "req_id is required" if !defined($req_id);
 
 use IPC::Cmd 0.36 qw( can_run run );
@@ -51 +51 @@
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
 
-#my $outputDataStoreRoot = "/var/www/html/ds/dsroot";
-
-my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_DATA_STORE_ROOT');
-exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot;
+my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
+exit ($PS_EXIT_CONFIG_ERROR) unless defined $outputDataStoreRoot; # lookup failure outputs a message
 my $defaultOutputRoot = $outputDataStoreRoot;
 
-
-#my $inputWorkdir = "/export/data1/bills/pstamp/work/$request_id";
-
-my $inputWorkdir = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_WORKDIR');
-exit ($PS_EXIT_CONFIG_ERROR) unless defined $inputWorkdir;
-$inputWorkdir .= "/$request_id";
+my $pstamp_workdir = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_WORKDIR');
+exit ($PS_EXIT_CONFIG_ERROR) unless defined $pstamp_workdir; # lookup failure outputs a message
+
+# workdir is where we download request files to and place any error output from the parser
+# NOTE: this location needs to be kept in sync with the web interface ( request.php )
+my $workdir = "$pstamp_workdir/$req_id";
+
+if (! -e $workdir ) {
+    mkdir $workdir or die "failed to create working directory $workdir for request id $req_id";
+}
+
+my $defaultDSProduct = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_DATA_STORE_PRODUCT');
+exit ($PS_EXIT_CONFIG_ERROR) unless defined $defaultDSProduct;
     
 my $missing_tools;
@@ -81 +86 @@
 #Look up the uri for the given request
 {
-    my $command = "$pstamptool -pendingreq -req_id $request_id";
+    my $command = "$pstamptool -pendingreq -req_id $req_id";
     $command .= " -dbname $dbname" if $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -90 +95 @@
 
     if (@$stdout_buf == 0) {
-        print STDERR "pending pstamp request id $request_id not found\n";
+        print STDERR "pending pstamp request id $req_id not found\n";
         exit 1;
     }
@@ -98 +103 @@
     my $requests = parse_md_list($metadata);
 
-    # TODO: We are assuming that we get an array of length 1 (or zero)
-    # should we fail if we get more? 
-    #$psrequest = ${@$requests}[0];
+    my $num = @$requests;
+    die "unexpected number of requests $num found" if $num > 1;
+
     $psrequest = $requests->[0];
 }
 
 if (!$psrequest) {
-    print STDERR "postage stamp request $request_id not found\n";
-    # TODO: is this always an error, what if the request is no longer pending?
+    print STDERR "pending postage stamp request $req_id not found\n";
     exit 1;
 }
@@ -112 +116 @@
 my $uri = $psrequest->{uri};
 
-# if the uri is remote, download it 
-# XXX how can we tell?
-# perhaps assume remote if the uri begins with http: 
-# or perhaps assume it is remote if it doesn't begin with '/' or path://
-# actually since we're using dsget which is an http program the former is the
-# way to go. Now it's easier than that, we can assume that dsget is needed if
-# request->{ds_id} != 0
+# if the uri is from a data store, download it 
 
 my $ds_id = $psrequest->{ds_id};
 
+my $new_uri;
 if ($ds_id) {
-    mkdir $inputWorkdir;
     {
-        my $new_uri = "$inputWorkdir/request.fits";
+        $new_uri = "$workdir/request.fits";
         my $command = "$dsget --uri $uri --filename $new_uri";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -138 +136 @@
 
 #  find the name of the output directory
-my $outdir;
+my $product;
 if ($ds_id) {
     my $command = "$pstamptool -datastore -ds_id $ds_id";
@@ -159 +157 @@
     my $dataStore = $dataStores->[0];
 
-    $outdir = "$outputDataStoreRoot/$psrequest->{outFileset}";
+    $product = $dataStore->{outProduct};
 } else {
-    $outdir = "$defaultOutputRoot/$psrequest->{outFileset}";
-}
-
-#  create the output directory
-if (! -e $outdir ) {
-    mkdir $outdir or die("unable to create output directory $outdir");
-} elsif (! -d $outdir) {
-    die("file with name of output directory $outdir exists but is not a directory");
-}
+    $product = $defaultDSProduct;
+}
+
+#
+# Here is one of the places that we assume that the output for a request goes directly into
+# the output fileset directory in the data store.
+#
+my $outProductDir = "$outputDataStoreRoot/$product";
+
+die "output product directory $outProductDir does not exist" if (! -e $outProductDir) ;
+die "file with name of output product directory $outProductDir exists but is not a directory" 
+    if (! -d $outProductDir);
+
 
 # run the appropriate parse command to parse the queue the jobs for this request
@@ -175 +177 @@
 my $request_type = find_request_type($uri);
 
-print STDERR "request_type is $request_type\n" if $verbose;
+print STDERR "request_type for $req_id is $request_type\n" if $verbose;
 
 my $parse_cmd;
+my $reqType;    # for the database
 if ($request_type eq "PS1_PS_REQUEST") {
-    $parse_cmd = $pstampparse . " -mode queue_job -req_id $request_id -out_dir $outdir -file $uri";
-    $parse_cmd .= " -dbname $dbname" if $dbname;
+    $reqType = 'pstamp';
+    $parse_cmd = $pstampparse . " --mode queue_job --req_id $req_id --product $product --out_dir $outProductDir --file $uri";
+    $parse_cmd .= " --dbname $dbname" if $dbname;
 } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {
-    $parse_cmd = $dqueryparse  . " --mode queue_job --req_id $request_id --out_dir $outdir --uri $uri";
+    $reqType = 'dquery';
+    $parse_cmd = $dqueryparse  . " --mode queue_job --req_id $req_id --out_dir $outProductDir --uri $uri";
 }
 
@@ -191 +196 @@
 }
 
+my $fault;
 {
+    my $error_file_name = "$workdir/parse_error.txt";
+    # get rid of any error file from previous run
+    unlink $error_file_name if (-e $error_file_name);
+
     my $command = "$parse_cmd";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-    unless ($success) {
-        if (!open OUT, ">$outdir/parse_error") {
-            die("unable to open parse_error file $outdir/parse_error");
+
+    # save the contents of stderr (if any) to a file. This is relevant if
+    # the file parsed properl but one of the rows in the request file generated an error
+    my $errbuf = join "", @$stderr_buf;
+    if ($errbuf) {
+        if (!open OUT, ">$error_file_name") {
+            die("unable to open parse_error file $error_file_name");
         }
-        print OUT "@$stderr_buf";
+        print OUT "$errbuf";
         close(OUT);
-    }
-}
-
-# XXX make sure that a job got actually got queued.
-# If not set the request result and set state to
-# run so that the "request finisher" can build the results file.
-
-#
-# update the state of this request from 'new' to 'run'
+        print STDERR $errbuf if $verbose;
+    }
+
+    unless ($success) {
+        $fault = $error_code >> 8;
+    }
+}
+
+#
+# update the state of this request from 'new' to 'run' and set the uri to the downloaded location
 #
 {
-    my $command = "$pstamptool -processedreq -req_id $request_id -state run";
-    $command .= " -dbname $dbname" if $dbname;
+    my $command = "$pstamptool -processedreq -req_id $req_id -state run";
+    $command   .= " -reqType $reqType" if $reqType;
+    $command   .= " -uri $new_uri" if $new_uri;
+    $command   .= " -fault $fault" if $fault;
+    $command   .= " -dbname $dbname" if $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -220 +238 @@
     }
 }
+
+# Note: We do not return $fault here. If there was a fatal error we've already exited.
+# If we got a fault it's due to bad input from the user we've set things up for this to be
+# handled by the pstamp.request.finish
 
 exit 0;