Changeset 27859 for trunk/pstamp/scripts/dqueryparse.pl

Timestamp:

May 4, 2010, 4:10:59 PM (16 years ago)

Author:

watersc1

Message:

Fix typos in pstamptool database queries.
Add pstampDependent.outdir entry to changes.txt.

Final tested code for detectability server.
Works on test cases that involve updates.

r

File:

• trunk/pstamp/scripts/dqueryparse.pl (modified) (8 diffs)

trunk/pstamp/scripts/dqueryparse.pl

@@ -2 +2 @@
 #
 # parse a MOPS_DETCTABILITY_QUERY table and create a results file
-#
-# Note: this file is currently only a placeholder which creates a fake response file
-# and adds a completed job to the database
 #
 
 use strict;
 use warnings;
-
+use Carp;
 use Getopt::Long qw( GetOptions );
 use Pod::Usage qw( pod2usage );
 use IPC::Cmd 0.36 qw( can_run run );
 
+use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
                        $PS_EXIT_UNKNOWN_ERROR
@@ -26 +25 @@
                        );
 
+my ($no_update, $imagedb, $label);
 my ($req_file, $req_id, $out_dir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps);
-
+my ($job_id,$rownum); # stuff from the post-update world
 #
 # parse args
@@ -35 +35 @@
         'file=s'          =>      \$req_file,
         'req_id=s'        =>      \$req_id,
+        'job_id=s'        =>      \$job_id,
+        'rownum=s'        =>      \$rownum,
         'out_dir=s'       =>      \$out_dir,
+        'label=s'         =>      \$label,
         'product=s'       =>      \$product,
         'mode=s'          =>      \$mode,
@@ -44 +47 @@
 ) or pod2usage(2);
 
-my $err = "";
-
-if (!$req_file) {
-    $err .= "--file is required\n";
-}
-if (!$req_id) {
-    $err .= "--req_id is required\n";
-}
-if (!$out_dir) {
-    $err .="--out_dir is required\n";
-}
-if (!$product) {
-    $err .="--product is required\n";
-}
-
-
-die $err if ($err);
-
+die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
+die "--file or --job_id is required" if !$req_file;
+
+if ($mode ne "list_uri") {
+    die "req_id is required" if !$req_id;
+    die "out_dir is required" if !$out_dir;
+    die "product is required" if !$product;
+}
 my $missing_tools;
 my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools =1);
-my $fakedresponse = can_run('fakedresponse.pl') or (warn "Can't find fakedresponse.pl" and $missing_tools =1);
+my $detectresponse = can_run('detectability_respond.pl') or 
+    (warn "Can't find detectability_respond.pl" and $missing_tools = 1);
 my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools =1);
 
@@ -77 +71 @@
 }
 
+# just deal with these arguments once and for all
+$pstamptool .= " -dbname $dbname" if $dbname;
+$pstamptool .= " -dbserver $dbserver" if $dbserver;
+$detectresponse .= " --dbname $dbname" if $dbname;
+
+$no_update = 1 if $mode eq "list_job";
+
+# Unless we're running as a job, write the parse arguments in case we need to rerun this parsing.
+if (!$job_id) {
+    my $argslist = "$out_dir/parse.args";
+    open ARGSLIST, ">$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+    print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --out_dir $out_dir --file $req_file\n";
+    close ARGSLIST;
+}
+
 # get the query id and check the extname and version from the header
-my $fields_output;
+my $fields_output; 
 {
     my $command = "echo $req_file | $fields -x 0 EXTNAME EXTVER QUERY_ID";
@@ -84 +93 @@
         run(command => $command, verbose => $verbose);
     
-#   fields doesn't return zero when it succeeds
-#    unless ($success) {
-#        print STDERR @$stderr_buf;
-#    }
     $fields_output = join "", @$stdout_buf;
 }
 my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
 
-die "$req_file is missing one of EXTNAME EXTVER or QUERY_ID" 
+my_die("$req_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR)
     if !(defined($extname) and defined($extver) and defined($req_name));
-
-die "$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table"
-                if $extname ne         "MOPS_DETECTABILITY_QUERY";
-die "$req_file is version $extver expecting 1" if $extver ne 1;
-
-if (0) {
-$out_dir .= "/$req_name";
-}
+my_die("$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR)
+    if $extname ne "MOPS_DETECTABILITY_QUERY";
+my_die("$req_file is version $extver expecting 1", $PS_EXIT_PROG_ERROR)
+    if $extver ne 1;
+
+# Set up the workdir for this query.
 if (! -e $out_dir ) {
-    mkdir $out_dir or die "cannot create output directory $out_dir";
+    mkdir $out_dir or my_die("cannot create output directory $out_dir", $PS_EXIT_PROG_ERROR);
 } elsif (! -d $out_dir ) {
-    die "output fileset directory $out_dir exists but is not a directory";
-}
-
-#
-# XXX Eventually we will parse the file here, to look up the list of input images to be processed
-# and queue a job for each image.
-# The request file will be one of the arguments. Each job will look at all rows and create entries
-# in the response file the coordinates that overlap the image.
-#
-# In the meantime we don't parse the file here, pass it to the program fakedresponse.
-# It creates a response file with a line for each row in the request file
-
-my $response_file = "$out_dir/response.fits";
+    my_die ("output fileset directory $out_dir exists but is not a directory", $PS_EXIT_PROG_ERROR);
+}
+
+
+# Pass along the request file to the response generator.
+my $response_file = "$out_dir/${req_name}.dresponse.${req_id}.fits";
 my $fault;
+my $data_to_update = '';
 {
-    my $command = "$fakedresponse --input $req_file --output $response_file --workdir $out_dir";
+    my $command = "$detectresponse --input $req_file --output $response_file --workdir $out_dir";
     $command .= " --save-temps" if $save_temps;
     $command .= " --verbose" if $verbose;
@@ -127 +124 @@
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        warn("Unable to perform $command error code: $error_code");
-    }
+        warn("Warn! Unable to perform $command error code: $error_code");
+    }
+    # Use the fault code to see if we can regenerate the missing data.
     $fault = $error_code >> 8;
-}
-
-my $job_id;
+    if ($fault == $PSTAMP_NOT_AVAILABLE) {
+        $data_to_update = (split /\n/, (join "", @$stdout_buf))[-1];    
+    }   
+}
+
 my $result;
-{
+# If we returned correctly with a valid response file, get a job ID
+# for the completed work, and move the response to a standardized name.
+if ($fault == 0) {
     my $command = "$pstamptool -addjob -req_id $req_id -outputBase $out_dir"; 
-    $command .= " -job_type detect_query -state stop -fault $fault";
+    $command .= " -job_type detect_query -state stop -fault 0";
     $command .= " -rownum 1";
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
     if ($success) {
         $job_id = join "", @$stdout_buf;
@@ -156 +155 @@
     }
 }
-
-
+else {
+    # Failed to run correctly, which means that we need to queue a job and flag data for updating.
+    # Get the dependency id for the data we're requesting be updated.
+    my $dep_id = queue_update_run($req_id,$job_id,$out_dir,$label,$data_to_update);
+
+    # Link this request to a job and link that job to any dependency
+    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $out_dir"; 
+    $command .= " -job_type detect_query -state run -fault 0";
+    $command .= " -rownum 1";
+    $command .= " -dep_id $dep_id" if $dep_id;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+
+    if ($success) {
+        $job_id = join "", @$stdout_buf;
+        chomp $job_id;
+        if ($job_id && -e $response_file) {
+            # We shouldn't have a response file at this stage.
+            rename $response_file, "$out_dir/response${job_id}.fits";
+        }
+        $result = 0;
+    } else {
+        warn("Unable to perform $command error code: $error_code");
+        $result = $error_code >> 8;
+    }
+}
+
+# This does not set the request state to stop.  That will happen with the request_finish.pl script, 
+# which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
 {
     my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name -outProduct $product";
     $command .= " -fault $result" if $result;
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        die "$command failed";
+        my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
     }
 }
  
 exit 0;
+
+
+# If we have to queue an update run, do so and create a new dependent
+sub queue_update_run {
+    my ($req_id, $job_id, $out_dir, $label, $data_to_update) = @_;
+
+    my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update;
+    
+    if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) {
+        # We should have received one of these states, so if not, signal that we have a problem.
+        my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR);
+    }
+    my $dep_id;
+    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component ";
+    $command .= " -outdir $out_dir";
+    $command .= " -need_magic" if $need_magic;
+
+    my $rlabel = "dq_ud_" . $label if $label;
+    $command .= " -rlabel $rlabel" if $rlabel;
+
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+        my $output = join "", @$stdout_buf;
+        chomp $output;
+        $dep_id = $output;
+
+        my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id;
+    } 
+    else {
+        print STDERR "skipping $command\n";
+        $dep_id = 42;
+    }
+
+    return($dep_id);
+}
+        
+    
+
+sub my_die {
+    my $msg = shift;
+    my $fault = shift;
+
+    carp $msg;
+
+    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
+    exit $fault;
+}