Changeset 18587

Timestamp:

Jul 16, 2008, 3:27:19 PM (18 years ago)

Author:

bills

Message:

Changes to get detectability query processing working with the new
request processing scheme and tables

Location:

trunk/pstamp/scripts

Files:

• Makefile.am (modified) (1 diff)
• detect_query_read (added)
• detect_response_create (added)
• dqueryparse.pl (modified) (7 diffs)
• pstamp_finish.pl (modified) (7 diffs)
• pstamp_parser_run.pl (modified) (4 diffs)
• pstamp_revert_request.pl (modified) (1 diff)
• pstamp_runcommand.sh (modified) (1 diff)
• pstampparse.pl (modified) (1 diff)

trunk/pstamp/scripts/Makefile.am

@@ -15 +15 @@
         pstamp_webrequest.pl \
         pstamp_get_image_job.pl \
-        dqueryparse.pl
+        request_finish.pl \
+        detect_query_read \
+        detect_response_create \
+        dquery_finish.pl \
+        dqueryparse.pl \
+        fakedresponse.pl
 
 install_SCRIPTS = $(install_files)

trunk/pstamp/scripts/dqueryparse.pl

@@ -5 +5 @@
 # Note: this file is currently only a placeholder which creates a fake response file
 # and adds a completed job to the database
-#
-# TODO: split off the actual generation of the response file into a standalone program
 #
 
@@ -28 +26 @@
                        );
 
-my ($uri, $out_dir, $mode, $req_id, $verbose, $save_temps);
+my ($req_file, $req_id, $out_dir, $product, $mode, $dbname, $verbose, $save_temps);
 
 #
@@ -35 +33 @@
 
 GetOptions(
-        'uri=s'           =>      \$uri,
+        'file=s'          =>      \$req_file,
+        'req_id=s'        =>      \$req_id,
         'out_dir=s'       =>      \$out_dir,
+        'product=s'       =>      \$product,
+        'mode=s'          =>      \$mode,
+        'dbname=s'        =>      \$dbname,
         'verbose'         =>      \$verbose,
-        'mode=s'          =>      \$mode,
-        'req_id=s'        =>      \$req_id,
         'save-temps'      =>      \$save_temps,
-        'verbose'         =>      \$verbose,
 ) or pod2usage(2);
 
 my $err = "";
 
-if (!$uri) {
-    $err .= "--uri is required\n";
+if (!$req_file) {
+    $err .= "--file is required\n";
+}
+if (!$req_id) {
+    $err .= "--req_id is required\n";
 }
 if (!$out_dir) {
     $err .="--out_dir is required\n";
 }
+if (!$product) {
+    $err .="--product is required\n";
+}
 
-if (!$req_id) {
-    $err .= "--req_id is required\n";
-}
 
 die $err if ($err);
@@ -62 +64 @@
 my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools =1);
 my $fakedresponse = can_run('fakedresponse.pl') or (warn "Can't find fakedresponse.pl" and $missing_tools =1);
+my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools =1);
 
 if ($missing_tools) {
@@ -68 +71 @@
 }
 
+# get the query id and check the extname and version from the header
+my $fields_output;
+{
+    my $command = "echo $req_file | $fields -x 0 EXTNAME EXTVER QUERY_ID";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    
+#   fields doesn't return zero when it succeeds
+#    unless ($success) {
+#        print STDERR @$stderr_buf;
+#    }
+    $fields_output = join "", @$stdout_buf;
+}
+my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
+
+die "$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table"
+                if $extname ne         "MOPS_DETECTABILITY_QUERY";
+die "$req_file does not have a QUERY_ID" if ! $req_name;
+die "$req_file is version $extver expecting 1" if $extver ne 1;
+
+$out_dir .= "/$req_name";
+if (! -e $out_dir ) {
+    mkdir $out_dir or die "cannot create output directory $out_dir";
+} elsif (! -d $out_dir ) {
+    die "output fileset directory $out_dir exists but is not a directory";
+}
+
+#
+# we don't parse the file here, pass it to fakedresponse it will create a response for each
+# row in the file
 my $response_file = "$out_dir/response.fits";
 my $fault;
 {
-    my $command = "$fakedresponse --input $uri --output $response_file --workdir $out_dir";
+    my $command = "$fakedresponse --input $req_file --output $response_file --workdir $out_dir";
     $command .= " --save-temps" if $save_temps;
     $command .= " --verbose" if $verbose;
@@ -84 +117 @@
 
 my $job_id;
+my $result;
 {
-    my $command = "$pstamptool -addjob -req_id $req_id -uri $uri -outputBase $out_dir -job_type detect_query";
-    $command .= " -state stop -fault $fault";
+    my $command = "$pstamptool -addjob -req_id $req_id -uri $req_file -outputBase $out_dir";
+    $command .= " -job_type detect_query -state stop -fault $fault";
+    $command .= " -rownum 1";
+    $command .= " -dbname $dbname" if $dbname;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -101 +137 @@
 }
 
-exit $result;
+{
+    my $command = "$pstamptool -processedreq -req_id $req_id -name $req_name -outProduct $product";
+    $command .= " -fault $result" if $result;
+    $command .= " -dbname $dbname" if $dbname;
 
-
-
-
-sub fake_dquery_response {
-    my $line = shift;
-    my ($rownum, $ra1, $dec1, $ra2, $dec2, $mag) = split " ", $line;
-
-    # todo perhaps think more about these values
-    my $n = rand(16);
-    my $f = rand(1);
-
-    return ($rownum, $n, $f);
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die "$command failed";
+    }
 }
+ 
+exit 0;

trunk/pstamp/scripts/pstamp_finish.pl

@@ -31 +31 @@
 
 
-my ( $dbname, $limit, $verbose, $save_temps );
+my ( $req_id, $req_name, $product, $dbname, $verbose, $save_temps );
 
 # the char to the right of the bar may be used as a single - alias for the longer name
 # for example --input and -i are equivalent
 GetOptions(
-           'limit=s'    => \$limit,
+           'req_id=s'   => \$req_id,
+           'req_name=s' => \$req_name,
+           'product=s'  => \$product,
            'dbname=s'   => \$dbname,
-           'verbose'  => \$verbose,
+           'verbose'    => \$verbose,
            'save-temps' => \$save_temps,
 ) or pod2usage( 2 );
@@ -62 +64 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my @requests;
 {
-    my $command = "$pstamptool -completedreq";
-    $command   .= " -limit $limit" if $limit;
-    $command   .= " -dbname $dbname" if $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        die("Unable to perform $command error code: $error_code");
-    }
-    my $output = join "", @$stdout_buf;
-    if (!$output) {
-        if ($verbose) {
-            print STDERR "no completed requests pending\n"
-        }
-        exit 0;
-    }
-    my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
-
-    my $requests = parse_md_list($metadata);
-
-    @requests = @$requests
-}
-
-foreach my $req (@requests) {
-    my $req_id = $req->{req_id};
-
-    my $product = $req->{outProduct};
-    die "outProduct in request is null" if !$product;
-
-    my $req_name = $req->{name};
-    die "request name is null" if !$req_name;
-
-
     # set the output fileset's name to the request name.
     my $fileset = $req_name;
@@ -156 +125 @@
             # including those that produced no jobs.
             # for now add an entry for rownum 1 and a phony error code.
-            #
-            # possible fix: have pstampparse add a dummy job with state=stop and a fault code
             # we've included parse_results.txt to the fileset if it exists
             #
@@ -169 +136 @@
             my $jobs = parse_md_list($metadata);
 
-            @jobs = @$jobs
+            @jobs = @$jobs;
         }
     }
@@ -201 +168 @@
             }
         } else {
-            die "Unknown jobType: $job_type";
+            print STDERR "Unknown jobType: $job_type";
+            next;
         }
     }
@@ -235 +203 @@
     # set the request's state to stop
     {
-        
         my $command = "$pstamptool -processedreq -req_id $req_id -state stop -fault $request_fault";
         $command   .= " -dbname $dbname" if $dbname;
@@ -245 +212 @@
     }
 }
+sub stop_request {
+    my $req_id = shift;
+    my $fault  = shift;
+    my $dbname = shift;
+
+    my $command = "$pstamptool -processedreq -req_id $req_id -state stop -fault $fault";
+    $command   .= " -dbname $dbname" if $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die("Unable to perform $command error code: $error_code");
+    }
+}

trunk/pstamp/scripts/pstamp_parser_run.pl

@@ -121 +121 @@
 
 my $new_uri;
-if ($ds_id) {
+if ($ds_id && ($uri =~ /^http:/)) {
     {
         $new_uri = "$workdir/request.fits";
@@ -183 +183 @@
 if ($request_type eq "PS1_PS_REQUEST") {
     $reqType = 'pstamp';
-    $parse_cmd = $pstampparse . " --mode queue_job --req_id $req_id --product $product --out_dir $outProductDir --file $uri";
-    $parse_cmd .= " --dbname $dbname" if $dbname;
+    $parse_cmd = $pstampparse;
 } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {
     $reqType = 'dquery';
-    $parse_cmd = $dqueryparse  . " --mode queue_job --req_id $req_id --out_dir $outProductDir --uri $uri";
+    $parse_cmd = $dqueryparse;
 }
 
 if (!$parse_cmd) {
-    # XXX TODO mark the error state for the job and set it to stop
-    die "null request type found in $uri" if !$request_type;
-    die "unknown request type $request_type found in $uri";
-}
+    print STDERR "No EXTNAME found in $uri" if !$request_type;
+    print STDERR "Unknown request type $request_type found in $uri";
+
+    my $command = "$pstamptool -processedreq -req_id $req_id -state run";
+    $command   .= " -reqType unknown";
+    $command   .= " -fault $PS_EXIT_DATA_ERROR";
+    $command   .= " -dbname $dbname" if $dbname;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        die("Unable to perform $command error code: $error_code");
+    }
+    exit $PS_EXIT_DATA_ERROR;
+}
+
+$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --out_dir $outProductDir --file $uri";
+$parse_cmd .= " --dbname $dbname" if $dbname;
 
 my $fault;
@@ -211 +223 @@
     if ($errbuf) {
         if (!open OUT, ">$error_file_name") {
-            die("unable to open parse_error file $error_file_name");
+            print STDERR ("unable to open parse_error file $error_file_name");
+        } else {
+            print OUT "$errbuf";
+            close(OUT);
         }
-        print OUT "$errbuf";
-        close(OUT);
         print STDERR $errbuf if $verbose;
     }
@@ -241 +254 @@
 # Note: We do not return $fault here. If there was a fatal error we've already exited.
 # If we got a fault it's due to bad input from the user we've set things up for this to be
-# handled by the pstamp.request.finish
+# handled by the task pstamp.request.finish
 
 exit 0;

trunk/pstamp/scripts/pstamp_revert_request.pl

@@ -91 +91 @@
 my $fileset = $req_name;
 
-# Here we invoke the assumption that the output for the request is placed in the
-# fileset directory directly
-my $out_dir = "$outputDataStoreRoot/$product/$fileset";
-
-print STDERR "product: $product  REQ_NAME: $req_name $out_dir\n" if $verbose;
+print STDERR "product: $product  REQ_NAME: $req_name\n" if $verbose;
 
 if ($product and $fileset) {

trunk/pstamp/scripts/pstamp_runcommand.sh

@@ -18 +18 @@
 # These variables need to be customized for a particular installation
 # XXX: why not pass these on on the command line or in the environment?
-export PSCONFDIR=/export/data0/bills/psconfig
+export PSCONFDIR=/home/panstarrs/bills/psconfig
 
-WORK_DIR=/export/data1/bills/pstamp/work
-CMD_DIR=/export/data0/bills/src/ipp/pstamp/scripts:/export/data0/bills/src/ipp/DataStoreServer/web/cgi
+WORK_DIR=/data/ipp004.0/pstamp-work
 
 #### END LOCAL_CONFIGURATION

trunk/pstamp/scripts/pstampparse.pl

@@ -88 +88 @@
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
-    unless ($success) {
-        print STDERR @$stderr_buf;
-    }
+    # fields doesn't return zero when it succeeds
+    #unless ($success) {
+    #    print STDERR @$stderr_buf;
+    #}
     $fields_output = join "", @$stdout_buf;
 }