Changeset 27859

Timestamp:

May 4, 2010, 4:10:59 PM (16 years ago)

Author:

watersc1

Message:

Fix typos in pstamptool database queries.
Add pstampDependent.outdir entry to changes.txt.

Final tested code for detectability server.
Works on test cases that involve updates.

r

Location:

trunk

Files:

• dbconfig/changes.txt (modified) (1 diff)
• ippTools/share/pstamptool_completedreq.sql (modified) (1 diff)
• ippTools/share/pstamptool_pendingdependent.sql (modified) (1 diff)
• pstamp/scripts/Makefile.am (modified) (1 diff)
• pstamp/scripts/detectability_respond.pl (modified) (21 diffs)
• pstamp/scripts/dquery_finish.pl (modified) (3 diffs)
• pstamp/scripts/dqueryparse.pl (modified) (8 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (1 diff)

trunk/dbconfig/changes.txt

@@ -1592 +1592 @@
 ALTER TABLE pstampDependent CHANGE COLUMN no_magic need_magic TINYINT;
 ALTER TABLE pstampDependent ADD COLUMN fault SMALLINT after need_magic;
+ALTER TABLE pstampDependent ADD COLUMN outdir VARCHAR(255) after imagedb;
 ALTER TABLE pstampDependent ADD KEY (state);
 ALTER TABLE pstampDependent ADD KEY (stage);

trunk/ippTools/share/pstamptool_completedreq.sql

@@ -1 +1 @@
 SELECT pstampRequest.* ,
-    IFNULL(labels.priority, 0) AS priority
+    IFNULL(Label.priority, 0) AS priority
 FROM pstampRequest 
-    LEFT JOIN labels USING(label) 
+    LEFT JOIN Label USING(label) 
 WHERE state = 'run'
     AND (

trunk/ippTools/share/pstamptool_pendingdependent.sql

@@ -4 +4 @@
 JOIN pstampJob USING(dep_id)
 JOIN pstampRequest USING(req_id)
-JOIN Label ON pstampRequest.label = Label.label
+LEFT JOIN Label ON pstampRequest.label = Label.label
 WHERE pstampDependent.state = 'new'
     AND pstampDependent.fault = 0

trunk/pstamp/scripts/Makefile.am

@@ -20 +20 @@
         pstamp_checkdependent.pl \
         request_finish.pl \
+        detectability_respond.pl \
         detect_query_read \
         detect_response_create \

trunk/pstamp/scripts/detectability_respond.pl

@@ -41 +41 @@
 my $EXTVER = 1.0;
 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
-my ($req_id,$req_name,$product,$need_magic,$missing_tools);
-my ($request_file,$output,$dbname,$verbose,$save_temps);
+my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project);
+my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps);
 GetOptions(
-    'input=s'          =>     \$request_file,
+    'input=s'         =>      \$request_file,
     'output=s'        =>      \$output,
+    'workdir=s'       =>      \$workdir,
     'dbname=s'        =>      \$dbname,
+    'dbserver=s'      =>      \$dbserver,
     'verbose'         =>      \$verbose,
     'save-temps'      =>      \$save_temps,
@@ -55 +57 @@
            -exitval => 3,
     ) unless
-    defined $request_file and defined $output and defined $dbname;
+    defined $request_file and defined $output and defined $workdir and defined $dbname;
 
 my $detect_query_read = can_run('detect_query_read') or (warn "Can't find detect_query_read" and $missing_tools = 1);
@@ -71 +73 @@
 
 my $ipprc = PS::IPP::Config->new();
-#my $tmp_dir = "/data/${host}.0/tmp/";
-
-my $project = resolve_project($ipprc,"project_name",$dbname);
+
+if (!$dbserver) {
+    $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER');
+}
+# This is hardcoded in for the moment.
+$project = resolve_project($ipprc,"gpc1",$dbname,$dbserver);
+my $imagedb = $project->{dbname};
+if (!$imagedb) {
+    carp("failed to find imagedb for project: $project");
+}
+
 #
 # Parse input request file using detect_query_read (as it's already written).
 #
-
 my $dqr_command = "$detect_query_read --input $request_file";
 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     run(command => $dqr_command, verbose => $verbose);
 unless ($success) {
-    my_die("Unable to perform $dqr_command error code: $error_code",-1,-1,-1,-1,-1,-1);
+    # This is a problem, because I'm not sure how we handle a failure to read something.
+    # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 
+    # we can't send that response back.
+    die("Unable to perform $dqr_command error code: $error_code");
 }
 my %query = ();
@@ -120 +132 @@
 my %image_list_hash;
 for (my $i = 1; $i < $Nrows; $i++) {
-#    print "$i $Nrows $query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]\n";
+    # This could use the fact that locate_images now accepts position arrays, but
+    # I'll save that for after I get the majority of things working.
     my $image_set_tmp  = find_image_set($query{HEADER}{FPA_ID}[0],$query{HEADER}{STAGE}[0],
                                         $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],
@@ -126 +139 @@
                                         $query{CONTENT}{ROWNUM}[$i],$verbose);
     unless (%$image_set_tmp) {
-        my_die("No images were returned.",$query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0],
-               $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0],
-               $query{HEADER}{STAGE}[0]);
-    }
-    print "=== $image_set_tmp->{IMAGE}\n    $image_set_tmp->{PSF}\n    $image_set_tmp->{MASK}\n    $image_set_tmp->{WEIGHT}\n    $image_set_tmp->{SKY_COORDINATES}\n    $image_set_tmp->{ROWNUM}\n";
-    # This appends, assuming that if we get an image, we also get the identical psf/mask/weight/etc.
+        # No images were returned, so create a dummy entry that 
+        $image_list_hash{'no_image'}{IMAGE}    = 'no_image';
+        $image_list_hash{'no_image'}{PSF}      = 'no_psf';
+        $image_list_hash{'no_image'}{MASK}     = 'no_mask';
+        $image_list_hash{'no_image'}{WEIGHT}   = 'no_weight';
+        $image_list_hash{'no_image'}{CATALOG}  = 'no_catalog';
+        $image_list_hash{'no_image'}{CLASS_ID} = 'no_class';
+        $image_list_hash{'no_image'}{ERROR}    = $PSTAMP_NO_IMAGE_MATCH;
+        push @{ $image_list_hash{'no_image'}{SKY_COORDINATES} }, "$query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]";
+        push @{ $image_list_hash{'no_image'}{ROWNUM} }, $query{CONTENT}{ROWNUM}[$i];
+        next;
+    }
+#     print "=== $image_set_tmp->{IMAGE}\n    $image_set_tmp->{PSF}\n";
+#     print "    $image_set_tmp->{MASK}\n    $image_set_tmp->{WEIGHT}\n";
+#     print "    $image_set_tmp->{SKY_COORDINATES}\n    $image_set_tmp->{ROWNUM}\n";
+
+    # This indexes the results for identical images into the same hash.
     $image_list_hash{$image_set_tmp->{IMAGE}}{IMAGE}    = $image_set_tmp->{IMAGE};
     $image_list_hash{$image_set_tmp->{IMAGE}}{PSF}      = $image_set_tmp->{PSF};
@@ -138 +162 @@
     $image_list_hash{$image_set_tmp->{IMAGE}}{CATALOG}  = $image_set_tmp->{CATALOG};
     $image_list_hash{$image_set_tmp->{IMAGE}}{CLASS_ID} = $image_set_tmp->{CLASS_ID};
+    $image_list_hash{$image_set_tmp->{IMAGE}}{ERROR}    = $image_set_tmp->{ERROR};
     push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{SKY_COORDINATES} }, $image_set_tmp->{SKY_COORDINATES};
     push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{ROWNUM} }, $image_set_tmp->{ROWNUM};
 }
+
 my $i = 0;
-
-
 foreach my $k (keys %image_list_hash) {
+    # If we errored out on finding an image, we need to not try to run psphot here.
+    if ($image_list_hash{$k}{ERROR} != 0) {
+        next;
+    }
     # Write coordinates of the requested targets to a file.
-    my ($coordfile,$coordname) = tempfile("/tmp/detect.coords.$i.XXXX", 
+    my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$i.XXXX", 
                                             UNLINK => !$save_temps);
-    my ($targetfile,$targetname) = tempfile("/tmp/detect.targets.$i.XXXX", 
+    my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$i.XXXX", 
                                             UNLINK => !$save_temps);
 
@@ -154 +182 @@
         print $coordfile "$image_list_hash{$k}{SKY_COORDINATES}[$j]\n";
     }
-    print "$k\n";
+#    print "$k\n";
     # Convert the sky coordinates to image coordinates with ppCoord.
     my $command = "ppCoord -astrom $image_list_hash{$k}{CATALOG} -radec $coordname";
@@ -172 +200 @@
     }
 
-    print "psphot $image_list_hash{$k}{PSF}\n";
+#     print "psphot $image_list_hash{$k}{PSF}\n";
     # Run psphotForced on the target list.
-    my $tmpdir  = tempdir("detect.$i.XXXX", DIR => "/tmp/", CLEANUP => !$save_temps);
+    my $tmpdir  = tempdir("detect.$i.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps);
     $image_list_hash{$k}{OUTROOT} = "$tmpdir/detectability.$query{HEADER}{STAGE}[0].$query{HEADER}{FPA_ID}[0]";
     
@@ -186 +214 @@
         run(command => $psphot_cmd, verbose => $verbose);
     unless ($success) {
-        my_die("Unable to perform $psphot_cmd. Error_code: $error_code",
-               $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0],
-               $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0],
-               $query{HEADER}{STAGE}[0],$error_code);
+        $image_list_hash{$k}{ERROR} = $PSTAMP_SYSTEM_ERROR;
     }
 }
@@ -197 +222 @@
 #
 my @rownums = ();
+my @out_errors = ();
 my @psphot_Npix = ();
 my @psphot_Qfact= ();
 my @psphot_flux = ();
+
 foreach my $k (keys %image_list_hash) {
-    my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf";
-    
-    my ($tmp_Npix,$tmp_Qfact,$tmp_flux) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE});
-
-    push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
-    push @psphot_Npix,    @{ $tmp_Npix };
-    push @psphot_Qfact,   @{ $tmp_Qfact };
-    push @psphot_flux,    @{ $tmp_flux };
+    if ($image_list_hash{$k}{ERROR} == 0) {
+        my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf";
+        
+        my ($tmp_Npix,$tmp_Qfact,$tmp_flux) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE});
+        
+        push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
+        push @out_errors,     (map { $image_list_hash{$k}{ERROR} }  @{ $image_list_hash{$k}{ROWNUM} });
+        push @psphot_Npix,    @{ $tmp_Npix };
+        push @psphot_Qfact,   @{ $tmp_Qfact };
+        push @psphot_flux,    @{ $tmp_flux };
+    }
+    else {
+        push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
+        push @out_errors,     (map { $image_list_hash{$k}{ERROR} }  @{ $image_list_hash{$k}{ROWNUM} });
+        push @psphot_Npix,    (map { 0 }  @{ $image_list_hash{$k}{ROWNUM} });
+        push @psphot_Qfact,   (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
+        push @psphot_flux,    (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
+    }   
 }
 
@@ -215 +252 @@
                     $query{HEADER}{MJD_OBS}[0],$query{HEADER}{filter}[0],
                     $query{HEADER}{obscode}[0],
-                    \@rownums, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux);
-print "Wrote response file $output\n";
+                    \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux);
+# print "Wrote response file $output\n";
 #
 # Add to datastore
 #
-# my $finish_command = "dquery_finish.pl --req_id $req_id --req_name $req_name --product $product";
-# ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-#     run(command => $finish_command, verbose => $verbose);
-# unless ($success) {
-#     warn ("Unable to perform $finish_command error code: $error_code");
-# }
-
+# Files are added to the datastore by dquery_finish.pl
 #
 # Cleanup
 #
 # Since everything is written to temporary files, there should be nothing to cleanup.
+
 #
 # Utilities
@@ -243 +275 @@
     my $index  = shift;
     my $verbose = shift;
-    
+
+    # This is the set of things that we need in order to run psphotForced
     my $option_mask |= 1;
     $option_mask |= $PSTAMP_SELECT_IMAGE;
@@ -253 +286 @@
     my $mjd_max = $mjd + 1;
 
-    # Construct a row list:
-
+    # Construct a row list. 
     my @rowList;
     $rowList[0]->{CENTER_X} = $ra;
@@ -261 +293 @@
     $rowList[0]->{STAGE} = $stage;
     $rowList[0]->{COORD_MASK} = 0;
-    print "$stage\n";
+
+    #    print "$stage\n";
     # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
-    my @images = locate_images($ipprc,$dbname,
+    my @images = locate_images($ipprc,$imagedb,
                                 \@rowList,
                                "bycoord",$stage,
@@ -284 +317 @@
             }
             elsif ($stage eq 'stack') {
-                # Stacks hide the exposure name very well, so I'm
-                # choosing the stack_id as the FPA_ID to match. This
-                # probably needs to be approved somehow.
+                # Stacks hide the exposure name very well, so we can only match against stage_id
                 if (${ $j }{stage_id} ne $FPA_ID) {
                     next;
@@ -299 +330 @@
             }
             # Debug prints of all the components of this image
-            foreach my $k (keys %{ $j }) {
-                print "$i $j $k ${ $j }{$k}\n";
-            }
-            
+#           foreach my $k (keys %{ $j }) {
+#               if ($k eq 'row_index') {
+#                   print "$i $j $k @{${ $j }{$k} }\n";
+#               }
+#               print "$i $j $k ${ $j }{$k}\n";
+#           }
+
+            # Check for existance of the images. Drawn mostly from pstampparse.pl
+            my $run_state = ${ $j }{state};
+            my $data_state = ${ $j }{data_state};
+            $data_state = $run_state if $stage eq 'stack';
+            my $fault = 0;
+            if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
+                ($run_state eq 'drop') or 
+                ($run_state eq 'error_cleaned') or 
+                ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
+                # image is gone and it's not coming back
+                $fault = $PSTAMP_GONE;
+            }
+            elsif  (($data_state ne 'full') or ($need_magic and (${ $j }{magicked} < 0))) {
+                if (($stage eq 'stack')||($stage eq 'diff')) {
+                    # updating stacks and diffs isn't implemented
+                    $fault = $PSTAMP_NOT_IMPLEMENTED;
+                }
+                if ($stage eq 'chip') {
+                    my $burntool_state = ${ $j }{burntool_state};
+                    if ($burntool_state and (abs($burntool_state) < 14)) {
+                        $fault = $PSTAMP_NOT_AVAILABLE;
+                    }
+                }
+                
+                if ($fault == 0) {
+                    # This bombs us out to dqueryparse, which will then flag a job for this run to be updated.
+                    my_die_for_update($data_state,$query{HEADER}{STAGE}[0],
+                                      ${ $j }{stage_id},${ $j }{class_id} || ${ $j }{skycell_id},
+                                      $need_magic,$imagedb,$PSTAMP_NOT_AVAILABLE);
+                }
+            }
+
             # This image matches, so we want to save the information into our output structure
             $image_info{ROWNUM} = $index;
@@ -309 +375 @@
             $image_info{MASK}   = ${ $j }{mask};
             $image_info{WEIGHT} = ${ $j }{weight};
+            $image_info{ERROR}  = $fault;
             $image_info{SKY_COORDINATES} = "$ra $dec";
             # To do sky->image coordinate transformations, we need to use the cmf/smf file. If 
@@ -419 +486 @@
     my $obscode = shift;
     my $rownum_ref = shift;
+    my $out_err_ref = shift;
     my $psphot_Npix_ref = shift;
     my $psphot_Qfact_ref = shift;
@@ -427 +495 @@
     my $columns = [
         # matching rownum from detectability original request
-        { name => 'ROWNUM',   type => '20A', writetype => TSTRING }, 
+        { name => 'ROWNUM',   type => 'V', writetype => TULONG }, 
+        # any errors that occurred during processing
+        { name => 'ERROR_CODE',   type => 'V', writetype => TULONG }, 
         # number of pixels used in hypothetical PSF for the query detection
         { name => 'DETECT_N', type => 'V',   writetype => TULONG },
@@ -469 +539 @@
     for (my $i = 0; $i < $numRows; $i++) {
         push @{$colData{'ROWNUM'}},      ${ $rownum_ref }[$i];
+        push @{$colData{'ERROR_CODE'}},  ${ $out_err_ref }[$i];
         push @{$colData{'DETECT_N'}},    ${ $psphot_Npix_ref }[$i];
         push @{$colData{'DETECT_F'}},    ${ $psphot_Qfact_ref }[$i];
@@ -543 +614 @@
 
 }
+
+sub my_die_for_update {
+    my $state = shift;
+    my $stage = shift;
+    my $stage_id = shift;
+    my $component = shift;
+    my $need_magic = shift;
+    my $imagedb = shift;
+    my $exit_code = shift;
+
+    print "$state $stage $stage_id $component $need_magic $imagedb\n";
+    print STDERR "$state $stage $stage_id $component $need_magic $imagedb\n";
+    exit($exit_code);
+}

trunk/pstamp/scripts/dquery_finish.pl

@@ -119 +119 @@
 # XXX: have the jobs produce the reglist as with postage stamp requests
 my ($REGLIST, $reg_list) = tempfile("$out_dir/reqlist.XXXX", UNLINK => !$save_temps);
+
 foreach my $job (@jobs) {
     my $job_id = $job->{job_id};
@@ -127 +128 @@
         # the job generated a response file put it into the Data Store
         print $REGLIST "$response_file|||table|\n";
+
+        # do the same if we have an error file.  Should the parse data be uploaded as well?
+        my $err_file = "parse_error.txt";
+        if (-e "$out_dir/$err_file") {
+            print $REGLIST "$err_file|||text|\n";
+        }
     } else  {
         print STDERR "detect_query response file for job $job_id not found\n" if $verbose;
@@ -138 +145 @@
     $command .= " --copy --datapath $out_dir";
     $command .= " --type MOPS_DETECTABILITY_RESPONSE";
+    $command .= " --ps0 $req_id";
     $command .= " --dbname $dbname" if $dbname;
 

trunk/pstamp/scripts/dqueryparse.pl

@@ -2 +2 @@
 #
 # parse a MOPS_DETCTABILITY_QUERY table and create a results file
-#
-# Note: this file is currently only a placeholder which creates a fake response file
-# and adds a completed job to the database
 #
 
 use strict;
 use warnings;
-
+use Carp;
 use Getopt::Long qw( GetOptions );
 use Pod::Usage qw( pod2usage );
 use IPC::Cmd 0.36 qw( can_run run );
 
+use PS::IPP::PStamp::RequestFile qw( :standard );
+use PS::IPP::PStamp::Job qw( :standard );
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
                        $PS_EXIT_UNKNOWN_ERROR
@@ -26 +25 @@
                        );
 
+my ($no_update, $imagedb, $label);
 my ($req_file, $req_id, $out_dir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps);
-
+my ($job_id,$rownum); # stuff from the post-update world
 #
 # parse args
@@ -35 +35 @@
         'file=s'          =>      \$req_file,
         'req_id=s'        =>      \$req_id,
+        'job_id=s'        =>      \$job_id,
+        'rownum=s'        =>      \$rownum,
         'out_dir=s'       =>      \$out_dir,
+        'label=s'         =>      \$label,
         'product=s'       =>      \$product,
         'mode=s'          =>      \$mode,
@@ -44 +47 @@
 ) or pod2usage(2);
 
-my $err = "";
-
-if (!$req_file) {
-    $err .= "--file is required\n";
-}
-if (!$req_id) {
-    $err .= "--req_id is required\n";
-}
-if (!$out_dir) {
-    $err .="--out_dir is required\n";
-}
-if (!$product) {
-    $err .="--product is required\n";
-}
-
-
-die $err if ($err);
-
+die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
+die "--file or --job_id is required" if !$req_file;
+
+if ($mode ne "list_uri") {
+    die "req_id is required" if !$req_id;
+    die "out_dir is required" if !$out_dir;
+    die "product is required" if !$product;
+}
 my $missing_tools;
 my $pstamptool = can_run('pstamptool') or (warn "Can't find pstamptool" and $missing_tools =1);
-my $fakedresponse = can_run('fakedresponse.pl') or (warn "Can't find fakedresponse.pl" and $missing_tools =1);
+my $detectresponse = can_run('detectability_respond.pl') or 
+    (warn "Can't find detectability_respond.pl" and $missing_tools = 1);
 my $fields = can_run('fields') or (warn "Can't find fields" and $missing_tools =1);
 
@@ -77 +71 @@
 }
 
+# just deal with these arguments once and for all
+$pstamptool .= " -dbname $dbname" if $dbname;
+$pstamptool .= " -dbserver $dbserver" if $dbserver;
+$detectresponse .= " --dbname $dbname" if $dbname;
+
+$no_update = 1 if $mode eq "list_job";
+
+# Unless we're running as a job, write the parse arguments in case we need to rerun this parsing.
+if (!$job_id) {
+    my $argslist = "$out_dir/parse.args";
+    open ARGSLIST, ">$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+    print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --out_dir $out_dir --file $req_file\n";
+    close ARGSLIST;
+}
+
 # get the query id and check the extname and version from the header
-my $fields_output;
+my $fields_output; 
 {
     my $command = "echo $req_file | $fields -x 0 EXTNAME EXTVER QUERY_ID";
@@ -84 +93 @@
         run(command => $command, verbose => $verbose);
     
-#   fields doesn't return zero when it succeeds
-#    unless ($success) {
-#        print STDERR @$stderr_buf;
-#    }
     $fields_output = join "", @$stdout_buf;
 }
 my (undef, $extname, $extver, $req_name) = split " ", $fields_output;
 
-die "$req_file is missing one of EXTNAME EXTVER or QUERY_ID" 
+my_die("$req_file is missing one of EXTNAME EXTVER or QUERY_ID", $PS_EXIT_PROG_ERROR)
     if !(defined($extname) and defined($extver) and defined($req_name));
-
-die "$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table"
-                if $extname ne         "MOPS_DETECTABILITY_QUERY";
-die "$req_file is version $extver expecting 1" if $extver ne 1;
-
-if (0) {
-$out_dir .= "/$req_name";
-}
+my_die("$req_file has EXTNAME $extname not MOPS_DETECTABILITY_QUERY table", $PS_EXIT_PROG_ERROR)
+    if $extname ne "MOPS_DETECTABILITY_QUERY";
+my_die("$req_file is version $extver expecting 1", $PS_EXIT_PROG_ERROR)
+    if $extver ne 1;
+
+# Set up the workdir for this query.
 if (! -e $out_dir ) {
-    mkdir $out_dir or die "cannot create output directory $out_dir";
+    mkdir $out_dir or my_die("cannot create output directory $out_dir", $PS_EXIT_PROG_ERROR);
 } elsif (! -d $out_dir ) {
-    die "output fileset directory $out_dir exists but is not a directory";
-}
-
-#
-# XXX Eventually we will parse the file here, to look up the list of input images to be processed
-# and queue a job for each image.
-# The request file will be one of the arguments. Each job will look at all rows and create entries
-# in the response file the coordinates that overlap the image.
-#
-# In the meantime we don't parse the file here, pass it to the program fakedresponse.
-# It creates a response file with a line for each row in the request file
-
-my $response_file = "$out_dir/response.fits";
+    my_die ("output fileset directory $out_dir exists but is not a directory", $PS_EXIT_PROG_ERROR);
+}
+
+
+# Pass along the request file to the response generator.
+my $response_file = "$out_dir/${req_name}.dresponse.${req_id}.fits";
 my $fault;
+my $data_to_update = '';
 {
-    my $command = "$fakedresponse --input $req_file --output $response_file --workdir $out_dir";
+    my $command = "$detectresponse --input $req_file --output $response_file --workdir $out_dir";
     $command .= " --save-temps" if $save_temps;
     $command .= " --verbose" if $verbose;
@@ -127 +124 @@
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        warn("Unable to perform $command error code: $error_code");
-    }
+        warn("Warn! Unable to perform $command error code: $error_code");
+    }
+    # Use the fault code to see if we can regenerate the missing data.
     $fault = $error_code >> 8;
-}
-
-my $job_id;
+    if ($fault == $PSTAMP_NOT_AVAILABLE) {
+        $data_to_update = (split /\n/, (join "", @$stdout_buf))[-1];    
+    }   
+}
+
 my $result;
-{
+# If we returned correctly with a valid response file, get a job ID
+# for the completed work, and move the response to a standardized name.
+if ($fault == 0) {
     my $command = "$pstamptool -addjob -req_id $req_id -outputBase $out_dir"; 
-    $command .= " -job_type detect_query -state stop -fault $fault";
+    $command .= " -job_type detect_query -state stop -fault 0";
     $command .= " -rownum 1";
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
     if ($success) {
         $job_id = join "", @$stdout_buf;
@@ -156 +155 @@
     }
 }
-
-
+else {
+    # Failed to run correctly, which means that we need to queue a job and flag data for updating.
+    # Get the dependency id for the data we're requesting be updated.
+    my $dep_id = queue_update_run($req_id,$job_id,$out_dir,$label,$data_to_update);
+
+    # Link this request to a job and link that job to any dependency
+    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $out_dir"; 
+    $command .= " -job_type detect_query -state run -fault 0";
+    $command .= " -rownum 1";
+    $command .= " -dep_id $dep_id" if $dep_id;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+
+    if ($success) {
+        $job_id = join "", @$stdout_buf;
+        chomp $job_id;
+        if ($job_id && -e $response_file) {
+            # We shouldn't have a response file at this stage.
+            rename $response_file, "$out_dir/response${job_id}.fits";
+        }
+        $result = 0;
+    } else {
+        warn("Unable to perform $command error code: $error_code");
+        $result = $error_code >> 8;
+    }
+}
+
+# This does not set the request state to stop.  That will happen with the request_finish.pl script, 
+# which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
 {
     my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name -outProduct $product";
     $command .= " -fault $result" if $result;
-    $command .= " -dbname $dbname" if $dbname;
-    $command .= " -dbserver $dbserver" if $dbserver;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
     unless ($success) {
-        die "$command failed";
+        my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
     }
 }
  
 exit 0;
+
+
+# If we have to queue an update run, do so and create a new dependent
+sub queue_update_run {
+    my ($req_id, $job_id, $out_dir, $label, $data_to_update) = @_;
+
+    my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update;
+    
+    if (($state ne 'cleaned') and ($state ne 'update') and ($state ne 'goto_cleaned')) {
+        # We should have received one of these states, so if not, signal that we have a problem.
+        my_die("$stage $stage_id is in unexpected state $state", $PS_EXIT_PROG_ERROR);
+    }
+    my $dep_id;
+    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component ";
+    $command .= " -outdir $out_dir";
+    $command .= " -need_magic" if $need_magic;
+
+    my $rlabel = "dq_ud_" . $label if $label;
+    $command .= " -rlabel $rlabel" if $rlabel;
+
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
+        }
+        my $output = join "", @$stdout_buf;
+        chomp $output;
+        $dep_id = $output;
+
+        my_die("pstamptool -getdependent returned invalid dep_id", $PS_EXIT_PROG_ERROR) if !$dep_id;
+    } 
+    else {
+        print STDERR "skipping $command\n";
+        $dep_id = 42;
+    }
+
+    return($dep_id);
+}
+        
+    
+
+sub my_die {
+    my $msg = shift;
+    my $fault = shift;
+
+    carp $msg;
+
+    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
+    exit $fault;
+}

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -74 +74 @@
 my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
 my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
+my $dqueryparse = can_run('dqueryparse.pl') or (warn "Can't find dqueryparse.pl" and $missing_tools = 1);
 
 if ($missing_tools) {
@@ -165 +166 @@
     }
 } elsif ($jobType eq "detect_query") {
-    my_die("detect_query jobs not supported yet", $job_id,$PS_EXIT_CONFIG_ERROR);
+    # detect_query jobs are basically holders to note that we need a file updated before we can continue.
+    # Load the argument list that dqueryparse should have created the first time it ran, so we know how to
+    # run it again the same way.
+    my $argslist = "$outputBase/parse.args";
+    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
+    my $argString = <ARGSLIST>;
+    close ARGSLIST;
+    chomp $argString;
+
+    # XXX: should we do any other sanity checking?
+    my_die("arglist file $argslist is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
+
+    my $command = "$dqueryparse --job_id $job_id $argString";
+    $command .= " --dbname $dbname" if $dbname;
+    $command .= " --dbserver $dbserver" if $dbserver;
+    $command .= " --verbose" if $verbose;
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+
+    # Although dqueryparse can potentially force more updates (and more jobs), it should still return success.
+    if ($success) {
+        $jobStatus = $PS_EXIT_SUCCESS;
+    } else {
+        $jobStatus = $error_code >> 8;
+        my_die("dqueryparse.pl failed with error code: $jobStatus", $job_id, $jobStatus);
+    }
 } else {
     my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR);

trunk/pstamp/scripts/pstamp_parser_run.pl

@@ -152 +152 @@
         } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {
             $reqType = 'dquery';
-            $parse_cmd = $dqueryparse;
+            $parse_cmd = "$dqueryparse";
+            $parse_cmd .= " --label $label" if $label;
         } else {
             print STDERR "Unknown request type $request_type found in $uri";