Changeset 19265

Timestamp:

Aug 28, 2008, 2:06:45 PM (18 years ago)

Author:

bills

Message:

remove reference to obsolete program

Location:

trunk/pstamp/scripts

Files:

• pstamp_finish.pl (modified) (7 diffs)
• pstamp_parser_run.pl (modified) (5 diffs)
• pstamp_queue_requests.pl (modified) (4 diffs)
• pstampparse.pl (modified) (1 diff)

trunk/pstamp/scripts/pstamp_finish.pl

@@ -9 +9 @@
 use Pod::Usage qw( pod2usage );
 
+use Time::Local;
 use Sys::Hostname;
 use IPC::Cmd 0.36 qw( can_run run );
@@ -82 +83 @@
     }
 
-
-                        # this function is PStamp::RequestFile::read_request_file
+    if (! -e $req_file ) {
+        print STDERR "request file $req_file is missing\n";
+        stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname);
+        exit  $PS_EXIT_CONFIG_ERROR;
+    }
+
+    # this function is PStamp::RequestFile::read_request_file
     my ($header, $rows) = read_request_file($req_file);
 
-    die "failed to read request_file $req_file" if !$header or !$rows;
+    if (!$header or !$rows) {
+        # Since a request got queued, the request file must have been readable at some
+        # point 
+        print STDERR "failed to read request_file $req_file" 
+        stop_request($req_id, $PS_EXIT_CONFIG_ERROR, $dbname);
+        exit  $PS_EXIT_CONFIG_ERROR;
+    }
 
     # at this point we need to find out what kind of request type it is and
@@ -144 +156 @@
         my $rownum = $job->{rownum};
         my $fault = $job->{fault};
-
-        my $row_info = get_request_info($rows, $rownum);
-        # rownum fault img_name job_id
+        my $exp_id = $job->{exp_id};
+
+        # XXX: get the image_db from a table in the database, or maybe save it in the job
+        my $image_db = "ps_simtest";     
+        my $exp_info = get_exposure_info($image_db, $exp_id);
+
+        # unless $exp_id is null (e.g. stack images) get the metadata for this exposure
+        my ($row, $req_info) = get_request_info($rows, $rownum) if $exp_id;
+
         if (($job_type eq "stamp") || ($job_type eq "get_image")) {
             my $jreglist = "$out_dir/reglist$job_id";
@@ -158 +176 @@
                     my ($img_name, undef) = split '\|', $line;
 
-                    # XXX TODO: need to get these from somewhere. Maybe the stamp.
-                    #
-                    # ra_deg and dec_deg are the coordinates of center of the stamp
-                    # the rest of the metadata come from the original FPA.
-                    # we could get these out of the rawExp table in the database....
-                    #
-                    # Much of those data do not apply to stamps from Stack images.
-                    my ($ra_deg, $dec_deg, $mjd_obs, $ra_obs, $dec_obs, $filter, $exp_time, $fpa_id) =
-                        (0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 );
-
                     # add line to the table definition file
                     print $tdf "$rownum|$fault|$img_name|$job_id|";
-                    print $tdf "$ra_deg|$dec_deg|$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id|";
-                    print $tdf "$row_info";
+
+                    # ra_deg and dec_deg are the coordinates of center of the stamp
+                    #
+                    # XXX  we need to get the center coordinate of the stamp.
+                    # if request used -skycenter then we have it in the request info
+
+                    print $tdf "0.0|0.0|";
+                    print $tdf "$exp_info|";
+                    print $tdf "$req_info|";
                     print $tdf "\n";
                 }
@@ -178 +193 @@
                 print STDERR "no reglist file for job $job_id\n" if $verbose;
                 print $tdf "$rownum|$fault|0|$job_id|";
-                print $tdf "0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|0|\n";
+                print $tdf "0|0|";       # center of (non-existent) stamp
+                print $tdf "$exp_info|";
+                print $tdf "$req_info|";
             }
         } else {
             # XXX do list jobs
+            # we can probably arange things to use the code as above and skip the fileset registration
             print STDERR "Unknown jobType: $job_type";
             next;
@@ -226 +244 @@
     }
 }
+
 sub stop_request {
     my $req_id = shift;
@@ -252 +271 @@
     $rowinfo   .= "$row->{ID}|$row->{CLASS_ID}|$row->{OPTION_MASK}|$row->{MJD_MIN}|$row->{MJD_MAX}|";
     $rowinfo   .= "$row->{REQFILT}|$row->{COORD_MASK}|$row->{CENTER_X}|$row->{CENTER_Y}|";
-    $rowinfo   .= "$row->{WIDTH}|$row->{HEIGHT}|";
-
-    return $rowinfo;
-}
+    $rowinfo   .= "$row->{WIDTH}|$row->{HEIGHT}";
+
+    return ($row, $rowinfo);
+}
+
+sub get_exposure_info {
+    my $image_db= shift;
+    my $exp_id = shift;
+
+    if (!$exp_id) {
+        # no exposure id just return zeros
+        # XXX: we could put a value in for filter, but we don't have a good place to find it
+
+        #"$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
+        return "0|0|0|0|0|0";
+    }
+
+    my $regtool = can_run('regtool') or (warn "Can't find regtool" and $missing_tools = 1);
+    if ($missing_tools) {
+        warn("Can't find required tools.");
+        exit ($PS_EXIT_CONFIG_ERROR);
+    }
+
+    my $command = "$regtool -processedexp -dbname $image_db -exp_id $exp_id";
+
+    # run the tool and parse the output
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $command, verbose => $verbose);
+    unless ($success) {
+        # not sure if we should die here
+        die "cannot get exposure information for $exp_id from image database $image_db";
+    }
+    my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+    my $output = join "", @$stdout_buf;
+    if (!$output) {
+        print STDERR "no output returned from $command\n" if $verbose;
+        return undef;
+    }
+    my $metadata = $mdcParser->parse($output) or die("Unable to parse metdata config doc");
+
+    my $exposures = parse_md_list($metadata);
+    my $numExp = @$exposures;
+
+    die "unexpected number of exposures $numExp found for exp_id: $exp_id in DB: $image_db" if $numExp != 1;
+    
+    my $exp = $exposures->[0];
+
+    #my $info = "$mjd_obs|$ra_obs|$dec_obs|$filter|$exp_time|$fpa_id";
+
+    use constant RADIANS_TO_DEGREES => 90. / atan2(1, 0);
+    my $ra_deg   = $exp->{ra} * RADIANS_TO_DEGREES;
+    my $decl_deg = $exp->{decl} * RADIANS_TO_DEGREES;
+    my $mjd_obs = dateobs_to_mjd($exp->{dateobs}, $exp->{exp_time});
+
+    my $info = "$mjd_obs|$ra_deg|$decl_deg|$exp->{filter}|$exp->{exp_time}|$exp->{exp_name}";
+            
+    return $info;
+}
+
+sub dateobs_to_mjd {
+    my $dateobs = shift;
+    my $exp_time = shift;
+
+    # dateobs is in format: 1970-01-01T00:00:00
+
+    my ($date, $time) = split "T", $dateobs;
+    my ($year, $mon, $day) = split "-", $date;
+    my ($hr, $min, $sec) = split ":", $time;
+
+    my $ticks = timegm($sec, $min, $hr, $day, $mon-1, $year-1900);
+
+    $ticks += $exp_time / 2.0;
+
+    return 40587.0 + ($ticks/86400.);
+}

trunk/pstamp/scripts/pstamp_parser_run.pl

@@ -3 +3 @@
 ### pstampparser_run.pl
 ###     Run the request parser for a given request id
+### This script should be called request_parser.pl since it handles more than postage
+### stamp requests
 ###
-
-#XXX see notes about error handling
 
 use warnings;
@@ -172 +172 @@
     if (! -d $outProductDir);
 
-
-# run the appropriate parse command to parse the queue the jobs for this request
-# first check the extension header to find the EXTNAME
-my $request_type = find_request_type($uri);
-
-print STDERR "request_type for $req_id is $request_type\n" if $verbose;
-
 my $parse_cmd;
+my $request_type;
 my $reqType;    # for the database
-if ($request_type eq "PS1_PS_REQUEST") {
-    $reqType = 'pstamp';
-    $parse_cmd = $pstampparse;
-} elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {
-    $reqType = 'dquery';
-    $parse_cmd = $dqueryparse;
+
+if (-r $uri) {
+    # run the appropriate parse command to parse the queue the jobs for this request
+    # first check the extension header to find the EXTNAME
+    $request_type = find_request_type($uri);
+
+    if ($request_type) {
+        print STDERR "request_type for $req_id is $request_type\n" if $verbose;
+        if ($request_type eq "PS1_PS_REQUEST") {
+            $reqType = 'pstamp';
+            $parse_cmd = $pstampparse;
+        } elsif ($request_type eq "MOPS_DETECTABILITY_QUERY") {
+            $reqType = 'dquery';
+            $parse_cmd = $dqueryparse;
+        } else {
+            print STDERR "Unknown request type $request_type found in $uri";
+        }
+    } else {
+        print STDERR "No EXTNAME found keyword in $uri";
+    }
+} else {
+    if (-e $uri) {
+        print STDERR "Request file $uri is not readable";
+    } else {
+        print STDERR "Request file $uri does not exist";
+    }
 }
 
 if (!$parse_cmd) {
-    print STDERR "No EXTNAME found in $uri" if !$request_type;
-    print STDERR "Unknown request type $request_type found in $uri";
+    # can't go any farther, set fault and set request state to run 
+    # request_finish.pl will clean up, perhaps notifiying the operator of the input data store
+    # that they sent us a request file that we don't understand
 
     my $command = "$pstamptool -updatereq -req_id $req_id -state run";
@@ -211 +226 @@
 {
     my $error_file_name = "$workdir/parse_error.txt";
-    # get rid of any error file from previous run
+    # get rid of any error file from previous attempt to parse this request
     unlink $error_file_name if (-e $error_file_name);
 
@@ -219 +234 @@
 
     # save the contents of stderr (if any) to a file. This is relevant if
-    # the file parsed properl but one of the rows in the request file generated an error
+    # the file was parseable but one or more of the rows in the request file generated an error
     my $errbuf = join "", @$stderr_buf;
     if ($errbuf) {
@@ -258 +273 @@
 exit 0;
 
+
 sub find_request_type {
     # find the EXTNAME in the input fits table
-    # TODO: do this right handling errors etc.
     my $file_name = shift;
     my $out = `echo $file_name | fields -x 0 EXTNAME`;
 
-    # output from fields is filename value
-    my ($dummy, $extname) = split " ", $out;
-
-    return $extname;
-}
+    if ($out) {
+        # output from fields is filename value
+        my ($dummy, $extname) = split " ", $out;
+
+        return $extname;
+    } else {
+        return undef;
+    }
+}

trunk/pstamp/scripts/pstamp_queue_requests.pl

@@ -22 +22 @@
     'verbose'       =>  \$verbose,
     'dbname=s'      =>  \$dbname,
-    'limit=s'       =>  \$limit,
+    'limit=i'       =>  \$limit,
 );
 
@@ -119 +119 @@
         my $out_buf = join("", @$stdout_buf);
         # split raw output into lines
-        # XXX: why do I think that i need the map?
-        #my @lines = map {split "\n"} $raw_output;
         @lines = split /^/, $out_buf;
     }
@@ -127 +125 @@
     # each line contains a fileset
     #
-    # XXX: we probably should set a limit on the number of these we process
-    # otherwise this could take a long time and the task could time out
+
+    # number that we've processed
+    my $numFilesets = 0;
     foreach my $line (@lines) {
         # parse the line into fields split by whitespace
         my ($uri, $fs_name, $date, $type) = split " ", $line;
 
+        # skip comment lines
         next if ( $uri =~ /^#.*/);
+
+        $numFilesets++;
 
         my @files;
@@ -163 +165 @@
                 my $command = "$pstamptool -addreq -uri $req_uri -ds_id $ds_id";
                 $command .= " -dbname $dbname" if $dbname;
+
                 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                         run(command => $command, verbose => $verbose);
-                    unless ($success) {
-                        # XXX: what do we do now?
-                        die("Unable to perform $command: $error_code");
-                    }
+
+                unless ($success) {
+                    die("Unable to perform $command: $error_code");
+                }
             }
-            $lastFileset = $fs_name;
         }
+        $lastFileset = $fs_name;
+        last if ($numFilesets >= $limit);
     }
 
     ## now update the last_fileset column in pstampDataStore
-    ## XXX: we should probably do this before we process the fileset. Otherwise 
-    ## if we get an error from pstamptool we'll get stuck processing it over and over.
+    ## XXX: we should perhaps do this while processing each fileset so that if a later
+    ## one has an error we don't get repeats.
+
     if ($lastFileset) {
         # print "last fileset: $lastFileset\n";

trunk/pstamp/scripts/pstampparse.pl

@@ -53 +53 @@
 
 my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
-my $locateimages = can_run('locateimages') or (warn "Can't find locateimages" and $missing_tools = 1);
 my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
 my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);