Changeset 28980 for branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read

Timestamp:

Aug 19, 2010, 6:50:13 PM (16 years ago)

Author:

eugene

Message:

updates from trunk

File:

• branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read (modified) (8 diffs)

branches/eam_branches/ipp-20100621/pstamp/scripts/detect_query_read

@@ -20 +20 @@
 use Math::Trig;
 use Data::Dumper;
+use IPC::Cmd 0.36 qw( can_run run );
 
 use constant EXTNAME => 'MOPS_DETECTABILITY_QUERY'; # Extension name for table
@@ -31 +32 @@
      $output,                   # Name of output table
      $save_temps,               # Save temporary files?
+     $dbname,                   # needed to do camtool lookups.
      );
+
+my $regtool = can_run('regtool') or (die "Can't find regtool");
+my $camtool = can_run('camtool') or (die "Can't find regtool");
+
 
 GetOptions(
            'input|i=s'    => \$input,
            'output|o=s'   => \$output,
+           'dbname=s'     => \$dbname,
            'nolabel|l'    => \$no_print_label,
            'noheader|h'   => \$no_print_header,
@@ -65 +72 @@
 
 # The keywords found in the header
-# my $header = {
-#         'EXTVER'   => { name => 'EXTVER',
-#                      writetype => TSTRING,
-#                      comment => 'Extension version',
-#                      value => undef
-#                       },
-#         'EXTNAME'  => { name => 'EXTNAME',
-#                      writetype => TSTRING,
-#                      comment => 'name of this binary table extension',
-#                      value => undef
-#                     },
-#         'QUERY_ID' => { name => 'QUERY_ID', 
-#                         writetype => TSTRING, 
-#                         comment => 'MOPS Query ID for this batch query',
-#                         value => undef
-#                       },
-#         'FPA_ID'   => { name => 'FPA_ID', 
-#                         writetype => TSTRING, 
-#                         comment => 'orginal FPA_ID used at ingest',
-#                         value => undef
-#                       }, 
-#         'MJD_OBS'  => { name => 'MJD-OBS', 
-#                         writetype => TDOUBLE, 
-#                         comment => 'starting time of the exposure, MJD',
-#                         value => undef
-#                       },
-#         'FILTER'   => { name => 'FILTER', 
-#                         writetype => TSTRING, 
-#                         comment => 'effective filter use for the exposure',
-#                         value => undef
-#                       },
-#         'OBSCODE'  => { name => 'OBSCODE', 
-#                         writetype => TSTRING, 
-#                         comment => 'site identifier (MPC observatory code)',
-#                         value => undef
-#                       },
-#       'STAGE'    => { 
-#                       name => 'STAGE',
-#                       writetype => TSTRING,
-#                       comment => 'processing stage to examine',
-#                       value => undef
-#                     }
-# };
 
 my $parse_error = 0;
-# Parse the header to determine what we expect to find.
+# Parse the header to determine what we have to work with.
+# Clean up empty space and remove unneeded single quotes.
 foreach my $header_key (keys %{ $inHeader }) {
     $inHeader->{$header_key} =~ s/\s+//g;
     $inHeader->{$header_key} =~ s/\'//g;
 }
-my ($EXTVER,$headerEXTNAME,$QUERY_ID,$FPA_ID,$MJD_OBS,$FILTER,$OBSCODE,$STAGE) = 
-    ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},$inHeader->{FPA_ID},
-     $inHeader->{'MJD-OBS'},$inHeader->{FILTER},$inHeader->{OBSCODE},$inHeader->{STAGE});
+my ($EXTVER,$headerEXTNAME,$QUERY_ID,
+    $FPA_ID,$MJD_OBS,$FILTER,
+    $OBSCODE,$STAGE) = 
+    ($inHeader->{EXTVER},$inHeader->{EXTNAME},$inHeader->{QUERY_ID},
+     $inHeader->{FPA_ID},$inHeader->{'MJD-OBS'},$inHeader->{FILTER},
+     $inHeader->{OBSCODE},$inHeader->{STAGE});
+
 unless(defined($EXTVER) && defined($headerEXTNAME) &&
        (($EXTVER == 1)||($EXTVER == 2)) && 
-       ($headerEXTNAME eq EXTNAME)) {
+       ($headerEXTNAME eq EXTNAME)&&
+       (defined($QUERY_ID))) {
     $parse_error = 1;
 }
@@ -143 +113 @@
         # apparent magnitude
         { name => 'MAG',      type => 'D',   writetype => TDOUBLE, version => 1 },
-    # v2 query_id: MOPS query ID for this batch query
+    # v2 query_id: needs to be here.
     { name => 'QUERY_ID', type => '20A', writetype => TSTRING, version => 2, default => $QUERY_ID },
     # v2 fpa_id: original FPA_ID used at ingest
@@ -172 +142 @@
 
     if ($col->{version} > $EXTVER) {
-        @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows);
+        @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1);
         next;
     }
-    $inFits->get_colnum(0, $col->{name}, $col_num, $status) and check_fitsio($status);
-    $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
-    $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status)
-                                                                    and check_fitsio($status);
-    $colData{$col->{name}} = $col_data;
-    if ($col->{name} eq 'MJD-OBS') {
-        print @{ $col_data } . "\n";
+    else {
+        if (defined($col->{default})) {
+            @{ $colData{$col->{name}} } = map { $col->{default} } (0 .. $numRows - 1);
+            next;
+        }        
+    }   
+
+    $inFits->get_colnum(0, $col->{name}, $col_num, $status);
+    if ($status == 0) {
+        $inFits->get_coltype($col_num, $col_type, undef, undef, $status) and check_fitsio($status);
+        $inFits->read_col($col_type, $col_num, 1, 1, $numRows, 0, $col_data, undef, $status)
+            and check_fitsio($status);
+        $colData{$col->{name}} = $col_data;
+        if ($col->{name} eq 'QUERY_ID') {
+            print @{ $col_data } . "\n";
+        }
+
+    }
+    elsif ($status == 219) {
+        @{ $colData{$col->{name}} } = map { "Not Set" } (0 .. $numRows - 1);
+        $status = 0;
+    }
+}
+
+# Set things that aren't set.
+for (my $i = 0; $i < $numRows; $i++) {
+# Simple stuff first.
+    my %known_filters = ();
+    if ($colData{STAGE}[$i] eq "Not Set") {
+        $colData{STAGE}[$i] = 'diff';
+    }
+    if ($colData{OBSCODE}[$i] eq "Not Set") {
+        $colData{OBSCODE}[$i] = 566;
+    }
+# Define filter and MJD from FPA_ID
+    if (($colData{FILTER}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
+        if (exists($known_filters{$colData{FPA_ID}[$i]})) {
+            $colData{FILTER}[$i] = $known_filters{$colData{FPA_ID}[$i]};
+        }
+        else {
+            my $cmd = "$regtool -processedexp -dbname $dbname -exp_name $colData{FPA_ID}[$i]";
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                run(command => $cmd, verbose => 0);
+            unless ($success) {
+                # This is a problem, because I'm not sure how we handle a failure to read something.
+                # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 
+                # we can't send that response back.
+                die("Unable to perform $cmd error code: $error_code");
+            }
+            foreach my $entry (split /\n/, (join "", @$full_buf)) {
+                $entry =~ s/^\s+//;
+                my @line = split /\s+/, $entry;
+                if (scalar(@line) != 3) {
+                    next;
+                }
+                my ($key,$type,$value) = @line;
+#               print "$entry => $key / $type / $value \n";
+                if ($key =~ /filter/i) {
+                    $value =~ s/.00000//;
+                    $colData{FILTER}[$i] = $value;
+                    $known_filters{$colData{FPA_ID}[$i]} = $value;
+                }
+            }
+        }
+    }
+    if (($colData{'MJD-OBS'}[$i] eq "Not Set")&&($colData{FPA_ID}[$i] ne "Not Set")) {
+        # HACK!
+        my $mjd = $colData{FPA_ID}[$i];
+        $mjd =~ s/o(....)g.*/$1/;
+        $mjd += 50000;
+        $colData{'MJD-OBS'}[$i] = $mjd;
+    }
+    if (($colData{FPA_ID}[$i] eq "Not Set")&&(($colData{'FILTER'}[$i] ne "Not Set")&&
+                                              ($colData{'MJD-OBS'}[$i] ne "Not Set"))) {
+        my $dateobs_begin = mjd_to_dateobs($colData{'MJD-OBS'}[$i]);
+        my $dateobs_end = mjd_to_dateobs($colData{'MJD-OBS'}[$i] + 1);
+        my $ra = $colData{'RA1_DEG'}[$i];
+        my $dec = $colData{'DEC1_DEG'}[$i];
+        my $filter = $colData{'FILTER'}[$i] . ".00000";
+        my $cmd = "$camtool -processedexp -dbname $dbname -filter $filter ";
+        $cmd .= " -dateobs_begin $dateobs_begin -dateobs_end $dateobs_end ";
+        $cmd .= " -ra $ra -decl $dec -radius 1.5 -limit 1";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $cmd, verbose => 0);
+        unless ($success) {
+            # This is a problem, because I'm not sure how we handle a failure to read something.
+            # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 
+            # we can't send that response back.
+            die("Unable to perform $cmd error code: $error_code");
+        }
+        foreach my $entry (split /\n/, (join "", @$full_buf)) {
+            $entry =~ s/^\s+//;
+            my @line = split /\s+/, $entry;
+            if (scalar(@line) != 3) {
+                next;
+            }
+            my ($key,$type,$value) = @line;
+#               print "$entry => $key / $type / $value \n";
+            if ($key =~ /exp_name/i) {
+                $colData{FPA_ID}[$i] = $value;
+            }
+        }
     }
 }
@@ -189 +254 @@
 foreach my $colName (@unique_fields) {
     my %counter = ();
+    my $i = 0;
     foreach my $row (@{ $colData{$colName} }) {
         $counter{$row} = 1;
@@ -232 +298 @@
         my $msg;                # Message to output
         Astro::FITS::CFITSIO::fits_get_errstatus( $status , $msg );
-        die "CFITSIO error: $msg\n";
+        die "CFITSIO error: $status => $msg\n";
     }
 }
@@ -252 +318 @@
 }
 
+# Stolen from PStamp/Job.pm.  Thanks, Bill.
+sub mjd_to_dateobs {
+    my $mjd = shift;
+
+    my $ticks = ($mjd - 40587.0) * 86400;
+
+    my ($sec, $min, $hr, $day, $mon, $year) = gmtime($ticks);
+
+    return sprintf "'%4d-%02d-%02dT%02d:%02d:%02dZ'", $year+1900, $mon+1, $day, $hr, $min, $sec;
+}
+