Changeset 8236

Timestamp:

Aug 8, 2006, 2:54:43 PM (20 years ago)

Author:

Paul Price

Message:

Fixed to do phase 0 processing on the exposure level, including
getting the backgrounds (measured at the imfile level) from the
database.

File:

• trunk/ippScripts/scripts/phase0exp.pl (modified) (6 diffs)

trunk/ippScripts/scripts/phase0exp.pl

@@ -10 +10 @@
 use constant RECIPE => "PPSTATS_PHASE0"; # Recipe to use for ppStats
 
+use constant TYPE => "OBSTYPE"; # Observation type keyword
+use constant DETRENDS => ( "bias", "dark", "flat", "fringe" ); # Observation types to mark as detrend
+use constant DETREND_FLAG => "-detrend"; # Flag to use to mark detrend exposure
+
+use constant IMFILE_URI => 'uri'; # Key for the URI out of p0search -pendingimfile
+use constant IMFILE_BG => 'background'; # Key for the background out of p0search -pendingimfile
+use constant IMFILE_BGSD => 'background_stdev'; # Key for the background standard deviation
+
+use constant EXP_BG => '-background'; # Switch to add background for exposure
+use constant EXP_BGSD => '-background_stdev'; # Switch to add background standard deviation
+use constant EXP_BGMEANSD => '-background_mean_stdev'; # Switch to add mean of background standard deviation
+
 # These values should be constant for all components
-use constant CONSTANTS => (
-                           "OBSTYPE", # Observation type
-                           "DETTEM", # Detector temperature
-                           "FPA.FILTER", # Filter used
-                           "FPA.RA", # Right ascension
-                           "FPA.DEC", # Declination
-                           "FPA.AIRMASS", # Airmass
-                           "TELALT", # Altitude
-                           "TELAZ", # Azimuth
-                           "FPA.POSANGLE" # Position angle
-    );
+use constant CONSTANTS => {
+                           "FPA.FILTER"   => "-filter", # Filter used
+                           "FPA.AIRMASS"  => "-airmass", # Airmass
+                           "FPA.RA"       => "-ra", # Right ascension
+                           "FPA.DEC"      => "-decl", # Declination
+                           "FPA.POSANGLE" => "-posang", # Position angle
+                           "TELALT"       => "-alt", # Altitude
+                           "TELAZ"        => "-az", # Azimuth
+                           "DETTEM"       => "-ccd_temp" # CCD temperature
+    };
 
 # These values may vary across components; we will take the average
-use constant VARIABLES => (
-                           "CELL.EXPOSURE", # Exposure time
+use constant VARIABLES => {
+                           "CELL.EXPOSURE" => "-exp_time", # Exposure time
 ###                        "CELL.TIME" # Time of exposure --- not yet implemented
-                           );
-if (scalar @ARGV == 0) {
-    die "Perform phase 0 processing on the component files of an FPA.\n\n" .
-        "Usage: $0 FPA_FILE_1.fits FPA_FILE_2.fits FPA_FILE_3.fits ...\n\n";
+                           };
+
+if (scalar @ARGV == 0 || scalar @ARGV >= 2) {
+    die "Perform phase 0 processing at the exposure level.\n\n" .
+        "Usage: $0 EXP_ID\n\n";
 }
 
@@ -36 +48 @@
 # Look for commands we need
 my $missing_tools;
+my $p0search = can_run('p0search')
+    or (warn "can't find p0search" and $missing_tools = 1);
 my $ppStats = can_run('ppStats')
     or (warn "can't find ppStats" and $missing_tools = 1);
 die "Can't find required tools.\n" if $missing_tools;
 
-my @filenames = @ARGV;          # Input files, all for one FPA
+my $expid = shift @ARGV;        # Exposure id
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# Get the list of filenames
+my $filenames;                  # Hash of input files, with their background and stdev
+{
+    my $command = "$p0search -pendingimfile -exp_id $expid";
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 0);
+    die "Unable to perform ppStats on exposure id $expid: $error_code\n" if not $success;
+    my $parsed = $mdcParser->parse(join "", @$stdout_buf); # Parsed metadata
+    $filenames = mdFileInfo($parsed); # Get filenames from the output
+}
+
 my %values;                     # Values to return
-
-my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
-foreach my $fileName (@filenames) {
+foreach my $fileName (keys %$filenames) {
     my $command = "$ppStats $fileName -recipe PPSTATS " . RECIPE; # Command to run
     print "Executing: $command\n";
@@ -56 +81 @@
 }
 
-foreach my $constant (CONSTANTS) {
+my $command = "$p0search -updateexp"; # Command to execute to update exposure parameters
+
+foreach my $constant (keys %{CONSTANTS()}) {
     if (not defined $values{$constant}) {
         die "Couldn't find value for $constant.\n";
     }
+    my $switch = CONSTANTS->{$constant}; # The switch to use on the value
     my $value = $values{$constant}; # The value of interest
-    print "$constant: $value\n";
-}
-
-foreach my $variable (VARIABLES) {
+    $command .= " $switch $value";
+}
+
+foreach my $variable (keys %{VARIABLES()}) {
     if (not defined $values{$variable}) {
         die "Couldn't find value for $variable.\n";
     }
+    my $switch = VARIABLES->{$variable}; # The switch to use on the value
     # Take the mean
     my $value = $values{$variable}->{value} / $values{$variable}->{num}; # Value of interest
-    print "$variable: $value\n";
-}
+    $command .= " $switch $value";
+}
+
+### Now to do the background: mean background, SD of backgrounds, mean of background SDs
+my $bgMean = 0;                 # Mean background
+my $bgsdMean = 0;               # Mean of background SDs
+foreach my $filename (keys %$filenames) {
+    my $stats = $filenames->{$filename}; # Statistics for that file
+    $bgMean += $stats->{bg};
+    $bgsdMean += $stats->{bgsd};
+}
+$bgMean /= scalar(keys %$filenames);
+$bgsdMean /= scalar(keys %$filenames);
+my $bgSD = 0;                   # SD of backgrounds
+foreach my $filename (keys %$filenames) {
+    my $stats = $filenames->{$filename}; # Statistics for that file
+    $bgSD += ($stats->{bg} - $bgMean)**2;
+}
+$bgSD = sqrt($bgSD / (scalar(keys %$filenames) - 1));
+
+$command .= " " . EXP_BGMEAN() . " $bgMean " . EXP_BGSD() . " $bgSD " . EXP_BGSDMEAN() . " $bgsdMean";
+
+print "$command";
+
+my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    run(command => $command, verbose => 0);
+die "Unable to run phase0 update for $expid: $error_code\n" if not $success;
+
 
 ### Pau.
+
+
+# Parse a metadata containing a list of files; return a hash keyed by the URI, containing bg and sd
+sub mdFileInfo
+{
+    my $mdItemArray = shift;    # Array of metadata items from "p0search -pendingimfile"
+
+    my %uriHash;                # Hash of URIs to return
+  IMFILE: foreach my $mdItem (@$mdItemArray) {
+      die "Unexpected format for filenames: does not consist solely of METADATAs.\n"
+          if $mdItem->{class} ne "metadata";
+      my %imfileInfo = values2hash($mdItem->{value});           # File information
+      die "Unexpected format for filenames: METADATAs do not contain " . IMFILE_URI . " element.\n"
+          if not defined $imfileInfo{IMFILE_URI};
+      die "Unexpected format for filenames: METADATAs do not contain " . IMFILE_BG . " element.\n"
+          if not defined $imfileInfo{IMFILE_BG};
+      die "Unexpected format for filenames: METADATAs do not contain " . IMFILE_BGSD . " element.\n"
+          if not defined $imfileInfo{IMFILE_BGSD};
+      $uriHash{$imfileInfo{IMFILE_URI}} = { 'bg' => $imfileInfo{IMFILE_BG},
+                                            'bgsd' => $imfileInfo{IMFILE_BGSD}
+                                        };
+  }
+    
+    return \%uriHash;
+}
+
 
 # Parse a metadata, looking for the appropriate values
@@ -89 +170 @@
       
       return if not defined $mdItem->{name};
+
+      if ($mdItem->{name} eq TYPE) {
+          foreach my $type (DETRENDS) {
+              if (lc($mdItem->{value}) eq lc($type)) {
+                  $values->{TYPE} = 1;
+                  next MDITEM;
+              }
+          }
+          # If we got here, it's not a detrend
+          $values->{TYPE} = 0;
+          next MDITEM;
+      }
       
-      foreach my $constant (CONSTANTS) {
+      foreach my $constant (keys %{CONSTANTS()}) {
           if ($mdItem->{name} eq $constant) {
               if (defined $values->{$constant} and $mdItem->{value} ne $values->{$constant}) {
@@ -100 +193 @@
       }
         
-      foreach my $variable (VARIABLES) {
+      foreach my $variable (keys %{VARIABLES()}) {
           if ($mdItem->{name} eq $variable) {
               $values->{$variable}->{value} += $mdItem->{value};
@@ -109 +202 @@
   }
 }
+
+
+# Convert values to a hash
+sub values2hash
+{
+    my $values = shift;
+ 
+    my %hash;
+    foreach my $data (@$values) {
+        $hash{$data->{name}} = $data->{value};
+    }
+ 
+    return \%hash;
+}