Index: trunk/ippScripts/scripts/phase0exp.pl
===================================================================
--- trunk/ippScripts/scripts/phase0exp.pl	(revision 8239)
+++ trunk/ippScripts/scripts/phase0exp.pl	(revision 8309)
@@ -6,7 +6,7 @@
 use IPC::Cmd qw( can_run run );
 use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
+use Statistics::Descriptive;
 use Data::Dumper;
-
-use constant RECIPE => "PPSTATS_PHASE0_EXP"; # Recipe to use for ppStats
 
 use constant TYPE => "OBSTYPE";	# Observation type keyword
@@ -14,7 +14,9 @@
 use constant DETREND_FLAG => "-detrend"; # Flag to use to mark detrend exposure
 
-use constant IMFILE_URI => 'uri'; # Key for the URI out of p0search -pendingimfile
-use constant IMFILE_BG => 'background';	# Key for the background out of p0search -pendingimfile
-use constant IMFILE_BGSD => 'background_stdev';	# Key for the background standard deviation
+use constant PHASE0_URI => 'uri'; # Key for the URI
+use constant PHASE0_CLASSID => 'class_id'; # Key for the class id
+use constant PHASE0_BG => 'bg';	# Key for the background
+use constant PHASE0_BG_STDEV => 'bg_stdev'; # Key for the background standard deviation
+use constant PHASE0_BG_MEAN_STDEV => 'bg_mean_stdev'; # Key for the mean of the background standard deviation
 
 use constant EXP_BG => '-background'; # Switch to add background for exposure
@@ -23,22 +25,20 @@
 
 # These values should be constant for all components
-# The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp"
-use constant CONSTANTS => {
-			   "FPA.FILTER"   => "-filter", # Filter used
-			   "FPA.AIRMASS"  => "-airmass", # Airmass
-			   "FPA.RA"       => "-ra", # Right ascension
-			   "FPA.DEC"      => "-decl", # Declination
-			   "FPA.POSANGLE" => "-posang", # Position angle
-			   "TELALT"       => "-alt", # Altitude
-			   "TELAZ"        => "-az", # Azimuth
-			   "DETTEM"       => "-ccd_temp" # CCD temperature
-    };
+use constant CONSTANTS => [
+			   "filter", # Filter used
+			   "airmass", # Airmass
+			   "ra", # Right ascension
+			   "decl", # Declination
+			   "posang", # Position angle
+			   "alt", # Altitude
+			   "az", # Azimuth
+			   "ccd_temp" # CCD temperature
+		       ];
 
 # These values may vary across components; we will take the average
-# The key is the name in the ppStats output; the value is the switch for "phase0 -updateexp"
-use constant VARIABLES => {
-			   "CELL.EXPOSURE" => "-exp_time", # Exposure time
-###			   "CELL.TIME" # Time of exposure --- not yet implemented
-			   };
+use constant VARIABLES => [
+			   "exp_time", # Exposure time
+###			   "time" # Time of exposure --- not yet implemented
+			   ];
 
 if (scalar @ARGV == 0 || scalar @ARGV >= 2) {
@@ -46,179 +46,121 @@
 	"Usage: $0 EXP_ID\n\n";
 }
+my $expid = shift @ARGV;	# Exposure id
 
 
 # Look for commands we need
 my $missing_tools;
-my $p0search = can_run('p0search')
-    or (warn "can't find p0search" and $missing_tools = 1);
+my $p0tool = can_run('p0tool')
+    or (warn "can't find p0tool" and $missing_tools = 1);
 my $ppStats = can_run('ppStats')
     or (warn "can't find ppStats" and $missing_tools = 1);
 die "Can't find required tools.\n" if $missing_tools;
 
-my $expid = shift @ARGV;	# Exposure id
-
 my $mdcParser = PS::IPP::Metadata::Config->new;	# Parser for metadata config files
 
-# Get the list of filenames
-my $filenames;			# Hash of input files, with their background and stdev
+# Get the list of imfiles
+my $imfiles;
 {
-    my $command = "$p0search -pendingimfile -exp_id $expid";
+    my $command = "$p0tool -rawimfile -exp_id $expid";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $command, verbose => 0);
-    die "Unable to perform p0search on exposure id $expid: $error_code\n" if not $success;
-    my $parsed = $mdcParser->parse(join "", @$stdout_buf); # Parsed metadata
-    $filenames = mdFileInfo($parsed); # Get filenames from the output
+	run(command => $command, verbose => 1);
+    die "Unable to perform p0tool on exposure id $expid: $error_code\n" if not $success;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf); # Parsed metadata
+    $imfiles = parse_md_list($metadata); # Data for imfiles
 }
 
-my %values;			# Values to return
-foreach my $fileName (keys %$filenames) {
-    my $command = "$ppStats $fileName -recipe PPSTATS " . RECIPE; # Command to run
-    print "Executing: $command\n";
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-	run(command => $command, verbose => 0);
-    die "Unable to perform ppStats on $fileName: $error_code\n" if not $success;
-    my $parsed = $mdcParser->parse(join "", @$stdout_buf); # Parsed metadata
-
-    mdParse($parsed, \%values);
+# Process the values
+my %values;			# Values to update
+foreach my $variable (@{VARIABLES()}) {
+    $values{$variable} = [];
+}
+my @backgrounds;		# Array of backgrounds
+my @stdevs;			# Array of background standard deviations\
+my $obsType;			# Observation type
+  
+foreach my $imfile (@$imfiles) {
+    foreach my $constant (@{CONSTANTS()}) {
+	my $value = get_value($imfile, $constant); # Value for imfile
+	if (not defined $values{$constant}) {
+	    $values{$constant} = $value;
+	} elsif ($values{$constant} != $value) {
+	    die "Value of $constant for " . $imfile->{PHASE0_CLASSID} .
+		" doesn't match previous value.\n";
+	}
+    }
+    
+    foreach my $variable (@{VARIABLES()}) {
+	my $value = get_value($imfile, $variable); # Value for imfile
+	my $array = $values{$variable};	# Array of data
+	push @$array, $value;
+    }
+    
+    my $bg = get_value($imfile, PHASE0_BG());
+    push @backgrounds, $bg;
+    
+    my $stdev = get_value($imfile, PHASE0_BG_MEAN_STDEV());
+    push @stdevs, $stdev;
+    
+    my $type = get_value($imfile, TYPE());
+    if (not defined $obsType) {
+	$obsType = $type;
+    } elsif ($obsType ne $type) {
+	die "Observation types differ.\n";
+    }
 }
 
-my $command = "$p0search -updateexp"; # Command to execute to update exposure parameters
+# Output results to the database
+{
+    my $command = "$p0tool -updateexp"; # Command to execute to update exposure parameters
+    
+    # Add the values of interest
+    foreach my $constant (@{CONSTANTS()}) {
+	$command .= " -" . $constant . " " . $values{$constant};
+    }
+    foreach my $variable (@{VARIABLES()}) {
+	my $array = $values{$variable};	# Array of values
+	my $stats = Statistics::Descriptive->new; # Statistics calculator
+	$stats->add_data(@$array);
+	$command .= " -" . $variable . " " . $stats->mean();
+    }
 
-foreach my $constant (keys %{CONSTANTS()}) {
-    if (not defined $values{$constant}) {
-	die "Couldn't find value for $constant.\n";
+    # Add the statistics
+    {
+	my $stats = Statistics::Descriptive->new; # Statistics calculator
+	$stats->add_data(@backgrounds);
+	$command .= " -" . PHASE0_BG() . " " . $stats->mean();
+	$command .= " -" . PHASE0_BG_STDEV() . " " . $stats->standard_deviation();
     }
-    my $switch = CONSTANTS->{$constant}; # The switch to use on the value
-    my $value = $values{$constant}; # The value of interest
-    $command .= " $switch $value";
+    {
+	my $stats = Statistics::Descriptive->new; # Statistics calculator
+	$stats->add_data(@stdevs);
+	$command .= " -" . PHASE0_BG_MEAN_STDEV() . " " . $stats->mean();
+    }
+    
+    # Add the detrend flag
+    foreach my $detrendType (@{DETRENDS()}) {
+	if (lc($obsType) eq lc($detrendType)) {
+	    $command .= " " . DETREND_FLAG;
+	    last;
+	}
+    }
+ 
+    print "$command\n";
+   
+#    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+#	run(command => $command, verbose => 1);
+#    die "Unable to run phase0 update for $expid: $error_code\n" if not $success;
 }
-
-foreach my $variable (keys %{VARIABLES()}) {
-    if (not defined $values{$variable}) {
-	die "Couldn't find value for $variable.\n";
-    }
-    my $switch = VARIABLES->{$variable}; # The switch to use on the value
-    # Take the mean
-    my $value = $values{$variable}->{value} / $values{$variable}->{num}; # Value of interest
-    $command .= " $switch $value";
-}
-
-# Check if it's a detrend
-if ($values{TYPE}) {
-    $command .= " " . DETREND_FLAG;
-}
-
-### Now to do the background: mean background, SD of backgrounds, mean of background SDs
-my $bgMean = 0;			# Mean background
-my $bgsdMean = 0;		# Mean of background SDs
-foreach my $filename (keys %$filenames) {
-    my $stats = $filenames->{$filename}; # Statistics for that file
-    $bgMean += $stats->{bg};
-    $bgsdMean += $stats->{bgsd};
-}
-$bgMean /= scalar(keys %$filenames);
-$bgsdMean /= scalar(keys %$filenames);
-my $bgSD = 0;			# SD of backgrounds
-foreach my $filename (keys %$filenames) {
-    my $stats = $filenames->{$filename}; # Statistics for that file
-    $bgSD += ($stats->{bg} - $bgMean)**2;
-}
-$bgSD = sqrt($bgSD / (scalar(keys %$filenames) - 1));
-
-$command .= " " . EXP_BGMEAN() . " $bgMean " . EXP_BGSD() . " $bgSD " . EXP_BGSDMEAN() . " $bgsdMean";
-
-print "$command";
-
-my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-    run(command => $command, verbose => 0);
-die "Unable to run phase0 update for $expid: $error_code\n" if not $success;
-
 
 ### Pau.
 
 
-# Parse a metadata containing a list of files; return a hash keyed by the URI, containing bg and sd
-sub mdFileInfo
-{
-    my $mdItemArray = shift;	# Array of metadata items from "p0search -pendingimfile"
+# Get the value for a particular imfile, along with a strong check for its existence
+sub get_value {
+    my $imfile = shift;		# The hash of values
+    my $name = shift;		# The name of the value to check
 
-    my %uriHash;		# Hash of URIs to return
-  IMFILE: foreach my $mdItem (@$mdItemArray) {
-      die "Unexpected format for filenames: does not consist solely of METADATAs.\n"
-	  if $mdItem->{class} ne "metadata";
-      my %imfileInfo = values2hash($mdItem->{value});		# File information
-      die "Unexpected format for filenames: METADATAs do not contain " . IMFILE_URI . " element.\n"
-	  if not defined $imfileInfo{IMFILE_URI};
-      die "Unexpected format for filenames: METADATAs do not contain " . IMFILE_BG . " element.\n"
-	  if not defined $imfileInfo{IMFILE_BG};
-      die "Unexpected format for filenames: METADATAs do not contain " . IMFILE_BGSD . " element.\n"
-	  if not defined $imfileInfo{IMFILE_BGSD};
-      $uriHash{$imfileInfo{IMFILE_URI}} = { 'bg' => $imfileInfo{IMFILE_BG},
-					    'bgsd' => $imfileInfo{IMFILE_BGSD}
-					};
-  }
-    
-    return \%uriHash;
+    my $source = $imfile->{PHASE0_CLASSID()}; # Where it comes from
+    die "Couldn't find value of $name for class_id=$source\n" if not defined $imfile->{$name};
+    return $imfile->{$name};
 }
-
-
-# Parse a metadata, looking for the appropriate values
-sub mdParse
-{
-    my $mdItemArray = shift;	# Array of metadata items
-    my $values = shift;		# The values
-
-  MDITEM:    foreach my $mdItem (@$mdItemArray) {
-      # Recurse if required
-      if ($mdItem->{class} eq "metadata") {
-	  mdParse($mdItem->{value}, $values);
-	  return;
-      }
-      
-      return if not defined $mdItem->{name};
-
-      if ($mdItem->{name} eq TYPE) {
-	  foreach my $type (DETRENDS) {
-	      if (lc($mdItem->{value}) eq lc($type)) {
-		  $values->{TYPE} = 1;
-		  next MDITEM;
-	      }
-	  }
-	  # If we got here, it's not a detrend
-	  $values->{TYPE} = 0;
-	  next MDITEM;
-      }
-      
-      foreach my $constant (keys %{CONSTANTS()}) {
-	  if ($mdItem->{name} eq $constant) {
-	      if (defined $values->{$constant} and $mdItem->{value} ne $values->{$constant}) {
-		  die "Differing values found for $constant\n";
-	      }
-	      $values->{$constant} = $mdItem->{value};
-	      next MDITEM;
-	  }
-      }
-	
-      foreach my $variable (keys %{VARIABLES()}) {
-	  if ($mdItem->{name} eq $variable) {
-	      $values->{$variable}->{value} += $mdItem->{value};
-	      $values->{$variable}->{num}++;
-	      next MDITEM;
-	  }
-      }
-  }
-}
-
-
-# Convert values to a hash
-sub values2hash
-{
-    my $values = shift;
- 
-    my %hash;
-    foreach my $data (@$values) {
-        $hash{$data->{name}} = $data->{value};
-    }
- 
-    return \%hash;
-}