Index: trunk/ippScripts/scripts/detrend_reject_imfile.pl
===================================================================
--- trunk/ippScripts/scripts/detrend_reject_imfile.pl	(revision 13989)
+++ trunk/ippScripts/scripts/detrend_reject_imfile.pl	(revision 14009)
@@ -22,7 +22,8 @@
 use IPC::Cmd 0.36 qw( can_run run );             # tools to run UNIX programs with control over I/O
 use PS::IPP::Metadata::Config;                   # tools to parse the psMetadataConfig files
-use PS::IPP::Metadata::List qw( parse_md_list ); # ?
+use PS::IPP::Metadata::List qw( parse_md_list ); # tools to parse a metadata into a hash list
 use Statistics::Descriptive;                     # tools for calculating basic statistical quantities
 use File::Temp qw( tempfile );                   # tools to construct temp files
+
 use PS::IPP::Config qw($PS_EXIT_SUCCESS          
 		       $PS_EXIT_UNKNOWN_ERROR
@@ -56,8 +57,7 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage(
-	  -msg => "Required options: --det_id --iteration --exp_tag --det_type --camera",
-	  -exitval => 3,
-	  ) unless defined $det_id
+pod2usage( -msg => "Required options: --det_id --iteration --exp_tag --det_type --camera",
+	   -exitval => 3)
+    unless defined $det_id
     and defined $iter
     and defined $exp_tag
@@ -68,25 +68,31 @@
 $ipprc->define_camera($camera);
 
-# Recipes to use, as a function of the detrend type
-use constant RECIPES => {
-    'bin1' => {		# We're creating a master --- already processed the input
-	'bias'     => 'PPIMAGE_J1_RESID_B',	# Bias only
-	'dark'     => 'PPIMAGE_J1_RESID_B',	# Dark only
-	'shutter'  => 'PPIMAGE_J1_RESID_F',	# Shutter only
-	'flat'     => 'PPIMAGE_J1_RESID_F',	# Flat-field only
-	'domeflat' => 'PPIMAGE_J1_RESID_F',	# Flat-field only
-	'skyflat'  => 'PPIMAGE_J1_RESID_F',	# Flat-field only
-	'fringe'   => 'PPIMAGE_J1_RESID_R',	# Fringe only
-    },
-    'bin2' => {		# We're checking the master --- input is not already processed
-	'bias'     => 'PPIMAGE_J2_RESID_B',	# Bias only
-	'dark'     => 'PPIMAGE_J2_RESID_B',	# Dark only
-	'shutter'  => 'PPIMAGE_J2_RESID_F',	# Shutter only
-	'flat'     => 'PPIMAGE_J2_RESID_F',	# Flat-field only
-	'domeflat' => 'PPIMAGE_J2_RESID_F',	# Flat-field only
-	'skyflat'  => 'PPIMAGE_J2_RESID_F',	# Flat-field only
-	'fringe'   => 'PPIMAGE_J2_RESID_R',	# Fringe only
-    },
-};
+# Recipes to use based on reduction class
+$reduction = 'DETREND' unless defined $reduction;
+
+my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_RESID' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
+
+my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_RESID' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
+
+# values to extract from output metadata and the stats to calculate
+# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
+# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
+# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
+my $STATS = 
+   [   
+       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+       { name => "bg",             type => "mean",  flag => "-bg" },
+       { name => "bg_mean_stdev",  type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
+       { name => "bin_stdev",      type => "rms",   flag => "-bin_stdev" },
+       { name => "fringe_0",       type => "mean",  flag => "-fringe_0" },
+       { name => "fringe_1",       type => "rms",   flag => "-fringe_1" },
+       { name => "fringe_0",       type => "stdev", flag => "-fringe_2" },
+       { name => "user_1",         type => "mean",  flag => "-user_1" }, # fringe residual
+       { name => "user_2",         type => "rms",   flag => "-user_2" }, # fringe residual
+       { name => "user_3",         type => "stdev", flag => "-user_1" }, # fringe residual
+   ];
 
 # Look for programs we need
@@ -98,7 +104,4 @@
     exit($PS_EXIT_CONFIG_ERROR); 
 }
-
-# Parser for psMetadataConfig files (used for output from dettool)
-my $mdcParser = PS::IPP::Metadata::Config->new;	
 
 # Get list of imfile files
@@ -120,9 +123,16 @@
     # XXX report an error message if stdout_buf is empty
 
+    # Parse the stdout buffer into a metadata
+    my $mdcParser = PS::IPP::Metadata::Config->new;	
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
 	&my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR);
 
+    # parse the file info in the metadata
     $files = parse_md_list($metadata) or
 	&my_die("Unable to parse metadata list", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR);
+
+    # Parse the statistics on the residual image
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
 }
 
@@ -136,9 +146,6 @@
 my $logFile;
 unless ($no_op) {
-    # XXX this will fail if the file exists (because of Nebulous rules)
     $logFile = $ipprc->file_create_append( $logName );
 }
-
-#### build the JPEG images ####
 
 # XXX in debug mode, unlink = 0
@@ -152,11 +159,6 @@
 close $list2File;
 
-# Recipes to use in processing
-my $recipe1 = RECIPES->{"bin1"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
-
-my $recipe2 = RECIPES->{"bin2"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
-
+
+# build the JPEG images
 unless ($no_op) {
 
@@ -206,18 +208,7 @@
 
 # storage variables
-my @means;			# Array of means
-my @fluxes;			# Array of means / exptimes
-my @variances;			# Array of variances
-my @binVariances;		# Array of binned variances
-my @meanStdevs;			# Array of mean stdevs
-my @names;			# Array of names (class_id)
-my @fringe_means;
-my @fringe_vars;
-my @dfringe_means;
-my @dfringe_vars;
 
 # load the arrays from the imfile output table
 foreach my $file (@$files) {
-    push @means,      	$file->{bg};                           # mean background counts
     if ($file->{exp_time} > 0.0) {
 	push @fluxes,     	$file->{bg} / $file->{exp_time};       # mean background counts / sec
@@ -225,13 +216,4 @@
 	push @fluxes,     	$file->{bg};
     }
-    push @meanStdevs, 	     $file->{bg_mean_stdev};                # stdev of the mean counts (for imfile components)
-    push @variances,  	     $file->{bg_stdev}*$file->{bg_stdev};   # total variance from all imfile components
-    push @binVariances,      $file->{bin_stdev}*$file->{bin_stdev}; # total variance of binned images over all imfile components
-    push @names,             $file->{class_id};                     # name of the component
-    push @fringe_means,      $file->{fringe_0};                     # fringe amplitude mean for imfile
-    push @fringe_vars,       $file->{fringe_1}*$file->{fringe_1};   # fringe variance
-    push @dfringe_means,     $file->{user_1};                       # fringe residual mean for imfile
-    push @dfringe_vars,      $file->{user_2}*$file->{user_2};       # fringe residual variance
-    # push @fringe_mean_stdev, $file->{fringe_2};                     # fringe amplitude stdev for imfile
 }
 
@@ -239,11 +221,24 @@
 # it is VALID to reject on more than one criterion
 &my_die("Number of means and number of variances differ!", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR) unless scalar @means == scalar @variances;
-for (my $i = 0; $i < scalar @means; $i++) {
-    my $flux = $fluxes[$i];	# Flux for this imfile
-    my $mean = $means[$i];	# Mean for this imfile
+
+# storage array
+my @fluxes;
+
+for (my $i = 0; $i < scalar @$files; $i++) {
+    my $file      = $files[$i];
+    my $name      = $file->{class_id};
+    my $mean      = $file->{bg};	# Mean for this imfile
+    my $stdev     = $file->{bg_stdev}; # Stdev for this imfile
+    my $meanStdev = $file->{bg_mean_stdev}; # Stdev of Means for this imfile
+    my $binStdev  = $file->{bin_stdev}; # Binned Stdev for this imfile
+
+    # calculate and save the fluxes
+    my $flux      = $mean / $file->{exp_time};
+    if ($file->{exp_time} == 0.0) {
+	push @fluxes, $mean;
+    }
+    push @fluxes, $flux;
+
     $mean -= $expected;
-    my $stdev = sqrt($variances[$i]);	# Stdev for this imfile
-    my $binStdev = sqrt($binVariances[$i]);	# Stdev for this imfile
-    my $name = $names[$i];
 
     last if $no_op;
@@ -301,5 +296,5 @@
     # component means is larger than the limit
     if ($reject_imfile_meanstdev > 0) {
-	if ($meanStdevs[$i] > $reject_imfile_meanstdev) {
+	if ($meanStdevs > $reject_imfile_meanstdev) {
 	    print $logFile "Rejecting exposure based on bad imfile mean stdev for $name: ";
 	    $reject = 1;
@@ -307,5 +302,5 @@
 	    print $logFile "Imfile mean stdev for $name meets requirements: ";
 	}
-	print $logFile "$meanStdevs[$i] vs $reject_imfile_meanstdev\n";
+	print $logFile "$meanStdevs vs $reject_imfile_meanstdev\n";
     } else {
 	print $logFile "No rejection on imfile mean stdev for $name\n";
@@ -356,54 +351,24 @@
 }
 
-# calculate the exposure ensemble statistics
-my $meanStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
-$meanStats->add_data(@means);
+# basic ensemble stats
+my $mean      	       = &STATS_value_for_flag ($STATS, "-bg");
+my $meanStdev 	       = &STATS_value_for_flag ($STATS, "-bg_mean_stdev");
+my $stdev     	       = &STATS_value_for_flag ($STATS, "-bg_stdev");
+my $binStdev  	       = &STATS_value_for_flag ($STATS, "-bin_stdev");
+my $fringe_mean        = &STATS_value_for_flag ($STATS, "-fringe_0");
+my $fringe_err         = &STATS_value_for_flag ($STATS, "-fringe_1");
+my $fringe_mean_stdev  = &STATS_value_for_flag ($STATS, "-fringe_2");
+my $dfringe_mean       = &STATS_value_for_flag ($STATS, "-user_1");
+my $dfringe_err        = &STATS_value_for_flag ($STATS, "-user_2");
+my $dfringe_mean_stdev = &STATS_value_for_flag ($STATS, "-user_3");
+
+# other stats (flux depends on bg and exp_time)
 my $fluxStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
 $fluxStats->add_data(@fluxes);
-my $varianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for variances
-$varianceStats->add_data(@variances);
-my $binVarianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for variances
-$binVarianceStats->add_data(@binVariances);
-
-# background stats
-my $mean = $meanStats->mean();	# Mean of the imfile means
 my $flux = $fluxStats->mean();	# Mean of the imfile means
-my $meanStdev = $meanStats->standard_deviation(); # Stdev of the imfile means
-if (not defined $meanStdev) {
-    # this is the case for Nimfile == 1
-    $meanStdev = 0;
-}
-my $stdev = sqrt($varianceStats->mean()); # Root-Mean-Square of the imfile stdevs (root mean of variances)
-my $binStdev = sqrt($binVarianceStats->mean()); # Root-Mean-Square of the imfile stdevs (root mean of variances)
+
+# other stats
 my $exp_sn = 0.0;
 if ($stdev > 0) { $exp_sn = $mean / $stdev; }
-
-# prepare fringe amplitude stats
-my $fringeMeanStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
-$fringeMeanStats->add_data(@fringe_means);
-my $fringeVarStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
-$fringeVarStats->add_data(@fringe_vars);
-
-# fringe amplitude stats
-my $fringe_mean = $fringeMeanStats->mean();
-my $fringe_err = sqrt($fringeVarStats->mean());
-my $fringe_mean_stdev = $fringeMeanStats->standard_deviation();
-if (not defined $fringe_mean_stdev) {
-    $fringe_mean_stdev = 0;
-}
-
-# prepare fringe residual stast 
-my $dfringeMeanStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
-$dfringeMeanStats->add_data(@dfringe_means);
-my $dfringeVarStats = Statistics::Descriptive::Sparse->new();	# Statistics calculator for means
-$dfringeVarStats->add_data(@dfringe_vars);
-
-# fringe amplitude stats
-my $dfringe_mean = $dfringeMeanStats->mean();
-my $dfringe_err = sqrt($dfringeVarStats->mean());
-my $dfringe_mean_stdev = $dfringeMeanStats->standard_deviation();
-if (not defined $dfringe_mean_stdev) {
-    $dfringe_mean_stdev = 0;
-}
 
 ## Reject based on the exposure ensemble stats
@@ -514,18 +479,17 @@
 }
 
-# Add the result into the database
-my $bg            = $mean;
-my $bg_stdev      = $stdev;
-my $bg_mean_stdev = $meanStdev;
-my $bin_stdev     = $binStdev;
+my $command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_tag $exp_tag";
+$command .= " -recip $recipe1,$recipe2 -path_base $outputRoot ";
+$command .= ' -reject' if $reject;
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
 
 unless ($no_update) {
-    my $command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_tag $exp_tag";
-    $command .= " -recip $recipe1,$recipe2 -path_base $outputRoot ";
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev -bin_stdev $bin_stdev";
-    $command .= " -fringe_0 $fringe_mean -fringe_1 $fringe_err -fringe_2 $fringe_mean_stdev";
-    $command .= " -user_1 $dfringe_mean -user_2 $dfringe_err -user_3 $dfringe_mean_stdev";
-    $command .= ' -reject' if $reject;
-    $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
 	run(command => $command, verbose => 1);
@@ -535,4 +499,6 @@
 	exit($error_code);
     }
+} else {
+    print "skipping command: $command\n";
 }
 
@@ -555,5 +521,9 @@
     carp($msg);
     if ($det_id and $iter and $exp_tag and not $no_update) {
-	my $command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_tag $exp_tag -code $exit_code";
+	my $command = "$dettool -addresidexp";
+	$command .= " -det_id $det_id";
+	$command .= " -iteration $iter";
+	$command .= " -exp_tag $exp_tag";
+	$command .= " -code $exit_code";
 	$command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);
@@ -561,5 +531,4 @@
     exit $exit_code;
 }
-
 
 # Retrieve the requested rejection limit, dying if not extant
@@ -579,6 +548,16 @@
 }
 
-
-
+sub STATS_value_for_flag 
+{
+    my $STATS = shift;
+    my $flag  = shift;
+
+    foreach my $entry (@$STATS) {
+	if ($flag eq $entry->{flag}) {
+	    return $entry->{value};
+	}
+    }
+    return 'NAN';
+}
 
 __END__