Changeset 14009

Timestamp:

Jul 4, 2007, 1:53:22 PM (19 years ago)

Author:

eugene

Message:

extensive changes to use new stats calculation tools; general cleanups

Location:

trunk/ippScripts/scripts

Files:

• chip_imfile.pl (modified) (6 diffs)
• detrend_norm_apply.pl (modified) (8 diffs)
• detrend_norm_calc.pl (modified) (7 diffs)
• detrend_norm_exp.pl (modified) (9 diffs)
• detrend_process_exp.pl (modified) (11 diffs)
• detrend_process_imfile.pl (modified) (7 diffs)
• detrend_reject_exp.pl (modified) (14 diffs)
• detrend_reject_imfile.pl (modified) (18 diffs)
• detrend_resid.pl (modified) (10 diffs)
• detrend_stack.pl (modified) (10 diffs)
• register_imfile.pl (modified) (2 diffs)

trunk/ippScripts/scripts/chip_imfile.pl

@@ -1 +1 @@
 #!/usr/bin/env perl
 
+use Carp;
 use warnings;
 use strict;
-use Carp;
 
 ## report the program and machine
@@ -11 +11 @@
 print "Starting script $0 on $host\n\n";
 
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::Stats;
-use Data::Dumper;
+
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
                        $PS_EXIT_UNKNOWN_ERROR
@@ -30 +33 @@
 
 # Parse the command-line arguments
-my ($exp_id,                    # Exposure identifier
-    $chip_id,                   # Chiptool identifier
-    $class_id,                  # Class identifier
-    $input,                     # Input FITS file
-    $camera,                    # Camera
-    $dbname,                    # Database name
-    $workdir,                   # Working directory, for output files
-    $reduction,                 # Reduction class
-    $no_update,                 # Don't update the database?
-    $no_op,                     # Don't do any operations?
-    );
+my ($exp_id, $chip_id, $class_id, $input_uri, $camera, $dbname, $workdir, $reduction, $no_update, $no_op);
 GetOptions(
-           'exp_id=s'      => \$exp_id,
-           'chip_id=s'     => \$chip_id,
-           'class_id=s'    => \$class_id,
-           'uri|u=s'       => \$input,
-           'camera|c=s'    => \$camera,
-           'dbname|d=s'    => \$dbname, # Database name
-           'workdir|w=s'   => \$workdir,
-           'reduction=s'   => \$reduction,
-           'no-update'     => \$no_update,
-           'no-op'         => \$no_op,
+           'exp_id=s'      => \$exp_id,    # Exposure identifier                       
+           'chip_id=s'     => \$chip_id,   # Chiptool identifier                       
+           'class_id=s'    => \$class_id,  # Class identifier                  
+           'input_uri|u=s' => \$input_uri, # Input FITS file                   
+           'camera|c=s'    => \$camera,    # Camera                            
+           'dbname|d=s'    => \$dbname,    # Database name
+           'workdir|w=s'   => \$workdir,   # Working directory, for output files  
+           'reduction=s'   => \$reduction, # Reduction class                   
+           'no-update'     => \$no_update, # Don't update the database?        
+           'no-op'         => \$no_op,     # Don't do any operations?             
            ) or pod2usage( 2 );
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage(
-    -msg => "Required options: --exp_id --chip_id --class_id --uri --camera",
-    -exitval => 3,
-) unless defined $exp_id
+pod2usage( -msg => "Required options: --exp_id --chip_id --class_id --uri --camera",
+           -exitval => 3) 
+    unless defined $exp_id
     and defined $chip_id 
     and defined $class_id 
-    and defined $input
+    and defined $input_uri
     and defined $camera;
 
 $ipprc->define_camera($camera);
+
+# Recipes to use based on reduction class
 $reduction = 'DEFAULT' unless defined $reduction;
 my $recipe = $ipprc->reduction($reduction, 'CHIP'); # Recipe to use
+
+# values to extract from output metadata and the stats to calculate
+my $STATS = 
+   [   
+       #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
+       { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg" },
+       { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev" },
+       { name => "FRINGE_0",       type => "rms",   flag => "-fringe_0" },
+       { name => "FRINGE_RESID_0", type => "rms",   flag => "-fringe_1" },
+       { name => "FRINGE_ERR_0",   type => "rms",   flag => "-fringe_2" },
+       { name => "OVER_VAL",       type => "mean",  flag => "-bias" },
+       { name => "FWHM",           type => "mean",  flag => "-fwhm" },
+       { name => "APMIFIT",        type => "mean",  flag => "-ap_resid" },
+       { name => "DAPMIFIT",       type => "rms",   flag => "-ap_resid_stdev" },
+       { name => "CERROR",         type => "rms",   flag => "-sigma_ra" },
+       { name => "CERROR",         type => "rms",   flag => "-sigma_dec" },
+       { name => "NSTARS",         type => "sum",   flag => "-n_stars" },
+       { name => "NASTRO",         type => "sum",   flag => "-n_astrom" },
+   ];
 
 # Look for programs we need
@@ -76 +90 @@
     exit($PS_EXIT_CONFIG_ERROR); 
 }
-$ppImage .= " -dbname $dbname" if defined $dbname;
-
-&my_die("Couldn't find input file: $input\n", $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input);
+
+&my_die("Couldn't find input file: $input_uri\n", $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri);
 
 $workdir = caturi( $workdir, $exp_id ) if defined $workdir;
 
-my $outputRoot = $ipprc->file_prepare( "$exp_id.chp$chip_id", $workdir, $input );
-my $outputImage = $ipprc->filename("PPIMAGE.CHIP", $outputRoot, $class_id);
-my $outputMask = $ipprc->filename("PPIMAGE.CHIP.MASK", $outputRoot, $class_id);
-my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id);
-my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id);
-my $outputStats = $outputRoot  . '.' . $class_id . '.stats';
-
-my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
+my $outputRoot  = $ipprc->file_prepare( "$exp_id.chp$chip_id", $workdir, $input_uri );
+
+my $outputImage = $ipprc->filename("PPIMAGE.CHIP",      $outputRoot, $class_id);
+my $outputMask  = $ipprc->filename("PPIMAGE.CHIP.MASK", $outputRoot, $class_id);
+my $outputBin1  = $ipprc->filename("PPIMAGE.BIN1",      $outputRoot, $class_id);
+my $outputBin2  = $ipprc->filename("PPIMAGE.BIN2",      $outputRoot, $class_id);
+my $outputStats = $ipprc->filename("PPIMAGE.STATS",     $outputRoot, $class_id);
+
+# Run ppImage
 unless ($no_op) {
-    # Run ppImage
-    print "outputImage: $outputImage\n";
-    print "outputBin1: $outputBin1\n";
-    print "outputStats: $outputStats\n";
-
-    my $command = "$ppImage -file $input $outputRoot";
+    my $command = "$ppImage -file $input_uri $outputRoot";
     $command .= " -recipe PPIMAGE $recipe";
     $command .= " -recipe PPSTATS CHIPSTATS";
-    $command .= " -stats $outputStats"; # Command to run ppImage
+    $command .= " -stats $outputStats";
+    $command .= " -dbname $dbname" if defined $dbname;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -119 +129 @@
     my @contents = <$statsFile>; # Contents of file
     close $statsFile;
-    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
-    my $metadata = $mdcParser->parse(join "", @contents) or
+
+    # parse the statistics MDC file
+    my $mdcParser = PS::IPP::Metadata::Config->new();   # Parser for metadata config files
+    my $metadata = $mdcParser->parse(join "", @contents);
+    unless ($metadata) {
         &my_die("Unable to parse metadata config doc", $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
-    $stats->parse($metadata) or
+    }
+
+    # extract the stats from the metadata
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+    unless ($stats->parse($metadata)) {
         &my_die("Unable to find all values in statistics output.\n", $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
+    }
+}
+
+# command to update database
+my $command = "$chiptool -addprocessedimfile";
+$command .= " -chip_id $chip_id";
+$command .= " -class_id $class_id";
+$command .= " -uri $outputImage";
+$command .= " -path_base $outputRoot";
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
 }
 
 # Add the processed file to the database
-my $bg            = $stats->bg_mean();
-my $bg_stdev      = $stats->bg_stdev();
-my $bg_mean_stdev = $stats->bg_mean_stdev();
-my $fringe_0      = ${$stats->fringe_mean()}[0];
-my $fringe_1      = ${$stats->fringe_err()}[0];
-my $fringe_2      = ${$stats->fringe_mean_stdev()}[0];
-my $dfringe_0     = ${$stats->dfringe_mean()}[0];
-my $dfringe_1     = ${$stats->dfringe_err()}[0];
-my $dfringe_2     = ${$stats->dfringe_mean_stdev()}[0];
-
 unless ($no_update) {
-    # Command to run chiptool
-    my $command = "$chiptool -addprocessedimfile";
-    $command .= " -chip_id $chip_id";
-    $command .= " -class_id $class_id";
-    $command .= " -uri $outputImage";
-    $command .= " -path_base $outputRoot";
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
-    $command .= " -fringe_0 $fringe_0 -fringe_1 $fringe_1 -fringe_2 $fringe_2";
-    $command .= " -user_1 $dfringe_0 -user_2 $dfringe_1 -user_3 $dfringe_2";
-    # XXX add sigma_ra, etc here
-    $command .= " -dbname $dbname" if defined $dbname;
-
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
@@ -157 +168 @@
         exit($error_code);
     }
-}
-
+} else {
+    print "skipping command: $command\n";
+}
 
 sub my_die

trunk/ippScripts/scripts/detrend_norm_apply.pl

@@ -11 +11 @@
 print "Starting script $0 on $host\n\n";
 
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
+use IPC::Cmd 0.36 qw( can_run run );
+use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::Stats;
-use IPC::Cmd 0.36 qw( can_run run );
-use Data::Dumper;
 
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
@@ -30 +33 @@
 
 # Parse the command-line
-my ($det_id,                     # Detrend ID
-    $iter,                      # Iteration
-    $class_id,                  # Class ID
-    $value,                     # Value to multiple (for normalisation)
-    $input,                     # Input file
-    $camera,                    # Camera
-    $det_type,                  # Detrend type
-    $dbname,                    # Database name
-    $workdir,                   # Working directory, for output files
-    $no_update,                 # Don't update the database
-    $no_op,                     # Don't do any operations
-    );
+my ($det_id, $iter, $class_id, $value, $input_uri, $camera, $det_type, $dbname, $workdir, $no_update, $no_op);
 GetOptions(
-    'det_id|d=s'        => \$det_id,
-    'iteration|n=s'     => \$iter,
-    'class_id|i=s'      => \$class_id,
-    'value|v=s'         => \$value,
-    'input_uri|u=s'     => \$input,
-    'camera|c=s'        => \$camera,
-    'det_type|t=s'      => \$det_type,
-    'dbname|d=s'        => \$dbname,
-    'workdir|w=s'       => \$workdir, # Working directory, for output files
-    'no-update'         => \$no_update,
-    'no-op'             => \$no_op,
+    'det_id|d=s'        => \$det_id,     # Detrend ID                           
+    'iteration|n=s'     => \$iter,       # Iteration                            
+    'class_id|i=s'      => \$class_id,   # Class ID                             
+    'value|v=s'         => \$value,      # Value to multiple (for normalisation)        
+    'input_uri|u=s'     => \$input_uri,  # Input file                           
+    'camera|c=s'        => \$camera,     # Camera                               
+    'det_type|t=s'      => \$det_type,   # Detrend type                         
+    'dbname|d=s'        => \$dbname,     # Database name                                
+    'workdir|w=s'       => \$workdir,    # Working directory, for output files  
+    'no-update'         => \$no_update,  # Don't update the database            
+    'no-op'             => \$no_op,      # Don't do any operations               
     ) or pod2usage( 2 );
     
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 pod2usage( -msg => "Required options: --det_id --iteration --class_id --value --input_uri --camera --det_type",
-    -exitval => 3,
-    ) unless defined $det_id
+           -exitval => 3) 
+    unless defined $det_id
     and defined $iter
     and defined $class_id
     and defined $value
-    and defined $input
+    and defined $input_uri
     and defined $camera
     and defined $det_type;
@@ -69 +61 @@
 $ipprc->define_camera($camera);
 
-
 my $RECIPE_PPIMAGE = 'PPIMAGE_N'; # Recipe to use with ppImage
 my $RECIPE_PPSTATS = 'CHIPSTATS'; # Recipe to use with ppStats
+
+# values to extract from output metadata and the stats to calculate
+my $STATS = 
+   [   
+       #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
+       { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg" },
+       { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev" },
+   ];
 
 # Look for programs we need
@@ -77 +77 @@
 my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
 my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
-
 if ($missing_tools) { 
     warn("Can't find required tools.");
@@ -83 +82 @@
 }
 
+&my_die("Couldn't find input file: $input_uri\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri);
+
 $workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir;
 
-my $outputRoot = $ipprc->file_prepare( "$camera.$det_type.norm.$det_id.$iter", $workdir, $input );
-my $output = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id);
-my $b1name = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id);
-my $b2name = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id);
-my $statsName = $outputRoot . '.' . $class_id . '.stats'; # Statistics file
+my $outputRoot = $ipprc->file_prepare( "$camera.$det_type.norm.$det_id.$iter", $workdir, $input_uri );
+
+my $output    = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id);
+my $b1name    = $ipprc->filename("PPIMAGE.BIN1",   $outputRoot, $class_id);
+my $b2name    = $ipprc->filename("PPIMAGE.BIN2",   $outputRoot, $class_id);
+my $statsName = $ipprc->filename("PPIMAGE.STATS",  $outputRoot, $class_id);
 
 # Run normalisation
-my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
 unless ($no_op) {
-    my $command = "$ppImage -file $input $outputRoot";
+    my $command = "$ppImage -file $input_uri $outputRoot";
     $command .= " -norm $value -stats $statsName";
     $command .= " -recipe PPIMAGE $RECIPE_PPIMAGE";
@@ -117 +118 @@
     my @contents = <$statsFile>; # Contents of file
     close $statsFile;
+
+    # parse the statistics MDC file
     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
-    my $metadata = $mdcParser->parse(join "", @contents)
-        or &my_die("Unable to parse metadata config", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
-    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
+    my $metadata = $mdcParser->parse(join "", @contents);
+    unless ($metadata) {
+        &my_die("Unable to parse metadata config", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
+    }
+
+    # extract the stats from the metadata
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+    unless ($stats->parse($metadata)) {
+        &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
+    }
 }
 
-# Update the database
-my $bg            = $stats->bg_mean();
-my $bg_stdev      = $stats->bg_stdev();
-my $bg_mean_stdev = $stats->bg_mean_stdev();
+# Command to update the database
+my $command = "$dettool -addnormalizedimfile";
+$command .= " -det_id $det_id";
+$command .= " -iteration $iter";
+$command .= " -class_id $class_id";
+$command .= " -uri $output";
+$command .= " -path_base $outputRoot";
+$command .= " -dbname $dbname" if defined $dbname;
 
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
+
+# Add the processed file to the database
 unless ($no_update) {
-    my $command = "$dettool -addnormalizedimfile -det_id $det_id -iteration $iter -class_id $class_id ".
-        "-uri $output -path_base $outputRoot"; # Command to run
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
-    $command .= " -dbname $dbname" if defined $dbname;
-
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
@@ -141 +158 @@
         exit($error_code);
     }
+} else {
+    print "skipping command: $command\n";
 }
 
@@ -153 +172 @@
     carp($msg);
     if ($det_id and $iter and $class_id and not $no_update) {
-        my $command = "$dettool -addnormalizedimfile -det_id $det_id -iteration $iter -class_id $class_id -code $exit_code";
+        my $command = "$dettool -addnormalizedimfile";
+        $command .= " -det_id $det_id";
+        $command .= " -iteration $iter";
+        $command .= " -class_id $class_id";
+        $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);

trunk/ippScripts/scripts/detrend_norm_calc.pl

@@ -11 +11 @@
 print "Starting script $0 on $host\n\n";
 
+use vars qw( $VERSION );
+$VERSION = '0.01';
+
 use IPC::Cmd 0.36 qw( can_run );
-use IPC::Run qw ( run );
+use IPC::Run 0.36 qw( run );
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
-use Data::Dumper;
-
-use PS::IPP::Config qw(
-    $PS_EXIT_SUCCESS
-    $PS_EXIT_UNKNOWN_ERROR
-    $PS_EXIT_SYS_ERROR
-    $PS_EXIT_CONFIG_ERROR
-    $PS_EXIT_PROG_ERROR
-    $PS_EXIT_DATA_ERROR
-    $PS_EXIT_TIMEOUT_ERROR
-    );
+
+use PS::IPP::Config qw($PS_EXIT_SUCCESS
+                       $PS_EXIT_UNKNOWN_ERROR
+                       $PS_EXIT_SYS_ERROR
+                       $PS_EXIT_CONFIG_ERROR
+                       $PS_EXIT_PROG_ERROR
+                       $PS_EXIT_DATA_ERROR
+                       $PS_EXIT_TIMEOUT_ERROR
+                       );
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
 use Pod::Usage qw( pod2usage );
 
-
 # Parse command-line arguments
-my ($det_id,                    # Detrend id
-    $iter,                      # Iteration
-    $detType,                   # Detrend type
-    $workdir,                   # Working directory for output files
-    $dbname,                    # Database name
-    $no_update,                 # Don't update the database?
-    $no_op,                     # Don't do operations
-    );
+my ($det_id, $iter, $detType, $workdir, $dbname, $no_update, $no_op);
 GetOptions(
-        'det_id|d=s'    => \$det_id,
-        'iteration|i=s' => \$iter,
-        'det_type|t=s'  => \$detType,
-        'workdir|w=s'   => \$workdir,
-        'dbname|d=s'    => \$dbname,# Database name
-        'no-update'     => \$no_update,
-        'no-op'         => \$no_op,
+        'det_id|d=s'    => \$det_id,    # Detrend id                         
+        'iteration|i=s' => \$iter,      # Iteration                          
+        'det_type|t=s'  => \$detType,   # Detrend type                       
+        'workdir|w=s'   => \$workdir,   # Working directory for output files 
+        'dbname|d=s'    => \$dbname,    # Database name                      
+        'no-update'     => \$no_update, # Don't update the database?         
+        'no-op'         => \$no_op,     # Don't do operations                
         ) or pod2usage( 2 );
 
@@ -53 +46 @@
 pod2usage( -msg => "Required options --det_id --iteration --det_type",
            -exitval => 3,
-           ) unless defined $det_id
+           ) 
+    unless defined $det_id
     and defined $iter
     and defined $detType;
 
-
 use constant STATISTIC => 'bg'; # Background statistic to use from the database
+
 # Define which detrend types we normalise
 use constant NORMALIZE => {
@@ -70 +64 @@
     }; 
 
-
 # Look for programs we need
 my $missing_tools;
@@ -80 +73 @@
 }
 
-&my_die("Unrecognised detrend type: $detType", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless
-    exists NORMALIZE()->{lc($detType)};
+&my_die("Unrecognised detrend type: $detType", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless exists NORMALIZE()->{lc($detType)};
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -172 +164 @@
 }
 
+my $commandBase = "$dettool -addnormalizedstat";
+$commandBase .= " -det_id $det_id";
+$commandBase .= " -iteration $iter";
+$commandBase .= " -dbname $dbname" if defined $dbname;
 
 # Process output normalisations
 unless ($no_update) {
     foreach my $normItem (@$norms) {
+
         my $className = $normItem->{name}; # Name of component
         my $normalisation = $normItem->{value}; # Normalisation for component
 
-        my $command = "$dettool -addnormalizedstat -det_id $det_id -iteration $iter -class_id $className ".
-            "-norm $normalisation"; # Command to run
-        $command .= " -dbname $dbname" if defined $dbname;
+        my $command = $commandBase;
+        $command .= " -class_id $className";
+        $command .= " -norm $normalisation";
+
         my @command = split /\s+/, $command;
 
@@ -192 +190 @@
         print $stdout . "\n";
     }
+} else {
+    print "skipping command: $commandBase\n";
 }
 
@@ -203 +203 @@
     carp($msg);
     if ($det_id and $iter and not $no_update) {
-        my $command = "$dettool -addnormalizedstat -det_id $det_id -iteration $iter -code $exit_code";
+        my $command = "$dettool -addnormalizedstat";
+        $command .= " -det_id $det_id";
+        $command .= " -iteration $iter";
+        $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);

trunk/ippScripts/scripts/detrend_norm_exp.pl

@@ -17 +17 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
-use Statistics::Descriptive;
 use File::Temp qw( tempfile );
 
@@ -47 +46 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage(
-    -msg => "Required options: --det_id --iteration --camera --det_type",
-    -exitval => 3,
-) unless defined $det_id
-     and defined $iter
-     and defined $camera
-     and defined $det_type;
+pod2usage( -msg => "Required options: --det_id --iteration --camera --det_type",
+           -exitval => 3)
+    unless defined $det_id
+    and defined $iter
+    and defined $camera
+    and defined $det_type;
 
 $ipprc->define_camera($camera);
 
-# Recipes to use, as a function of the detrend type
-use constant RECIPES => {
-    'bin1' => {         # We're creating a master --- already processed the input
-        'bias'     => 'PPIMAGE_J1_IMAGE_B',     # Bias only
-        'dark'     => 'PPIMAGE_J1_IMAGE_B',     # Dark only
-        'shutter'  => 'PPIMAGE_J1_IMAGE_F',     # Shutter only
-        'flat'     => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
-        'domeflat' => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
-        'skyflat'  => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
-        'fringe'   => 'PPIMAGE_J1_IMAGE_R',     # Fringe only
-    },
-    'bin2' => {         # We're checking the master --- input is not already processed
-        'bias'     => 'PPIMAGE_J2_IMAGE_B',     # Bias only
-        'dark'     => 'PPIMAGE_J2_IMAGE_B',     # Dark only
-        'shutter'  => 'PPIMAGE_J2_IMAGE_F',     # Shutter only
-        'flat'     => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
-        'domeflat' => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
-        'skyflat'  => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
-        'fringe'   => 'PPIMAGE_J2_IMAGE_R',     # Fringe only
-    },
-};
+# Recipes to use based on reduction class
+$reduction = 'DETREND' unless defined $reduction;
+
+my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
+
+my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
+
+# values to extract from output metadata and the stats to calculate
+# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
+# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
+# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
+my $STATS = 
+   [   
+       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+       { name => "bg",             type => "mean",  flag => "-bg" },
+       { name => "bg",             type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
+       # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
+   ];
 
 # Look for programs we need
@@ -87 +85 @@
     exit($PS_EXIT_CONFIG_ERROR); 
 }
-
-my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 # Get list of component files
@@ -101 +97 @@
         &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $error_code);
     }
+
+    # convert stdout to a metadata
+    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR);
+
+    # parse the file info in the metadata
     $files = parse_md_list($metadata) or
         &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR);
-}
-
-# Gather the statistics
-my ($bg, $bg_stdev, $bg_mean_stdev); # The statistics triplet
-{
-    my @backgrounds;            # Array of backgrounds in each component
-    my @variances;    # Array of variances for each component
-    foreach my $file (@$files) {
-        &my_die("Unable to find class id", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $file->{class_id};
-        my $class_id = $file->{class_id};
-        &my_die("Unable to find bg for class_id=$class_id", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $file->{bg};
-        &my_die("Unable to find bg_mean_stdev for class_id=$class_id", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $file->{bg_mean_stdev};
-        push @backgrounds, $file->{bg};
-        push @variances, $file->{bg_stdev}**2;
-    }
-
-    {
-        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
-        $stats->add_data(@backgrounds);
-        $bg = ($stats->mean() or 'NAN');
-        $bg_mean_stdev = ($stats->standard_deviation() or 'NAN');
-    }
-    {
-        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
-        $stats->add_data(@variances);
-        $bg_stdev = (sqrt( $stats->mean() ) or 'NAN');
+
+    # parse the stats in the metadata
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+    unless ($stats->parse($metadata)) {
+        &my_die("Unable to find all values in statistics output.\n", $det_id, $iter, $PS_EXIT_PROG_ERROR);
     }
 }
@@ -136 +116 @@
 my ($list1File, $list1Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b1.list.XXXX", UNLINK => 1 );
 my ($list2File, $list2Name) = tempfile( "$camera.$det_type.norm.$det_id.$iter.b2.list.XXXX", UNLINK => 1 );
-my @means;                      # Array of means
-my @stdevs;                     # Array of stdevs
 foreach my $file (@$files) {
     print $list1File ( $ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
     print $list2File ( $ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
-    push @means, $file->{bg};
-    push @stdevs, $file->{bg_stdev};
 }
 close $list1File;
@@ -152 +128 @@
 my $jpeg1Name = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1
 my $jpeg2Name = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2
-
-# Recipes to use in processing
-my $recipe1 = RECIPES->{"bin1"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
-
-my $recipe2 = RECIPES->{"bin2"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
 
 unless ($no_op) {
@@ -178 +147 @@
 }
 
+# command to update the database
+my $command = "$dettool -addnormalizedexp";
+$command .= " -det_id $det_id";
+$command .= " -iteration $iter";
+$command .= " -recip $recipe1,$recipe2";
+$command .= " -path_base $outputRoot ";
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
+
+# Add the processed file to the database
 unless ($no_update) {
-    my $command = "$dettool -addnormalizedexp -det_id $det_id -iteration $iter";
-    $command .= " -recip $recipe1,$recipe2 -path_base $outputRoot ";
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
-    $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
@@ -189 +170 @@
         &my_die("Unable to perform dettool -addnormalizedexp: $error_code", $det_id, $iter, $error_code);
     }
-}
-
+} else {
+    print "skipping command: $command\n";
+}
 
 sub my_die
@@ -201 +183 @@
     carp($msg);
     if ($det_id and $iter and not $no_update) {
-        my $command = "$dettool -addprocessedimfile -det_id $det_id -iter $iter -code $exit_code";
+        my $command = "$dettool -addprocessedimfile";
+        $command .= " -det_id $det_id";
+        $command .= " -iter $iter";
+        $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);

trunk/ippScripts/scripts/detrend_process_exp.pl

@@ -17 +17 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
-use Statistics::Descriptive;
+use File::Temp qw( tempfile );
 
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
@@ -29 +29 @@
                        );
 my $ipprc = PS::IPP::Config->new(); # IPP configuration
-use File::Temp qw( tempfile );
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -47 +46 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage(
-    -msg => "Required options: --det_id --det_type --exp_tag --camera",
-    -exitval => 3,
-) unless defined $det_id
+pod2usage( -msg => "Required options: --det_id --det_type --exp_tag --camera",
+           -exitval => 3) 
+    unless defined $det_id
     and defined $det_type
     and defined $exp_tag
@@ -57 +55 @@
 $ipprc->define_camera($camera);
 
-# Recipes to use, as a function of the detrend type
-use constant RECIPES => {
-    'bin1' => {         # We're creating a master --- already processed the input
-        'bias'     => 'PPIMAGE_J1_IMAGE_B',     # Bias only
-        'dark'     => 'PPIMAGE_J1_IMAGE_B',     # Dark only
-        'shutter'  => 'PPIMAGE_J1_IMAGE_F',     # Shutter only
-        'flat'     => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
-        'domeflat' => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
-        'skyflat'  => 'PPIMAGE_J1_IMAGE_F',     # Flat-field only
-        'fringe'   => 'PPIMAGE_J1_IMAGE_R',     # Fringe only
-    },
-    'bin2' => {         # We're checking the master --- input is not already processed
-        'bias'     => 'PPIMAGE_J2_IMAGE_B',     # Bias only
-        'dark'     => 'PPIMAGE_J2_IMAGE_B',     # Dark only
-        'shutter'  => 'PPIMAGE_J2_IMAGE_F',     # Shutter only
-        'flat'     => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
-        'domeflat' => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
-        'skyflat'  => 'PPIMAGE_J2_IMAGE_F',     # Flat-field only
-        'fringe'   => 'PPIMAGE_J2_IMAGE_R',     # Fringe only
-    },
-};
+# Recipes to use based on reduction class
+$reduction = 'DETREND' unless defined $reduction;
+
+my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
+
+my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
+
+# values to extract from output metadata and the stats to calculate
+# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
+# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
+# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
+my $STATS = 
+   [   
+       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+       { name => "bg",             type => "mean",  flag => "-bg" },
+       { name => "bg",             type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
+       # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
+   ];
 
 # Look for programs we need
@@ -87 +85 @@
     exit($PS_EXIT_CONFIG_ERROR); 
 }
-
-my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
 # Get list of component files
@@ -101 +97 @@
         &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $exp_tag, $error_code);
     }
+
+    # convert stdout to a metadata
+    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR);
+
+    # parse the file info in the metadata
     $files = parse_md_list($metadata) or
         &my_die("Unable to parse metadata list", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR);
-}
-
-# Gather the statistics
-my ($bg, $bg_stdev, $bg_mean_stdev); # The statistics triplet
-{
-    my @backgrounds;            # Array of backgrounds in each component
-    my @variances;    # Array of variances for each component
-    foreach my $file (@$files) {
-        &my_die("Unable to find class id", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{class_id};
-        my $class_id = $file->{class_id};
-        &my_die("Unable to find bg for class_id=$class_id", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg};
-        &my_die("Unable to find bg_mean_stdev for class_id=$class_id", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless defined $file->{bg_mean_stdev};
-        push @backgrounds, $file->{bg};
-        push @variances, $file->{bg_stdev}**2;
-    }
-
-    {
-        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
-        $stats->add_data(@backgrounds);
-        $bg = ($stats->mean() or 'NAN');
-        $bg_mean_stdev = ($stats->standard_deviation() or 'NAN');
-    }
-    {
-        my $stats = Statistics::Descriptive::Sparse->new; # Statistics calculator
-        $stats->add_data(@variances);
-        $bg_stdev = (sqrt( $stats->mean() ) or 'NAN');
+
+    # parse the stats in the metadata
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+    unless ($stats->parse($metadata)) {
+        &my_die("Unable to find all values in statistics output.\n", $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
     }
 }
@@ -137 +117 @@
 my ($list1File, $list1Name) = tempfile( "$exp_tag.detproc.$det_id.b1.list.XXXX", UNLINK => 1 );
 my ($list2File, $list2Name) = tempfile( "$exp_tag.detproc.$det_id.b2.list.XXXX", UNLINK => 1 );
-my @means;                      # Array of means
-my @stdevs;                     # Array of stdevs
 foreach my $file (@$files) {
     print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
     print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
-    push @means, $file->{bg};
-    push @stdevs, $file->{bg_stdev};
 }
 close $list1File;
@@ -153 +129 @@
 my $jpeg1 = $ipprc->filename("PPIMAGE.JPEG1", $outputRoot); # Binned JPEG #1
 my $jpeg2 = $ipprc->filename("PPIMAGE.JPEG2", $outputRoot); # Binned JPEG #2
-
-# Recipes to use in processing
-my $recipe1 = RECIPES->{"bin1"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
-
-my $recipe2 = RECIPES->{"bin2"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
 
 unless ($no_op) {
@@ -187 +156 @@
 }
 
+# Command to update the database
+my $command = "$dettool -addprocessedexp";
+$command .= " -det_id $det_id";
+$command .= " -exp_tag $exp_tag";
+$command .= " -recip $recipe1,$recipe2 -path_base $outputRoot";
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
+
+# Add the processed file to the database
 unless ($no_update) {
-    my $command = "$dettool -addprocessedexp -det_id $det_id -exp_tag $exp_tag";
-    $command .= " -recip $recipe1,$recipe2 -path_base $outputRoot";
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
-    $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
@@ -199 +179 @@
         exit($error_code);
     }
-}
-
+} else {
+    print "skipping command: $command\n";
+}
 
 sub my_die
@@ -211 +192 @@
     carp($msg);
     if ($det_id and $exp_tag and not $no_update) {
-        my $command = "$dettool -addprocessedexp -det_id $det_id -exp_tag $exp_tag -code $exit_code";
+        my $command = "$dettool -addprocessedexp";
+        $command .= " -det_id $det_id";
+        $command .= " -exp_tag $exp_tag";
+        $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);

trunk/ippScripts/scripts/detrend_process_imfile.pl

@@ -48 +48 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage(
-    -msg => "Required options: --det_id --exp_tag --class_id --det_type --input_uri --camera",
-    -exitval => 3,
-) unless defined $det_id
+pod2usage( -msg => "Required options: --det_id --exp_tag --class_id --det_type --input_uri --camera",
+           -exitval => 3)
+    unless defined $det_id
     and defined $exp_tag
     and defined $class_id
@@ -61 +60 @@
 $ipprc->define_camera($camera);
 
+# Recipes to use as a function of detrend type
 $reduction = "DETREND" unless defined $reduction;
 my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_PROCESS'); # Recipe name to use
+
+# values to extract from output metadata and the stats to calculate
+my $STATS = 
+   [   
+       #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
+       { name => "ROBUST_MEDIAN",  type => "mean",  flag => "-bg" },
+       { name => "ROBUST_MEDIAN",  type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "ROBUST_STDEV",   type => "rms",   flag => "-bg_stdev" },
+   ];
 
 # Look for programs we need
@@ -74 +83 @@
 $ppImage .= " -dbname $dbname" if defined $dbname;
 
+&my_die("Couldn't find input file: $input_uri\n", $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri);
+
 $workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir;
 
-my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri );
+my $outputRoot  = $ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri );
+
 my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id);
-my $outputBin1 = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id);
-my $outputBin2 = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id);
-my $outputStats = $outputRoot . '.' . $class_id . '.stats';
+my $outputBin1  = $ipprc->filename("PPIMAGE.BIN1",   $outputRoot, $class_id);
+my $outputBin2  = $ipprc->filename("PPIMAGE.BIN2",   $outputRoot, $class_id);
+my $outputStats = $ipprc->filename("PPIMAGE.STATS",  $outputRoot, $class_id);
 
 # Run ppImage
-my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
 unless ($no_op) {
     my $command = "$ppImage -file $input_uri $outputRoot";
@@ -96 +107 @@
         &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_tag, $class_id, $error_code);
     }
+
     &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage);
     &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats);
@@ -106 +118 @@
     my @contents = <$statsFile>; # Contents of file
     close $statsFile;
+
+    # parse the statistics MDC file
     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     my $metadata = $mdcParser->parse(join "", @contents)
         or &my_die("Unable to parse metadata config", $det_id, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+
+    # extract the stats from the metadata
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
     $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
 }
 
-my $bg            = $stats->bg_mean();
-my $bg_stdev      = $stats->bg_stdev();
-my $bg_mean_stdev = $stats->bg_mean_stdev();
+# command to update database
+my $command = "$dettool -addprocessedimfile";
+$command .= " -det_id $det_id";
+$command .= " -exp_tag $exp_tag";
+$command .= " -class_id $class_id";
+$command .= " -recip $reduction";
+$command .= " -uri $outputImage -path_base $outputRoot";
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
 
 # Add the processed file to the database
 unless ($no_update) {
-    my $command = "$dettool -addprocessedimfile";
-    $command .= " -det_id $det_id";
-    $command .= " -exp_tag $exp_tag";
-    $command .= " -class_id $class_id";
-    $command .= " -recip $recipe";
-    $command .= " -uri $outputImage -path_base $outputRoot";
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
-    $command .= " -dbname $dbname" if defined $dbname;
-
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
@@ -134 +154 @@
         exit($error_code);
     }
+} else {
+    print "skipping command: $command\n";
 }
 
@@ -146 +168 @@
     carp($msg);
     if ($det_id and $exp_tag and $class_id and not $no_update) {
-        my $command = "$dettool -addprocessedimfile -det_id $det_id -exp_tag $exp_tag -class_id $class_id -code $exit_code";
+        my $command = "$dettool -addprocessedimfile";
+        $command .= " -det_id $det_id";
+        $command .= " -exp_tag $exp_tag"; 
+        $command .= " -class_id $class_id";
+        $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);

trunk/ippScripts/scripts/detrend_reject_exp.pl

@@ -47 +47 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage(
-    -msg => "Required options: --det_id --iteration --det_type --camera",
-    -exitval => 3,
-) unless defined $det_id
+pod2usage( -msg => "Required options: --det_id --iteration --det_type --camera",
+           -exitval => 3)
+    unless defined $det_id
     and defined $iter
     and defined $det_type
     and defined $camera;
 
+# values to extract from output metadata and the stats to calculate
+# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
+# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
+# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
+my $STATS = 
+   [   
+       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+       { name => "bg",             type => "mean",  flag => "-bg" },
+       { name => "bg_mean_stdev",  type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
+   ];
+
+my $REJSTATS = 
+   [   
+       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+       { name => "bg",             type => "mean",   flag => "ensMeanMean" },
+       { name => "bg",             type => "stdev",  flag => "ensMeanStdev" },
+       { name => "bg_mean_stdev",  type => "mean",   flag => "ensMeanStdevMean" },
+       { name => "bg_mean_stdev",  type => "stdev",  flag => "ensMeanStdevStdev" },
+       { name => "bg_stdev",       type => "mean",   flag => "ensStdevMean" },
+       { name => "bg_stdev",       type => "stdev",  flag => "ensStdevStdev" },
+   ];
 
 # Look for programs we need
@@ -64 +85 @@
 }
 
-my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
 # Get list of component files
 my $exposures;                  # Array of exposures
 {
-    my $command = "$dettool -residexp -det_id $det_id -iteration $iter"; # Command to run
+    # dettool command to select exp data for this det_run
+    my $command = "$dettool -residexp";
+    $command .= " -det_id $det_id";
+    $command .= " -iteration $iter";
     $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -78 +100 @@
     }
 
+    # Parse the stdout buffer into a metadata
+    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", $det_id, $iter, $PS_EXIT_PROG_ERROR);
-    # XXX need to test that the result list is 0 length and return with a different error
+
+    # parse the file info in the metadata
     $exposures = parse_md_list($metadata) or
         &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR);
+
+    # Parse the statistics on the residual image
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR);
+
+    # Parse the statistics for rejections
+    my $rejstats = PS::IPP::Metadata::Stats->new($REJSTATS); # Stats parser
+    $rejstats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $PS_EXIT_PROG_ERROR);
 }
 
 my @expTags;                    # Array of exposure IDs
-my @means;                      # Array of means
-my @stdevs;                     # Array of standard deviations
-my @meanStdevs;                 # Array of standard deviations of the mean (normalised by the mean)
-my @variances;                  # Array of variances
 my @accept;                     # Array of accept flags
 my @include;                    # Array of include flags
 foreach my $exposure (@$exposures) {
     &my_die("Unable to find exposure id.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $exposure->{exp_tag};
-    &my_die("Unable to find mean.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $exposure->{bg};
-    &my_die("Unable to find stdev.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $exposure->{bg_stdev};
-    &my_die("Unable to find mean stdev.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $exposure->{bg_mean_stdev};
     &my_die("Unable to find accept.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $exposure->{accept};
     &my_die("Unable to find include.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $exposure->{include};
-    push @expTags, $exposure->{exp_tag};
-    push @means, $exposure->{bg};
-    push @stdevs, $exposure->{bg_stdev};
-    # XXX why are we using mean_stdev / bd?
-    # push @meanStdevs, $exposure->{bg_mean_stdev} / $exposure->{bg};
-    push @meanStdevs, $exposure->{bg_mean_stdev};
-    push @variances, $exposure->{bg_stdev}**2;
-    push @accept, $exposure->{accept};
-    push @include, $exposure->{include};
-}
-my $meanStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
-$meanStats->add_data(@means);
-my $stdevStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
-$stdevStats->add_data(@stdevs);
-my $meanStdevStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
-$meanStdevStats->add_data(@meanStdevs);
+}
+
+my $ensMeanMean       = &STATS_value_for_flag ($REJSTATS, "ensMeanMean");      
+my $ensMeanStdev      = &STATS_value_for_flag ($REJSTATS, "ensMeanStdev");     
+my $ensMeanStdevMean  = &STATS_value_for_flag ($REJSTATS, "ensMeanStdevMean"); 
+my $ensMeanStdevStdev = &STATS_value_for_flag ($REJSTATS, "ensMeanStdevStdev");
+my $ensStdevMean      = &STATS_value_for_flag ($REJSTATS, "ensStdevMean");     
+my $ensStdevStdev     = &STATS_value_for_flag ($REJSTATS, "ensStdevStdev");    
 
 $ipprc->define_camera($camera);
 # Rejection thresholds
-my $reject_mean = rejection_limit( 'ENSEMBLE.MEAN', $det_type, $filter );
-my $reject_stdev = rejection_limit( 'ENSEMBLE.STDEV', $det_type, $filter );
+my $reject_mean      = rejection_limit( 'ENSEMBLE.MEAN',      $det_type, $filter );
+my $reject_stdev     = rejection_limit( 'ENSEMBLE.STDEV',     $det_type, $filter );
 my $reject_meanstdev = rejection_limit( 'ENSEMBLE.MEANSTDEV', $det_type, $filter );
 
@@ -129 +147 @@
 unless ($no_op) {
     $logFile = $ipprc->file_create_append( $logName );
-    print $logFile "Ensemble mean " . $meanStats->mean() . " +/- " . $meanStats->standard_deviation .
-        ", stdev " . $stdevStats->mean() . " +/- " . $stdevStats->standard_deviation() . "\n\n";
+    print $logFile "Ensemble mean $meanEnsemble +/- $stdevEnsemble, stdev $meanStdevEnsemble\n\n";
 }
 
@@ -136 +153 @@
 my $numChanges = 0;             # Number of exposures with changed status
 my $numReject = 0;              # Number of exposures rejected
-my $command;
-my $reject;
-my $expTag;
-for (my $i = 0; $i < scalar @means; $i++) {
-    $expTag = $expTags[$i];     # Exposure ID
-    $command = "$dettool -updateresidexp -det_id $det_id -iteration $iter -exp_tag $expTag"; # Command to run
+
+for (my $i = 0; $i < scalar @$exposures; $i++) {
+    my $file      = $files[$i];
+    my $mean      = $file->{bg};        # Mean for this exposure
+    my $stdev     = $file->{bg_stdev}; # Stdev for this exposure
+    my $meanStdev = $file->{bg_mean_stdev}; # Stdev of Means for this exposure
+
+    my $expTag  = $exposure->{exp_tag};
+    my $accept  = $exposure->{accept};
+    my $include = $exposure->{include};
+
+    my $reject = 0;             # Reject this exposure?
+
+    my $command = "$dettool -updateresidexp";
+    $command .= " -det_id $det_id";
+    $command .= " -iteration $iter";
+    $command .= " -exp_tag $expTag";
     $command .= " -dbname $dbname" if defined $dbname;
     
-    $reject = 0;                # Reject this exposure?
-
-    if (not $accept[$i]) {
+    if (not $accept) {
         # Rejected this at an earlier stage
         unless ($no_op) {
@@ -165 +191 @@
     }
     
-    if ($reject_mean > 0 and defined $meanStats->standard_deviation() ) {
-        my $dMean = abs($means[$i] - $meanStats->mean()) ;
-        if ($dMean > ($reject_mean * $meanStats->standard_deviation())) {
+    if ($reject_mean > 0 and $ensMeanStdev > 0) {
+        my $delta = abs($mean - $ensMeanMean);
+        if ($delta > ($reject_mean * $ensMeanStdev)) {
             print $logFile "Rejecting $expTag based on ensemble mean value: ";
             $reject = 1;
@@ -174 +200 @@
             print $logFile "$expTag OK against ensemble mean: ";
         }
-        print $logFile "$means[$i] --> $dMean vs " . $reject_mean * $meanStats->standard_deviation() . "\n";
+        print $logFile "$mean --> $delta vs " . $reject_mean * $ensMeanStdev . "\n";
     } else {
         print $logFile "No rejection of $expTag for ensemble mean\n";
     }
 
-    if ($reject_stdev > 0 and defined $stdevStats->standard_deviation() > 0) {
-        my $dMean = abs($stdevs[$i] - $stdevStats->mean());
-        if ($dMean > ($reject_stdev * $stdevStats->standard_deviation())) {
+    if ($reject_stdev > 0 and $ensStdevStdev > 0) {
+        my $delta = abs($stdev - $ensStdevMean);
+        if ($delta > ($reject_stdev * $ensStdevStdev)) {
             print $logFile "Rejecting $expTag based on ensemble stdev: ";
             $reject = 1;
@@ -188 +214 @@
             print $logFile "$expTag OK against ensemble stdev: ";
         }
-        print $logFile "$stdevs[$i] --> $dMean sigma vs " . $reject_stdev  * $stdevStats->standard_deviation() . "\n";
+        print $logFile "$stdev --> $delta sigma vs " . $reject_stdev * $ensStdevStdev . "\n";
     } else {
         print $logFile "No rejection of $expTag for ensemble stdev\n";
     }
     
-    if ($reject_meanstdev > 0 and defined $meanStdevStats->standard_deviation() > 0) {
-        my $dMean = abs($meanStdevs[$i] - $meanStdevStats->mean());
-        if ($dMean > ($reject_meanstdev * $meanStdevStats->standard_deviation())) {
+    if ($reject_meanstdev > 0 and $ensMeanStdevStdev > 0) {
+        my $delta = abs($meanStdev - $ensMeanStdevMean);
+        if ($delta > ($reject_meanstdev * $ensMeanStdevStdev)) {
             print $logFile "Rejecting $expTag based on ensemble mean stdev: ";
             $reject = 1;
@@ -202 +228 @@
             print $logFile "$expTag OK against ensemble mean stdev: ";
         }
-        print $logFile "$meanStdevs[$i] --> $dMean sigma vs " . $reject_meanstdev . "\n";
+        print $logFile "$meanStdev --> $delta sigma vs " . $reject_meanstdev * $ensMeanStdevStdev. "\n";
     } else {
         print $logFile "No rejection of $expTag for ensemble mean stdev\n";
@@ -214 +240 @@
     
     # Check for status changes
-    if ((not $include[$i] and not $reject) or ($include[$i] and $reject)) {
+    if ((not $include and not $reject) or ($include and $reject)) {
         unless ($no_op) {
             print $logFile "Status of $expTag has changed.\n";
@@ -253 +279 @@
 }
 
+my $command = "$dettool -adddetrunsummary";
+$command .= " -det_id $det_id";
+$command .= " -iteration $iter";
+$command .= " -accept" if $master;
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
+
 # Put results into the database
 unless ($no_update) {
-
-    {
-        # Add the summary
-        my $varianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
-        $varianceStats->add_data(@variances);
-        
-        my $bg = ($meanStats->mean() or 'NAN');
-        my $bg_stdev = (sqrt( $varianceStats->mean() ) or 'NAN');
-        my $bg_mean_stdev = ($meanStats->standard_deviation() or 'NAN');
-        
-        my $command = "$dettool -adddetrunsummary -det_id $det_id -iteration $iter";
-        $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
-        $command .= " -accept" if $master;
-        $command .= " -dbname $dbname" if defined $dbname;
-        
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 1);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            warn("Unable to perform dettool -adddetrunsummary: $error_code");
-            exit($error_code);
-        }
-    }
-
-    # Re-run processing if required
-    {
-        my $command = "$dettool -updatedetrun -det_id $det_id";
-        if ($stop) {
-            $command .= ' -state stop';
-        } else {
-            $command .= ' -again';
-        }
-        
-        $command .= " -dbname $dbname" if defined $dbname;
-        
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 1);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            warn("Unable to perform dettool -updatedetrun: $error_code");
-            exit($error_code);
-        }
-    }
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 1);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("Unable to perform dettool -adddetrunsummary: $error_code");
+        exit($error_code);
+    }
+
+    $command = "$dettool -updatedetrun -det_id $det_id";
+    if ($stop) {
+        $command .= ' -state stop';
+    } else {
+        $command .= ' -again';
+    }
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 1);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("Unable to perform dettool -updatedetrun: $error_code");
+        exit($error_code);
+    }
+} else {
+    print "skipping command: $command\n";
 }
 
@@ -317 +338 @@
     carp($msg);
     if ($det_id and $iter and not $no_update) {
-        my $command = "$dettool -adddetrunsummary -det_id $det_id -iteration $iter -code $exit_code";
+        my $command = "$dettool -adddetrunsummary";
+        $command .= " -det_id $det_id";
+        $command .= " -iteration $iter";
+        $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);
@@ -323 +347 @@
     exit $exit_code;
 }
-
-
 
 # Retrieve the requested rejection limit, dying unless extant
@@ -342 +364 @@
 }
 
+sub STATS_value_for_flag 
+{
+    my $STATS = shift;
+    my $flag  = shift;
+
+    foreach my $entry (@$STATS) {
+        if ($flag eq $entry->{flag}) {
+            return $entry->{value};
+        }
+    }
+    return 'NAN';
+}
 
 __END__

trunk/ippScripts/scripts/detrend_reject_imfile.pl

@@ -22 +22 @@
 use IPC::Cmd 0.36 qw( can_run run );             # tools to run UNIX programs with control over I/O
 use PS::IPP::Metadata::Config;                   # tools to parse the psMetadataConfig files
-use PS::IPP::Metadata::List qw( parse_md_list ); # ?
+use PS::IPP::Metadata::List qw( parse_md_list ); # tools to parse a metadata into a hash list
 use Statistics::Descriptive;                     # tools for calculating basic statistical quantities
 use File::Temp qw( tempfile );                   # tools to construct temp files
+
 use PS::IPP::Config qw($PS_EXIT_SUCCESS          
                        $PS_EXIT_UNKNOWN_ERROR
@@ -56 +57 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage(
-          -msg => "Required options: --det_id --iteration --exp_tag --det_type --camera",
-          -exitval => 3,
-          ) unless defined $det_id
+pod2usage( -msg => "Required options: --det_id --iteration --exp_tag --det_type --camera",
+           -exitval => 3)
+    unless defined $det_id
     and defined $iter
     and defined $exp_tag
@@ -68 +68 @@
 $ipprc->define_camera($camera);
 
-# Recipes to use, as a function of the detrend type
-use constant RECIPES => {
-    'bin1' => {         # We're creating a master --- already processed the input
-        'bias'     => 'PPIMAGE_J1_RESID_B',     # Bias only
-        'dark'     => 'PPIMAGE_J1_RESID_B',     # Dark only
-        'shutter'  => 'PPIMAGE_J1_RESID_F',     # Shutter only
-        'flat'     => 'PPIMAGE_J1_RESID_F',     # Flat-field only
-        'domeflat' => 'PPIMAGE_J1_RESID_F',     # Flat-field only
-        'skyflat'  => 'PPIMAGE_J1_RESID_F',     # Flat-field only
-        'fringe'   => 'PPIMAGE_J1_RESID_R',     # Fringe only
-    },
-    'bin2' => {         # We're checking the master --- input is not already processed
-        'bias'     => 'PPIMAGE_J2_RESID_B',     # Bias only
-        'dark'     => 'PPIMAGE_J2_RESID_B',     # Dark only
-        'shutter'  => 'PPIMAGE_J2_RESID_F',     # Shutter only
-        'flat'     => 'PPIMAGE_J2_RESID_F',     # Flat-field only
-        'domeflat' => 'PPIMAGE_J2_RESID_F',     # Flat-field only
-        'skyflat'  => 'PPIMAGE_J2_RESID_F',     # Flat-field only
-        'fringe'   => 'PPIMAGE_J2_RESID_R',     # Fringe only
-    },
-};
+# Recipes to use based on reduction class
+$reduction = 'DETREND' unless defined $reduction;
+
+my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_RESID' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
+
+my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_RESID' . uc($det_type); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
+
+# values to extract from output metadata and the stats to calculate
+# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
+# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
+# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
+my $STATS = 
+   [   
+       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+       { name => "bg",             type => "mean",  flag => "-bg" },
+       { name => "bg_mean_stdev",  type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
+       { name => "bin_stdev",      type => "rms",   flag => "-bin_stdev" },
+       { name => "fringe_0",       type => "mean",  flag => "-fringe_0" },
+       { name => "fringe_1",       type => "rms",   flag => "-fringe_1" },
+       { name => "fringe_0",       type => "stdev", flag => "-fringe_2" },
+       { name => "user_1",         type => "mean",  flag => "-user_1" }, # fringe residual
+       { name => "user_2",         type => "rms",   flag => "-user_2" }, # fringe residual
+       { name => "user_3",         type => "stdev", flag => "-user_1" }, # fringe residual
+   ];
 
 # Look for programs we need
@@ -98 +104 @@
     exit($PS_EXIT_CONFIG_ERROR); 
 }
-
-# Parser for psMetadataConfig files (used for output from dettool)
-my $mdcParser = PS::IPP::Metadata::Config->new; 
 
 # Get list of imfile files
@@ -120 +123 @@
     # XXX report an error message if stdout_buf is empty
 
+    # Parse the stdout buffer into a metadata
+    my $mdcParser = PS::IPP::Metadata::Config->new;     
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR);
 
+    # parse the file info in the metadata
     $files = parse_md_list($metadata) or
         &my_die("Unable to parse metadata list", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR);
+
+    # Parse the statistics on the residual image
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
+    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
 }
 
@@ -136 +146 @@
 my $logFile;
 unless ($no_op) {
-    # XXX this will fail if the file exists (because of Nebulous rules)
     $logFile = $ipprc->file_create_append( $logName );
 }
-
-#### build the JPEG images ####
 
 # XXX in debug mode, unlink = 0
@@ -152 +159 @@
 close $list2File;
 
-# Recipes to use in processing
-my $recipe1 = RECIPES->{"bin1"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
-
-my $recipe2 = RECIPES->{"bin2"}->{lc($det_type)};
-&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
-
+
+# build the JPEG images
 unless ($no_op) {
 
@@ -206 +208 @@
 
 # storage variables
-my @means;                      # Array of means
-my @fluxes;                     # Array of means / exptimes
-my @variances;                  # Array of variances
-my @binVariances;               # Array of binned variances
-my @meanStdevs;                 # Array of mean stdevs
-my @names;                      # Array of names (class_id)
-my @fringe_means;
-my @fringe_vars;
-my @dfringe_means;
-my @dfringe_vars;
 
 # load the arrays from the imfile output table
 foreach my $file (@$files) {
-    push @means,        $file->{bg};                           # mean background counts
     if ($file->{exp_time} > 0.0) {
         push @fluxes,           $file->{bg} / $file->{exp_time};       # mean background counts / sec
@@ -225 +216 @@
         push @fluxes,           $file->{bg};
     }
-    push @meanStdevs,        $file->{bg_mean_stdev};                # stdev of the mean counts (for imfile components)
-    push @variances,         $file->{bg_stdev}*$file->{bg_stdev};   # total variance from all imfile components
-    push @binVariances,      $file->{bin_stdev}*$file->{bin_stdev}; # total variance of binned images over all imfile components
-    push @names,             $file->{class_id};                     # name of the component
-    push @fringe_means,      $file->{fringe_0};                     # fringe amplitude mean for imfile
-    push @fringe_vars,       $file->{fringe_1}*$file->{fringe_1};   # fringe variance
-    push @dfringe_means,     $file->{user_1};                       # fringe residual mean for imfile
-    push @dfringe_vars,      $file->{user_2}*$file->{user_2};       # fringe residual variance
-    # push @fringe_mean_stdev, $file->{fringe_2};                     # fringe amplitude stdev for imfile
 }
 
@@ -239 +221 @@
 # it is VALID to reject on more than one criterion
 &my_die("Number of means and number of variances differ!", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR) unless scalar @means == scalar @variances;
-for (my $i = 0; $i < scalar @means; $i++) {
-    my $flux = $fluxes[$i];     # Flux for this imfile
-    my $mean = $means[$i];      # Mean for this imfile
+
+# storage array
+my @fluxes;
+
+for (my $i = 0; $i < scalar @$files; $i++) {
+    my $file      = $files[$i];
+    my $name      = $file->{class_id};
+    my $mean      = $file->{bg};        # Mean for this imfile
+    my $stdev     = $file->{bg_stdev}; # Stdev for this imfile
+    my $meanStdev = $file->{bg_mean_stdev}; # Stdev of Means for this imfile
+    my $binStdev  = $file->{bin_stdev}; # Binned Stdev for this imfile
+
+    # calculate and save the fluxes
+    my $flux      = $mean / $file->{exp_time};
+    if ($file->{exp_time} == 0.0) {
+        push @fluxes, $mean;
+    }
+    push @fluxes, $flux;
+
     $mean -= $expected;
-    my $stdev = sqrt($variances[$i]);   # Stdev for this imfile
-    my $binStdev = sqrt($binVariances[$i]);     # Stdev for this imfile
-    my $name = $names[$i];
 
     last if $no_op;
@@ -301 +296 @@
     # component means is larger than the limit
     if ($reject_imfile_meanstdev > 0) {
-        if ($meanStdevs[$i] > $reject_imfile_meanstdev) {
+        if ($meanStdevs > $reject_imfile_meanstdev) {
             print $logFile "Rejecting exposure based on bad imfile mean stdev for $name: ";
             $reject = 1;
@@ -307 +302 @@
             print $logFile "Imfile mean stdev for $name meets requirements: ";
         }
-        print $logFile "$meanStdevs[$i] vs $reject_imfile_meanstdev\n";
+        print $logFile "$meanStdevs vs $reject_imfile_meanstdev\n";
     } else {
         print $logFile "No rejection on imfile mean stdev for $name\n";
@@ -356 +351 @@
 }
 
-# calculate the exposure ensemble statistics
-my $meanStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for means
-$meanStats->add_data(@means);
+# basic ensemble stats
+my $mean               = &STATS_value_for_flag ($STATS, "-bg");
+my $meanStdev          = &STATS_value_for_flag ($STATS, "-bg_mean_stdev");
+my $stdev              = &STATS_value_for_flag ($STATS, "-bg_stdev");
+my $binStdev           = &STATS_value_for_flag ($STATS, "-bin_stdev");
+my $fringe_mean        = &STATS_value_for_flag ($STATS, "-fringe_0");
+my $fringe_err         = &STATS_value_for_flag ($STATS, "-fringe_1");
+my $fringe_mean_stdev  = &STATS_value_for_flag ($STATS, "-fringe_2");
+my $dfringe_mean       = &STATS_value_for_flag ($STATS, "-user_1");
+my $dfringe_err        = &STATS_value_for_flag ($STATS, "-user_2");
+my $dfringe_mean_stdev = &STATS_value_for_flag ($STATS, "-user_3");
+
+# other stats (flux depends on bg and exp_time)
 my $fluxStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for means
 $fluxStats->add_data(@fluxes);
-my $varianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for variances
-$varianceStats->add_data(@variances);
-my $binVarianceStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for variances
-$binVarianceStats->add_data(@binVariances);
-
-# background stats
-my $mean = $meanStats->mean();  # Mean of the imfile means
 my $flux = $fluxStats->mean();  # Mean of the imfile means
-my $meanStdev = $meanStats->standard_deviation(); # Stdev of the imfile means
-if (not defined $meanStdev) {
-    # this is the case for Nimfile == 1
-    $meanStdev = 0;
-}
-my $stdev = sqrt($varianceStats->mean()); # Root-Mean-Square of the imfile stdevs (root mean of variances)
-my $binStdev = sqrt($binVarianceStats->mean()); # Root-Mean-Square of the imfile stdevs (root mean of variances)
+
+# other stats
 my $exp_sn = 0.0;
 if ($stdev > 0) { $exp_sn = $mean / $stdev; }
-
-# prepare fringe amplitude stats
-my $fringeMeanStats = Statistics::Descriptive::Sparse->new();   # Statistics calculator for means
-$fringeMeanStats->add_data(@fringe_means);
-my $fringeVarStats = Statistics::Descriptive::Sparse->new();    # Statistics calculator for means
-$fringeVarStats->add_data(@fringe_vars);
-
-# fringe amplitude stats
-my $fringe_mean = $fringeMeanStats->mean();
-my $fringe_err = sqrt($fringeVarStats->mean());
-my $fringe_mean_stdev = $fringeMeanStats->standard_deviation();
-if (not defined $fringe_mean_stdev) {
-    $fringe_mean_stdev = 0;
-}
-
-# prepare fringe residual stast 
-my $dfringeMeanStats = Statistics::Descriptive::Sparse->new();  # Statistics calculator for means
-$dfringeMeanStats->add_data(@dfringe_means);
-my $dfringeVarStats = Statistics::Descriptive::Sparse->new();   # Statistics calculator for means
-$dfringeVarStats->add_data(@dfringe_vars);
-
-# fringe amplitude stats
-my $dfringe_mean = $dfringeMeanStats->mean();
-my $dfringe_err = sqrt($dfringeVarStats->mean());
-my $dfringe_mean_stdev = $dfringeMeanStats->standard_deviation();
-if (not defined $dfringe_mean_stdev) {
-    $dfringe_mean_stdev = 0;
-}
 
 ## Reject based on the exposure ensemble stats
@@ -514 +479 @@
 }
 
-# Add the result into the database
-my $bg            = $mean;
-my $bg_stdev      = $stdev;
-my $bg_mean_stdev = $meanStdev;
-my $bin_stdev     = $binStdev;
+my $command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_tag $exp_tag";
+$command .= " -recip $recipe1,$recipe2 -path_base $outputRoot ";
+$command .= ' -reject' if $reject;
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
 
 unless ($no_update) {
-    my $command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_tag $exp_tag";
-    $command .= " -recip $recipe1,$recipe2 -path_base $outputRoot ";
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev -bin_stdev $bin_stdev";
-    $command .= " -fringe_0 $fringe_mean -fringe_1 $fringe_err -fringe_2 $fringe_mean_stdev";
-    $command .= " -user_1 $dfringe_mean -user_2 $dfringe_err -user_3 $dfringe_mean_stdev";
-    $command .= ' -reject' if $reject;
-    $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
@@ -535 +499 @@
         exit($error_code);
     }
+} else {
+    print "skipping command: $command\n";
 }
 
@@ -555 +521 @@
     carp($msg);
     if ($det_id and $iter and $exp_tag and not $no_update) {
-        my $command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_tag $exp_tag -code $exit_code";
+        my $command = "$dettool -addresidexp";
+        $command .= " -det_id $det_id";
+        $command .= " -iteration $iter";
+        $command .= " -exp_tag $exp_tag";
+        $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);
@@ -561 +531 @@
     exit $exit_code;
 }
-
 
 # Retrieve the requested rejection limit, dying if not extant
@@ -579 +548 @@
 }
 
-
-
+sub STATS_value_for_flag 
+{
+    my $STATS = shift;
+    my $flag  = shift;
+
+    foreach my $entry (@$STATS) {
+        if ($flag eq $entry->{flag}) {
+            return $entry->{value};
+        }
+    }
+    return 'NAN';
+}
 
 __END__

trunk/ippScripts/scripts/detrend_resid.pl

@@ -17 +17 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::Stats;
-use Data::Dumper;
 
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
@@ -33 +32 @@
 use Pod::Usage qw( pod2usage );
 
-my ($det_id, $iter, $exp_tag, $class_id, $det_type, $detrend,
-        $input_uri, $camera, $mode, $dbname, $workdir, $reduction, $no_update, $no_op);
+my ($det_id, $iter, $exp_tag, $class_id, $det_type, $detrend, $input_uri, $camera, $mode, $dbname, $workdir, $reduction, $no_update, $no_op);
 GetOptions(
     'det_id|d=s'        => \$det_id,
@@ -53 +51 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage(
-    -msg => "Required options: --det_id --iteration --exp_tag --class_id --det_type --camera --input_uri --mode --detrend (not for 'verify' mode)",
-    -exitval => 3,
-) unless defined $det_id
+pod2usage( -msg => "Required options: --det_id --iteration --exp_tag --class_id --det_type --camera --input_uri --mode --detrend (not for 'verify' mode)",
+           -exitval => 3)
+    unless defined $det_id
     and defined $iter
     and defined $exp_tag
@@ -68 +65 @@
 $ipprc->define_camera($camera);
 
+# Recipes to use as a function of detrend type and mode
 $reduction = 'DETREND' unless defined $reduction;
 my $recipe;                     # Name of recipe to use
@@ -78 +76 @@
 }
 $recipe = $ipprc->reduction($reduction, $recipe);
+
+# values to extract from output metadata and the stats to calculate
+my $STATS = 
+   [   
+       #          PPSTATS KEYWORD         STATISTIC          CHIPTOOL FLAG
+       { name => "ROBUST_MEDIAN",      type => "mean",  flag => "-bg" },
+       { name => "ROBUST_MEDIAN",      type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "ROBUST_STDEV",       type => "rms",   flag => "-bg_stdev" },
+       { name => "FRINGE_0",           type => "mean",  flag => "-fringe_0" },
+       { name => "FRINGE_ERR_0",       type => "rms",   flag => "-fringe_1" },
+       { name => "FRINGE_0",           type => "stdev", flag => "-fringe_2" },
+       { name => "FRINGE_RESID_0",     type => "mean",  flag => "-user_1" },
+       { name => "FRINGE_RESID_ERR_0", type => "rms",   flag => "-user_2" },
+       { name => "FRINGE_RESID_0",     type => "stdev", flag => "-user_3" },
+   ];
+my $BINNED_STATS = 
+   [   
+       { name => "ROBUST_STDEV",   type => "rms",   flag => "-bin_stdev" },
+   ];
 
 # Flags to specify the particular detrend to use
@@ -106 +123 @@
 
 my $outputRoot = $ipprc->file_prepare( "$exp_tag/$exp_tag.detresid.$det_id.$iter", $workdir, $input_uri );
-my $outputName = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id);
-my $bin1Name =  $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id);
-my $bin2Name =  $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id);
-my $outputStats = $outputRoot . '.' . $class_id . '.stats';
+
+my $outputName  = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id);
+my $bin1Name    = $ipprc->filename("PPIMAGE.BIN1", $outputRoot, $class_id);
+my $bin2Name    = $ipprc->filename("PPIMAGE.BIN2", $outputRoot, $class_id);
+my $outputStats = $ipprc->filename("PPIMAGE.STATS", $outputRoot, $class_id);
 
 # Run ppImage & ppStats
-my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
-my $binnedStats = PS::IPP::Metadata::Stats->new(); # Stats parser
 unless ($no_op) {
     my $command = "$ppImage -file $input_uri $outputRoot";
@@ -144 +160 @@
     close $statsFile;
 
-    # Parse the statistics on the residual image
+    # Parse the stats file contents into a metadata
     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     my $metadata = $mdcParser->parse(join "", @contents) or &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+
+    # Parse the statistics on the residual image
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
     $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
 
@@ -160 +179 @@
     # parse the binned image statistics
     my $binnedMetadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata output", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+
+    my $binnedStats = PS::IPP::Metadata::Stats->new($BINNED_STATS); # Stats parser
     $binnedStats->parse($binnedMetadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
 }
 
+# Command to update the database
+my $command = "$dettool -addresidimfile";
+$command .= " -det_id $det_id";
+$command .= " -iteration $iter";
+$command .= " -exp_tag $exp_tag";
+$command .= " -class_id $class_id";
+$command .= " -recip $recipe";
+$command .= " -uri $outputName";
+$command .= " -path_base $outputRoot";
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS @$BINNED_STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
+
 # Add the processed file to the database
-# these value must be defined (NAN if not valid), or we raise a programming error
-my $bg            = $stats->bg_mean();
-my $bg_stdev      = $stats->bg_stdev();
-my $bg_mean_stdev = $stats->bg_mean_stdev();
-my $bin_stdev     = $binnedStats->bg_stdev();
-my $fringe_0      = ${$stats->fringe_mean()}[0];
-my $fringe_1      = ${$stats->fringe_err()}[0];
-my $fringe_2      = ${$stats->fringe_mean_stdev()}[0];
-my $dfringe_0     = ${$stats->dfringe_mean()}[0];
-my $dfringe_1     = ${$stats->dfringe_err()}[0];
-my $dfringe_2     = ${$stats->dfringe_mean_stdev()}[0];
-
-print "bg_mean_stdev: " . $bg_mean_stdev . "\n";
-print "stats : bg_mean_stdev: " . $stats->bg_mean_stdev() . "\n";
-
 unless ($no_update) {
-    my $command = "$dettool -addresidimfile -det_id $det_id -iteration $iter";
-    $command .= " -exp_tag $exp_tag -class_id $class_id";
-    $command .= " -recip $recipe -uri $outputName -path_base $outputRoot";
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
-    $command .= " -bin_stdev $bin_stdev";
-    $command .= " -fringe_0 $fringe_0 -fringe_1 $fringe_1 -fringe_2 $fringe_2";
-    $command .= " -user_1 $dfringe_0 -user_2 $dfringe_1 -user_3 $dfringe_2";
-    $command .= " -dbname $dbname" if defined $dbname;
-
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
@@ -196 +211 @@
         exit($error_code);
     }
+} else {
+    print "skipping command: $command\n";
 }
 
@@ -209 +226 @@
     carp($msg);
     if ($det_id and $iter and $exp_tag and not $no_update) {
-        my $command = "$dettool -addresidimfile -det_id $det_id -iteration $iter -exp_tag $exp_tag -class_id $class_id -code $exit_code";
+        my $command = "$dettool -addresidimfile";
+        $command .= " -det_id $det_id";
+        $command .= " -iteration $iter";
+        $command .= " -exp_tag $exp_tag";
+        $command .= " -class_id $class_id";
+        $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);

trunk/ippScripts/scripts/detrend_stack.pl

@@ -16 +16 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::Stats;
 use PS::IPP::Metadata::List qw( parse_md_list );
-use PS::IPP::Metadata::Stats;
 
 use PS::IPP::Config qw($PS_EXIT_SUCCESS
@@ -41 +41 @@
     'camera|c=s'        => \$camera,
     'dbname|d=s'        => \$dbname, # Database name
-    'workdir|w=s'       => \$workdir,   # Working directory, for output files
+    'workdir|w=s'       => \$workdir, # Working directory, for output files
     'reduction=s'       => \$reduction, # Reduction class for processing
     'no-update'         => \$no_update,
@@ -49 +49 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --det_id --iteration --class_id --det_type --camera", -exitval => 3 ) 
+pod2usage( -msg => "Required options: --det_id --iteration --class_id --det_type --camera", 
+           -exitval => 3) 
     unless defined $det_id
     and defined $iter
@@ -64 +65 @@
 $reduction = "DETREND" unless defined $reduction;
 my $recipe = $ipprc->reduction($reduction, uc($det_type) . '_STACK'); # Recipe name to use
+
+# values to extract from output metadata and the stats to calculate
+# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
+# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
+# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
+my $STATS = 
+   [   
+       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+       { name => "bg",             type => "mean",  flag => "-bg" },
+       { name => "bg",             type => "stdev", flag => "-bg_mean_stdev" },
+       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
+       # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
+   ];
 
 # Look for programs we need
@@ -74 +88 @@
 }
 
-my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir;
+
+my $outputRoot  = $ipprc->file_prepare( "$camera.$det_type.$det_id.$iter.$class_id", $workdir, ${$files}[0]->{uri} );
+my $outputStack = $outputRoot . '.fits'; # Output name
+my $outputStats = $outputRoot . '.stats'; # Statistics name
 
 # Get list of files to stack
 my $files;                      # Array of files to be stacked
 {
-    my $command = "$dettool -processedimfile -det_id $det_id -class_id $class_id -included"; # Command to run
+    my $command = "$dettool -processedimfile -included";
+    $command .= " -det_id $det_id";
+    $command .= " -class_id $class_id";
     $command .= " -dbname $dbname" if defined $dbname;
+
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -87 +108 @@
         &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $iter, $class_id, $error_code);
     }
+
+    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
         &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
@@ -93 +116 @@
 }
 
-$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir;
-
-my $outputRoot = $ipprc->file_prepare( "$camera.$det_type.$det_id.$iter.$class_id", $workdir, ${$files}[0]->{uri} );
-my $outputStack = $outputRoot . '.fits'; # Output name
-my $outputStats = $outputRoot . '.stats'; # Statistics name
+my $command = "$ppMerge $outputStack"; # Command to run
+foreach my $file (@$files) {
+    $command .= ' ' . $file->{uri};
+}
+$command .= " -recipe PPMERGE $recipe";
+$command .= ' -type ' . uc($det_type); # Type of stacking to perform
+$command .= " -stats $outputStats";     # Statistics output filename
+$command .= " -recipe PPSTATS CHIPSTATS";
 
 # Stack the files
-my $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
 unless ($no_op) {
-    my $command = "$ppMerge $outputStack"; # Command to run
-    foreach my $file (@$files) {
-        $command .= ' ' . $file->{uri};
-    }
-    $command .= " -recipe PPMERGE $recipe";
-    $command .= ' -type ' . uc($det_type); # Type of stacking to perform
-    $command .= " -stats $outputStats"; # Statistics output filename
-    $command .= " -recipe PPSTATS CHIPSTATS";
     
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -129 +146 @@
     my $metadata = $mdcParser->parse($contents) or
         &my_die("Unable to parse metadata config doc", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR);
-    $stats = PS::IPP::Metadata::Stats->new(); # Stats parser
+
+    my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
     $stats->parse($metadata)  or
         &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR);
 }
 
-my $bg            = $stats->bg_mean();
-my $bg_stdev      = $stats->bg_stdev();
-my $bg_mean_stdev = $stats->bg_mean_stdev();
+# Command to update the database
+$command  = "$dettool -addstacked";
+$command .= " -det_id $det_id -iteration $iter";
+$command .= " -class_id $class_id";
+$command .= " -uri $outputStack";
+$command .= " -recip $recipe";
+$command .= " -dbname $dbname" if defined $dbname;
+
+# add in the elements from the selected stats above
+foreach my $entry (@$STATS) {
+    my $value = $entry->{value};
+    my $flag = $entry->{flag};
+    $command .= " $flag $value";
+}
 
 # Add the resultant into the database
 unless ($no_update) {
-    my $command = "$dettool -addstacked";
-    $command .= " -det_id $det_id -iteration $iter";
-    $command .= " -class_id $class_id";
-    $command .= " -uri $outputStack";
-    $command .= " -recip $recipe";
-    $command .= " -bg $bg -bg_stdev $bg_stdev -bg_mean_stdev $bg_mean_stdev";
-    $command .= " -dbname $dbname" if defined $dbname;
-
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -155 +176 @@
         exit($error_code);
     }
-}
-
+} else {
+    print "skipping command: $command\n";
+}
 
 sub my_die
@@ -168 +190 @@
     carp($msg);
     if ($det_id and $iter and $class_id and not $no_update) {
-        my $command = "$dettool -addstacked -det_id $det_id -iteration $iter -class_id $class_id -code $exit_code";
+        my $command = "$dettool -addstacked";
+        $command .= " -det_id $det_id";
+        $command .= " -iteration $iter";
+        $command .= " -class_id $class_id";
+        $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;
 ###        system ($command);

trunk/ippScripts/scripts/register_imfile.pl

@@ -98 +98 @@
 
 # setup cache interface
-
+# XXX why is this being cached?
 my $c = Cache::File->new(
     cache_root => File::Spec->catdir($ENV{'HOME'}, '.pxtools', basename($0)),
@@ -125 +125 @@
     my @constants = keys %{CONSTANTS()}; # List of constants to parse out
     my @variables = keys %{VARIABLES()}; # List of variables to parse out
+
     $stats = PS::IPP::Metadata::Stats->new(\@constants, \@variables); # Stats parser
     unless ($stats->parse($metadata)) {
         warn ("Unable to find all values");
         &my_die ($exp_tag, $class_id, $PS_EXIT_CONFIG_ERROR);
-    }
-
-    if (0) {
-        # XXX for a test, randomly declare a failure and return to pantasks
-        my $rnd = rand(1);
-        if ($rnd > 0.5) {
-            warn ("random failure");
-            &my_die ($exp_tag, $class_id, $PS_EXIT_DATA_ERROR);
-        }
     }
 }