Changeset 14048

Timestamp:

Jul 6, 2007, 3:44:38 PM (19 years ago)

Author:

eugene

Message:

rework for consistency with the changes to the database and PS-IPP-Config

Location:

trunk/ippScripts/scripts

Files:

• detrend_norm_exp.pl (modified) (7 diffs)
• detrend_process_exp.pl (modified) (9 diffs)
• detrend_process_imfile.pl (modified) (8 diffs)
• detrend_reject_exp.pl (modified) (16 diffs)
• detrend_reject_imfile.pl (modified) (16 diffs)
• detrend_resid.pl (modified) (10 diffs)
• detrend_stack.pl (modified) (3 diffs)
• register_exp.pl (modified) (3 diffs)
• register_imfile.pl (modified) (6 diffs)

trunk/ippScripts/scripts/detrend_norm_exp.pl

@@ -16 +16 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::Stats;
+
 use PS::IPP::Metadata::List qw( parse_md_list );
 use File::Temp qw( tempfile );
@@ -33 +35 @@
 use Pod::Usage qw( pod2usage );
 
-my ($det_id, $iter, $det_type, $camera, $dbname, $workdir, $no_update, $no_op);
+my ($det_id, $iter, $det_type, $camera, $dbname, $workdir, $reduction, $no_update, $no_op);
 GetOptions(
-    'det_id|d=s'        => \$det_id,
-    'iteration|i=s'     => \$iter,
-    'camera|c=s'        => \$camera,
-    'det_type|t=s'      => \$det_type,
-    'dbname|d=s'        => \$dbname, # Database name
-    'workdir|w=s'       => \$workdir,   # Working directory, for output files
-    'no-update'         => \$no_update,
-    'no-op'             => \$no_op,
-) or pod2usage( 2 );
+           'det_id|d=s'        => \$det_id,
+           'iteration|i=s'     => \$iter,
+           'camera|c=s'        => \$camera,
+           'det_type|t=s'      => \$det_type,
+           'dbname|d=s'        => \$dbname, # Database name
+           'workdir|w=s'       => \$workdir,    # Working directory, for output files
+           'reduction|=s'      => \$reduction,
+           'no-update'         => \$no_update,
+           'no-op'             => \$no_op,
+           ) or pod2usage( 2 );
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
@@ -58 +61 @@
 $reduction = 'DETREND' unless defined $reduction;
 
-my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE' . uc($det_type); # Recipe to use
+my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use
 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
 
-my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE' . uc($det_type); # Recipe to use
+my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use
 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
 
@@ -69 +72 @@
 # XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
 my $STATS = 
-   [   
-       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
-       { name => "bg",             type => "mean",  flag => "-bg" },
-       { name => "bg",             type => "stdev", flag => "-bg_mean_stdev" },
-       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
-       # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
-   ];
+    [   
+        #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+        { name => "bg",             type => "mean",  flag => "-bg" },
+        { name => "bg",             type => "stdev", flag => "-bg_mean_stdev" },
+        { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
+        # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
+        ];
 
 # Look for programs we need
@@ -87 +90 @@
 
 # Get list of component files
-my $files;                      # Array of component files
+my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
 {
-    my $command = "$dettool -normalizedimfile -det_id $det_id -iteration $iter"; # Command to run
+    $command  = "$dettool -normalizedimfile -det_id $det_id -iteration $iter"; # Command to run
     $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
     unless ($success) {
@@ -131 +134 @@
 unless ($no_op) {
     # Make the jpeg for binning 1
-    {
-        my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 1);
-        &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name);
-    }
+    $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 1);
+    &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name);
     
     # Make the jpeg for binning 2
-    {
-        my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 1);
-        &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name);
-    }
+    $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 1);
+    &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name);
+
 }
 
 # command to update the database
-my $command = "$dettool -addnormalizedexp";
+$command  = "$dettool -addnormalizedexp";
 $command .= " -det_id $det_id";
 $command .= " -iteration $iter";
@@ -164 +164 @@
 # Add the processed file to the database
 unless ($no_update) {
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
     unless ($success) {

trunk/ippScripts/scripts/detrend_process_exp.pl

@@ -16 +16 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::Stats;
+
 use PS::IPP::Metadata::List qw( parse_md_list );
 use File::Temp qw( tempfile );
@@ -33 +35 @@
 use Pod::Usage qw( pod2usage );
 
-my ($det_id, $det_type, $exp_tag, $camera, $dbname, $workdir, $no_update, $no_op);
+my ($det_id, $exp_id, $det_type, $exp_tag, $camera, $dbname, $workdir, $reduction, $no_update, $no_op);
 GetOptions(
     'det_id|d=s'        => \$det_id,
     'det_type|t=s'      => \$det_type,
-    'exp_tag|e=s'       => \$exp_tag,
+    'exp_id|d=s'        => \$exp_id,
+    'exp_tag|=s'        => \$exp_tag,
     'camera|c=s'        => \$camera,
     'dbname|d=s'        => \$dbname, # Database name
     'workdir|w=s'       => \$workdir,   # Working directory, for output files
+    'reduction|=s'      => \$reduction,
     'no-update'         => \$no_update,
     'no-op'             => \$no_op,
@@ -46 +50 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --det_id --det_type --exp_tag --camera",
+pod2usage( -msg => "Required options: --det_id --det_type --exp_id --exp_tag --camera",
            -exitval => 3) 
     unless defined $det_id
     and defined $det_type
+    and defined $exp_id
     and defined $exp_tag
     and defined $camera;
@@ -58 +63 @@
 $reduction = 'DETREND' unless defined $reduction;
 
-my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE' . uc($det_type); # Recipe to use
-&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
-
-my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE' . uc($det_type); # Recipe to use
-&my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
+my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_IMAGE_' . uc($det_type)); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
+
+my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_IMAGE_' . uc($det_type)); # Recipe to use
+&my_die("Unrecognised detrend type: $det_type", $det_id, $exp_id, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
 
 # values to extract from output metadata and the stats to calculate
@@ -87 +92 @@
 
 # Get list of component files
-my $files;                      # Array of component files
+my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
 {
-    my $command = "$dettool -processedimfile -det_id $det_id -exp_tag $exp_tag"; # Command to run
+    $command = "$dettool -processedimfile -det_id $det_id -exp_id $exp_id"; # Command to run
     $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => 1);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $exp_tag, $error_code);
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 1);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform dettool -processedimfile: $error_code", $det_id, $exp_id, $error_code);
     }
 
@@ -101 +106 @@
     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to parse metadata config doc", $det_id, $exp_id, $PS_EXIT_PROG_ERROR);
 
     # parse the file info in the metadata
     $files = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $det_id, $exp_tag, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to parse metadata list", $det_id, $exp_id, $PS_EXIT_PROG_ERROR);
 
     # parse the stats in the metadata
     my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
     unless ($stats->parse($metadata)) {
-        &my_die("Unable to find all values in statistics output.\n", $chip_id, $class_id, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to find all values in statistics output.\n", $det_id, $exp_id, $PS_EXIT_PROG_ERROR);
     }
 }
@@ -132 +137 @@
 unless ($no_op) {
     # Make the jpeg for binning 1
-    {
-        my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 1);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_tag, $error_code);
-        }
-        &my_die("Unable to find expected output file: $jpeg1", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1);
-    }
+    $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 1);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code);
+    }
+    &my_die("Unable to find expected output file: $jpeg1", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1);
     
     # Make the jpeg for binning 2
-    {
-        my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 1);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_tag, $error_code);
-        }
-        &my_die("Unable to find expected output file: $jpeg2", $det_id, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2);
-    }
+    $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 1);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $error_code);
+    }
+    &my_die("Unable to find expected output file: $jpeg2", $det_id, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2);
 }
 
 # Command to update the database
-my $command = "$dettool -addprocessedexp";
+$command  = "$dettool -addprocessedexp";
 $command .= " -det_id $det_id";
-$command .= " -exp_tag $exp_tag";
+$command .= " -exp_id $exp_id";
 $command .= " -recip $recipe1,$recipe2 -path_base $outputRoot";
 $command .= " -dbname $dbname" if defined $dbname;
@@ -172 +173 @@
 # Add the processed file to the database
 unless ($no_update) {
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
     unless ($success) {
@@ -187 +188 @@
     my $msg = shift; # Warning message on die
     my $det_id = shift;         # Detrend identifier
-    my $exp_tag = shift; # Exposure tag
+    my $exp_id = shift; # Exposure tag
     my $exit_code = shift; # Exit code to add
 
     carp($msg);
-    if ($det_id and $exp_tag and not $no_update) {
+    if ($det_id and $exp_id and not $no_update) {
         my $command = "$dettool -addprocessedexp";
         $command .= " -det_id $det_id";
-        $command .= " -exp_tag $exp_tag";
+        $command .= " -exp_id $exp_id";
         $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;

trunk/ippScripts/scripts/detrend_process_imfile.pl

@@ -32 +32 @@
 use Pod::Usage qw( pod2usage );
 
-my ($det_id, $exp_tag, $class_id, $det_type, $input_uri, $camera, $dbname, $workdir, $reduction, $no_update, $no_op);
+my ($det_id, $exp_id, $class_id, $det_type, $exp_tag, $input_uri, $camera, $dbname, $workdir, $reduction, $no_update, $no_op);
 GetOptions(
     'det_id|d=s'        => \$det_id,
-    'exp_tag|e=s'       => \$exp_tag,
+    'exp_id|e=s'        => \$exp_id,
     'class_id|i=s'      => \$class_id,
     'det_type|t=s'      => \$det_type,
+    'exp_tag|=s'        => \$exp_tag,
     'input_uri|u=s'     => \$input_uri,
     'camera|c=s'        => \$camera,
@@ -48 +49 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --det_id --exp_tag --class_id --det_type --input_uri --camera",
+pod2usage( -msg => "Required options: --det_id --exp_id --class_id --det_type --exp_tag --input_uri --camera",
            -exitval => 3)
     unless defined $det_id
-    and defined $exp_tag
+    and defined $exp_id
     and defined $class_id
     and defined $det_type
+    and defined $exp_tag
     and defined $input_uri
     and defined $camera;
@@ -83 +85 @@
 $ppImage .= " -dbname $dbname" if defined $dbname;
 
-&my_die("Couldn't find input file: $input_uri\n", $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri);
+&my_die("Couldn't find input file: $input_uri\n", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($input_uri);
 
 $workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir;
@@ -89 +91 @@
 my $outputRoot  = $ipprc->file_prepare( "$exp_tag/$exp_tag.detproc.$det_id", $workdir, $input_uri );
 
-my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id);
-my $outputBin1  = $ipprc->filename("PPIMAGE.BIN1",   $outputRoot, $class_id);
-my $outputBin2  = $ipprc->filename("PPIMAGE.BIN2",   $outputRoot, $class_id);
-my $outputStats = $ipprc->filename("PPIMAGE.STATS",  $outputRoot, $class_id);
+my $outputImage = $ipprc->filename("PPIMAGE.OUTPUT", $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
+my $outputBin1  = $ipprc->filename("PPIMAGE.BIN1",   $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
+my $outputBin2  = $ipprc->filename("PPIMAGE.BIN2",   $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
+my $outputStats = $ipprc->filename("PPIMAGE.STATS",  $outputRoot, $class_id) or &my_die("Missing entry from camera config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 
 # Run ppImage
@@ -105 +107 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_tag, $class_id, $error_code);
+        &my_die("Unable to perform ppImage: $error_code", $det_id, $exp_id, $class_id, $error_code);
     }
 
-    &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage);
-    &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats);
-    &my_die("Couldn't find expected output file: $outputBin1", $det_id, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin1);
-    &my_die("Couldn't find expected output file: $outputBin2", $det_id, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin2);
+    &my_die("Couldn't find expected output file: $outputImage", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputImage);
+    &my_die("Couldn't find expected output file: $outputStats", $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats);
+    &my_die("Couldn't find expected output file: $outputBin1",  $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin1);
+    &my_die("Couldn't find expected output file: $outputBin2",  $det_id, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputBin2);
 
     # Get the statistics on the processed image
@@ -122 +124 @@
     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
     my $metadata = $mdcParser->parse(join "", @contents)
-        or &my_die("Unable to parse metadata config", $det_id, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+        or &my_die("Unable to parse metadata config", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 
     # extract the stats from the metadata
     my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
-    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 }
 
@@ -132 +134 @@
 my $command = "$dettool -addprocessedimfile";
 $command .= " -det_id $det_id";
-$command .= " -exp_tag $exp_tag";
+$command .= " -exp_id $exp_id";
 $command .= " -class_id $class_id";
 $command .= " -recip $reduction";
@@ -162 +164 @@
     my $msg = shift; # Warning message on die
     my $det_id = shift;         # Detrend identifier
-    my $exp_tag = shift; # Exposure tag
+    my $exp_id = shift; # Exposure tag
     my $class_id = shift; # Class identifier
     my $exit_code = shift; # Exit code to add
 
     carp($msg);
-    if ($det_id and $exp_tag and $class_id and not $no_update) {
+    if ($det_id and $exp_id and $class_id and not $no_update) {
         my $command = "$dettool -addprocessedimfile";
         $command .= " -det_id $det_id";
-        $command .= " -exp_tag $exp_tag"; 
+        $command .= " -exp_id $exp_id"; 
         $command .= " -class_id $class_id";
         $command .= " -code $exit_code";

trunk/ippScripts/scripts/detrend_reject_exp.pl

@@ -16 +16 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::Stats;
+
 use PS::IPP::Metadata::List qw( parse_md_list );
 use Statistics::Descriptive;
@@ -86 +88 @@
 
 # Get list of component files
-my $exposures;                  # Array of exposures
+my ($exposures, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
 {
     # dettool command to select exp data for this det_run
-    my $command = "$dettool -residexp";
+    $command = "$dettool -residexp";
     $command .= " -det_id $det_id";
     $command .= " -iteration $iter";
     $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
     unless ($success) {
@@ -118 +120 @@
 }
 
-my @expTags;                    # Array of exposure IDs
-my @accept;                     # Array of accept flags
-my @include;                    # Array of include flags
-foreach my $exposure (@$exposures) {
-    &my_die("Unable to find exposure id.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $exposure->{exp_tag};
-    &my_die("Unable to find accept.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $exposure->{accept};
-    &my_die("Unable to find include.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $exposure->{include};
-}
-
+# we use the statistics of the ensemble to accept/reject exposurs
 my $ensMeanMean       = &STATS_value_for_flag ($REJSTATS, "ensMeanMean");      
 my $ensMeanStdev      = &STATS_value_for_flag ($REJSTATS, "ensMeanStdev");     
@@ -147 +141 @@
 unless ($no_op) {
     $logFile = $ipprc->file_create_append( $logName );
-    print $logFile "Ensemble mean $meanEnsemble +/- $stdevEnsemble, stdev $meanStdevEnsemble\n\n";
+    print $logFile "Ensemble mean $ensMeanMean +/- $ensMeanStdevMean, stdev $ensStdevMean\n\n";
 }
 
@@ -153 +147 @@
 my $numChanges = 0;             # Number of exposures with changed status
 my $numReject = 0;              # Number of exposures rejected
-
-for (my $i = 0; $i < scalar @$exposures; $i++) {
-    my $file      = $files[$i];
-    my $mean      = $file->{bg};        # Mean for this exposure
-    my $stdev     = $file->{bg_stdev}; # Stdev for this exposure
-    my $meanStdev = $file->{bg_mean_stdev}; # Stdev of Means for this exposure
-
-    my $expTag  = $exposure->{exp_tag};
-    my $accept  = $exposure->{accept};
-    my $include = $exposure->{include};
+my $firstElement = 1;
+
+foreach my $exposure (@$exposures) {
+    my $mean      = $exposure->{bg};    # Mean for this exposure
+    my $stdev     = $exposure->{bg_stdev}; # Stdev for this exposure
+    my $meanStdev = $exposure->{bg_mean_stdev}; # Stdev of Means for this exposure
+    my $expID     = $exposure->{exp_id};
+    my $accept    = $exposure->{accept};
+    my $include   = $exposure->{include};
+
+    &my_die("Unable to find exposure id.\n", $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $expID;
+    &my_die("Unable to find accept.\n",      $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $accept;
+    &my_die("Unable to find include.\n",     $det_id, $iter, $PS_EXIT_SYS_ERROR) unless defined $include;
 
     my $reject = 0;             # Reject this exposure?
 
-    my $command = "$dettool -updateresidexp";
+    $command  = "$dettool -updateresidexp";
     $command .= " -det_id $det_id";
     $command .= " -iteration $iter";
-    $command .= " -exp_tag $expTag";
+    $command .= " -exp_id $expID";
     $command .= " -dbname $dbname" if defined $dbname;
     
@@ -175 +172 @@
         # Rejected this at an earlier stage
         unless ($no_op) {
-            print $logFile "Rejecting $expTag based on earlier determination.\n";
+            print $logFile "Rejecting $expID based on earlier determination.\n";
         }
         $reject = 1;
@@ -185 +182 @@
         # Make sure something gets rejected (just once!), just so that
         # we can trace the full range of the workflow
-        if ($i == 0 and $iter == 0) {
+        if ($firstElement and $iter == 0) {
             $reject = 1;
         }
@@ -194 +191 @@
         my $delta = abs($mean - $ensMeanMean);
         if ($delta > ($reject_mean * $ensMeanStdev)) {
-            print $logFile "Rejecting $expTag based on ensemble mean value: ";
+            print $logFile "Rejecting $expID based on ensemble mean value: ";
             $reject = 1;
             #goto UPDATE;
         } else {
-            print $logFile "$expTag OK against ensemble mean: ";
+            print $logFile "$expID OK against ensemble mean: ";
         }
         print $logFile "$mean --> $delta vs " . $reject_mean * $ensMeanStdev . "\n";
     } else {
-        print $logFile "No rejection of $expTag for ensemble mean\n";
+        print $logFile "No rejection of $expID for ensemble mean\n";
     }
 
@@ -208 +205 @@
         my $delta = abs($stdev - $ensStdevMean);
         if ($delta > ($reject_stdev * $ensStdevStdev)) {
-            print $logFile "Rejecting $expTag based on ensemble stdev: ";
+            print $logFile "Rejecting $expID based on ensemble stdev: ";
             $reject = 1;
             #goto UPDATE;
         } else {
-            print $logFile "$expTag OK against ensemble stdev: ";
+            print $logFile "$expID OK against ensemble stdev: ";
         }
         print $logFile "$stdev --> $delta sigma vs " . $reject_stdev * $ensStdevStdev . "\n";
     } else {
-        print $logFile "No rejection of $expTag for ensemble stdev\n";
+        print $logFile "No rejection of $expID for ensemble stdev\n";
     }
     
@@ -222 +219 @@
         my $delta = abs($meanStdev - $ensMeanStdevMean);
         if ($delta > ($reject_meanstdev * $ensMeanStdevStdev)) {
-            print $logFile "Rejecting $expTag based on ensemble mean stdev: ";
+            print $logFile "Rejecting $expID based on ensemble mean stdev: ";
             $reject = 1;
             #goto UPDATE;
         } else {
-            print $logFile "$expTag OK against ensemble mean stdev: ";
+            print $logFile "$expID OK against ensemble mean stdev: ";
         }
         print $logFile "$meanStdev --> $delta sigma vs " . $reject_meanstdev * $ensMeanStdevStdev. "\n";
     } else {
-        print $logFile "No rejection of $expTag for ensemble mean stdev\n";
+        print $logFile "No rejection of $expID for ensemble mean stdev\n";
     }
 
@@ -242 +239 @@
     if ((not $include and not $reject) or ($include and $reject)) {
         unless ($no_op) {
-            print $logFile "Status of $expTag has changed.\n";
+            print $logFile "Status of $expID has changed.\n";
         }
         $numChanges++;
@@ -249 +246 @@
     unless ($no_update) {
         # Update 
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => 1);
         unless ($success) {
@@ -268 +265 @@
 
 # Rejecting everything --- stop before something bad happens!
-if ($numReject == scalar @means) {
+if ($numReject == scalar @$exposures) {
     $master = 0;
     $stop = 1;
@@ -279 +276 @@
 }
 
-my $command = "$dettool -adddetrunsummary";
+$command = "$dettool -adddetrunsummary";
 $command .= " -det_id $det_id";
 $command .= " -iteration $iter";
@@ -294 +291 @@
 # Put results into the database
 unless ($no_update) {
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
     unless ($success) {
@@ -310 +307 @@
     $command .= " -dbname $dbname" if defined $dbname;
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
     unless ($success) {

trunk/ippScripts/scripts/detrend_reject_imfile.pl

@@ -22 +22 @@
 use IPC::Cmd 0.36 qw( can_run run );             # tools to run UNIX programs with control over I/O
 use PS::IPP::Metadata::Config;                   # tools to parse the psMetadataConfig files
+use PS::IPP::Metadata::Stats;
+
 use PS::IPP::Metadata::List qw( parse_md_list ); # tools to parse a metadata into a hash list
 use Statistics::Descriptive;                     # tools for calculating basic statistical quantities
@@ -41 +43 @@
 
 # parse the command-line options
-my ($det_id, $iter, $exp_tag, $det_type, $camera, $filter, $reject, $dbname, $workdir, $no_update, $no_op);
+my ($det_id, $iter, $exp_id, $exp_tag, $det_type, $camera, $filter, $reject, $dbname, $workdir, $reduction, $no_update, $no_op);
 GetOptions(
            'det_id|d=s'        => \$det_id,
            'iteration=s'       => \$iter,
-           'exp_tag|e=s'       => \$exp_tag,
+           'exp_id|e=s'        => \$exp_id,
+           'exp_tag|=s'        => \$exp_tag,
            'det_type|t=s'      => \$det_type,
            'camera=s'          => \$camera,
@@ -52 +55 @@
            'dbname|d=s'        => \$dbname, # Database name
            'workdir|w=s'       => \$workdir, # Working directory, for output files
+           'reduction|=s'      => \$reduction,
            'no-update'         => \$no_update,
            'no-op'             => \$no_op,
@@ -57 +61 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --det_id --iteration --exp_tag --det_type --camera",
+pod2usage( -msg => "Required options: --det_id --iteration --exp_id --exp_tag --det_type --camera",
            -exitval => 3)
     unless defined $det_id
     and defined $iter
+    and defined $exp_id
     and defined $exp_tag
     and defined $det_type
@@ -71 +76 @@
 $reduction = 'DETREND' unless defined $reduction;
 
-my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_RESID' . uc($det_type); # Recipe to use
+my $recipe1 = $ipprc->reduction($reduction, 'JPEG_BIN1_RESID_' . uc($det_type)); # Recipe to use
 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe1;
 
-my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_RESID' . uc($det_type); # Recipe to use
+my $recipe2 = $ipprc->reduction($reduction, 'JPEG_BIN2_RESID_' . uc($det_type)); # Recipe to use
 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe2;
 
@@ -82 +87 @@
 # XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
 my $STATS = 
-   [   
-       #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
-       { name => "bg",             type => "mean",  flag => "-bg" },
-       { name => "bg_mean_stdev",  type => "stdev", flag => "-bg_mean_stdev" },
-       { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
-       { name => "bin_stdev",      type => "rms",   flag => "-bin_stdev" },
-       { name => "fringe_0",       type => "mean",  flag => "-fringe_0" },
-       { name => "fringe_1",       type => "rms",   flag => "-fringe_1" },
-       { name => "fringe_0",       type => "stdev", flag => "-fringe_2" },
-       { name => "user_1",         type => "mean",  flag => "-user_1" }, # fringe residual
-       { name => "user_2",         type => "rms",   flag => "-user_2" }, # fringe residual
-       { name => "user_3",         type => "stdev", flag => "-user_1" }, # fringe residual
-   ];
+    [   
+        #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
+        { name => "bg",             type => "mean",  flag => "-bg" },
+        { name => "bg_mean_stdev",  type => "stdev", flag => "-bg_mean_stdev" },
+        { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev" },
+        { name => "bin_stdev",      type => "rms",   flag => "-bin_stdev" },
+        { name => "fringe_0",       type => "mean",  flag => "-fringe_0" },
+        { name => "fringe_1",       type => "rms",   flag => "-fringe_1" },
+        { name => "fringe_0",       type => "stdev", flag => "-fringe_2" },
+        { name => "user_1",         type => "mean",  flag => "-user_1" }, # fringe residual
+        { name => "user_2",         type => "rms",   flag => "-user_2" }, # fringe residual
+        { name => "user_3",         type => "stdev", flag => "-user_1" }, # fringe residual
+        ];
 
 # Look for programs we need
@@ -106 +111 @@
 
 # Get list of imfile files
-my $files;                      # Array of imfile files
+my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
 {
-    # dettool command to select imfile data for this exp_tag
-    my $command = "$dettool -residimfile";
+    # dettool command to select imfile data for this exp_id
+    $command  = "$dettool -residimfile";
     $command .= " -det_id $det_id";
     $command .= " -iteration $iter";
-    $command .= " -exp_tag $exp_tag";
+    $command .= " -exp_id $exp_id";
     $command .= " -dbname $dbname" if defined $dbname;
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
     unless ($success) {
@@ -126 +131 @@
     my $mdcParser = PS::IPP::Metadata::Config->new;     
     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
 
     # parse the file info in the metadata
     $files = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
 
     # Parse the statistics on the residual image
     my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
-    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
 }
 
@@ -162 +167 @@
 # build the JPEG images
 unless ($no_op) {
-
     # Make the jpeg for binning 1
-    {
-        my $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 1);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_tag, $error_code);
-        }
-        &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name);
-    }
+    $command = "$ppImage -list $list1Name $outputRoot -recipe PPIMAGE $recipe1"; # Command to run
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 1);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code);
+    }
+    &my_die("Unable to find expected output file: $jpeg1Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg1Name);
     
     # Make the jpeg for binning 2
-    {
-        my $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => 1);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_tag, $error_code);
-        }
-        &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $exp_tag, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name);
-    }
+    $command = "$ppImage -list $list2Name $outputRoot -recipe PPIMAGE $recipe2"; # Command to run
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => 1);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to run ppImage: $error_code", $det_id, $iter, $exp_id, $error_code);
+    }
+    &my_die("Unable to find expected output file: $jpeg2Name", $det_id, $iter, $exp_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($jpeg2Name);
 }
 
@@ -207 +207 @@
 my $reject_exp_bin_sn       = rejection_limit( 'EXP.BIN.SN',       $det_type, $filter );
 
-# storage variables
-
-# load the arrays from the imfile output table
-foreach my $file (@$files) {
-    if ($file->{exp_time} > 0.0) {
-        push @fluxes,           $file->{bg} / $file->{exp_time};       # mean background counts / sec
-    } else {
-        push @fluxes,           $file->{bg};
-    }
-}
-
-# Reject based on the stats of the imfiles
-# it is VALID to reject on more than one criterion
-&my_die("Number of means and number of variances differ!", $det_id, $iter, $exp_tag, $PS_EXIT_PROG_ERROR) unless scalar @means == scalar @variances;
-
 # storage array
 my @fluxes;
 
-for (my $i = 0; $i < scalar @$files; $i++) {
-    my $file      = $files[$i];
+foreach my $file (@$files) {
     my $name      = $file->{class_id};
     my $mean      = $file->{bg};        # Mean for this imfile
@@ -234 +218 @@
 
     # calculate and save the fluxes
-    my $flux      = $mean / $file->{exp_time};
+    my $flux;
     if ($file->{exp_time} == 0.0) {
-        push @fluxes, $mean;
+        $flux = $mean;
+    } else {
+        $flux = $mean / $file->{exp_time};
     }
     push @fluxes, $flux;
@@ -296 +282 @@
     # component means is larger than the limit
     if ($reject_imfile_meanstdev > 0) {
-        if ($meanStdevs > $reject_imfile_meanstdev) {
+        if ($meanStdev > $reject_imfile_meanstdev) {
             print $logFile "Rejecting exposure based on bad imfile mean stdev for $name: ";
             $reject = 1;
@@ -302 +288 @@
             print $logFile "Imfile mean stdev for $name meets requirements: ";
         }
-        print $logFile "$meanStdevs vs $reject_imfile_meanstdev\n";
+        print $logFile "$meanStdev vs $reject_imfile_meanstdev\n";
     } else {
         print $logFile "No rejection on imfile mean stdev for $name\n";
@@ -479 +465 @@
 }
 
-my $command = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_tag $exp_tag";
+$command  = "$dettool -addresidexp -det_id $det_id -iteration $iter -exp_id $exp_id";
 $command .= " -recip $recipe1,$recipe2 -path_base $outputRoot ";
 $command .= ' -reject' if $reject;
@@ -492 +478 @@
 
 unless ($no_update) {
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => 1);
     unless ($success) {
@@ -516 +502 @@
     my $det_id = shift;         # Detrend identifier
     my $iter = shift;           # Iteration
-    my $exp_tag = shift; # Exposure tag
+    my $exp_id = shift; # Exposure tag
     my $exit_code = shift; # Exit code to add
 
     carp($msg);
-    if ($det_id and $iter and $exp_tag and not $no_update) {
+    if ($det_id and $iter and $exp_id and not $no_update) {
         my $command = "$dettool -addresidexp";
         $command .= " -det_id $det_id";
         $command .= " -iteration $iter";
-        $command .= " -exp_tag $exp_tag";
+        $command .= " -exp_id $exp_id";
         $command .= " -code $exit_code";
         $command .= " -dbname $dbname" if defined $dbname;

trunk/ippScripts/scripts/detrend_resid.pl

@@ -32 +32 @@
 use Pod::Usage qw( pod2usage );
 
-my ($det_id, $iter, $exp_tag, $class_id, $det_type, $detrend, $input_uri, $camera, $mode, $dbname, $workdir, $reduction, $no_update, $no_op);
+my ($det_id, $iter, $exp_id, $exp_tag, $class_id, $det_type, $detrend, $input_uri, $camera, $mode, $dbname, $workdir, $reduction, $no_update, $no_op);
 GetOptions(
     'det_id|d=s'        => \$det_id,
     'iteration=s'       => \$iter,
-    'exp_tag|e=s'       => \$exp_tag,
+    'exp_id|e=s'        => \$exp_id,
+    'exp_tag|=s'        => \$exp_tag,
     'class_id|i=s'      => \$class_id,
     'det_type|t=s'      => \$det_type,
@@ -51 +52 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --det_id --iteration --exp_tag --class_id --det_type --camera --input_uri --mode --detrend (not for 'verify' mode)",
+pod2usage( -msg => "Required options: --det_id --iteration --exp_id --exp_tag --class_id --det_type --camera --input_uri --mode --detrend (not for 'verify' mode)",
            -exitval => 3)
     unless defined $det_id
     and defined $iter
+    and defined $exp_id
     and defined $exp_tag
     and defined $class_id
@@ -73 +75 @@
     $recipe = uc($det_type) . '_VERIFY';
 } else {
-    &my_die("Unrecognised mode: $mode", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+    &my_die("Unrecognised mode: $mode", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 }
 $recipe = $ipprc->reduction($reduction, $recipe);
@@ -139 +141 @@
     if (lc($mode) ne 'verify') {
         my $detFlag = DETRENDS->{lc($det_type)};
-        &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR) unless defined $detFlag;
+        &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR) unless defined $detFlag;
         $command .= " $detFlag $detrend";
     }
@@ -147 +149 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $exp_tag, $class_id, $error_code);
-    }
-    &my_die("Couldn't find expected output file: $outputName", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputName);
-    &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats);
-    &my_die("Couldn't find expected output file: $bin1Name", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin1Name);
-    &my_die("Couldn't find expected output file: $bin2Name", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin2Name);
+        &my_die("Unable to perform ppImage: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code);
+    }
+    &my_die("Couldn't find expected output file: $outputName", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputName);
+    &my_die("Couldn't find expected output file: $outputStats", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($outputStats);
+    &my_die("Couldn't find expected output file: $bin1Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin1Name);
+    &my_die("Couldn't find expected output file: $bin2Name", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($bin2Name);
 
     # Load the raw output stats file
     my $statsFile;              # File handle
-    open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_SYS_ERROR);
+    open $statsFile, $ipprc->file_resolve($outputStats) or &my_die("Can't open statistics file $outputStats: $!", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_SYS_ERROR);
     my @contents = <$statsFile>; # Contents of file
     close $statsFile;
@@ -162 +164 @@
     # Parse the stats file contents into a metadata
     my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
-    my $metadata = $mdcParser->parse(join "", @contents) or &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+    my $metadata = $mdcParser->parse(join "", @contents) or &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 
     # Parse the statistics on the residual image
     my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
-    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+    $stats->parse($metadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 
     # run ppStats on the binned image
@@ -174 +176 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform ppStats: $error_code", $det_id, $iter, $exp_tag, $class_id, $error_code);
+        &my_die("Unable to perform ppStats: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code);
     }
 
     # parse the binned image statistics
-    my $binnedMetadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata output", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+    my $binnedMetadata = $mdcParser->parse(join "", @$stdout_buf) or &my_die("Unable to parse metadata output", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 
     my $binnedStats = PS::IPP::Metadata::Stats->new($BINNED_STATS); # Stats parser
-    $binnedStats->parse($binnedMetadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_tag, $class_id, $PS_EXIT_PROG_ERROR);
+    $binnedStats->parse($binnedMetadata) or &my_die("Unable to find all values in statistics output.", $det_id, $iter, $exp_id, $class_id, $PS_EXIT_PROG_ERROR);
 }
 
@@ -188 +190 @@
 $command .= " -det_id $det_id";
 $command .= " -iteration $iter";
-$command .= " -exp_tag $exp_tag";
+$command .= " -exp_id $exp_id";
 $command .= " -class_id $class_id";
 $command .= " -recip $recipe";
@@ -196 +198 @@
 
 # add in the elements from the selected stats above
-foreach my $entry (@$STATS @$BINNED_STATS) {
+foreach my $entry (@$STATS, @$BINNED_STATS) {
     my $value = $entry->{value};
     my $flag = $entry->{flag};
@@ -220 +222 @@
     my $det_id = shift;         # Detrend identifier
     my $iter = shift;           # Iteration
-    my $exp_tag = shift; # Exposure tag
+    my $exp_id = shift; # Exposure tag
     my $class_id = shift; # Class identifier
     my $exit_code = shift; # Exit code to add
 
     carp($msg);
-    if ($det_id and $iter and $exp_tag and not $no_update) {
+    if ($det_id and $iter and $exp_id and not $no_update) {
         my $command = "$dettool -addresidimfile";
         $command .= " -det_id $det_id";
         $command .= " -iteration $iter";
-        $command .= " -exp_tag $exp_tag";
+        $command .= " -exp_id $exp_id";
         $command .= " -class_id $class_id";
         $command .= " -code $exit_code";

trunk/ippScripts/scripts/detrend_stack.pl

@@ -88 +88 @@
 }
 
-$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir;
-
-my $outputRoot  = $ipprc->file_prepare( "$camera.$det_type.$det_id.$iter.$class_id", $workdir, ${$files}[0]->{uri} );
-my $outputStack = $outputRoot . '.fits'; # Output name
-my $outputStats = $outputRoot . '.stats'; # Statistics name
-
 # Get list of files to stack
-my $files;                      # Array of files to be stacked
+my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
 {
-    my $command = "$dettool -processedimfile -included";
+    $command = "$dettool -processedimfile -included";
     $command .= " -det_id $det_id";
     $command .= " -class_id $class_id";
     $command .= " -dbname $dbname" if defined $dbname;
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
     unless ($success) {
@@ -116 +110 @@
 }
 
-my $command = "$ppMerge $outputStack"; # Command to run
+$workdir = caturi( $workdir, "$camera.$det_type.$det_id" ) if defined $workdir;
+
+my $outputRoot  = $ipprc->file_prepare( "$camera.$det_type.$det_id.$iter.$class_id", $workdir, ${$files}[0]->{uri} );
+my $outputStack = $outputRoot . '.fits'; # Output name
+my $outputStats = $outputRoot . '.stats'; # Statistics name
+
+$command = "$ppMerge $outputStack"; # Command to run
 foreach my $file (@$files) {
     $command .= ' ' . $file->{uri};
@@ -128 +128 @@
 unless ($no_op) {
     
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
     unless ($success) {

trunk/ippScripts/scripts/register_exp.pl

@@ -19 +19 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::List qw( parse_md_list );
-use Statistics::Descriptive;
+use PS::IPP::Metadata::Stats;
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -62 +61 @@
        { name => "exp_name",       type => "constant",   flag => "-exp_name" }, # File level
        { name => "telescope",      type => "constant",   flag => "-telescope" }, # File level
-       { name => "inst",           type => "constant",   flag => "-inst" }, # File level
+       { name => "camera",         type => "constant",   flag => "-inst" }, # File level
        { name => "filelevel",      type => "constant",   flag => "-filelevel" }, # File level
        { name => "object",         type => "constant",   flag => "-object" },
@@ -120 +119 @@
     my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
     unless ($stats->parse($metadata)) {
-        &my_die ("Unable to find all values", $exp_id, $class_id, $PS_EXIT_CONFIG_ERROR);
-    }
-}
+        &my_die ("Unable to find all values", $exp_id, $PS_EXIT_CONFIG_ERROR);
+    }
+}
+
+# we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type
+if (uc(&STATS_value_for_flag ($STATS, "-telescope")) eq "NAN") { &my_die ("telescope not found", $exp_id, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&STATS_value_for_flag ($STATS, "-inst"))      eq "NAN") { &my_die ("inst      not found", $exp_id, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&STATS_value_for_flag ($STATS, "-filelevel")) eq "NAN") { &my_die ("filelevel not found", $exp_id, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&STATS_value_for_flag ($STATS, "-exp_type"))  eq "NAN") { &my_die ("exp_type  not found", $exp_id, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&STATS_value_for_flag ($STATS, "-exp_name"))  eq "NAN") { &my_die ("exp_name  not found", $exp_id, $PS_EXIT_CONFIG_ERROR); }
 
 my $command = "$regtool -addprocessedexp";

trunk/ippScripts/scripts/register_imfile.pl

@@ -20 +20 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::Stats;
-use Data::Dumper;
 
 use PS::IPP::Config qw(
@@ -38 +37 @@
 use Pod::Usage qw( pod2usage );
 
-my ($cache, $exp_id, $tmp_class_id, $exp_name, $uri, $workdir, $dbname, $no_update, $no_op);
+my ($cache, $exp_id, $tmp_class_id, $tmp_exp_name, $uri, $workdir, $dbname, $no_update, $no_op);
 GetOptions(
     'caches'           => \$cache,
     'exp_id|e=s'       => \$exp_id,
     'tmp_class_id|i=s' => \$tmp_class_id,
-    'exp_name|n=s'     => \$exp_name,
+    'tmp_exp_name|n=s' => \$tmp_exp_name,
     'uri|u=s'          => \$uri,
     'workdir|w=s'      => \$workdir, # Working directory for output files
@@ -52 +51 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --exp_id --tmp_class_id --exp_name --uri",
+pod2usage( -msg => "Required options: --exp_id --tmp_class_id --tmp_exp_name --uri",
            -exitval => 3) unless
     defined $exp_id and
     defined $tmp_class_id and
-    defined $exp_name and
+    defined $tmp_exp_name and
     defined $uri;
 
@@ -114 +113 @@
     unless ($success) { 
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die ("Unable to perform ppStats on exposure id $exp_id: $error_code", $exp_id, $exp_name, $tmp_class_id, $error_code);
+        &my_die ("Unable to perform ppStats on exposure id $exp_id: $error_code", $exp_id, $tmp_exp_name, $tmp_class_id, $error_code);
     }
     
@@ -121 +120 @@
     my $metadata = $mdcParser->parse(join "", @$stdout_buf); # XXX is this join necessary?
     unless ($metadata) {
-        &my_die ("Unable to parse metadata config doc", $exp_id, $exp_name, $tmp_class_id, $PS_EXIT_PROG_ERROR);
+        &my_die ("Unable to parse metadata config doc", $exp_id, $tmp_exp_name, $tmp_class_id, $PS_EXIT_PROG_ERROR);
     }
 
@@ -127 +126 @@
     my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
     unless ($stats->parse($metadata)) {
-        &my_die ("Unable to find all values", $exp_id, $exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR);
+        &my_die ("Unable to find all values", $exp_id, $tmp_exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR);
     }
 }
 
 # we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type
-if (uc(&STATS_value_for_flag ($STATS, "-telescope")) eq "NAN") { &my_die ("telescope not found", $exp_id, $exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&STATS_value_for_flag ($STATS, "-inst"))      eq "NAN") { &my_die ("inst      not found", $exp_id, $exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&STATS_value_for_flag ($STATS, "-filelevel")) eq "NAN") { &my_die ("filelevel not found", $exp_id, $exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&STATS_value_for_flag ($STATS, "-class_id"))  eq "NAN") { &my_die ("class_id  not found", $exp_id, $exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
-if (uc(&STATS_value_for_flag ($STATS, "-exp_type"))  eq "NAN") { &my_die ("exp_type  not found", $exp_id, $exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&STATS_value_for_flag ($STATS, "-telescope")) eq "NAN") { &my_die ("telescope not found", $exp_id, $tmp_exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&STATS_value_for_flag ($STATS, "-inst"))      eq "NAN") { &my_die ("inst      not found", $exp_id, $tmp_exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&STATS_value_for_flag ($STATS, "-filelevel")) eq "NAN") { &my_die ("filelevel not found", $exp_id, $tmp_exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&STATS_value_for_flag ($STATS, "-class_id"))  eq "NAN") { &my_die ("class_id  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
+if (uc(&STATS_value_for_flag ($STATS, "-exp_type"))  eq "NAN") { &my_die ("exp_type  not found", $exp_id, $tmp_exp_name, $tmp_class_id, $PS_EXIT_CONFIG_ERROR); }
 
 my $command = "$regtool -addprocessedimfile";
 $command .= " -exp_id $exp_id";
-$command .= " -exp_name $exp_name"; # keep the supplied exp_name (could be derived from the file)
+$command .= " -exp_name $tmp_exp_name"; # keep the supplied exp_name (could be derived from the file)
 $command .= " -tmp_class_id $tmp_class_id"; # the original class_id supplied by the user, replace by ppStats CLASS.ID
 $command .= " -dbname $dbname" if defined $dbname;