Changeset 19782

Timestamp:

Sep 28, 2008, 11:40:31 AM (18 years ago)

Author:

eugene

Message:

convert to ppStatsFromMetadata for robust stats

Location:

trunk/ippScripts/scripts

Files:

• detrend_norm_apply.pl (modified) (5 diffs)
• detrend_norm_exp.pl (modified) (6 diffs)

trunk/ippScripts/scripts/detrend_norm_apply.pl

@@ -16 +16 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::Stats;
 use PS::IPP::Config 1.01 qw( :standard );
 
@@ -88 +87 @@
 my $det_type_real = DETTYPE()->{lc($det_type)};
 
-# values to extract from output metadata and the stats to calculate
-my $STATS =
-   [
-       #          PPSTATS KEYWORD  STATISTIC            DETTOOL FLAG
-       { name => "ROBUST_MEDIAN",  type => "clipmean",  flag => "-bg",            dtype => "float" },
-       { name => "ROBUST_MEDIAN",  type => "clipstdev", flag => "-bg_mean_stdev", dtype => "float" },
-       { name => "ROBUST_STDEV",   type => "rms",       flag => "-bg_stdev",      dtype => "float" },
-   ];
-my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
-
 # Look for programs we need
 my $missing_tools;
 my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
 my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
+my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -126 +116 @@
 my $statsName = $ipprc->filename("PPIMAGE.STATS", $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR);
 my $traceDest = $ipprc->filename("TRACE.IMFILE",  $outroot, $class_id) or &my_die("Missing entry from camera config", $det_id, $iter, $class_id, $PS_EXIT_CONFIG_ERROR);
+
+my $cmdflags;
 
 # Run normalisation
@@ -157 +149 @@
     &my_die("Can't find expected output file: $b1name",    $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b1name);
     &my_die("Can't find expected output file: $b2name",    $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($b2name);
-    &my_die("Can't find expected output file: $statsName", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $ipprc->file_resolve($statsName);
-
-    # Get the statistics on the normalised image
-    my $statsFile; # File handle
-    open $statsFile, $ipprc->file_resolve($statsName) or &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR);
-    &my_die("Can't open statistics file $statsName: $!\n", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless defined $statsFile;
-    my @contents = <$statsFile>; # Contents of file
-    close $statsFile;
-
-    # parse the statistics MDC file
-    my $mdcParser = PS::IPP::Metadata::Config->new;     # Parser for metadata config files
-    my $metadata = $mdcParser->parse(join "", @contents);
-    unless ($metadata) {
-        &my_die("Unable to parse metadata config", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
-    }
-
-    # extract the stats from the metadata
-    unless ($stats->parse($metadata)) {
-        &my_die("Unable to find all values in statistics output.", $det_id, $iter, $class_id, $PS_EXIT_PROG_ERROR);
-    }
+
+    my $statsNameReal = $ipprc->file_resolve($statsName);
+    &my_die("Can't find expected output file: $statsName", $det_id, $iter, $class_id, $PS_EXIT_SYS_ERROR) unless -f $statsNameReal;
+
+    # ppStatsFromMetadata $outputStats - DETREND_RESID_IMFILE
+    $command = "$ppStatsFromMetadata $statsNameReal - DETREND_RESID_IMFILE";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppStatsFromMetadata: $error_code", $det_id, $iter, $exp_id, $class_id, $error_code);
+    }
+    foreach my $line (@$stdout_buf) {
+        $cmdflags .= " $line";
+    }
+    chomp $cmdflags;
 }
 
@@ -187 +175 @@
 $command .= " -path_base $outroot";
 $command .= " -dbname $dbname" if defined $dbname;
-$command .= $stats->cmdflags();
+$command .= " $cmdflags";
 
 # Add the processed file to the database

trunk/ippScripts/scripts/detrend_norm_exp.pl

@@ -16 +16 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
-use PS::IPP::Metadata::Stats;
 use PS::IPP::Config 1.01 qw( :standard );
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -62 +61 @@
 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe;
 
-# values to extract from output metadata and the stats to calculate
-# XXX -bg_mean_stdev should take rms of bg_mean_stdev if bg_mean_stdev != 0 (A)
-# XXX -bg_mean_stdev should take stdev of bg_mean if bg_mean_stdev == 0     (B)
-# XXX  (A) if imfile.Ncomp > 1, (B) if imfile.Ncomp == 1
-my $STATS =
-    [
-        #          KEYWORD                 STATISTIC          CHIPTOOL FLAG
-        { name => "bg",             type => "mean",  flag => "-bg",            dtype => "float" },
-        { name => "bg",             type => "stdev", flag => "-bg_mean_stdev", dtype => "float" },
-        { name => "bg_stdev",       type => "rms",   flag => "-bg_stdev",      dtype => "float" },
-        # { name => "bg_mean_stdev",  type => "rms",   flag => "-bg_mean_stdev" },
-        ];
-my $stats = PS::IPP::Metadata::Stats->new($STATS); # Stats parser
-
 # Look for programs we need
 my $missing_tools;
 my $dettool = can_run('dettool') or (warn "Can't find dettool" and $missing_tools = 1);
 my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
+my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -86 +72 @@
 
 # Get list of component files
+my $cmdflags;
 my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
 {
@@ -98 +85 @@
         &my_die("Unable to get list of normalized imfiles from dettool: $error_code", $det_id, $iter, $error_code);
     }
+    # XXX report an error message if stdout_buf is empty
 
     # convert stdout to a metadata
@@ -108 +96 @@
         &my_die("Unable to parse metadata list", $det_id, $iter, $PS_EXIT_PROG_ERROR);
 
-    # parse the stats in the metadata
-    unless ($stats->parse($metadata)) {
-        &my_die("Unable to find all values in statistics output.\n", $det_id, $iter, $PS_EXIT_PROG_ERROR);
-    }
+    # since I can't figure out how to do input and output within PERL, I'm writing to a temp file
+    my ($statFile, $statName) = tempfile( "/tmp/$camera.$det_type.norm.$det_id.$iter.stats.XXXX", UNLINK => !$save_temps );
+    foreach my $line (@$stdout_buf) {
+        print $statFile $line;
+    }
+    close $statFile;
+
+    $command = "$ppStatsFromMetadata $statName - DETREND_NORM_EXP";
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("Unable to perform ppStatsFromMetadata: $error_code\n");
+        exit($error_code);
+    }
+
+    foreach my $line (@$stdout_buf) {
+        $cmdflags .= " $line";
+    }
+    print "cmdflags: $cmdflags\n";
 }
 
@@ -162 +166 @@
 $command .= " -path_base $outroot ";
 $command .= " -dbname $dbname" if defined $dbname;
-$command .= $stats->cmdflags();
+$command .= " $cmdflags";
 
 # Add the processed file to the database