Changeset 23209

Timestamp:

Mar 6, 2009, 11:22:53 AM (17 years ago)

Author:

eugene

Message:

modify the detrend sequence to generate the detResidImfiles before generating the detNormalizedStat and supply a per-imfile normalization

Location:

branches/eam_branches/eam_branch_20090303/ippScripts/scripts

Files:

• detrend_norm_apply.pl (modified) (1 diff)
• detrend_norm_calc.pl (modified) (2 diffs)
• detrend_resid_exp.pl (modified) (7 diffs)

branches/eam_branches/eam_branch_20090303/ippScripts/scripts/detrend_norm_apply.pl

@@ -40 +40 @@
     'iteration|n=s'     => \$iter,       # Iteration
     'class_id|i=s'      => \$class_id,   # Class ID
-    'value|v=s'         => \$value,      # Value to multiple (for normalisation)
+    'value|v=s'         => \$value,      # Value to apply (for normalisation)
     'input_uri|u=s'     => \$input_uri,  # Input file
     'camera|c=s'        => \$camera,     # Camera

branches/eam_branches/eam_branch_20090303/ippScripts/scripts/detrend_norm_calc.pl

@@ -95 +95 @@
 my @files;                      # The input files
 {
-    my $command = "$dettool -processedimfile";
-    $command .= " -det_id $det_id"; # Command to run
+    my $command = "$dettool -residimfile";
+    $command .= " -det_id $det_id";
+    $command .= " -iteration $iter";
     $command .= " -included"; # only use the inputs for this detrend run to calculate the norm
     $command .= " -dbname $dbname" if defined $dbname;
@@ -103 +104 @@
     print "Running [$command]...\n" if $verbose;
     if (not run(\@command, \$stdin, \$stdout, \$stderr)) {
-        &my_die("Unable to perform dettool -processedimfile on detrend $det_id/$iter: $?",
+        &my_die("Unable to perform dettool -residimfile on detrend $det_id/$iter: $?",
                 $det_id, $iter, $PS_EXIT_SYS_ERROR);
     }

branches/eam_branches/eam_branch_20090303/ippScripts/scripts/detrend_resid_exp.pl

@@ -96 +96 @@
 &my_die("Unrecognised detrend type: $det_type", $det_id, $iter, $PS_EXIT_PROG_ERROR) unless defined $recipe;
 
+# variables used for I/O
+my ($command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
+
+# Get list of normalizations by class_id : stored as $norms; save to temp file for ppImage runs below
+my (%norms, $normsName);
+{
+    # dettool command to select imfile data for this exp_id
+    $command  = "$dettool -normalizedstat";
+    $command .= " -det_id $det_id";
+    $command .= " -iteration $iter";
+    $command .= " -dbname $dbname" if defined $dbname;
+    ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        warn("Unable to perform dettool -residimfile: $error_code\n");
+        exit($error_code);
+    }
+    if (@$stdout_buf == 0) {
+        &my_die("No normalizations were found", $det_id, $iter, $PS_EXIT_PROG_ERROR);
+    }
+
+    # Parse the stdout buffer into a metadata
+    my $mdcParser = PS::IPP::Metadata::Config->new;
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
+
+    # parse the file info in the metadata
+    my $normsMD = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
+
+
+    # write the normalizations to a file as a metadata config file in the form: class_id F32 value
+    # XXX a possible optimization: if there is only one imfile, skip normalization
+    my $normsFile;
+    ($normsFile, $normsName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.norms.XXXX", UNLINK => !$save_temps );
+    print "saving norms to $normsName\n";
+    foreach my $norm (@$normsMD) {
+        my $class_id = $norm->{class_id};
+        my $normalization = $norm->{norm};
+
+        $norms{$class_id} = $normalization;
+        printf $normsFile "$class_id F32 $normalization\n", 
+    }
+    close $normsFile;
+}
+
 # Get list of imfile files
 my $cmdflags;
-my ($files, $command, $success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
+my (@files);
 {
     # dettool command to select imfile data for this exp_id
@@ -113 +160 @@
         exit($error_code);
     }
-    # XXX report an error message if stdout_buf is empty
+    if (@$stdout_buf == 0) {
+        &my_die("No imfiles were found", $det_id, $iter, $PS_EXIT_PROG_ERROR);
+    }
 
     # Parse the stdout buffer into a metadata
@@ -120 +169 @@
         &my_die("Unable to parse metadata config doc", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
 
+    # since I can't figure out how to do input and output within PERL, I'm writing the (modified) metadata to a temp file
+    my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.stats.XXXX", UNLINK => !$save_temps );
+
     # parse the file info in the metadata
-    $files = parse_md_list($metadata) or
-        &my_die("Unable to parse metadata list", $det_id, $iter, $exp_id, $PS_EXIT_PROG_ERROR);
-
-    # since I can't figure out how to do input and output within PERL, I'm writing to a temp file
-    my ($statFile, $statName) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.stats.XXXX", UNLINK => !$save_temps );
-    print "saving stats to $statName\n";
-    foreach my $line (@$stdout_buf) {
-        print $statFile $line;
+    # as we parse the list of files and their stats, apply the normalization to the relevant fields
+    # also, write out the modified metadata set
+    foreach my $mdItem (@$metadata) {
+        if ($mdItem->{class} ne "metadata") {
+            carp "MD element ", $mdItem->{name}, " isn't of type METADATA --- ignored.\n";
+            next;
+        }
+        my %hash;               # Hash element
+        my $mdComponents = $mdItem->{value}; # Components of the metadata
+
+        # determine the class_id for this block:
+        my $class_id;
+        foreach my $data (@$mdComponents) {
+            unless ($data->{name} eq "class_id") { next; }
+            $class_id = $data->{value};
+            last;
+        }
+
+        # a new metadata block
+        print $statFile "rawResidImfile  METADATA\n";
+
+        # modify and save the data in this block:
+        foreach my $data (@$mdComponents) {
+            my $norm = $norms{$class_id};
+
+            # fields to modify by the normalization:
+            if ($data->{name} eq "bg")            { $data->{value} *= $norm; }
+            if ($data->{name} eq "bg_stdev")      { $data->{value} *= $norm; }
+            if ($data->{name} eq "bg_mean_stdev") { $data->{value} *= $norm; }
+            if ($data->{name} eq "bg_skewness")   { $data->{value} *= $norm; }
+            if ($data->{name} eq "bg_kurtosis")   { $data->{value} *= $norm; }
+            if ($data->{name} eq "bin_stdev")     { $data->{value} *= $norm; }
+
+            # write out the metadata, save on the array of hashes
+            print $statFile "  $data->{name}  $data->{type}  $data->{value}\n";
+            $hash{$data->{name}} = $data->{value};
+        }
+        print $statFile "END\n";
+        push @files, \%hash;
     }
     close $statFile;
@@ -162 +245 @@
 my ($list1File, $list1Name) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.b1.list.XXXX", UNLINK => !$save_temps );
 my ($list2File, $list2Name) = tempfile( "/tmp/$exp_tag.detresid.$det_id.$iter.b2.list.XXXX", UNLINK => !$save_temps );
-foreach my $file (@$files) {
+foreach my $file (@files) {
     print $list1File ($ipprc->filename( "PPIMAGE.BIN1", $file->{path_base}, $file->{class_id} ) . "\n");
     print $list2File ($ipprc->filename( "PPIMAGE.BIN2", $file->{path_base}, $file->{class_id} ) . "\n");
@@ -173 +256 @@
 unless ($no_op) {
     # Make the jpeg for binning 1
+    # XXX EAM : supply the collection of normalizations as a metadata
     $command = "$ppImage -list $list1Name $outroot"; # Command to run
     $command .= " -recipe PPIMAGE PPIMAGE_J1";
     $command .= " -recipe JPEG $recipe";
+    $command .= " -normlist $normsName";
     $command .= " -dbname $dbname" if defined $dbname;
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -186 +271 @@
 
     # Make the jpeg for binning 2
+    # XXX EAM : supply the collection of normalizations as a metadata
     $command = "$ppImage -list $list2Name $outroot"; # Command to run
     $command .= " -recipe PPIMAGE PPIMAGE_J2";
     $command .= " -recipe JPEG $recipe";
+    $command .= " -normlist $normsName";
     $command .= " -dbname $dbname" if defined $dbname;
     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -223 +310 @@
 my @fluxes;
 
-foreach my $file (@$files) {
+foreach my $file (@files) {
     my $name      = $file->{class_id};
     my $mean      = $file->{bg};        # Mean for this imfile