Changeset 9454

Timestamp:

Oct 9, 2006, 10:20:43 PM (20 years ago)

Author:

eugene

Message:

clarified some of the operations,
but added comments to address concerns about the goals.

File:

• trunk/ippScripts/scripts/detrend_reject_imfile.pl (modified) (8 diffs)

trunk/ippScripts/scripts/detrend_reject_imfile.pl

@@ -41 +41 @@
 use constant RECIPE2 => 'PPIMAGE_J2'; # Recipe to use for ppImage to make JPEGs
 
+#### XXXXX these values must come from the config system, and may depend on filter!!!
 # The expected mean, as a function of detrend type
 use constant EXPECTED_MEAN => {
@@ -46 +47 @@
     'dark' => 0,                # Dark should be zero
     'shutter' => undef,         # Shutter could be anything (depends on exposure level)
-    'flat' => 0         # Flat could be anything (depends on exposure level)
+    'flat' => 0                 # Flat could be anything (depends on exposure level)
     };
 
 # Rejection threshold for the mean, in terms of the standard deviation
 # This measures how close it is to what's expected
-use constant REJECT_INDIVIDUAL_MEAN => {
+# XXX it is valid to reject on more than one criterion
+use constant REJECT_COMPONENT_MEAN => {
     'bias' => 0,                # Should be fairly flat; some CRs
     'dark' => 0,                # Lots of CRs
@@ -59 +61 @@
 
 # Rejection threshold for the standard deviation, in ADUs
-# This measures how much variation there is in each component
-use constant REJECT_INDIVIDUAL_STDEV => {
+# This measures how much variation there is in each imfile component
+use constant REJECT_COMPONENT_STDEV => {
     'bias' => 0,                # Should be fairly flat; some CRs
     'dark' => 0,                # Lots of CRs
@@ -77 +79 @@
 
 # Rejection threshold for the stdev of the sample, in ADUs
-# This measures how much variation there is across the components
+# This measures how much variation there is across the imfile components
 use constant REJECT_SAMPLE_STDEV => {
+    'bias' => 0,                # Should be little variation between chips
+    'dark' => 0,                # Could be some glow on some chips
+    'shutter' => undef,         # Can be significant structure
+    'flat' => 0                 # Could be features on some chips, but all should be about the same
+    };
+
+# Rejection threshold for the stdev of the sample, in ADUs
+# This measures how much variation there is across the imfile components
+use constant REJECT_SAMPLE_MEAN_STDEV => {
     'bias' => 0,                # Should be little variation between chips
     'dark' => 0,                # Could be some glow on some chips
@@ -151 +162 @@
 die "Unknown expected mean for detrend type $det_type\n"
     if not exists EXPECTED_MEAN->{$det_type};
-die "Unknown individual mean rejection level for detrend type $det_type\n"
-    if not exists REJECT_INDIVIDUAL_MEAN->{$det_type};
-die "Unknown individual stdev rejection level for detrend type $det_type\n"
-    if not exists REJECT_INDIVIDUAL_STDEV->{$det_type};
+die "Unknown component mean rejection level for detrend type $det_type\n"
+    if not exists REJECT_COMPONENT_MEAN->{$det_type};
+die "Unknown component stdev rejection level for detrend type $det_type\n"
+    if not exists REJECT_COMPONENT_STDEV->{$det_type};
 die "Unknown sample mean rejection level for detrend type $det_type\n"
     if not exists REJECT_SAMPLE_MEAN->{$det_type};
@@ -160 +171 @@
     if not exists REJECT_SAMPLE_STDEV->{$det_type};
 
-# Reject based on the individual stats
+# Reject based on the stats of the component imfile
 die "Number of means and number of stdevs differ!\n" if scalar @means != scalar @stdevs;
 for (my $i = 0; $i < scalar @means; $i++) {
     my $mean = $means[$i];      # Mean for this component
     $mean -= EXPECTED_MEAN->{$det_type} if defined EXPECTED_MEAN->{$det_type};
-    my $stdev = $stdevs[$i];    # Stdev for this component
-    my $not_reject;
-
-    if (defined REJECT_INDIVIDUAL_MEAN->{$det_type}
-        and $stdev > 0 and
-        $mean / $stdev > REJECT_INDIVIDUAL_MEAN->{$det_type}) {
-        print "Rejecting exposure based on bad individual mean for component $i: " .
-            ($mean / $stdev) . " vs " . REJECT_INDIVIDUAL_MEAN->{$det_type} . "\n";
-        $not_reject = 1;
-        last;
-    } elsif (defined REJECT_INDIVIDUAL_STDEV->{$det_type} and
-             $stdev > REJECT_INDIVIDUAL_STDEV->{$det_type}) {
-        print "Rejecting exposure based on bad individual stdev for component $i: " .
-            $stdev . " vs " . REJECT_INDIVIDUAL_STDEV->{$det_type} . "\n";
-        $not_reject = 1;
-        last;
+    my $stdev = sqrt($stdevs[$i]);      # Stdev for this component
+
+    ## IT IS VALID to reject on more than one criterion
+    if (REJECT_COMPONENT_MEAN->{$det_type} && ($stdev > 0) ) {
+        if ($mean / $stdev > REJECT_COMPONENT_MEAN->{$det_type}) {
+            print "Rejecting exposure based on bad component mean for component $i: " .
+                ($mean / $stdev) . " vs " . REJECT_COMPONENT_MEAN->{$det_type} . "\n";
+            $reject = 1;
+            last;
+        }
+    } 
+    if (REJECT_COMPONENT_STDEV->{$det_type}) {
+        if ($stdev > REJECT_COMPONENT_STDEV->{$det_type}) {
+            print "Rejecting exposure based on bad component stdev for component $i: " .
+                $stdev . " vs " . REJECT_COMPONENT_STDEV->{$det_type} . "\n";
+            $reject = 1;
+            last;
+        }
     }
 }
 
-# Reject based on the sample stats
+# calculate the imfile ensemble statistics
 my $meanStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator for means
 $meanStats->add_data(@means);
@@ -190 +203 @@
 $stdevStats->add_data(@stdevs);
 my $mean = $meanStats->mean();  # Mean of the sample of means
-my $stdev = $meanStats->standard_deviation(); # Stdev of the sample of means
+my $meanStdev = $meanStats->standard_deviation(); # Stdev of the sample of means
 if (not defined $stdev) {
     $stdev = 0;
 }
-my $meanStdev = sqrt($stdevStats->mean()); # Root-Mean-Square of the sample of stdevs
-my $not_reject;
+my $stdev = sqrt($stdevStats->mean()); # Root-Mean-Square of the sample of stdevs
 print "calculating root-mean-square of the sample stdevs: $meanStdev\n";
-if (defined REJECT_SAMPLE_MEAN->{$det_type} and
-    $stdev != 0 and
-    $mean / $stdev > REJECT_SAMPLE_MEAN->{$det_type}) {
-    print "Rejecting exposure based on bad mean: " . ($mean / $stdev) . " vs " .
-        REJECT_SAMPLE_MEAN->{$det_type} . "\n";
-    $not_reject = 1;
-} elsif (defined REJECT_SAMPLE_STDEV->{$det_type} and
-         $meanStdev > REJECT_SAMPLE_STDEV->{$det_type}) {
-    print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
-        REJECT_SAMPLE_STDEV->{$det_type} . "\n";
-    $not_reject = 1;
+
+## Reject based on the imfile ensemble stats
+# reject if the imfile ensemble 
+if (REJECT_SAMPLE_MEAN->{$det_type} && ($stdev > 0)) {
+    if ($mean / $stdev > REJECT_SAMPLE_MEAN->{$det_type}) {
+        print "Rejecting exposure based on bad mean: " . ($mean / $stdev) . " vs " .
+            REJECT_SAMPLE_MEAN->{$det_type} . "\n";
+        $reject = 1;
+    } 
+}
+if (REJECT_SAMPLE_STDEV->{$det_type}) {
+    if ($stdev > REJECT_SAMPLE_STDEV->{$det_type}) {
+        print "Rejecting exposure based on bad mean stdev: " . $stdev . " vs " .
+            REJECT_SAMPLE_STDEV->{$det_type} . "\n";
+        $reject = 1;
+    }
+}
+if (REJECT_SAMPLE_MEAN_STDEV->{$det_type}) {
+    if ($meanStdev > REJECT_SAMPLE_MEAN_STDEV->{$det_type}) {
+        print "Rejecting exposure based on bad mean stdev: " . $meanStdev . " vs " .
+            REJECT_SAMPLE_MEAN_STDEV->{$det_type} . "\n";
+        $reject = 1;
+    }
 }
 
@@ -229 +253 @@
 
 __END__
+
+####
+
+## this function is not well though out.  it loads a collection of
+## values (background mean and stdev) for a collection of imfile
+## subcomponents.  it then measures the mean background, the
+## background stdev (rms of ensemble stdevs), and the background mean
+## stdev (rms of the means).  it then tries to reject based on the
+## following criteria:
+## (background / stdev > limit) for a single component
+##    ---> this will ACCEPT for a poor images (large stdev)
+## (stdev > limit)
+## (mean background / stdev > limit) same problem
+## (mean stdev > limit)
+
+### I would suggest the following:
+## 1) calculate the ensemble mean and stdev
+## 2) for each component, is the (back - mean) / mean stdev > limit?
+##    (ie, is it an outlier?)
+## 3) is the component stdev > limit? (absolute test)
+## 4) calculate the background stdev for the ensemble (excluding rejects)
+## 5) is the mean stdev > limit (poor images)
+## 6) calculate the ensemble stdev (rms) of remaining (excluding rejects)
+## 7) is the ensemble stdev > limit 
+
+## for flats, the value used for stdev should be the fractional stdev
+## (stdev/mean) and no rejection should be performed on the basis of
+## the mean value, but could still be done on the mean stdev
+
+## the same logical cuts above can be applied to components in an
+## imfile, imfiles in an exposure, and exposures in a stack