Changeset 9457 for trunk/ippScripts/scripts/detrend_reject_exp.pl

Timestamp:

Oct 10, 2006, 8:52:00 AM (20 years ago)

Author:

eugene

Message:

updating bg_stdev, bg_mean_stdev tests

File:

• trunk/ippScripts/scripts/detrend_reject_exp.pl (modified) (7 diffs)

trunk/ippScripts/scripts/detrend_reject_exp.pl

@@ -78 +78 @@
 my @expTags;                    # Array of exposure IDs
 my @means;                      # Array of means
-my @stdevs;                     # Array of stdevs
+my @variances;                  # Array of variances
 my @meanStdevs;                 # Array of mean stdevs
 my @accept;                     # Array of accept flags
@@ -91 +91 @@
     push @expTags, $exposure->{exp_tag};
     push @means, $exposure->{bg};
-    push @stdevs, $exposure->{bg_stdev};
-    push @meanStdevs, $exposure->{bg_mean_stdev};
+    push @variances, ($exposure->{bg_stdev}*$exposure->{bg_stdev});
+    push @meanStdevs, $exposure->{bg_mean_stdev}; ### XXX are we keeping this or stdev(mean)?
     push @accept, $exposure->{accept};
     push @include, $exposure->{include};
@@ -98 +98 @@
 my $meanStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
 $meanStats->add_data(@means);
-my $stdevStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
-$stdevStats->add_data(@stdevs);
+my $variancestats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
+$variancestats->add_data(@variances);
 my $meanStdevStats = Statistics::Descriptive::Sparse->new(); # Statistics calculator
 $meanStdevStats->add_data(@meanStdevs);
@@ -111 +111 @@
     if not exists REJECT_MEAN_STDEV->{$det_type};
 
+# rejections based on comparison with ensemble statistics
+my $mean = $meanStats->mean();
+my $meanStdev = $meanStats->standard_deviation();
+if (not defined $meanStdev) { $meanStdev = 0; }
+
+my $var = $variances->mean();
+my $varStdev = $variances->standard_deviation();
+
 # Go through again to do rejection, and update the database for each exposure
 my $numChanges = 0;             # Number of exposures with changed status
@@ -117 +125 @@
     my $command = "$dettool -updateresidexp -det_id $det_id -iteration $iter -exp_tag $expTag"; # Command to run
     my $reject = 0;             # Reject this exposure?
-    my $not_reject = 0;         # Reject this exposure?
+
     if (not $accept[$i]) {
         # Rejected this at an earlier stage
         print "Rejecting $expTag based on earlier determination.\n";
         $reject = 1;
-    } elsif (defined REJECT_MEAN->{$det_type} and
-             defined $meanStats->standard_deviation() and
-             $meanStats->standard_deviation() > 0 and
-             ($means[$i] - $meanStats->mean()) / $meanStats->standard_deviation() > REJECT_MEAN->{$det_type}) {
-        print "Rejecting $expTag based on bad mean: " .
-            (($means[$i] - $meanStats->mean()) / $meanStats->standard_deviation()) .
-            " vs " . REJECT_MEAN->{$det_type} . "\n";
-        $not_reject = 1;
-    } elsif (defined REJECT_STDEV->{$det_type} and
-             defined $stdevStats->standard_deviation() and
-             $stdevStats->standard_deviation() > 0 and
-             ($stdevs[$i] - $stdevStats->mean()) / $stdevStats->standard_deviation() >
-             REJECT_STDEV->{$det_type}) {
-        print "Rejecting $expTag based on bad stdev: " .
-            (($stdevs[$i] - $stdevStats->mean()) / $stdevStats->standard_deviation()) .
-            " vs " . REJECT_STDEV->{$det_type} . "\n";
-        $not_reject = 1;
-    } elsif (defined REJECT_MEAN_STDEV->{$det_type} and
-             defined $meanStdevStats->standard_deviation() and 
-             $meanStdevStats->standard_deviation() > 0 and
-             ($meanStdevs[$i] - $meanStdevStats->mean()) / $meanStdevStats->standard_deviation() >
-             REJECT_MEAN_STDEV->{$det_type}) {
-        print "Rejecting $expTag based on bad stdev: " .
-            (($meanStdevs[$i] - $meanStdevStats->mean()) / $meanStdevStats->standard_deviation()) .
-            " vs " . REJECT_MEAN_STDEV->{$det_type} . "\n";
-        $not_reject = 1;
-    }
-    
+        goto UPDATE;
+    } 
+    if (REJECT_MEAN->{$det_type} && ($meanStdev > 0)) {
+        my $nSigma = abs($means[$i] - $mean) / $meanStdev;
+        if ($nSigma > REJECT_MEAN->{$det_type}) {
+            print "Rejecting $expTag based on outlier mean value: " .
+                "$means[$i] is $nSigma vs " . REJECT_MEAN->{$det_type} . "\n";
+            $reject = 1;
+            goto UPDATE;
+        }
+    } else {
+        print "no rejection for exposure mean\n";
+    }
+    if (defined REJECT_STDEV->{$det_type} && ($varStdev >0)) {
+        my $nSigma = abs($var[$i] - $var) / $varStdev;
+        if ($nSigma > REJECT_STDEV->{$det_type}) {
+            print "Rejecting $expTag based on outlier stdev: " .
+                sqrt($variances[$i]) " is $nSigma vs " . REJECT_STDEV->{$det_type} . "\n";
+            $reject = 1;
+            goto UPDATE;
+        }
+    } else {
+        print "no rejection for exposure stdev\n";
+    }
+    
+  UPDATE:
     if ($reject) {
         $command .= ' -reject';
@@ -159 +165 @@
         $numChanges++;
     }
-    
+
     unless ($no_update) {
         # Update 
@@ -182 +188 @@
 # Put the result into the database
 unless ($no_update) {
-    my $command = "$dettool -adddetrunsummary -det_id $det_id -iteration $iter " .
-        "-bg " . $meanStats->mean() . " -bg_stdev " . $stdevStats->mean() .
-        " -bg_mean_stdev " . $meanStdevStats->mean();
+    my $command = "$dettool -adddetrunsummary -det_id $det_id -iteration $iter" .
+        " -bg " . $mean . " -bg_stdev " . sqrt($var) .
+        " -bg_mean_stdev " . $meanStdev;
     $command .= " -accept" if $master;