Changeset 31634 for trunk/psModules/src/imcombine/pmStack.c

Timestamp:

Jun 15, 2011, 6:10:15 PM (15 years ago)

Author:

watersc1

Message:

merge from czw branch: improved stacking using KMM outlier identification.

File:

• trunk/psModules/src/imcombine/pmStack.c (modified) (14 diffs)

trunk/psModules/src/imcombine/pmStack.c

@@ -22 +22 @@
 #include <pslib.h>
 
+#include <gsl/gsl_cdf.h>
+
 #include "pmHDU.h"
 #include "pmFPA.h"
@@ -35 +37 @@
 
 
-# if (0)
+//# if (0)
 #define TESTING                         // Enable test output
-#define TEST_X 3145                       // x coordinate to examine
-#define TEST_Y 2334                       // y coordinate to examine
+/* #define TEST_X 5745                       // x coordinate to examine */
+/* #define TEST_Y 5331                       // y coordinate to examine */
+#define TEST_X 25
+#define TEST_Y 25
 #define TEST_RADIUS 0.5                 // Radius to examine
-# endif
+//# endif
 
 # ifdef TESTING
@@ -100 +104 @@
     return;
 }
+
+// KMM functions to do bimodality rejection of pixels
+float gaussian(float x, float m, float s) {
+  return(pow(s * sqrt(2 * M_PI),-1) * exp(-0.5 * pow( (x - m) / s, 2)));
+}
+
+static void KMMcalculate(const psVector *values,
+                         float *Punimodal,int *iter,
+                         float *mU, float *sU,
+                         float *pi1, float *m1, float *s1,
+                         float *pi2, float *m2, float *s2) {
+  assert(values);
+  assert(values->type.type == PS_TYPE_F32);
+  
+  double logL_bimodal = 0, logL_unimodal;
+  psVector *P1 = psVectorAlloc(values->n,PS_TYPE_F32);
+  psVector *P2 = psVectorAlloc(values->n,PS_TYPE_F32);
+  int i;
+/*   int debug = 0; */
+  
+  // Calculate unimodal properties
+  *mU = 0;
+  *sU = 0;
+  logL_unimodal = 0;
+  for (i = 0; i < values->n; i++) { // Calculate mean
+    *mU += values->data.F32[i];
+  }
+  *mU /= values->n;
+  for (i = 0; i < values->n; i++) { // Calculate sigma
+    *sU += pow(values->data.F32[i] - *mU,2);
+  }
+  *sU = sqrt(*sU / values->n);
+  for (i = 0; i < values->n; i++) { // Calculate log likelihood
+    logL_unimodal += log(gaussian(values->data.F32[i],*mU,*sU));
+  }
+
+  // Do EM loop
+  float dL = 0;
+  float oldL = -999;
+  *iter = 0;
+  logL_bimodal = logL_unimodal;
+  *m1 = *mU - 3 * *sU;
+  *m2 = *mU + 3 * *sU;
+  *s1 = *sU / 2;
+  *s2 = *sU / 2;
+  *pi1 = 0.5;
+  *pi2 = 0.5;
+
+  float g1,g2,norm;
+  float w1,w2;
+
+  // These should be options.
+  float KMM_TOLERANCE = 1e-6;
+  int KMM_MAX_ITERATIONS = 500;
+  float KMM_SMALL_NUMBER = 1e-5;
+  while (((dL > KMM_TOLERANCE)||(*iter < 3))&&(*iter < KMM_MAX_ITERATIONS)) {
+    *iter += 1;
+    dL = fabs(logL_bimodal - oldL);
+    oldL = logL_bimodal;
+/*     if (debug == 1) { */
+/*       fprintf(stderr,"KMM: %d %f %f %f %f (%f %f %f) (%f %f %f)\n", */
+/*            *iter,logL_unimodal,logL_bimodal,oldL,dL, */
+/*            *m1,*s1,*pi1, */
+/*            *m2,*s2,*pi2); */
+/*     } */
+    // Expectation/P-stage
+    for (i = 0; i < values->n; i++) { // Calculate probabilities for each mode
+      g1 = gaussian(values->data.F32[i],*m1,*s1);
+      g2 = gaussian(values->data.F32[i],*m2,*s2);
+      norm = (*pi1 * g1 + *pi2 * g2);
+      P1->data.F32[i] = (*pi1 * g1) / norm;
+      P2->data.F32[i] = (*pi2 * g2) / norm;
+    }
+    // Maximization/M-stage
+    logL_bimodal = 0;
+    w1 = 0;
+    w2 = 0;
+    for (i = 0; i < values->n; i++) { // Calculate log likelihood
+      if (!((*pi1 == 0)||(*pi2 == 0))) {
+        logL_bimodal += log(*pi1 * gaussian(values->data.F32[i],*m1,*s1) +
+                            *pi2 * gaussian(values->data.F32[i],*m2,*s2));
+      }
+    }
+    *m1 = 0;
+    *m2 = 0;
+    *s1 = 0;
+    *s2 = 0;
+    for (i = 0; i < values->n; i++) { // Calculate new means and weights
+      *m1 += values->data.F32[i] * P1->data.F32[i];
+      *m2 += values->data.F32[i] * P2->data.F32[i];
+
+      w1 += P1->data.F32[i];
+      w2 += P2->data.F32[i];
+    }
+    *m1 /= w1;
+    *m2 /= w2;
+    for (i = 0; i < values->n; i++) { // Calculate new sigmas
+      *s1 += pow(values->data.F32[i] - *m1,2) * P1->data.F32[i];
+      *s2 += pow(values->data.F32[i] - *m2,2) * P2->data.F32[i];
+    }
+    *s1 = sqrt(*s1 / w1);
+    *s2 = sqrt(*s2 / w2);
+
+    *pi1 = w1 / values->n;
+    *pi2 = w2 / values->n;
+
+    if (!isfinite(*pi1)) { // finite checks
+      *pi1 = 0.0;
+    }
+    if (!isfinite(*pi2)) { // finite checks
+      *pi2 = 0.0;
+    }
+    if (*s1 == 0) { // sigma may not be zero
+      *s1 = KMM_SMALL_NUMBER * *m1;
+    }
+    if (*s2 == 0) { // sigma may not be zero
+      *s2 = KMM_SMALL_NUMBER * *m2;
+    }
+  } // End EM phase
+
+  // Calculate Punimodal
+  double lambda = -2.0 * (logL_unimodal - logL_bimodal);
+  int    df     = 2 + 2 * 1; // I can't find my reference on this. 
+  if (lambda > 0) {
+    *Punimodal = gsl_cdf_chisq_Q(lambda,df);
+  }
+  else { // If lambda <= 0, then logL_unimodal > logL_bimodal, so Punimodal must be by definition 1.0
+    *Punimodal = 1.0;
+  }
+  psFree(P1);
+  psFree(P2);
+}
+
+static void KMMFindPopular(const psVector *values, float *Punimodal, float *mean, float *sigma, float *pi) {
+  float KMM_MINIMUM_PVALUE = 0.05; // Should be an option.
+  float mU,sU;
+  float pi1,m1,s1,pi2,m2,s2;
+  int iter;
+
+  assert(values);
+  assert(values->type.type == PS_TYPE_F32);
+  
+  KMMcalculate(values,Punimodal,&iter,
+               &mU,&sU,
+               &pi1,&m1,&s1,
+               &pi2,&m2,&s2);
+
+  if (*Punimodal > KMM_MINIMUM_PVALUE) {
+    // Is unimodal
+    *mean = mU;
+    *sigma = sU;
+    *pi = 1.0;
+  }
+  else {
+    // Is bimodal. Select most popular mode.
+    if (pi1 >= pi2) {
+      *mean = m1;
+      *sigma = s1;
+      *pi = pi1;
+    }
+    else {
+      *mean = m2;
+      *sigma = s2;
+      *pi = pi2;
+    }
+  }  
+}
+
+                               
 
 
@@ -285 +458 @@
     return;
 }
-
+#if (0)
+// Currently unused function to reject inputs for a given pixel(x,y) based on the selecting the
+// most popular mode after running the KMM test, and rejecting all inputs that belong to the
+// least popular mode.
+static void KMMRejectUnpopular(const psArray *inputs, int x, int y) {
+  float KMM_MINIMUM_PVALUE = 0.05;
+  float mU,sU;
+  float Punimodal,pi1,m1,s1,pi2,m2,s2;
+  int iter;
+  int j,k;
+
+  psVector *values = psVectorAlloc(inputs->n, PS_TYPE_F32);
+  k = 0;
+  for (j = 0; j < inputs->n; j++) {
+    pmStackData *data = inputs->data[j]; // Stack data of interest
+    if (!data) {
+      k++;
+      continue;
+    }
+    psImage *image = data->readout->image; // Image of interest
+    int xIn = x - data->readout->col0, yIn = y - data->readout->row0; // Coordinates on input readout
+    values->data.F32[j - k] = image->data.F32[yIn][xIn];
+  }
+  
+  KMMcalculate(values,&Punimodal,&iter,
+               &mU,&sU,
+               &pi1,&m1,&s1,
+               &pi2,&m2,&s2);
+
+   CHECKPIX(x, y, 
+          "KMM Unpopular Test: %d,%d: Puni: %g in %d",x,y,Punimodal,iter);  
+  if (Punimodal < KMM_MINIMUM_PVALUE) {
+    int i;
+    float g1,g2,norm;
+    float P1,P2;
+
+    for (i = 0; i < values->n; i++) { // Calculate probabilities for each mode
+      g1 = gaussian(values->data.F32[i],m1,s1);
+      g2 = gaussian(values->data.F32[i],m2,s2);
+      norm = (pi1 * g1 + pi2 * g2);
+      P1 = (pi1 * g1) / norm;
+      P2 = (pi2 * g2) / norm;
+
+      CHECKPIX(x, y, "KMM Unpopular Rejection: %d,%d: %f(%d): %d %f %f:(%f %f %f ) %f:(%f %f %f) rejection? %d %d\n",
+               x, y,
+               Punimodal,iter,
+               i, values->data.F32[i],
+               P1,m1,s1,pi1,
+               P2,m2,s2,pi2,
+               (pi1 > pi2)&&(P1 < P2),
+               (pi1 < pi2)&&(P1 > P2));
+      if ((pi1 > pi2)&&(P1 < P2)) { // mode 1 is more popular, but this element belongs to mode 2
+        combineMarkReject(inputs,x,y,i);
+      }
+      if ((pi1 < pi2)&&(P1 > P2)) { // mode 2 is more popular, but this element belongs to mode 1
+        combineMarkReject(inputs,x,y,i);
+      }
+    }
+  }
+  psFree(values);
+  // else do nothing.
+}
+
+// Currently unused function to reject inputs for a given pixel(x,y) based on the selecting the
+// faintest mode as determined by the KMM test, and rejecting all inputs that belong to the brighest.
+static void KMMRejectBright(const psArray *inputs, int x, int y) {
+  float KMM_MINIMUM_PVALUE = 0.05;
+  float mU,sU;
+  float Punimodal,pi1,m1,s1,pi2,m2,s2;
+  int iter;
+  int j;
+
+  psVector *values = psVectorAlloc(inputs->n, PS_TYPE_F32);
+  for (j = 0; j < inputs->n; j++) {
+    pmStackData *data = inputs->data[j]; // Stack data of interest
+    psImage *image = data->readout->image; // Image of interest
+    int xIn = x - data->readout->col0, yIn = y - data->readout->row0; // Coordinates on input readout
+    values->data.F32[j] = image->data.F32[yIn][xIn];
+  }
+  
+  KMMcalculate(values,&Punimodal,&iter,
+               &mU,&sU,
+               &pi1,&m1,&s1,
+               &pi2,&m2,&s2);
+  if (Punimodal < KMM_MINIMUM_PVALUE) {
+    int i;
+    float g1,g2,norm;
+    float P1,P2;
+
+    for (i = 0; i < values->n; i++) { // Calculate probabilities for each mode
+      g1 = gaussian(values->data.F32[i],m1,s1);
+      g2 = gaussian(values->data.F32[i],m2,s2);
+      norm = (pi1 * g1 + pi2 * g2);
+      P1 = (pi1 * g1) / norm;
+      P2 = (pi2 * g2) / norm;
+
+      if ((m1 > m2)&&(P1 > P2)) { // m1 is larger, and this element belongs to mode 1
+        combineMarkReject(inputs,x,y,i);
+      }
+      if ((m1 < m2)&&(P1 < P2)) { // m2 is larger, and this element belongs to mode 2
+        combineMarkReject(inputs,x,y,i);
+      }
+    }
+  }
+  psFree(values);
+  // else do nothing.
+}
+#endif // End if(0) to prevent KMMReject{Unpopular|Bright} from being defined.
 
 // Extract vectors for simple combination/rejection operations
@@ -332 +612 @@
     memset (nGoodBits, 0, 16*sizeof(int));
 
-    // Extract the pixel and mask data
+
+    // Extract the pixel and mask data    
     int numGood = 0;                    // Number of good pixels
     for (int i = 0, j = 0; i < inputs->n; i++) {
@@ -364 +645 @@
             continue;
         }
-
+        
         int xIn = x - data->readout->col0, yIn = y - data->readout->row0; // Coordinates on input readout
         psImage *mask = data->readout->mask; // Mask of interest
@@ -400 +681 @@
         CHECKPIX(x, y, "keep: %d : %x (badMask = %x)\n", i, mask->data.PS_TYPE_IMAGE_MASK_DATA[yIn][xIn], *badMask);
     }
+    
     pixelData->n = numGood;
     if (variance) {
@@ -426 +708 @@
     if (PS_SQR(x - TEST_X) + PS_SQR(y - TEST_Y) <= PS_SQR(TEST_RADIUS)) {
         for (int i = 0; i < numGood; i++) {
-            fprintf(stderr, "Input %d, pixel %d,%d (%" PRIu16 "): %f %f (%f) %f %f %d %x %x -> %x %x\n",
+            fprintf(stderr,"Input %d, pixel %d,%d (%" PRIu16 "): %f %f (%f) %f %f %d %x %x -> %x %x\n",
                     i, x, y, pixelSources->data.U16[i], pixelData->data.F32[i], pixelVariances->data.F32[i],
                     addVariance->data.F32[i], pixelWeights->data.F32[i], pixelExps->data.F32[i],
@@ -567 +849 @@
     psVector *pixelLimits = buffer->limits; // Is the pixel suspect?
 
+    // KMM values;
+    float Punimodal = 1.0,KMMmean = NAN,KMMsigma = NAN,KMMpi = NAN;
+    int KMM_MINIMUM_INPUTS = 6;
+    bool useKMM = false;
+    if (num >= KMM_MINIMUM_INPUTS) {
+      useKMM = true;
+    }
+    
     // Set up rejection limits
     float rej2 = PS_SQR(rej); // Rejection level squared
@@ -573 +863 @@
         // Using squared rejection limit because it's cheaper than sqrts
         double sumWeights = 0.0;
+
+        // Determine the systematic error from the most popular population in the sample
+        // This should probably be an option
+        if (useKMM) {
+          KMMFindPopular(pixelData,&Punimodal,&KMMmean,&KMMsigma,&KMMpi);
+          CHECKPIX(x,y,"KMM Popularity Contest: (%d,%d) Puni: %g Mean: %f Sigma %f Pi: %f\n",
+                   x,y,Punimodal,KMMmean,KMMsigma,KMMpi);
+        }
         for (int i = 0; i < num; i++) {
             sumWeights += pixelWeights->data.F32[i];
         }
         for (int i = 0; i < num; i++) {
-            // Systematic error contributes to the rejection level
-            float sysVar = PS_SQR(sys * pixelData->data.F32[i]);
+          // Systematic error contributes to the rejection level
+          float sysVar;
+          if (useKMM) { // If we can trust KMM results, set the systematic variance
+            sysVar = PS_SQR(KMMsigma);
+          }
+          else { // Otherwise, use the 10% systematic variance we've done in the past.
+            sysVar = PS_SQR(sys * pixelData->data.F32[i]);
+          }
+
             CHECKPIX(x, y, "Variance %d (%d), pixel %d,%d: %f %f %f\n", 
                      i, pixelSources->data.U16[i], x, y, 
@@ -727 +1032 @@
           default: {
               if (useVariance) {
-                  float median = combinationWeightedOlympic(pixelData, pixelWeights,
-                                                            olympic, buffer->sort); // Median for stack
+                float median;
+                if ((useKMM)&&(Punimodal < 0.05)) {
+                  median = KMMmean;
+                }
+                else {
+                  median = combinationWeightedOlympic(pixelData, pixelWeights,
+                                                      olympic, buffer->sort); // Median for stack
+                }
+                
                   CHECKPIX(x, y, "Flag with variance pixel %d,%d: median = %f\n", x, y, median);
                   float worst = -INFINITY; // Largest deviation
                   for (int j = 0; j < num; j++) {
-                      float diff = pixelData->data.F32[j] - median; // Difference from expected
+                      float diff = pixelData->data.F32[j] - median; // Difference from expected
                       CHECKPIX(x, y, "Testing input %d for pixel %d,%d: %f\n", j, x, y, diff);
 
@@ -738 +1050 @@
                       // pixelVariances includes the rejection limit, from above
                       float diff2 = PS_SQR(diff); // Square difference
+                      CHECKPIX(x,y, "Input %d, pixel %d,%d (%" PRIu16 "): %f %f (%f) %f %f :: %f %f %f %f\n",
+                              i, x, y, pixelSources->data.U16[j], pixelData->data.F32[j], pixelVariances->data.F32[j],
+                              1.0, pixelWeights->data.F32[j], 1.0,
+                              pixelLimits->data.F32[j], diff2, diff2 / pixelLimits->data.F32[j],worst);
+
                       if (diff2 > pixelLimits->data.F32[j]) {
                           float dev = diff2 / pixelLimits->data.F32[j]; // Deviation
@@ -1131 +1448 @@
     }
 
+/*     // Pre-reject inputs using KMM bimodality test. */
+/*     if (1)  { */
+/* /\*       KMMRejectUnpopular(input,x,y); *\/ */
+/*       rejection = true; */
+/*     } */
+   
     // Set up rejection list
     psArray *pixelMap = NULL;           // Map of pixels to source
@@ -1141 +1464 @@
         for (int x = minInputCols; x < maxInputCols; x++) {
             CHECKPIX(x, y, "Combining pixel %d,%d: %x %x %f %f %f %f %d %d %d\n", x, y, badMaskBits, blankMaskBits, iter, rej, sys, olympic, useVariance, safe, rejection);
+
+/*          // Pre-reject inputs using KMM bimodality test. */
+/*        if (1)  { */
+/*          KMMRejectUnpopular(input,x,y); */
+/* /\*      rejection = true; *\/ */
+/*        } */
+/*        else { */
+/*          KMMRejectBright(input,x,y); */
+/*        } */
 
 #ifdef TESTING