Changeset 10550 for trunk/psLib/src/math/psStats.c
- Timestamp:
- Dec 8, 2006, 1:38:54 AM (19 years ago)
- File:
-
- 1 edited
-
trunk/psLib/src/math/psStats.c (modified) (47 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/psLib/src/math/psStats.c
r10475 r10550 7 7 * on those data structures. 8 8 * 9 * @author GLG, MHPCC 10 * 11 * XXX: The following stats members are never used, or set in this code. 12 * stats->clippedNvalues 9 * @author GLG (MHPCC), EAM (IfA) 13 10 * 14 11 * XXX: Must do … … 16 13 * use ->min and ->max (PS_STAT_USE_RANGE) 17 14 * 18 * @version $Revision: 1.19 3$ $Name: not supported by cvs2svn $19 * @date $Date: 2006-12-0 5 20:05:57$15 * @version $Revision: 1.194 $ $Name: not supported by cvs2svn $ 16 * @date $Date: 2006-12-08 11:38:54 $ 20 17 * 21 * Copyright 200 4 Maui High Performance Computing Center, University of Hawaii18 * Copyright 2006 IfA, University of Hawaii 22 19 */ 23 20 … … 35 32 #include "psMemory.h" 36 33 #include "psAbort.h" 37 #include "psImage.h"34 //#include "psImage.h" 38 35 #include "psVector.h" 39 36 #include "psTrace.h" 40 37 #include "psLogMsg.h" 41 38 #include "psError.h" 39 #include "psHistogram.h" 42 40 #include "psStats.h" 43 41 #include "psMinimizeLMM.h" … … 64 62 #define PS_BIN_MIDPOINT(HISTOGRAM, BIN_NUM) \ 65 63 (0.5 * (HISTOGRAM->bounds->data.F32[(BIN_NUM)] + HISTOGRAM->bounds->data.F32[(BIN_NUM)+1])) 64 66 65 /*****************************************************************************/ 67 66 /* TYPE DEFINITIONS */ … … 87 86 NOTE: it is assumed that any call to these statistical functions will have 88 87 been preceded by a call to the psVectorStats() function. Various sanity tests 89 will only be performed in psVectorStats(). Should we perform the sanity 90 checks in each routine anyway? 88 will only be performed in psVectorStats(). 91 89 92 XXX: For many of these private stats routines, what should be done if there 93 are no acceptable elements in the input vector (if no elements lie within 94 range, or there are no unmasked elements, or the input vector is NULL)? 95 Currently we set the value to NAN. 96 97 XXX: Optimization: many routines have an "empty" boolean variable which keeps 98 track of whether or not the vector has any valid elements. This code can 99 possibly be optimized away. 100 *****************************************************************************/ 101 /****************************************************************************** 102 vectorSampleMean(myVector, maskVector, maskVal, stats): calculates the 103 mean of the input vector. If there was a problem with the mean calculation, 104 this routine sets stats->sampleMean to NAN. 105 *****************************************************************************/ 106 # if (0) 107 static bool vectorSampleMean(const psVector* myVector, 108 const psVector* errors, 109 const psVector* maskVector, 110 psMaskType maskVal, 111 psStats* stats) 112 { 113 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__); 114 115 psF32 mean = 0.0; // The mean 116 long count = 0; // Number of points contributing to this mean 117 118 psF32 *data = myVector->data.F32; // Dereference 119 int numData = myVector->n; // Number of data points 120 121 // If PS_STAT_USE_RANGE is requested, then we enter a slightly different loop. 122 if (!errors) { 123 if (stats->options & PS_STAT_USE_RANGE) { 124 if (maskVector) { 125 // No errors, check range, check mask 126 psU8 *maskData = maskVector->data.U8; 127 for (long i = 0; i < numData; i++) { 128 // Check if the data is with the specified range 129 if (!(maskVal & maskData[i]) && stats->min <= data[i] && data[i] <= stats->max) { 130 mean += data[i]; 131 count++; 132 } 133 } 134 } else { 135 // No errors, check range, no mask 136 for (long i = 0; i < numData; i++) { 137 if (stats->min <= data[i] && data[i] <= stats->max) { 138 mean += data[i]; 139 count++; 140 } 141 } 142 } 143 } else { 144 if (maskVector) { 145 // No errors, no range check, check mask 146 psU8 *maskData = maskVector->data.U8; 147 for (long i = 0; i < numData; i++) { 148 if (!(maskVal & maskData[i])) { 149 mean += data[i]; 150 count++; 151 } 152 } 153 154 } else { 155 // No errors, no range check, no mask 156 for (long i = 0; i < numData; i++) { 157 mean += data[i]; 158 } 159 count = numData; 160 } 161 } 162 163 if (count > 0) { 164 mean /= (psF32)count; 165 } else { 166 mean = NAN; 167 } 168 169 } else { 170 psF32 sumWeights = 0.0; // The sum of the weights 171 psF32 *errorsData = errors->data.F32; 172 if (stats->options & PS_STAT_USE_RANGE) { 173 if (maskVector) { 174 psU8 *maskData = maskVector->data.U8; 175 for (long i = 0; i < numData; i++) { 176 // Check if the data is with the specified range 177 if (!(maskVal & maskData[i]) && stats->min <= data[i] && data[i] <= stats->max) { 178 float weight = 1.0 / PS_SQR(errorsData[i]); 179 mean += data[i] * weight; 180 sumWeights += weight; 181 count++; 182 } 183 } 184 } else { 185 for (long i = 0; i < myVector->n; i++) { 186 if (stats->min <= data[i] && data[i] <= stats->max) { 187 float weight = 1.0 / PS_SQR(errorsData[i]); 188 mean += data[i] * weight; 189 sumWeights += weight; 190 count++; 191 } 192 } 193 } 194 } else { 195 if (maskVector) { 196 psU8 *maskData = maskVector->data.U8; 197 for (long i = 0; i < myVector->n; i++) { 198 if (!(maskVal & maskData[i])) { 199 float weight = 1.0 / PS_SQR(errorsData[i]); 200 mean += data[i] * weight; 201 sumWeights += weight; 202 count++; 203 } 204 } 205 } else { 206 for (long i = 0; i < myVector->n; i++) { 207 float weight = 1.0 / PS_SQR(errorsData[i]); 208 mean += data[i] * weight; 209 sumWeights += weight; 210 } 211 count = myVector->n; 212 } 213 } 214 215 if (count > 0) { 216 mean /= sumWeights; 217 } else { 218 mean = NAN; 219 } 220 } 221 222 stats->sampleMean = mean; 223 if (isnan(mean)) { 224 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 225 return false; 226 } 227 228 psTrace("psLib.math", 4, "---- %s(true) end ----\n", __func__); 229 return true; 230 } 231 #endif 90 For many of these private stats routines, it is possible that there are no acceptable elements 91 in the input vector (if no elements lie within range, or there are no unmasked elements, or the 92 input vector is NULL). In such cases, we set the desired value to NAN and return an error. 93 The calling function may clear the error. 94 *****************************************************************************/ 95 96 // static prototypes: 97 static psF32 minimizeLMChi2Gauss1D(psVector *deriv, const psVector *params, const psVector *coords); 98 static psF32 fitQuadraticSearchForYThenReturnX(const psVector *xVec, psVector *yVec, psS32 binNum, psF32 yVal); 232 99 233 100 /****************************************************************************** … … 236 103 this routine sets stats->sampleMean to NAN. 237 104 238 XXX : using the method below, with a single loop for various options 239 costs only a small amount and is much easier to debug. running 10000 tests 240 of 1000 point vectors for the two methods gives: 241 separate single 242 (mask: 0, range: 0): 0.067 sec 0.073 sec 243 (mask: 1, range: 0): 0.098 sec 0.102 sec 244 (mask: 0, range: 0): 0.136 sec 0.162 sec 245 (mask: 1, range: 0): 0.177 sec 0.181 sec 246 *****************************************************************************/ 105 using the method below, with a single loop for various options costs only a small amount and is 106 much easier to debug. running 10000 tests of 1000 point vectors for the two methods gives: 107 108 separate single w/errors 109 (mask: 0, range: 0): 0.067 sec 0.073 sec (10%) 0.072 sec 110 (mask: 1, range: 0): 0.098 sec 0.102 sec ( 4%) 0.119 sec 111 (mask: 0, range: 0): 0.136 sec 0.162 sec (20%) 0.170 sec 112 (mask: 1, range: 0): 0.177 sec 0.181 sec ( 2%) 0.198 sec 113 114 (effect of errors not tested) 115 116 To optmize this, use a macro and ifdef in or out the three states (errors, mask, range) 117 *****************************************************************************/ 247 118 static bool vectorSampleMean(const psVector* myVector, 248 119 const psVector* errors, … … 253 124 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__); 254 125 126 long count = 0; // Number of points contributing to this mean 255 127 psF32 mean = 0.0; // The mean 256 long count = 0; // Number of points contributing to this mean128 psF32 weight; 257 129 258 130 psF32 *data = myVector->data.F32; // Dereference … … 262 134 bool useRange = stats->options & PS_STAT_USE_RANGE; 263 135 264 // If errors exist, we enter a slightly different loop. 265 // XXX collapse to a single loop? 266 if (!errors) { 267 for (long i = 0; i < numData; i++) { 268 // Check if the data is with the specified range 269 if (useRange && (data[i] < stats->min)) 270 continue; 271 if (useRange && (data[i] > stats->max)) 272 continue; 273 if (maskData && (maskData[i] & maskVal)) 274 continue; 275 mean += data[i]; 276 count++; 277 } 278 mean = (count > 0) ? mean / (psF32) count : NAN; 279 } else { 280 psF32 sumWeights = 0.0; // The sum of the weights 281 psF32 *errorsData = errors->data.F32; 282 for (long i = 0; i < numData; i++) { 283 // Check if the data is with the specified range 284 if (useRange && (data[i] < stats->min)) 285 continue; 286 if (useRange && (data[i] > stats->max)) 287 continue; 288 if (maskData && (maskData[i] & maskVal)) 289 continue; 290 291 float weight = 1.0 / PS_SQR(errorsData[i]); 136 psF32 sumWeights = 0.0; // The sum of the weights 137 psF32 *errorsData = (errors == NULL) ? NULL : errors->data.F32; 138 139 for (long i = 0; i < numData; i++) { 140 // Check if the data is with the specified range 141 if (useRange && (data[i] < stats->min)) 142 continue; 143 if (useRange && (data[i] > stats->max)) 144 continue; 145 if (maskData && (maskData[i] & maskVal)) 146 continue; 147 if (errors) { 148 weight = (errorsData[i] == 0) ? 0.0 : PS_SQR(errorsData[i]); 292 149 mean += data[i] * weight; 293 150 sumWeights += weight; 294 count++; 295 } 151 } else { 152 mean += data[i]; 153 } 154 count++; 155 } 156 if (errors) { 296 157 mean = (count > 0) ? mean / sumWeights : NAN; 297 } 298 158 } else { 159 mean = (count > 0) ? mean / count : NAN; 160 } 299 161 stats->sampleMean = mean; 300 162 if (isnan(mean)) { 163 // XXX raise an error here? 301 164 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 302 165 return false; 303 166 } 304 167 168 stats->results |= PS_STAT_SAMPLE_MEAN; 305 169 psTrace("psLib.math", 4, "---- %s(true) end ----\n", __func__); 306 170 return true; … … 319 183 (mask: 0, range: 0): 0.179 sec 0.208 sec 320 184 (mask: 1, range: 0): 0.200 sec 0.244 sec 321 *****************************************************************************/185 *****************************************************************************/ 322 186 static long vectorMinMax(const psVector* myVector, 323 187 const psVector* maskVector, … … 350 214 } 351 215 216 // XXX save numValid in psStats? 352 217 if (numValid == 0) { 353 218 stats->max = NAN; … … 356 221 stats->max = max; 357 222 stats->min = min; 223 stats->results |= PS_STAT_MIN; 224 stats->results |= PS_STAT_MAX; 358 225 } 359 226 psTrace("psLib.math", 4, "---- %s(%d) end ----\n", __func__, numValid); 360 227 return numValid; 361 228 } 362 363 #if 0364 /******************************************************************************365 vectorCheckNonEmpty(myVector, maskVector, maskVal, stats): This routine returns366 "true" if the inputPsVector has 1 or more valid elements (a valid element is an367 unmasked element within the specified min/max range). Otherwise, return368 "false".369 *****************************************************************************/370 static bool vectorCheckNonEmpty(const psVector* myVector,371 const psVector* maskVector,372 psMaskType maskVal,373 psStats* stats)374 {375 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__);376 PS_ASSERT_VECTOR_NON_NULL(myVector, false);377 PS_ASSERT_PTR_NON_NULL(stats, false);378 379 if (stats->options & PS_STAT_USE_RANGE) {380 if (maskVector) {381 for (long i = 0; i < myVector->n; i++) {382 if (!(maskVal & maskVector->data.U8[i]) &&383 (stats->min <= myVector->data.F32[i]) &&384 (myVector->data.F32[i] <= stats->max)) {385 psTrace("psLib.math", 4, "---- %s(true) end ----\n", __func__);386 return true;387 }388 }389 } else {390 for (long i = 0; i < myVector->n; i++) {391 if ((stats->min <= myVector->data.F32[i]) &&392 (myVector->data.F32[i] <= stats->max)) {393 psTrace("psLib.math", 4, "---- %s(true) end ----\n", __func__);394 return true;395 }396 }397 }398 } else {399 if (maskVector != NULL) {400 for (long i = 0; i < myVector->n; i++) {401 if (!(maskVal & maskVector->data.U8[i])) {402 psTrace("psLib.math", 4, "---- %s(true) end ----\n", __func__);403 return true;404 }405 }406 } else {407 if (myVector->n > 0) {408 psTrace("psLib.math", 4, "---- %s(true) end ----\n", __func__);409 return true;410 }411 }412 }413 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__);414 return(false);415 }416 #endif417 229 418 230 /****************************************************************************** 419 231 vectorSampleMedian(myVector, maskVector, maskVal, stats): calculates the median 420 232 and quartiles of the input vector. Returns true on success (including if there 421 were no valid input vector elements). 422 *****************************************************************************/423 static bool vectorSampleMedian(const psVector* myVector,233 were no valid input vector elements). Expects F32 vector for input. 234 *****************************************************************************/ 235 static bool vectorSampleMedian(const psVector* inVector, 424 236 const psVector* maskVector, 425 237 psMaskType maskVal, … … 428 240 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__); 429 241 242 bool useRange = stats->options & PS_STAT_USE_RANGE; 243 psU8 *maskData = (maskVector == NULL) ? NULL : maskVector->data.U8; // Dereference the vector 244 psF32 *input = inVector->data.F32; // Dereference the vector 245 430 246 // Allocate temporary vectors for the data. 431 psVector* vector = psVectorAllocEmpty(myVector->n, PS_TYPE_F32); // Vector containing the unmasked values 247 psVector *outVector = psVectorAllocEmpty(inVector->n, PS_TYPE_F32); // Vector containing the unmasked values 248 psF32 *output = outVector->data.F32; // Dereference the vector 249 432 250 long count = 0; // Number of valid entries 433 251 434 // Determine if we must only use data points within a min/max range. 435 if (stats->options & PS_STAT_USE_RANGE) { 436 // Store all non-masked data points within the min/max range 437 // into the temporary vectors. 438 if (maskVector != NULL) { 439 for (long i = 0; i < myVector->n; i++) { 440 if (!(maskVal & maskVector->data.U8[i]) && 441 (stats->min <= myVector->data.F32[i]) && 442 (myVector->data.F32[i] <= stats->max)) { 443 vector->data.F32[count] = myVector->data.F32[i]; 444 count++; 445 } 446 } 447 } else { 448 for (long i = 0; i < myVector->n; i++) { 449 if ((stats->min <= myVector->data.F32[i]) && 450 (myVector->data.F32[i] <= stats->max)) { 451 vector->data.F32[count] = myVector->data.F32[i]; 452 count++; 453 } 454 } 455 } 456 } else { 457 // Store all non-masked data points into the temporary vectors. 458 if (maskVector) { 459 for (long i = 0; i < myVector->n; i++) { 460 if (!(maskVal & maskVector->data.U8[i])) { 461 vector->data.F32[count] = myVector->data.F32[i]; 462 count++; 463 } 464 } 465 } else { 466 vector = psVectorCopy(vector, myVector, PS_TYPE_F32); 467 count = myVector->n; 468 } 469 } 470 vector->n = count; 252 // Store all non-masked data points within the min/max range 253 // into the temporary vectors. 254 for (long i = 0; i < inVector->n; i++) { 255 if (useRange && (input[i] < stats->min)) 256 continue; 257 if (useRange && (input[i] > stats->max)) 258 continue; 259 if (maskData && (maskData[i] & maskVal)) 260 continue; 261 262 output[count] = input[i]; 263 count++; 264 } 265 outVector->n = count; 266 471 267 if (count == 0) { 472 268 psError(PS_ERR_BAD_PARAMETER_SIZE, true, "No valid data in input vector.\n"); 473 psFree(vector); 269 psFree(outVector); 270 stats->sampleUQ = NAN; 271 stats->sampleLQ = NAN; 474 272 stats->sampleMedian = NAN; 475 273 return false; … … 477 275 478 276 // Sort the temporary vector. 479 vector = psVectorSort(vector, vector); // Sort in-place (since it's a copy, it's OK) 480 if (!vector) { 481 psError(PS_ERR_UNEXPECTED_NULL, 482 false, 483 _("Failed to sort input data.")); 484 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 485 psFree(vector); 277 // XXX this is messed up: should psVectorSort free on error or not? 278 if (!psVectorSort(outVector, outVector)) { // Sort in-place (since it's a copy, it's OK) 279 psError(PS_ERR_UNEXPECTED_NULL, false, _("Failed to sort input data.")); 280 stats->sampleUQ = NAN; 281 stats->sampleLQ = NAN; 282 stats->sampleMedian = NAN; 486 283 return false; 487 284 } 488 285 489 // Calculate the median exactly. Use the average if the number of samples 490 // is even. 286 // Calculate the median. Use the average if the number of samples if even. 491 287 if (count % 2 == 0) { 492 stats->sampleMedian = 0.5 * (vector->data.F32[(count / 2) - 1] + 493 vector->data.F32[count / 2]); 288 stats->sampleMedian = 0.5 * (output[(count/2) - 1] + output[count/2]); 494 289 } else { 495 stats->sampleMedian = vector->data.F32[count /2];290 stats->sampleMedian = output[count/2]; 496 291 } 497 292 498 293 // Calculate the quartile points exactly. 499 stats->sampleUQ = vector->data.F32[3 * (count / 4)]; 500 stats->sampleLQ = vector->data.F32[count / 4]; 294 stats->sampleUQ = output[(int)(0.75*count)]; 295 stats->sampleLQ = output[(int)(0.25*count)]; 296 297 stats->results |= PS_STAT_SAMPLE_MEDIAN; 298 stats->results |= PS_STAT_SAMPLE_QUARTILE; 501 299 502 300 // Free the temporary data structures. 503 psFree( vector);301 psFree(outVector); 504 302 505 303 // Return "true" on success. 506 304 psTrace("psLib.math", 4, "---- %s(true) end ----\n", __func__); 507 305 return true; 508 }509 510 /******************************************************************************511 vectorSmoothHistGaussian(): This routine smoothes the data in the input512 robustHistogram with a Gaussian of width sigma. It returns a psVector of the513 smoothed data.514 *****************************************************************************/515 psVector *vectorSmoothHistGaussian(psHistogram *histogram,516 psF32 sigma)517 {518 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__);519 psTrace("psLib.math", 5, "(histogram->nums->n, sigma) is (%d, %.2f\n", (int) histogram->nums->n, sigma);520 PS_ASSERT_PTR_NON_NULL(histogram, NULL);521 PS_ASSERT_PTR_NON_NULL(histogram->bounds, NULL);522 PS_ASSERT_PTR_NON_NULL(histogram->nums, NULL);523 if (psTraceGetLevel("psLib.math") >= 8) {524 PS_VECTOR_PRINT_F32(histogram->nums);525 }526 527 long numBins = histogram->nums->n; // Number of histogram bins528 psVector *smooth = psVectorAlloc(numBins, PS_TYPE_F32); // Smoothed version of histogram bins529 const psVector *bounds = histogram->bounds; // The bounds for the histogram bins530 531 if (!histogram->uniform) {532 //533 // We get here if the histogram is non-uniform. This code is not tested.534 // However, it is also not used anywhere, yet.535 //536 psLogMsg(__func__, PS_LOG_WARN, "WARNING: vectorSmoothHistGaussian() on non-uniform histograms has not been tested or used.\n");537 538 for (long i = 0; i < numBins; i++) {539 // Determine the midpoint of bin i.540 float iMid = PS_BIN_MIDPOINT(histogram, i);541 542 //543 // We determine the bin numbers (jMin:jMax) corresponding to a544 // range of data values surrounding iMid. The range is of size:545 // 2*PS_GAUSS_WIDTH*sigma546 long jMin, jMax;547 psF32 x = iMid - (PS_GAUSS_WIDTH * sigma);548 for (jMin = i; jMin > 0 && bounds->data.F32[jMin] > x; jMin--)549 ;550 x = iMid + (PS_GAUSS_WIDTH * sigma);551 for (jMax = i; jMax < bounds->n - 1 && bounds->data.F32[jMax + 1] > x; jMax++)552 ;553 554 //555 // Loop from jMin to jMax, computing the gaussian of data i.556 //557 smooth->data.F32[i] = 0.0;558 for (long j = jMin ; j <= jMax ; j++) {559 float jMid = PS_BIN_MIDPOINT(histogram, j);560 smooth->data.F32[i] +=561 histogram->nums->data.F32[j] * psGaussian(jMid, iMid, sigma, true);562 }563 }564 } else {565 //566 // We get here if the histogram is uniform.567 //568 for (long i = 0; i < numBins; i++) {569 psF32 binSize = bounds->data.F32[1] - bounds->data.F32[0];570 psS32 gaussWidth = ((PS_GAUSS_WIDTH * sigma) / binSize);571 572 //573 // XXX: The following is wrong. However, in practice, the sigma was too574 // large compared to the binSize. This meant that the smoothing was done575 // over 500 bins in the robust stats algorithm. This mean that the smoothing576 // took way too long.577 //578 #if 0579 580 gaussWidth = 10;581 #endif582 //583 // We determine the bin numbers (jMin:jMax) corresponding to a584 // range of data values surrounding iMid. The range is of size:585 // 2*PS_GAUSS_WIDTH*sigma586 //587 psS32 jMin = PS_MAX(i - gaussWidth, 0);588 psS32 jMax = PS_MIN(i + gaussWidth, bounds->n - 1);589 590 //591 // Loop from jMin to jMax, computing the gaussian of data i.592 //593 smooth->data.F32[i] = 0.0;594 float iMid = PS_BIN_MIDPOINT(histogram, i);595 for (long j = jMin ; j <= jMax ; j++) {596 float jMid = PS_BIN_MIDPOINT(histogram, j);597 smooth->data.F32[i] +=598 histogram->nums->data.F32[j] * psGaussian(jMid, iMid, sigma, true);599 }600 }601 }602 603 if (psTraceGetLevel("psLib.math") >= 8) {604 PS_VECTOR_PRINT_F32(smooth);605 }606 psTrace("psLib.math", 4, "---- %s() end ----\n", __func__);607 return(smooth);608 306 } 609 307 … … 619 317 NULL 620 318 621 *****************************************************************************/ 319 using the method below, with a single loop for various options costs only a small amount and is 320 much easier to debug. running 10000 tests of 1000 point vectors for the two methods gives: 321 322 single 323 (mask: 0, range: 0): 0.193 sec 324 (mask: 1, range: 0): 0.257 sec 325 (mask: 0, range: 0): 0.349 sec 326 (mask: 1, range: 0): 0.401 sec 327 328 *****************************************************************************/ 622 329 static bool vectorSampleStdev(const psVector* myVector, 623 330 const psVector* errors, … … 630 337 // This procedure requires the mean. If it has not been already 631 338 // calculated, then call vectorSampleMean() 632 if ( isnan(stats->sampleMean)) {339 if (!(stats->results & PS_STAT_SAMPLE_MEAN)) { 633 340 vectorSampleMean(myVector, errors, maskVector, maskVal, stats); 634 341 } 342 635 343 // If the mean is NAN, then generate a warning and set the stdev to NAN. 636 344 if (isnan(stats->sampleMean)) { … … 641 349 } 642 350 351 psF32 *data = myVector->data.F32; // Dereference 352 psU8 *maskData = (maskVector == NULL) ? NULL : maskVector->data.U8; 353 bool useRange = stats->options & PS_STAT_USE_RANGE; 354 psF32 *errorsData = (errors == NULL) ? NULL : errors->data.F32; 355 643 356 // Accumulate the sums 644 357 double mean = stats->sampleMean; // The mean … … 647 360 double errorDivisor = 0.0; // Division for errors 648 361 long count = 0; // Number of data points being used 649 if (stats->options & PS_STAT_USE_RANGE) { 650 if (maskVector) { 651 for (long i = 0; i < myVector->n; i++) { 652 if (!(maskVal & maskVector->data.U8[i]) && 653 (stats->min <= myVector->data.F32[i]) && 654 (myVector->data.F32[i] <= stats->max)) { 655 double diff = myVector->data.F32[i] - mean; 656 sumSquares += PS_SQR(diff); 657 sumDiffs += diff; 658 count++; 659 if (errors) { 660 errorDivisor += 1.0 / PS_SQR(errors->data.F32[i]); 661 } 662 } 663 } 664 } else { 665 for (long i = 0; i < myVector->n; i++) { 666 if ((stats->min <= myVector->data.F32[i]) && (myVector->data.F32[i] <= stats->max)) { 667 double diff = myVector->data.F32[i] - mean; 668 sumSquares += PS_SQR(diff); 669 sumDiffs += diff; 670 count++; 671 if (errors) { 672 errorDivisor += 1.0 / PS_SQR(errors->data.F32[i]); 673 } 674 } 675 } 676 count = myVector->n; 677 } 678 } else { 679 if (maskVector) { 680 for (long i = 0; i < myVector->n; i++) { 681 if (!(maskVal & maskVector->data.U8[i])) { 682 double diff = myVector->data.F32[i] - mean; 683 sumSquares += PS_SQR(diff); 684 sumDiffs += diff; 685 count++; 686 if (errors) { 687 errorDivisor += 1.0 / PS_SQR(errors->data.F32[i]); 688 } 689 } 690 } 691 } else { 692 for (long i = 0; i < myVector->n; i++) { 693 double diff = myVector->data.F32[i] - mean; 694 sumSquares += PS_SQR(diff); 695 sumDiffs += diff; 696 count++; 697 if (errors) { 698 errorDivisor += 1.0 / PS_SQR(errors->data.F32[i]); 699 } 700 } 701 count = myVector->n; 362 for (long i = 0; i < myVector->n; i++) { 363 // Check if the data is with the specified range 364 if (useRange && (data[i] < stats->min)) 365 continue; 366 if (useRange && (data[i] > stats->max)) 367 continue; 368 if (maskData && (maskData[i] & maskVal)) 369 continue; 370 371 double diff = data[i] - mean; 372 sumSquares += PS_SQR(diff); 373 sumDiffs += diff; 374 count++; 375 if (errors) { 376 errorDivisor += 1.0 / PS_SQR(errorsData[i]); 702 377 } 703 378 } … … 717 392 stats->sampleStdev = (1.0 / sqrtf(errorDivisor)); 718 393 } else { 719 stats->sampleStdev = sqrt((sumSquares - (sumDiffs * sumDiffs / (float)count)) / 720 (float)(count - 1)); 721 } 394 stats->sampleStdev = sqrt((sumSquares - (sumDiffs * sumDiffs / (float)count)) / (float)(count - 1)); 395 } 396 stats->results |= PS_STAT_SAMPLE_STDEV; 397 722 398 psTrace("psLib.math", 4, "---- %s() end ----\n", __func__); 723 399 … … 736 412 Returns 737 413 true for success; false otherwise 738 *****************************************************************************/414 *****************************************************************************/ 739 415 static bool vectorClippedStats(const psVector* myVector, 740 416 const psVector* errors, … … 757 433 PS_CLIPPED_SIGMA_UB, -1); 758 434 759 // Allocate a psStats structure for calculating the mean, median, and 760 // stdev. 761 psStats *statsTmp = psStatsAlloc(PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); 435 // unless we succeed, these will have NAN values 436 stats->clippedMean = NAN; 437 stats->clippedStdev = NAN; 438 stats->clippedNvalues = 0; 762 439 763 440 // We copy the mask vector, to preserve the original … … 773 450 } 774 451 775 // 1. Compute the sample median .776 vectorSampleMedian(myVector, tmpMask, maskVal, stats Tmp);777 if (isnan(stats Tmp->sampleMedian)) {452 // 1. Compute the sample median, which we save for output 453 vectorSampleMedian(myVector, tmpMask, maskVal, stats); 454 if (isnan(stats->sampleMedian)) { 778 455 psLogMsg(__func__, PS_LOG_WARN, "Call to vectorSampleMedian returned NAN\n"); 779 stats->clippedMean = NAN;780 stats->clippedStdev = NAN;781 456 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 782 457 psFree(tmpMask); 783 psFree(statsTmp);784 458 return false; 785 459 } 786 psTrace("psLib.math", 6, "The initial sample median is %f\n", stats Tmp->sampleMedian);787 788 // 2. Compute the sample standard deviation .789 vectorSampleStdev(myVector, errors, tmpMask, maskVal, stats Tmp);790 if (isnan(stats Tmp->sampleStdev)) {460 psTrace("psLib.math", 6, "The initial sample median is %f\n", stats->sampleMedian); 461 462 // 2. Compute the sample standard deviation, which we save for output 463 vectorSampleStdev(myVector, errors, tmpMask, maskVal, stats); 464 if (isnan(stats->sampleStdev)) { 791 465 psLogMsg(__func__, PS_LOG_WARN, "Call to vectorSampleStdev returned NAN\n"); 792 stats->clippedMean = NAN;793 stats->clippedStdev = NAN;794 466 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 795 467 psFree(tmpMask); 796 psFree(statsTmp);797 468 return false; 798 469 } 799 psTrace("psLib.math", 6, "The initial sample stdev is %f\n", stats Tmp->sampleStdev);470 psTrace("psLib.math", 6, "The initial sample stdev is %f\n", stats->sampleStdev); 800 471 801 472 // 3. Use the sample median as the first estimator of the mean X. 802 psF32 clippedMean = stats Tmp->sampleMedian;473 psF32 clippedMean = stats->sampleMedian; 803 474 804 475 // 4. Use the sample stdev as the first estimator of the mean stdev. 805 psF32 clippedStdev = stats Tmp->sampleStdev;476 psF32 clippedStdev = stats->sampleStdev; 806 477 807 478 // 5. Repeat N (stats->clipIter) times: … … 836 507 837 508 // b) compute new mean and stdev 509 // Allocate a psStats structure for calculating the mean and stdev. 510 psStats *statsTmp = psStatsAlloc(PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); 838 511 vectorSampleMean(myVector, errors, tmpMask, maskVal, statsTmp); 839 512 vectorSampleStdev(myVector, errors, tmpMask, maskVal, statsTmp); … … 853 526 clippedStdev = statsTmp->sampleStdev; 854 527 } 528 psFree(statsTmp); 855 529 } 856 530 … … 859 533 860 534 // 7. The last calcuated value of x is the clipped mean. 861 if (stats->options & PS_STAT_CLIPPED_MEAN) {862 stats->clippedMean = clippedMean;863 psTrace("psLib.math", 6, "The final clipped mean is %f\n", clippedMean);864 }865 535 // 8. The last calcuated value of stdev is the clipped stdev. 866 if (stats->options & PS_STAT_CLIPPED_STDEV) { 867 stats->clippedStdev = clippedStdev; 868 psTrace("psLib.math", 6, "The final clipped stdev is %f\n", clippedStdev); 869 } 536 // we always return both stats even if only one was requested 537 stats->clippedMean = clippedMean; 538 stats->clippedStdev = clippedStdev; 539 540 stats->results |= PS_STAT_CLIPPED_MEAN; 541 stats->results |= PS_STAT_CLIPPED_STDEV; 542 543 psTrace("psLib.math", 6, "The final clipped mean is %f\n", clippedMean); 544 psTrace("psLib.math", 6, "The final clipped stdev is %f\n", clippedStdev); 870 545 871 546 psFree(tmpMask); 872 psFree(statsTmp);873 547 psTrace("psLib.math", 4, "---- %s(true) end ----\n", __func__); 874 548 return true; 875 549 } 876 877 static psF32 QuadraticInverse(psF32 a,878 psF32 b,879 psF32 c,880 psF32 y,881 psF32 xLo,882 psF32 xHi883 )884 {885 psF64 tmp = sqrt((y - c)/a + (b*b)/(4.0 * a * a));886 887 psF64 x1 = -b/(2.0*a) + tmp;888 psF64 x2 = -b/(2.0*a) - tmp;889 890 #if 0891 892 psF64 y1 = (a * x1 * x1) + (b * x1) + c;893 psF64 y2 = (a * x2 * x2) + (b * x2) + c;894 printf("QuadraticInverse: %fx^2 + %fx + %f\n", a, b, c);895 printf("QuadraticInverse: y is %f\n", y);896 printf("QuadraticInverse: (x1, x2) is (%f %f)\n", x1, x2);897 printf("QuadraticInverse: (y1, y2) is (%f %f)\n", y1, y2);898 #endif899 900 if (xLo <= x1 && x1 <= xHi) {901 return x1;902 }903 if (xLo <= x2 && x2 <= xHi) {904 return x2;905 }906 return 0.5 * (xLo + xHi);907 }908 909 910 /******************************************************************************911 fitQuadraticSearchForYThenReturnX(*xVec, *yVec, binNum, yVal): A general912 routine which fits a quadratic to three points and returns the x-value913 corresponding to the input y-value. This routine takes psVectors of x/y pairs914 as input, and fits a quadratic to the 3 points surrounding element binNum in915 the vectors (the midpoint between element i and i+1 is used for x[i]). It916 then determines for what value x does that quadratic f(x) = yVal (the input917 parameter).918 919 *****************************************************************************/920 static psF32 fitQuadraticSearchForYThenReturnX(const psVector *xVec,921 psVector *yVec,922 psS32 binNum,923 psF32 yVal924 )925 {926 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__);927 psTrace("psLib.math", 5, "binNum, yVal is (%d, %f)\n", binNum, yVal);928 if (psTraceGetLevel("psLib.math") >= 8) {929 PS_VECTOR_PRINT_F32(xVec);930 PS_VECTOR_PRINT_F32(yVec);931 }932 933 PS_ASSERT_VECTOR_NON_NULL(xVec, NAN);934 PS_ASSERT_VECTOR_NON_NULL(yVec, NAN);935 PS_ASSERT_VECTOR_TYPE(xVec, PS_TYPE_F32, NAN);936 PS_ASSERT_VECTOR_TYPE(yVec, PS_TYPE_F32, NAN);937 // PS_ASSERT_VECTORS_SIZE_EQUAL(xVec, yVec, NAN);938 PS_ASSERT_INT_WITHIN_RANGE(binNum, 0, (int)(xVec->n - 1), NAN);939 PS_ASSERT_INT_WITHIN_RANGE(binNum, 0, (int)(yVec->n - 1), NAN);940 941 psVector *x = psVectorAlloc(3, PS_TYPE_F64);942 psVector *y = psVectorAlloc(3, PS_TYPE_F64);943 psF32 tmpFloat = 0.0f;944 945 if ((binNum >= 1) && (binNum < (yVec->n - 2)) && (binNum < (xVec->n - 2))) {946 // The general case. We have all three points.947 x->data.F64[0] = (psF64) (0.5 * (xVec->data.F32[binNum - 1] + xVec->data.F32[binNum]));948 x->data.F64[1] = (psF64) (0.5 * (xVec->data.F32[binNum] + xVec->data.F32[binNum+1]));949 x->data.F64[2] = (psF64) (0.5 * (xVec->data.F32[binNum+1] + xVec->data.F32[binNum+2]));950 y->data.F64[0] = yVec->data.F32[binNum - 1];951 y->data.F64[1] = yVec->data.F32[binNum];952 y->data.F64[2] = yVec->data.F32[binNum + 1];953 psTrace("psLib.math", 6, "x vec (orig) is (%f %f %f %f)\n", xVec->data.F32[binNum - 1],954 xVec->data.F32[binNum], xVec->data.F32[binNum+1], xVec->data.F32[binNum+2]);955 psTrace("psLib.math", 6, "x data is (%f %f %f)\n", x->data.F64[0], x->data.F64[1], x->data.F64[2]);956 psTrace("psLib.math", 6, "y data is (%f %f %f)\n", y->data.F64[0], y->data.F64[1], y->data.F64[2]);957 958 //959 // Ensure that the y value lies within range of the y values.960 //961 if (! (((y->data.F64[0] <= yVal) && (yVal <= y->data.F64[2])) ||962 ((y->data.F64[2] <= yVal) && (yVal <= y->data.F64[0]))) ) {963 psError(PS_ERR_BAD_PARAMETER_VALUE, true,964 _("Specified yVal, %g, is not within y-range, %g to %g."),965 (psF64)yVal, y->data.F64[0], y->data.F64[2]);966 }967 968 //969 // Ensure that the y values are monotonic.970 //971 if (((y->data.F64[0] < y->data.F64[1]) && !(y->data.F64[1] <= y->data.F64[2])) ||972 ((y->data.F64[0] > y->data.F64[1]) && !(y->data.F64[1] >= y->data.F64[2]))) {973 psError(PS_ERR_UNKNOWN, true,974 "This routine must be called with monotonically increasing or decreasing data points.\n");975 psFree(x);976 psFree(y);977 psTrace("psLib.math", 5, "---- %s() end ----\n", __func__);978 return NAN;979 }980 981 // Determine the coefficients of the polynomial.982 psPolynomial1D *myPoly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2);983 myPoly = psVectorFitPolynomial1D(myPoly, NULL, 0, y, NULL, x);984 if (myPoly == NULL) {985 psError(PS_ERR_UNEXPECTED_NULL, false,986 _("Failed to fit a 1-dimensional polynomial to the three specified data points. Returning NAN."));987 psFree(x);988 psFree(y);989 psTrace("psLib.math", 5, "---- %s(NAN) end ----\n", __func__);990 return NAN;991 }992 psTrace("psLib.math", 6, "myPoly->coeff[0] is %f\n", myPoly->coeff[0]);993 psTrace("psLib.math", 6, "myPoly->coeff[1] is %f\n", myPoly->coeff[1]);994 psTrace("psLib.math", 6, "myPoly->coeff[2] is %f\n", myPoly->coeff[2]);995 psTrace("psLib.math", 6, "Fitted y vec is (%f %f %f)\n",996 (psF32) psPolynomial1DEval(myPoly, (psF64) x->data.F64[0]),997 (psF32) psPolynomial1DEval(myPoly, (psF64) x->data.F64[1]),998 (psF32) psPolynomial1DEval(myPoly, (psF64) x->data.F64[2]));999 1000 psTrace("psLib.math", 6, "We fit the polynomial, now find x such that f(x) equals %f\n", yVal);1001 tmpFloat = QuadraticInverse(myPoly->coeff[2], myPoly->coeff[1], myPoly->coeff[0], yVal,1002 x->data.F64[0], x->data.F64[2]);1003 psFree(myPoly);1004 1005 if (isnan(tmpFloat)) {1006 psError(PS_ERR_UNEXPECTED_NULL,1007 false, _("Failed to determine the median of the fitted polynomial. Returning NAN."));1008 psFree(x);1009 psFree(y);1010 psTrace("psLib.math", 5, "---- %s(NAN) end ----\n", __func__);1011 return(NAN);1012 }1013 } else {1014 // These are special cases where the bin is at the beginning or end of the vector.1015 if (binNum == 0) {1016 // We have two points only at the beginning of the vectors x and y.1017 tmpFloat = 0.5 * (xVec->data.F32[binNum] +1018 xVec->data.F32[binNum + 1]);1019 } else if (binNum == (xVec->n - 1)) {1020 // The special case where we have two points only at the end of1021 // the vectors x and y.1022 // XXX: Is this right?1023 tmpFloat = xVec->data.F32[binNum];1024 } else if (binNum == (xVec->n - 2)) {1025 // XXX: Is this right?1026 tmpFloat = 0.5 * (xVec->data.F32[binNum] + xVec->data.F32[binNum + 1]);1027 }1028 }1029 1030 psTrace("psLib.math", 6, "FIT: return %f\n", tmpFloat);1031 psFree(x);1032 psFree(y);1033 1034 psTrace("psLib.math", 5, "---- %s(%f) end ----\n", __func__, tmpFloat);1035 return tmpFloat;1036 }1037 1038 /******************************************************************************1039 NOTE: We assume unnormalized gaussians.1040 *****************************************************************************/1041 static psF32 minimizeLMChi2Gauss1D(psVector *deriv,1042 const psVector *params,1043 const psVector *coords1044 )1045 {1046 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__);1047 PS_ASSERT_VECTOR_NON_NULL(params, NAN);1048 PS_ASSERT_VECTOR_SIZE(params, (long)2, NAN);1049 PS_ASSERT_VECTOR_TYPE(params, PS_TYPE_F32, NAN);1050 PS_ASSERT_VECTOR_NON_NULL(coords, NAN);1051 PS_ASSERT_VECTOR_SIZE(coords, (long)1, NAN);1052 PS_ASSERT_VECTOR_TYPE(coords, PS_TYPE_F32, NAN);1053 1054 psF32 x = coords->data.F32[0];1055 psF32 mean = params->data.F32[0];1056 psF32 var = params->data.F32[1];1057 psF32 dx = (x - mean);1058 1059 psF32 gauss = exp (-0.5*PS_SQR(dx)/var);1060 if (deriv) {1061 PS_ASSERT_VECTOR_SIZE(deriv, (long)2, NAN);1062 PS_ASSERT_VECTOR_TYPE(deriv, PS_TYPE_F32, NAN);1063 psF32 tmp = dx * gauss;1064 deriv->data.F32[0] = tmp / var;1065 deriv->data.F32[1] = tmp * dx / (var * var);1066 }1067 1068 1069 psTrace("psLib.math", 4, "---- %s() end ----\n", __func__);1070 return gauss;1071 }1072 1073 /*1074 * vectorFittedStats requires guess for fittedMean and fittedStdev1075 * robustN50 should also be set1076 */1077 static bool vectorFittedStats (const psVector* myVector,1078 const psVector* errors,1079 psVector* mask,1080 psMaskType maskVal,1081 psStats* stats)1082 {1083 1084 psTrace("psLib.math", 6, "The guess mean is %f.\n", stats->fittedMean);1085 psTrace("psLib.math", 6, "The guess stdev is %f.\n", stats->fittedStdev);1086 1087 psStats *statsMinMax = psStatsAlloc(PS_STAT_MIN | PS_STAT_MAX); // Statistics for min and max1088 1089 psScalar tmpScalar; // Temporary scalar variable, for p_psVectorBinDisect1090 tmpScalar.type.type = PS_TYPE_F32;1091 1092 psF32 binSize = 1;1093 if (stats->options & PS_STAT_USE_BINSIZE) {1094 // Set initial bin size to the specified value.1095 binSize = stats->binsize;1096 psTrace("psLib.math", 6, "Setting initial robust bin size to %.2f\n", binSize);1097 } else {1098 // construct a histogram with (sigma/10 < binsize < sigma)1099 // set roughly so that the lowest bins have about 2 cnts1100 // Nsmallest ~ N50 / (4*dN))1101 psF32 dN = PS_MAX (1, PS_MIN (10, stats->robustN50 / 8));1102 binSize = stats->fittedStdev / dN;1103 }1104 1105 // Determine the min/max of the vector (which prior outliers masked out)1106 int numValid = vectorMinMax(myVector, mask, maskVal, statsMinMax); // Number of values1107 float min = statsMinMax->min;1108 float max = statsMinMax->max;1109 if (numValid == 0 || isnan(min) || isnan(max)) {1110 psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n");1111 psFree(statsMinMax);1112 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__);1113 return false;1114 }1115 1116 // Calculate the number of bins.1117 long numBins = (max - min) / binSize;1118 psTrace("psLib.math", 6, "The new min/max values are (%f, %f).\n", min, max);1119 psTrace("psLib.math", 6, "The new bin size is %f.\n", binSize);1120 psTrace("psLib.math", 6, "The numBins is %ld\n", numBins);1121 1122 psHistogram *histogram = psHistogramAlloc(min, max, numBins); // A new histogram (without outliers)1123 histogram = psVectorHistogram(histogram, myVector, errors, mask, maskVal);1124 if (psTraceGetLevel("psLib.math") >= 8) {1125 PS_VECTOR_PRINT_F32(histogram->nums);1126 }1127 1128 long binMin = 0; // Low bin to check1129 long binMax = 0; // High bin to check1130 1131 // Fit a Gaussian to the bins in the requested range about the guess mean1132 // min,max overrides clipSigma1133 psF32 maxFitSigma = 2.0;1134 if (isfinite(stats->clipSigma)) {1135 maxFitSigma = fabs(stats->clipSigma);1136 }1137 if (isfinite(stats->max)) {1138 maxFitSigma = fabs(stats->max);1139 }1140 1141 psF32 minFitSigma = 2.0;1142 if (isfinite(stats->clipSigma)) {1143 minFitSigma = fabs(stats->clipSigma);1144 }1145 if (isfinite(stats->min)) {1146 minFitSigma = fabs(stats->min);1147 }1148 1149 tmpScalar.data.F32 = stats->fittedMean - minFitSigma*stats->fittedStdev;1150 if (tmpScalar.data.F32 < min) {1151 binMin = 0;1152 } else {1153 binMin = p_psVectorBinDisect(histogram->bounds, &tmpScalar);1154 }1155 tmpScalar.data.F32 = stats->fittedMean + maxFitSigma*stats->fittedStdev;1156 if (tmpScalar.data.F32 > max) {1157 binMax = histogram->bounds->n - 1;1158 } else {1159 binMax = p_psVectorBinDisect(histogram->bounds, &tmpScalar);1160 }1161 if (binMin < 0 || binMax < 0) {1162 psError(PS_ERR_UNKNOWN, false, "Failed to calculate the -%f sigma : +%f sigma bins\n", minFitSigma, maxFitSigma);1163 psFree(statsMinMax);1164 psFree(histogram);1165 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__);1166 return false;1167 }1168 1169 // Generate the variables that will be used in the Gaussian fitting1170 // XXX EAM : we should test / guarantee that there are at least 3 (right?) bins1171 psVector *y = psVectorAlloc((1 + (binMax - binMin)), PS_TYPE_F32); // Vector of coordinates1172 psArray *x = psArrayAlloc((1 + (binMax - binMin))); // Array of ordinates1173 for (long i = binMin, j = 0; i <= binMax ; i++, j++) {1174 y->data.F32[j] = histogram->nums->data.F32[i];1175 psVector *ordinate = psVectorAlloc(1, PS_TYPE_F32); // The ordinate value1176 ordinate->data.F32[0] = PS_BIN_MIDPOINT(histogram, i);1177 x->data[j] = ordinate;1178 }1179 if (psTraceGetLevel("psLib.math") >= 8) {1180 // XXX: Print the x array somehow.1181 PS_VECTOR_PRINT_F32(y);1182 }1183 psTrace("psLib.math", 6, "The clipped numBins is %ld\n", y->n);1184 1185 // Normalize y to [0.0:1.0] (since the psMinimizeLMChi2Gauss1D() functions is [0.0:1.0])1186 // XXX EAM : why not just fit the normalization as well??1187 p_psNormalizeVectorRange(y, 0.0, 1.0);1188 1189 // Fit a Gaussian to the data.1190 psMinimization *minimizer = psMinimizationAlloc(100, 0.01); // The minimizer information1191 psVector *params = psVectorAlloc(2, PS_TYPE_F32); // Parameters for the Gaussian1192 // Initial guess for the mean (index 0) and var (index 1).1193 params->data.F32[0] = stats->fittedMean;1194 params->data.F32[1] = PS_SQR(stats->fittedStdev);1195 if (psTraceGetLevel("psLib.math") >= 8) {1196 PS_VECTOR_PRINT_F32(params);1197 PS_VECTOR_PRINT_F32(y);1198 }1199 if (!psMinimizeLMChi2(minimizer, NULL, params, NULL, x, y, NULL, minimizeLMChi2Gauss1D)) {1200 psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n");1201 psFree(statsMinMax);1202 psFree(histogram);1203 psFree(x);1204 psFree(y);1205 psFree(minimizer);1206 psFree(params);1207 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__);1208 return false;1209 }1210 if (psTraceGetLevel("psLib.math") >= 8) {1211 PS_VECTOR_PRINT_F32(params);1212 }1213 1214 // The fitted mean is the Gaussian mean.1215 stats->fittedMean = params->data.F32[0];1216 psTrace("psLib.math", 6, "The fitted mean is %f.\n", stats->fittedMean);1217 1218 // The fitted standard deviation1219 stats->fittedStdev = sqrt(params->data.F32[1]);1220 psTrace("psLib.math", 6, "The fitted stdev is %f.\n", stats->fittedStdev);1221 1222 // Clean up after fitting1223 psFree (histogram);1224 psFree (x);1225 psFree (y);1226 psFree (minimizer);1227 psFree (params);1228 psFree (statsMinMax);1229 return true;1230 }1231 1232 550 1233 551 /****************************************************************************** … … 1278 596 } 1279 597 598 // statsMinMax is only applied to a subset of the data points 1280 599 psStats *statsMinMax = psStatsAlloc(PS_STAT_MIN | PS_STAT_MAX); // Statistics for min and max 1281 600 psHistogram *histogram = NULL; // Histogram of the data … … 1287 606 // Iterate to get the best bin size 1288 607 for (int iterate = 1; iterate > 0; iterate++) { 1289 psTrace("psLib.math", 6, 1290 "-------------------- Iterating on Bin size. Iteration number %d --------------------\n", 1291 iterate); 608 psTrace("psLib.math", 6, "-------------------- Iterating on Bin size. Iteration number %d --------------------\n", iterate); 1292 609 1293 610 // Get the minimum and maximum values … … 1308 625 if (fabs(max - min) <= FLT_EPSILON) { 1309 626 psTrace("psLib.math", 5, "All data points have the same value: %f.\n", min); 1310 if (stats->options & PS_STAT_ROBUST_MEDIAN) { 1311 stats->robustMedian = min; 1312 } 1313 if (stats->options & PS_STAT_ROBUST_QUARTILE) { 1314 stats->robustUQ = min; 1315 stats->robustLQ = min; 1316 } 627 stats->robustMedian = min; 628 stats->robustUQ = min; 629 stats->robustLQ = min; 1317 630 stats->robustN50 = numValid; 1318 631 psFree(statsMinMax); … … 1433 746 return false; 1434 747 } 1435 // XXXX we can probably just use the bin values to get sigma without interpolating... 1436 // ADD step 4b: Interpolate Sigma to find these two positions exactly: these are the 1sigma positions. 1437 #if 0 1438 // XXX: I am overriding the ADD for now. The following code implements the ADD exactly and is having 1439 // problems fitting a 2nd-order polynomial to data y-values suchs as (1, 1, 100). Therefore, I 1440 // commented out the code and am doing simply linear interpolation. 1441 1442 // Value for the 15.8655% mark 1443 float binLoF32 = fitQuadraticSearchForYThenReturnX(cumulative->bounds, cumulative->nums, binLo, 1444 totalDataPoints * 0.158655f); 1445 // Value for the 84.1345% mark 1446 float binHiF32 = fitQuadraticSearchForYThenReturnX(cumulative->bounds, cumulative->nums, binHi, 1447 totalDataPoints * 0.841345f); 1448 psTrace("psLib.math", 6, "The exact 15.8655%% and 84.1345%% data point positions are: (%f, %f)\n", 1449 binLoF32, binHiF32); 1450 #else 1451 // This code basically interpolates to find the positions exactly. 748 749 // ADD step 4b: Interpolate Sigma (linearly) to find these two positions exactly: these are the 1sigma positions. 1452 750 psTrace("psLib.math", 6, "binLo is %ld. Nums at that bin and the next are (%.2f, %.2f)\n", 1453 751 binLo, cumulative->nums->data.F32[binLo], cumulative->nums->data.F32[binLo+1]); … … 1455 753 binHi, cumulative->nums->data.F32[binHi], cumulative->nums->data.F32[binHi+1]); 1456 754 1457 float deltaBounds = cumulative->bounds->data.F32[binLo+1] - cumulative->bounds->data.F32[binLo]; 1458 float deltaNums; 1459 float prevPixels; 755 float deltaBounds, deltaNums, prevPixels; 756 float percentNums, base, binLoF32; 757 758 // find the -1 sigma points with linear interpolation 759 deltaBounds = cumulative->bounds->data.F32[binLo+1] - cumulative->bounds->data.F32[binLo]; 1460 760 if (binLo == 0) { 1461 761 deltaNums = cumulative->nums->data.F32[0]; … … 1465 765 prevPixels = cumulative->nums->data.F32[binLo-1]; 1466 766 } 1467 floatpercentNums = (totalDataPoints * 0.158655f) - prevPixels;1468 floatbase = cumulative->bounds->data.F32[binLo];1469 floatbinLoF32 = base + (deltaBounds / deltaNums) * percentNums; // Value for the 15.8655% mark767 percentNums = (totalDataPoints * 0.158655f) - prevPixels; 768 base = cumulative->bounds->data.F32[binLo]; 769 binLoF32 = base + (deltaBounds / deltaNums) * percentNums; // Value for the 15.8655% mark 1470 770 psTrace("psLib.math", 6, 1471 771 "(base, deltaBounds, deltaNums, prevPixels, percentNums) is (%.2f %.2f %.2f %.2f %.2f)\n", 1472 772 base, deltaBounds, deltaNums, prevPixels, percentNums); 1473 773 774 // find the +1 sigma points with linear interpolation 1474 775 deltaBounds = cumulative->bounds->data.F32[binHi+1] - cumulative->bounds->data.F32[binHi]; 1475 776 if (binHi == 0) { … … 1486 787 "(base, deltaBounds, deltaNums, prevPixels, percentNums) is (%.2f %.2f %.2f %.2f %.2f)\n", 1487 788 base, deltaBounds, deltaNums, prevPixels, percentNums); 1488 psTrace("psLib.math", 6, 789 790 // report +/- 1 sigma points 791 psTrace("psLib.math", 5, 1489 792 "The exact 15.8655 and 84.1345 percent data point positions are: (%f, %f)\n", 1490 793 binLoF32, binHiF32); 1491 #endif1492 794 1493 795 // ADD step 5: Determine SIGMA as 1/2 of the distance between these positions. … … 1585 887 psTrace("psLib.math", 6, "The robustN50 is %ld.\n", N50); 1586 888 1587 // Fitted statistics1588 if (stats->options & PS_STAT_FITTED_MEAN || stats->options & PS_STAT_FITTED_STDEV) {1589 stats->fittedStdev = stats->robustStdev; // pass the guess sigma1590 stats->fittedMean = stats->robustMedian; // pass the guess mean1591 if (!vectorFittedStats(myVector, errors, mask, maskVal, stats)) {1592 psError(PS_ERR_UNKNOWN, false, "Unable to estimate statistics (pass 1)");1593 psFree(statsMinMax);1594 psFree(mask);1595 1596 return false;1597 }1598 // if there is a large swing in sigma, try a second time1599 if ((stats->fittedStdev/stats->robustStdev) < 0.75) {1600 if (!vectorFittedStats (myVector, errors, mask, maskVal, stats)) {1601 psError(PS_ERR_UNKNOWN, false, "Unable to estimate statistics (pass 2)");1602 psFree(statsMinMax);1603 psFree(mask);1604 1605 return false;1606 }1607 }1608 }1609 1610 889 // Clean up 1611 890 psFree(statsMinMax); 1612 891 psFree(mask); 1613 892 893 stats->results |= PS_STAT_ROBUST_MEDIAN; 894 stats->results |= PS_STAT_ROBUST_STDEV; 895 1614 896 psTrace("psLib.math", 4, "---- %s(0) end ----\n", __func__); 1615 897 return true; 1616 898 } 1617 899 900 /* 901 * vectorFittedStats requires guess for fittedMean and fittedStdev 902 * robustN50 should also be set 903 */ 904 static bool vectorFittedStats (const psVector* myVector, 905 const psVector* errors, 906 psVector* mask, 907 psMaskType maskVal, 908 psStats* stats) 909 { 910 911 // This procedure requires the mean. If it has not been already 912 // calculated, then call vectorSampleMean() 913 if (!(stats->results & PS_STAT_ROBUST_MEDIAN)) { 914 vectorRobustStats(myVector, errors, mask, maskVal, stats); 915 } 916 917 // If the mean is NAN, then generate a warning and set the stdev to NAN. 918 if (isnan(stats->robustMedian)) { 919 stats->fittedStdev = NAN; 920 stats->fittedStdev = NAN; 921 psTrace("psLib.math", 4, "---- %s() end ----\n", __func__); 922 return false; 923 } 924 925 float guessStdev = stats->robustStdev; // pass the guess sigma 926 float guessMean = stats->robustMedian; // pass the guess mean 927 928 psTrace("psLib.math", 6, "The guess mean is %f.\n", guessMean); 929 psTrace("psLib.math", 6, "The guess stdev is %f.\n", guessStdev); 930 931 bool done = false; 932 for (int iteration = 0; !done && (iteration < 2); iteration ++) { 933 psStats *statsMinMax = psStatsAlloc(PS_STAT_MIN | PS_STAT_MAX); // Statistics for min and max 934 935 psScalar tmpScalar; // Temporary scalar variable, for p_psVectorBinDisect 936 tmpScalar.type.type = PS_TYPE_F32; 937 938 psF32 binSize = 1; 939 if (stats->options & PS_STAT_USE_BINSIZE) { 940 // Set initial bin size to the specified value. 941 binSize = stats->binsize; 942 psTrace("psLib.math", 6, "Setting initial robust bin size to %.2f\n", binSize); 943 } else { 944 // construct a histogram with (sigma/10 < binsize < sigma) 945 // set roughly so that the lowest bins have about 2 cnts 946 // Nsmallest ~ N50 / (4*dN)) 947 psF32 dN = PS_MAX (1, PS_MIN (10, stats->robustN50 / 8)); 948 binSize = guessStdev / dN; 949 } 950 951 // Determine the min/max of the vector (which prior outliers masked out) 952 int numValid = vectorMinMax(myVector, mask, maskVal, statsMinMax); // Number of values 953 float min = statsMinMax->min; 954 float max = statsMinMax->max; 955 if (numValid == 0 || isnan(min) || isnan(max)) { 956 psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n"); 957 psFree(statsMinMax); 958 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 959 return false; 960 } 961 962 // Calculate the number of bins. 963 // XXX can we calculate the binMin, binMax **before** building this histogram? 964 long numBins = (max - min) / binSize; 965 psTrace("psLib.math", 6, "The new min/max values are (%f, %f).\n", min, max); 966 psTrace("psLib.math", 6, "The new bin size is %f.\n", binSize); 967 psTrace("psLib.math", 6, "The numBins is %ld\n", numBins); 968 969 psHistogram *histogram = psHistogramAlloc(min, max, numBins); // A new histogram (without outliers) 970 histogram = psVectorHistogram(histogram, myVector, errors, mask, maskVal); 971 if (psTraceGetLevel("psLib.math") >= 8) { 972 PS_VECTOR_PRINT_F32(histogram->nums); 973 } 974 975 long binMin = 0; // Low bin to check 976 long binMax = 0; // High bin to check 977 978 // Fit a Gaussian to the bins in the requested range about the guess mean 979 // min,max overrides clipSigma 980 psF32 maxFitSigma = 2.0; 981 if (isfinite(stats->clipSigma)) { 982 maxFitSigma = fabs(stats->clipSigma); 983 } 984 if (isfinite(stats->max)) { 985 maxFitSigma = fabs(stats->max); 986 } 987 988 psF32 minFitSigma = 2.0; 989 if (isfinite(stats->clipSigma)) { 990 minFitSigma = fabs(stats->clipSigma); 991 } 992 if (isfinite(stats->min)) { 993 minFitSigma = fabs(stats->min); 994 } 995 996 tmpScalar.data.F32 = guessMean - minFitSigma*guessStdev; 997 if (tmpScalar.data.F32 < min) { 998 binMin = 0; 999 } else { 1000 binMin = p_psVectorBinDisect(histogram->bounds, &tmpScalar); 1001 } 1002 tmpScalar.data.F32 = guessMean + maxFitSigma*guessStdev; 1003 if (tmpScalar.data.F32 > max) { 1004 binMax = histogram->bounds->n - 1; 1005 } else { 1006 binMax = p_psVectorBinDisect(histogram->bounds, &tmpScalar); 1007 } 1008 if (binMin < 0 || binMax < 0) { 1009 psError(PS_ERR_UNKNOWN, false, "Failed to calculate the -%f sigma : +%f sigma bins\n", minFitSigma, maxFitSigma); 1010 psFree(statsMinMax); 1011 psFree(histogram); 1012 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 1013 return false; 1014 } 1015 1016 // Generate the variables that will be used in the Gaussian fitting 1017 // XXX EAM : we should test / guarantee that there are at least 3 (right?) bins 1018 psVector *y = psVectorAlloc((1 + (binMax - binMin)), PS_TYPE_F32); // Vector of coordinates 1019 psArray *x = psArrayAlloc((1 + (binMax - binMin))); // Array of ordinates 1020 for (long i = binMin, j = 0; i <= binMax ; i++, j++) { 1021 y->data.F32[j] = histogram->nums->data.F32[i]; 1022 psVector *ordinate = psVectorAlloc(1, PS_TYPE_F32); // The ordinate value 1023 ordinate->data.F32[0] = PS_BIN_MIDPOINT(histogram, i); 1024 x->data[j] = ordinate; 1025 } 1026 if (psTraceGetLevel("psLib.math") >= 8) { 1027 // XXX: Print the x array somehow. 1028 PS_VECTOR_PRINT_F32(y); 1029 } 1030 psTrace("psLib.math", 6, "The clipped numBins is %ld\n", y->n); 1031 psTrace("psLib.math", 6, "The clipped min is %f (%ld)\n", PS_BIN_MIDPOINT(histogram, binMin), binMin); 1032 psTrace("psLib.math", 6, "The clipped max is %f (%ld)\n", PS_BIN_MIDPOINT(histogram, binMax), binMax); 1033 1034 // Normalize y to [0.0:1.0] (since the psMinimizeLMChi2Gauss1D() functions is [0.0:1.0]) 1035 // XXX EAM : why not just fit the normalization as well?? 1036 p_psNormalizeVectorRange(y, 0.0, 1.0); 1037 1038 // Fit a Gaussian to the data. 1039 psMinimization *minimizer = psMinimizationAlloc(100, 0.01); // The minimizer information 1040 psVector *params = psVectorAlloc(2, PS_TYPE_F32); // Parameters for the Gaussian 1041 // Initial guess for the mean (index 0) and var (index 1). 1042 params->data.F32[0] = guessMean; 1043 params->data.F32[1] = PS_SQR(guessStdev); 1044 if (psTraceGetLevel("psLib.math") >= 8) { 1045 PS_VECTOR_PRINT_F32(params); 1046 PS_VECTOR_PRINT_F32(y); 1047 } 1048 if (!psMinimizeLMChi2(minimizer, NULL, params, NULL, x, y, NULL, minimizeLMChi2Gauss1D)) { 1049 psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n"); 1050 psFree(params); 1051 psFree(minimizer); 1052 psFree(x); 1053 psFree(y); 1054 psFree(histogram); 1055 psFree(statsMinMax); 1056 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 1057 return false; 1058 } 1059 if (psTraceGetLevel("psLib.math") >= 8) { 1060 PS_VECTOR_PRINT_F32(params); 1061 } 1062 1063 guessMean = params->data.F32[0]; 1064 guessStdev = sqrt(params->data.F32[1]); 1065 if (guessStdev > 0.75*stats->robustStdev) { 1066 done = true; 1067 } 1068 1069 // Clean up after fitting 1070 psFree (params); 1071 psFree (minimizer); 1072 psFree (x); 1073 psFree (y); 1074 psFree (histogram); 1075 psFree (statsMinMax); 1076 } 1077 1078 // The fitted mean is the Gaussian mean. 1079 stats->fittedMean = guessMean; 1080 psTrace("psLib.math", 6, "The fitted mean is %f.\n", stats->fittedMean); 1081 1082 // The fitted standard deviation 1083 stats->fittedStdev = guessStdev; 1084 psTrace("psLib.math", 6, "The fitted stdev is %f.\n", stats->fittedStdev); 1085 1086 stats->results |= PS_STAT_FITTED_MEAN; 1087 stats->results |= PS_STAT_FITTED_STDEV; 1088 1089 return true; 1090 } 1091 1092 1093 /******************** 1094 * vectorFittedStats_v2 requires guess for fittedMean and fittedStdev 1095 * robustN50 should also be set 1096 * gaussian fit is performed using 2D polynomial to ln(y) 1097 ********************/ 1098 static bool vectorFittedStats_v2 (const psVector* myVector, 1099 const psVector* errors, 1100 psVector* mask, 1101 psMaskType maskVal, 1102 psStats* stats) 1103 { 1104 1105 // This procedure requires the mean. If it has not been already 1106 // calculated, then call vectorSampleMean() 1107 if (!(stats->results & PS_STAT_ROBUST_MEDIAN)) { 1108 vectorRobustStats(myVector, errors, mask, maskVal, stats); 1109 } 1110 1111 // If the mean is NAN, then generate a warning and set the stdev to NAN. 1112 if (isnan(stats->robustMedian)) { 1113 stats->fittedStdev = NAN; 1114 stats->fittedStdev = NAN; 1115 psTrace("psLib.math", 4, "---- %s() end ----\n", __func__); 1116 return false; 1117 } 1118 1119 float guessStdev = stats->robustStdev; // pass the guess sigma 1120 float guessMean = stats->robustMedian; // pass the guess mean 1121 1122 psTrace("psLib.math", 6, "The guess mean is %f.\n", guessMean); 1123 psTrace("psLib.math", 6, "The guess stdev is %f.\n", guessStdev); 1124 1125 bool done = false; 1126 for (int iteration = 0; !done && (iteration < 2); iteration ++) { 1127 psStats *statsMinMax = psStatsAlloc(PS_STAT_MIN | PS_STAT_MAX); // Statistics for min and max 1128 1129 psScalar tmpScalar; // Temporary scalar variable, for p_psVectorBinDisect 1130 tmpScalar.type.type = PS_TYPE_F32; 1131 1132 psF32 binSize = 1; 1133 if (stats->options & PS_STAT_USE_BINSIZE) { 1134 // Set initial bin size to the specified value. 1135 binSize = stats->binsize; 1136 psTrace("psLib.math", 6, "Setting initial robust bin size to %.2f\n", binSize); 1137 } else { 1138 // construct a histogram with (sigma/10 < binsize < sigma) 1139 // set roughly so that the lowest bins have about 2 cnts 1140 // Nsmallest ~ N50 / (4*dN)) 1141 psF32 dN = PS_MAX (1, PS_MIN (10, stats->robustN50 / 8)); 1142 binSize = guessStdev / dN; 1143 } 1144 1145 // Determine the min/max of the vector (which prior outliers masked out) 1146 int numValid = vectorMinMax(myVector, mask, maskVal, statsMinMax); // Number of values 1147 float min = statsMinMax->min; 1148 float max = statsMinMax->max; 1149 if (numValid == 0 || isnan(min) || isnan(max)) { 1150 psError(PS_ERR_UNKNOWN, false, "Failed to calculate the min/max of the input vector.\n"); 1151 psFree(statsMinMax); 1152 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 1153 return false; 1154 } 1155 1156 // Calculate the number of bins. 1157 // XXX can we calculate the binMin, binMax **before** building this histogram? 1158 long numBins = (max - min) / binSize; 1159 psTrace("psLib.math", 6, "The new min/max values are (%f, %f).\n", min, max); 1160 psTrace("psLib.math", 6, "The new bin size is %f.\n", binSize); 1161 psTrace("psLib.math", 6, "The numBins is %ld\n", numBins); 1162 1163 psHistogram *histogram = psHistogramAlloc(min, max, numBins); // A new histogram (without outliers) 1164 histogram = psVectorHistogram(histogram, myVector, errors, mask, maskVal); 1165 if (psTraceGetLevel("psLib.math") >= 8) { 1166 PS_VECTOR_PRINT_F32(histogram->nums); 1167 } 1168 1169 long binMin = 0; // Low bin to check 1170 long binMax = 0; // High bin to check 1171 1172 // Fit a Gaussian to the bins in the requested range about the guess mean 1173 // min,max overrides clipSigma 1174 psF32 maxFitSigma = 2.0; 1175 if (isfinite(stats->clipSigma)) { 1176 maxFitSigma = fabs(stats->clipSigma); 1177 } 1178 if (isfinite(stats->max)) { 1179 maxFitSigma = fabs(stats->max); 1180 } 1181 1182 psF32 minFitSigma = 2.0; 1183 if (isfinite(stats->clipSigma)) { 1184 minFitSigma = fabs(stats->clipSigma); 1185 } 1186 if (isfinite(stats->min)) { 1187 minFitSigma = fabs(stats->min); 1188 } 1189 1190 tmpScalar.data.F32 = guessMean - minFitSigma*guessStdev; 1191 if (tmpScalar.data.F32 < min) { 1192 binMin = 0; 1193 } else { 1194 binMin = p_psVectorBinDisect(histogram->bounds, &tmpScalar); 1195 } 1196 tmpScalar.data.F32 = guessMean + maxFitSigma*guessStdev; 1197 if (tmpScalar.data.F32 > max) { 1198 binMax = histogram->bounds->n - 1; 1199 } else { 1200 binMax = p_psVectorBinDisect(histogram->bounds, &tmpScalar); 1201 } 1202 if (binMin < 0 || binMax < 0) { 1203 psError(PS_ERR_UNKNOWN, false, "Failed to calculate the -%f sigma : +%f sigma bins\n", minFitSigma, maxFitSigma); 1204 psFree(statsMinMax); 1205 psFree(histogram); 1206 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 1207 return false; 1208 } 1209 1210 // Generate the variables that will be used in the Gaussian fitting 1211 // XXX EAM : we should test / guarantee that there are at least 3 (right?) bins 1212 psVector *y = psVectorAllocEmpty((1 + (binMax - binMin)), PS_TYPE_F32); // Vector of coordinates 1213 psVector *x = psVectorAllocEmpty((1 + (binMax - binMin)), PS_TYPE_F32); // Vector of ordinates 1214 long j = 0; 1215 for (long i = binMin; i <= binMax ; i++) { 1216 if (histogram->nums->data.F32[i] <= 0.0) 1217 continue; 1218 x->data.F32[j] = PS_BIN_MIDPOINT(histogram, i); 1219 y->data.F32[j] = log(histogram->nums->data.F32[i]); 1220 // XXX note this is the natural log: expected distribution is A exp(-(x-xo)^2/2sigma^2) 1221 j++; 1222 } 1223 y->n = x->n = j; 1224 if (psTraceGetLevel("psLib.math") >= 8) { 1225 // XXX: Print the x array somehow. 1226 PS_VECTOR_PRINT_F32(y); 1227 } 1228 psTrace("psLib.math", 6, "The clipped numBins is %ld\n", y->n); 1229 psTrace("psLib.math", 6, "The clipped min is %f (%ld)\n", PS_BIN_MIDPOINT(histogram, binMin), binMin); 1230 psTrace("psLib.math", 6, "The clipped max is %f (%ld)\n", PS_BIN_MIDPOINT(histogram, binMax), binMax); 1231 1232 // fit 2nd order polynomial to ln(y) = -(x-xo)^2/2sigma^2 1233 psPolynomial1D *poly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); 1234 1235 // XXX how can we protect against some extreme outliers? the robust histogram 1236 // should have already dealt with those, but we are again sensitive to them... 1237 // psStats *fitStats = psStatsAlloc (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV); 1238 // fitStats->clipIter = 3.0; 1239 // fitStats->clipSigma = 3.0; 1240 // psVector *fitMask = psVectorAlloc(y->n, PS_TYPE_U8); 1241 // psVectorInit (fitMask, 0); 1242 1243 if (!psVectorFitPolynomial1D (poly, NULL, 0, y, NULL, x)) { 1244 psError(PS_ERR_UNKNOWN, false, "Failed to fit a gaussian to the robust histogram.\n"); 1245 psFree(x); 1246 psFree(y); 1247 psFree(poly); 1248 // psFree(fitStats); 1249 // psFree(fitMask); 1250 psFree(histogram); 1251 psFree(statsMinMax); 1252 psTrace("psLib.math", 4, "---- %s(false) end ----\n", __func__); 1253 return false; 1254 } 1255 1256 guessStdev = sqrt(-0.5/poly->coeff[2]); 1257 guessMean = poly->coeff[1]*PS_SQR(guessStdev); 1258 if (guessStdev > 0.75*stats->robustStdev) { 1259 done = true; 1260 } else { 1261 psTrace("psLib.math", 6, "The new guess mean is %f.\n", guessMean); 1262 psTrace("psLib.math", 6, "The new guess stdev is %f.\n", guessStdev); 1263 } 1264 1265 // Clean up after fitting 1266 psFree (x); 1267 psFree (y); 1268 psFree (poly); 1269 // psFree (fitStats); 1270 // psFree (fitMask); 1271 psFree (histogram); 1272 psFree (statsMinMax); 1273 } 1274 1275 // The fitted mean is the Gaussian mean. 1276 stats->fittedMean = guessMean; 1277 psTrace("psLib.math", 6, "The fitted mean is %f.\n", stats->fittedMean); 1278 1279 // The fitted standard deviation 1280 stats->fittedStdev = guessStdev; 1281 psTrace("psLib.math", 6, "The fitted stdev is %f.\n", stats->fittedStdev); 1282 1283 stats->results |= PS_STAT_FITTED_MEAN_V2; 1284 stats->results |= PS_STAT_FITTED_STDEV_V2; 1285 1286 return true; 1287 } 1288 1289 1290 /****************************************************************************** 1291 vectorSmoothHistGaussian(): This routine smoothes the data in the input 1292 robustHistogram with a Gaussian of width sigma. It returns a psVector of the 1293 smoothed data. 1294 1295 XXX this function is unused 1296 *****************************************************************************/ 1297 psVector *vectorSmoothHistGaussian(psHistogram *histogram, 1298 psF32 sigma) 1299 { 1300 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__); 1301 psTrace("psLib.math", 5, "(histogram->nums->n, sigma) is (%d, %.2f\n", (int) histogram->nums->n, sigma); 1302 PS_ASSERT_PTR_NON_NULL(histogram, NULL); 1303 PS_ASSERT_PTR_NON_NULL(histogram->bounds, NULL); 1304 PS_ASSERT_PTR_NON_NULL(histogram->nums, NULL); 1305 if (psTraceGetLevel("psLib.math") >= 8) { 1306 PS_VECTOR_PRINT_F32(histogram->nums); 1307 } 1308 1309 long numBins = histogram->nums->n; // Number of histogram bins 1310 psVector *smooth = psVectorAlloc(numBins, PS_TYPE_F32); // Smoothed version of histogram bins 1311 const psVector *bounds = histogram->bounds; // The bounds for the histogram bins 1312 1313 if (!histogram->uniform) { 1314 // 1315 // We get here if the histogram is non-uniform. This code is not tested. 1316 // However, it is also not used anywhere, yet. 1317 // 1318 psLogMsg(__func__, PS_LOG_WARN, "WARNING: vectorSmoothHistGaussian() on non-uniform histograms has not been tested or used.\n"); 1319 1320 for (long i = 0; i < numBins; i++) { 1321 // Determine the midpoint of bin i. 1322 float iMid = PS_BIN_MIDPOINT(histogram, i); 1323 1324 // 1325 // We determine the bin numbers (jMin:jMax) corresponding to a 1326 // range of data values surrounding iMid. The range is of size: 1327 // 2*PS_GAUSS_WIDTH*sigma 1328 long jMin, jMax; 1329 psF32 x = iMid - (PS_GAUSS_WIDTH * sigma); 1330 for (jMin = i; jMin > 0 && bounds->data.F32[jMin] > x; jMin--) 1331 ; 1332 x = iMid + (PS_GAUSS_WIDTH * sigma); 1333 for (jMax = i; jMax < bounds->n - 1 && bounds->data.F32[jMax + 1] > x; jMax++) 1334 ; 1335 1336 // 1337 // Loop from jMin to jMax, computing the gaussian of data i. 1338 // 1339 smooth->data.F32[i] = 0.0; 1340 for (long j = jMin ; j <= jMax ; j++) { 1341 float jMid = PS_BIN_MIDPOINT(histogram, j); 1342 smooth->data.F32[i] += 1343 histogram->nums->data.F32[j] * psGaussian(jMid, iMid, sigma, true); 1344 } 1345 } 1346 } else { 1347 // 1348 // We get here if the histogram is uniform. 1349 // 1350 for (long i = 0; i < numBins; i++) { 1351 psF32 binSize = bounds->data.F32[1] - bounds->data.F32[0]; 1352 psS32 gaussWidth = ((PS_GAUSS_WIDTH * sigma) / binSize); 1353 1354 // 1355 // XXX: The following is wrong. However, in practice, the sigma was too 1356 // large compared to the binSize. This meant that the smoothing was done 1357 // over 500 bins in the robust stats algorithm. This mean that the smoothing 1358 // took way too long. 1359 // 1360 #if 0 1361 1362 gaussWidth = 10; 1363 #endif 1364 // 1365 // We determine the bin numbers (jMin:jMax) corresponding to a 1366 // range of data values surrounding iMid. The range is of size: 1367 // 2*PS_GAUSS_WIDTH*sigma 1368 // 1369 psS32 jMin = PS_MAX(i - gaussWidth, 0); 1370 psS32 jMax = PS_MIN(i + gaussWidth, bounds->n - 1); 1371 1372 // 1373 // Loop from jMin to jMax, computing the gaussian of data i. 1374 // 1375 smooth->data.F32[i] = 0.0; 1376 float iMid = PS_BIN_MIDPOINT(histogram, i); 1377 for (long j = jMin ; j <= jMax ; j++) { 1378 float jMid = PS_BIN_MIDPOINT(histogram, j); 1379 smooth->data.F32[i] += 1380 histogram->nums->data.F32[j] * psGaussian(jMid, iMid, sigma, true); 1381 } 1382 } 1383 } 1384 1385 if (psTraceGetLevel("psLib.math") >= 8) { 1386 PS_VECTOR_PRINT_F32(smooth); 1387 } 1388 psTrace("psLib.math", 4, "---- %s() end ----\n", __func__); 1389 return(smooth); 1390 } 1391 1618 1392 /*****************************************************************************/ 1619 1393 … … 1621 1395 1622 1396 /*****************************************************************************/ 1623 1624 static void histogramFree(psHistogram* myHist);1625 1397 1626 1398 // We keep statsFree so that we can identify statistics pointers from the memblock … … 1632 1404 /****************************************************************************** 1633 1405 psStatsAlloc(): This routine must create a new psStats data structure. 1634 *****************************************************************************/1406 *****************************************************************************/ 1635 1407 psStats* psStatsAlloc(psStatsOptions options) 1636 1408 { … … 1675 1447 } 1676 1448 1677 1678 1679 /******************************************************************************1680 psHistogramAlloc(lower, upper, n): allocate a uniform histogram structure1681 with the specifed upper and lower limits, and the specifed number of bins.1682 This routine will also set the bounds for each of the bins.1683 1684 Input:1685 lower1686 upper1687 n1688 Returns:1689 The histogram structure1690 *****************************************************************************/1691 psHistogram* psHistogramAlloc(float lower, float upper, int n)1692 {1693 psTrace("psLib.math", 3, "---- %s() begin ----\n", __func__);1694 psTrace("psLib.math", 5, "(lower, upper, n) is (%f, %f, %d)\n", lower, upper, n);1695 PS_ASSERT_INT_POSITIVE(n, NULL);1696 PS_ASSERT_FLOAT_LARGER_THAN_OR_EQUAL(upper, lower, NULL);1697 1698 // Allocate memory for the new histogram structure. If there are N bins, then there are N+1 bounds to1699 // those bins.1700 psHistogram *newHist = (psHistogram* ) psAlloc(sizeof(psHistogram)); // The new histogram structure1701 psMemSetDeallocator(newHist, (psFreeFunc) histogramFree);1702 psVector* newBounds = psVectorAlloc(n + 1, PS_TYPE_F32);1703 newHist->bounds = newBounds;1704 1705 // Calculate the bounds for each bin.1706 psF32 binSize = (upper - lower) / (psF32)n; // The histogram bin size1707 // XXX: Is the following necessary? It prevents the max data point from being in a non-existant bin.1708 binSize += FLT_EPSILON;1709 for (long i = 0; i < n + 1; i++) {1710 newBounds->data.F32[i] = lower + (binSize * (psF32)i);1711 }1712 1713 // Allocate the bins, and initialize them to zero.1714 newHist->nums = psVectorAlloc(n, PS_TYPE_F32);1715 psVectorInit(newHist->nums, 0.0);1716 1717 // Initialize the other members.1718 newHist->minNum = 0;1719 newHist->maxNum = 0;1720 newHist->uniform = true;1721 1722 psTrace("psLib.math", 3, "---- %s() end ----\n", __func__);1723 return newHist;1724 }1725 1726 /******************************************************************************1727 psHistogramAllocGeneric(bounds): allocate a non-uniform histogram structure1728 with the specifed bounds.1729 1730 Input:1731 bounds1732 Returns:1733 The histogram structure1734 *****************************************************************************/1735 psHistogram* psHistogramAllocGeneric(const psVector* bounds)1736 {1737 psTrace("psLib.math", 3, "---- %s() begin ----\n", __func__);1738 PS_ASSERT_VECTOR_NON_NULL(bounds, NULL);1739 PS_ASSERT_VECTOR_TYPE(bounds, PS_TYPE_F32, NULL);1740 PS_ASSERT_LONG_LARGER_THAN_OR_EQUAL(bounds->n, (long)2, NULL);1741 1742 // Allocate memory for the new histogram structure.1743 psHistogram *newHist = (psHistogram* ) psAlloc(sizeof(psHistogram)); // The new histogram structure1744 psMemSetDeallocator(newHist, (psFreeFunc) histogramFree);1745 psVector* newBounds = psVectorCopy(NULL, bounds, PS_TYPE_F32);1746 newHist->bounds = newBounds;1747 1748 // Allocate the bins, and initialize them to zero. If there are N bounds,1749 // then there are N-1 bins.1750 newHist->nums = psVectorAlloc((bounds->n) - 1, PS_TYPE_F32);1751 psVectorInit(newHist->nums, 0.0);1752 1753 // Initialize the other members.1754 newHist->minNum = 0;1755 newHist->maxNum = 0;1756 newHist->uniform = false;1757 1758 psTrace("psLib.math", 3, "---- %s() end ----\n", __func__);1759 return (newHist);1760 }1761 1762 static void histogramFree(psHistogram* myHist)1763 {1764 psFree(myHist->bounds);1765 psFree(myHist->nums);1766 }1767 1768 1769 bool psMemCheckHistogram(psPtr ptr)1770 {1771 PS_ASSERT_PTR(ptr, false);1772 return ( psMemGetDeallocator(ptr) == (psFreeFunc)histogramFree );1773 }1774 1775 1776 1777 /*****************************************************************************1778 UpdateHistogramBins(binNum, out, data, error): This routine is to be used when1779 updating the histogram in the presence of errors in the input data. We treat1780 the data point as a boxcar PDF and update a range of points surrounding the1781 histogram bin which contains the point. The width of that boxcar is defined1782 as 2.35 * error.1783 1784 XXX: Must test this.1785 *****************************************************************************/1786 static bool UpdateHistogramBins(long binNum, // Bin number of the data point1787 psHistogram* out, // The histogram to be updated1788 psF32 data, // The data point value1789 psF32 error // The error in the data point1790 )1791 {1792 psTrace("psLib.math", 3, "---- %s() begin ----\n", __func__);1793 PS_ASSERT_PTR_NON_NULL(out, false);1794 PS_ASSERT_PTR_NON_NULL(out->bounds, false);1795 PS_ASSERT_PTR_NON_NULL(out->nums, false);1796 PS_ASSERT_LONG_WITHIN_RANGE(binNum, (long)0, (long)((out->nums->n)-1), false);1797 PS_ASSERT_FLOAT_LARGER_THAN_OR_EQUAL(error, 0.0, false);1798 PS_ASSERT_FLOAT_WITHIN_RANGE(data, out->bounds->data.F32[0],1799 out->bounds->data.F32[(out->bounds->n)-1], false);1800 1801 psF32 boxcarWidth = 2.35 * error; // Width of the boxcar1802 psF32 boxcarCenter = (out->bounds->data.F32[binNum] +1803 out->bounds->data.F32[binNum+1]) / 2.0; // Centre of the boxcar1804 psF32 boxcarLeft = boxcarCenter - (boxcarWidth / 2.0); // Left endpoint of the boxcar for the PDF1805 psF32 boxcarRight = boxcarCenter + (boxcarWidth / 2.0); // Right endpoint of the boxcar for the PDF1806 psS32 boxcarLeftBinNum = 0; // Bin number for left endpoint1807 psS32 boxcarRightBinNum = 0; // Bin number for right endpoint1808 1809 // Determine the left endpoint of the boxcar for the PDF.1810 for (long bin = binNum; bin >= 0; bin--) {1811 if (out->nums->data.F32[bin] <= boxcarLeft) {1812 boxcarLeftBinNum = bin;1813 break;1814 }1815 }1816 1817 // Determine the right endpoint of the boxcar for the PDF.1818 for (long bin = binNum; bin < out->nums->n; bin++) {1819 if (out->nums->data.F32[bin] >= boxcarRight) {1820 boxcarRightBinNum = bin;1821 break;1822 }1823 }1824 1825 // If the boxcar fits entirely inside this bin, then simply add 1.0 to the1826 // bin and return.1827 if (boxcarLeftBinNum == boxcarRightBinNum) {1828 out->nums->data.F32[binNum] += 1.0;1829 psTrace("psLib.math", 3, "---- %s(true) end ----\n", __func__);1830 return true;1831 }1832 1833 // If we get here, multiple bins must be updated. We handle the left-most endpoint, and right-most1834 // endpoints differently.1835 out->nums->data.F32[boxcarLeftBinNum] +=1836 (out->bounds->data.F32[boxcarLeftBinNum+1] - boxcarLeft) / boxcarWidth;1837 1838 // Loop through the center bins, if any.1839 for (long bin = boxcarLeftBinNum + 1; bin < (boxcarRightBinNum - 1); bin++) {1840 out->nums->data.F32[bin] +=1841 (out->bounds->data.F32[bin+1] - out->bounds->data.F32[bin]) / boxcarWidth;1842 }1843 1844 // Handle the right endpoint differently.1845 out->nums->data.F32[boxcarRightBinNum]+=1846 (boxcarRight - out->bounds->data.F32[boxcarRightBinNum]) / boxcarWidth;1847 1848 psTrace("psLib.math", 3, "---- %s(true) end ----\n", __func__);1849 return true;1850 }1851 1852 1853 /*****************************************************************************1854 psVectorHistogram(out, in, errors, mask, maskVal): this procedure takes as1855 input a preallocated and initialized histogram structure. It fills the bins1856 in that histogram structure in accordance with the input data "in" and the,1857 possibly NULL, mask vector.1858 1859 Inputs:1860 out1861 in1862 mask1863 maskVal1864 Returns:1865 The histogram structure "out".1866 *****************************************************************************/1867 psHistogram* psVectorHistogram(psHistogram* out,1868 const psVector* values,1869 const psVector* errors,1870 const psVector* mask,1871 psMaskType maskVal)1872 {1873 psTrace("psLib.math", 3, "---- %s() begin ----\n", __func__);1874 PS_ASSERT_PTR_NON_NULL(out, NULL);1875 PS_ASSERT_VECTOR_NON_NULL(out->bounds, NULL);1876 PS_ASSERT_VECTOR_TYPE(out->bounds, PS_TYPE_F32, NULL);1877 PS_ASSERT_INT_NONNEGATIVE(out->bounds->n, NULL);1878 PS_ASSERT_VECTOR_NON_NULL(out->nums, NULL);1879 PS_ASSERT_VECTOR_TYPE(out->nums, PS_TYPE_F32, NULL);1880 PS_ASSERT_INT_NONNEGATIVE(out->nums->n, NULL);1881 PS_ASSERT_VECTOR_NON_NULL(values, out);1882 if (mask) {1883 PS_ASSERT_VECTORS_SIZE_EQUAL(values, mask, NULL);1884 PS_ASSERT_VECTOR_TYPE(mask, PS_TYPE_U8, NULL);1885 }1886 if (errors) {1887 PS_ASSERT_VECTORS_SIZE_EQUAL(values, errors, NULL);1888 PS_ASSERT_VECTOR_TYPE(errors, values->type.type, NULL);1889 }1890 1891 long binNum = 0; // A temporary bin number1892 long numBins = out->nums->n; // The total number of bins1893 psScalar tmpScalar;1894 tmpScalar.type.type = PS_TYPE_F32;1895 1896 // Convert input and errors vectors to F32 if necessary.1897 psVector* inF32 = NULL; // F32 version of input vector1898 if (values->type.type == PS_TYPE_F32) {1899 inF32 = psMemIncrRefCounter((psPtr)values);1900 } else {1901 inF32 = psVectorCopy(NULL, values, PS_TYPE_F32);1902 }1903 psVector* errorsF32 = NULL; // F32 version of errors vector1904 if (errors) {1905 if (errors->type.type == PS_TYPE_F32) {1906 errorsF32 = psMemIncrRefCounter((psPtr)errors);1907 } else {1908 errorsF32 = psVectorCopy(NULL, errors, PS_TYPE_F32);1909 }1910 }1911 1912 for (long i = 0; i < inF32->n; i++) {1913 // Check if this pixel is masked, and if so, skip it.1914 if (!mask || (mask && (!(mask->data.U8[i] & maskVal)))) {1915 if (inF32->data.F32[i] < out->bounds->data.F32[0]) {1916 // If this pixel is below minimum value, count it, then skip.1917 out->minNum++;1918 } else if (inF32->data.F32[i] > out->bounds->data.F32[numBins]) {1919 // If this pixel is above maximum value, count it, then skip.1920 out->maxNum++;1921 } else {1922 // If this is a uniform histogram, determining the correct1923 // number is trivial.1924 if (out->uniform == true) {1925 float binSize = out->bounds->data.F32[1] - out->bounds->data.F32[0]; // Histogram bin size1926 binNum = (inF32->data.F32[i] - out->bounds->data.F32[0]) / binSize;1927 if (errorsF32) {1928 if (!UpdateHistogramBins(binNum, out, inF32->data.F32[i], errorsF32->data.F32[i])) {1929 psLogMsg(__func__, PS_LOG_WARN, "WARNING: Failed to update the histogram "1930 "bins with the errors vector.\n");1931 }1932 } else {1933 // This if-statement really shouldn't be necessary.1934 // However, due to numerical lack of precision, we1935 // occasionally produce a binNum outside the range.1936 if (binNum >= out->nums->n) {1937 binNum = out->nums->n - 1;1938 }1939 (out->nums->data.F32[binNum])+= 1.0;1940 }1941 1942 } else {1943 // If this is a non-uniform histogram, determining the1944 // correct bin number requires a bit more work.1945 tmpScalar.data.F32 = inF32->data.F32[i];1946 binNum = p_psVectorBinDisect( *(psVector* *)&out->bounds, &tmpScalar);1947 if (binNum < 0) {1948 psLogMsg(__func__, PS_LOG_WARN,1949 "WARNING: psVectorHistogram(): element outside histogram bounds.\n");1950 } else {1951 if (errorsF32 != NULL) {1952 if (!UpdateHistogramBins(binNum, out, inF32->data.F32[i], errors->data.F32[i])) {1953 psLogMsg(__func__, PS_LOG_WARN, "WARNING: Failed to update the histogram "1954 "bins with the errors vector.\n");1955 }1956 } else {1957 out->nums->data.F32[binNum] += 1.0;1958 }1959 }1960 }1961 }1962 }1963 }1964 1965 psFree(inF32);1966 psFree(errorsF32);1967 1968 psTrace("psLib.math", 3, "---- %s() end ----\n", __func__);1969 return (out);1970 }1971 1972 1449 /****************************************************************************** 1973 1450 psVectorStats(in, mask, maskVal, stats): this is the public API … … 1984 1461 1985 1462 XXX: Should we free stats if the asserts fail? NO; we don't own it (RHL). 1986 *****************************************************************************/ 1987 psStats* psVectorStats( 1988 psStats* stats, 1989 const psVector* in, 1990 const psVector* errors, 1991 const psVector* mask, 1992 psMaskType maskVal) 1463 *****************************************************************************/ 1464 bool psVectorStats(psStats* stats, 1465 const psVector* in, 1466 const psVector* errors, 1467 const psVector* mask, 1468 psMaskType maskVal) 1993 1469 { 1994 1470 psTrace("psLib.math", 3,"---- %s() begin ----\n", __func__); … … 2036 1512 } 2037 1513 1514 // init the value of stats->results: this is used internally to check if 1515 // prior functions have been called 1516 stats->results = PS_STAT_NONE; 1517 bool status = true; 1518 2038 1519 // ************************************************************************ 2039 1520 if (stats->options & PS_STAT_SAMPLE_MEAN) { 2040 1521 if (!vectorSampleMean(inF32, errorsF32, maskU8, maskVal, stats)) { 2041 ps LogMsg(__func__, PS_LOG_WARN, "WARNING: vectorSampleMean() returned an error.\n");2042 stat s->sampleMean = NAN;1522 psError(PS_ERR_UNKNOWN, false, "Failed to calculate vector sample mean"); 1523 status &= false; 2043 1524 } 2044 1525 } … … 2047 1528 if (stats->options & (PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_QUARTILE)) { 2048 1529 if (!vectorSampleMedian(inF32, maskU8, maskVal, stats)) { 2049 psLogMsg(__func__, PS_LOG_WARN, "WARNING: vectorSampleMedian() returned an error.\n"); 2050 stats->sampleMedian = NAN; 2051 stats->sampleUQ = NAN; 2052 stats->sampleLQ = NAN; 1530 psError(PS_ERR_UNKNOWN, false, "Failed to calculate sample median"); 1531 status &= false; 2053 1532 } 2054 1533 } … … 2056 1535 // ************************************************************************ 2057 1536 if (stats->options & PS_STAT_SAMPLE_STDEV) { 2058 if (!vectorSampleMean(inF32, errorsF32, maskU8, maskVal, stats)) { 2059 psLogMsg(__func__, PS_LOG_WARN, "WARNING: vectorSampleMean() returned an error.\n"); 2060 stats->sampleMean = NAN; 2061 } else { 2062 vectorSampleStdev(inF32, errorsF32, maskU8, maskVal, stats); 2063 } 2064 } 2065 2066 // ************************************************************************ 2067 if (stats->options & (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_ROBUST_QUARTILE | 2068 PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV)) { 2069 if (!vectorRobustStats(inF32, errorsF32, maskU8, maskVal, stats)) { 2070 psError(PS_ERR_UNKNOWN, false, _("Failed to calculate the specified statistic.")); 2071 psFree(inF32); 2072 psFree(errorsF32); 2073 psTrace("psLib.math", 3,"---- %s(NULL) end ----\n", __func__); 2074 return(NULL); 2075 } 2076 } 2077 2078 // ************************************************************************ 2079 if ((stats->options & PS_STAT_CLIPPED_MEAN) || (stats->options & PS_STAT_CLIPPED_STDEV)) { 2080 if (!vectorClippedStats(inF32, errorsF32, maskU8, maskVal, stats)) { 2081 psError(PS_ERR_UNKNOWN, false, "Failed to calculate clipped statistics for input psVector.\n"); 2082 stats->clippedMean = NAN; 2083 stats->clippedStdev = NAN; 1537 if (!vectorSampleStdev(inF32, errorsF32, maskU8, maskVal, stats)) { 1538 psError(PS_ERR_UNKNOWN, false, "Failed to calculate sample stdev"); 1539 status &= false; 2084 1540 } 2085 1541 } … … 2089 1545 if (vectorMinMax(inF32, maskU8, maskVal, stats) == 0) { 2090 1546 psError(PS_ERR_UNKNOWN, false, "Failed to calculate vector minimum and maximum"); 1547 status &= false; 1548 } 1549 } 1550 1551 // ************************************************************************ 1552 if (stats->options & (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV | PS_STAT_ROBUST_QUARTILE)) { 1553 if (!vectorRobustStats(inF32, errorsF32, maskU8, maskVal, stats)) { 1554 psError(PS_ERR_UNKNOWN, false, _("Failed to calculate robust statistics")); 1555 status &= false; 1556 } 1557 } 1558 1559 // ************************************************************************ 1560 if (stats->options & (PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV)) { 1561 if (!vectorFittedStats(inF32, errorsF32, maskU8, maskVal, stats)) { 1562 psError(PS_ERR_UNKNOWN, false, _("Failed to calculate fitted statistics")); 1563 status &= false; 1564 } 1565 } 1566 1567 // ************************************************************************ 1568 if (stats->options & (PS_STAT_FITTED_MEAN_V2 | PS_STAT_FITTED_STDEV_V2)) { 1569 if (stats->options & (PS_STAT_FITTED_MEAN | PS_STAT_FITTED_STDEV)) { 1570 psAbort ("stats", "you may not specify both FITTED_MEAN and FITTED_MEAN_V2"); 1571 } 1572 if (!vectorFittedStats_v2(inF32, errorsF32, maskU8, maskVal, stats)) { 1573 psError(PS_ERR_UNKNOWN, false, _("Failed to calculate fitted statistics")); 1574 status &= false; 1575 } 1576 } 1577 1578 // ************************************************************************ 1579 if ((stats->options & PS_STAT_CLIPPED_MEAN) || (stats->options & PS_STAT_CLIPPED_STDEV)) { 1580 if (!vectorClippedStats(inF32, errorsF32, maskU8, maskVal, stats)) { 1581 psError(PS_ERR_UNKNOWN, false, "Failed to calculate clipped statistics\n"); 1582 status &= false; 2091 1583 } 2092 1584 } … … 2096 1588 psFree(maskU8); 2097 1589 psTrace("psLib.math", 3,"---- %s() end ----\n", __func__); 2098 return (stat s);1590 return (status); 2099 1591 } 2100 1592 … … 2106 1598 } 2107 1599 2108 READ_STAT("MEAN", PS_STAT_SAMPLE_MEAN);2109 READ_STAT("STDEV", PS_STAT_SAMPLE_STDEV);2110 READ_STAT("MEDIAN", PS_STAT_SAMPLE_MEDIAN);2111 READ_STAT("QUARTILE", PS_STAT_SAMPLE_QUARTILE);1600 READ_STAT("MEAN", PS_STAT_SAMPLE_MEAN); 1601 READ_STAT("STDEV", PS_STAT_SAMPLE_STDEV); 1602 READ_STAT("MEDIAN", PS_STAT_SAMPLE_MEDIAN); 1603 READ_STAT("QUARTILE", PS_STAT_SAMPLE_QUARTILE); 2112 1604 READ_STAT("SAMPLE_MEAN", PS_STAT_SAMPLE_MEAN); 2113 1605 READ_STAT("SAMPLE_STDEV", PS_STAT_SAMPLE_STDEV); … … 2118 1610 READ_STAT("ROBUST_STDEV", PS_STAT_ROBUST_STDEV); 2119 1611 READ_STAT("ROBUST_QUARTILE", PS_STAT_ROBUST_QUARTILE); 2120 READ_STAT("FITTED", PS_STAT_FITTED_MEAN); 2121 READ_STAT("FITTED_MEAN", PS_STAT_FITTED_MEAN); 2122 READ_STAT("FITTED_STDEV", PS_STAT_ROBUST_STDEV); 2123 READ_STAT("CLIPPED", PS_STAT_CLIPPED_MEAN); 2124 READ_STAT("CLIPPED_MEAN", PS_STAT_CLIPPED_MEAN); 2125 READ_STAT("CLIPPED_STDEV", PS_STAT_CLIPPED_STDEV); 1612 READ_STAT("FITTED", PS_STAT_FITTED_MEAN); 1613 READ_STAT("FITTED_MEAN", PS_STAT_FITTED_MEAN); 1614 READ_STAT("FITTED_STDEV", PS_STAT_FITTED_STDEV); 1615 READ_STAT("FITTED_V2", PS_STAT_FITTED_MEAN_V2); 1616 READ_STAT("FITTED_MEAN_V2", PS_STAT_FITTED_MEAN_V2); 1617 READ_STAT("FITTED_STDEV_V2", PS_STAT_FITTED_STDEV_V2); 1618 READ_STAT("CLIPPED", PS_STAT_CLIPPED_MEAN); 1619 READ_STAT("CLIPPED_MEAN", PS_STAT_CLIPPED_MEAN); 1620 READ_STAT("CLIPPED_STDEV", PS_STAT_CLIPPED_STDEV); 2126 1621 2127 1622 psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to parse statistic: %s\n", string); … … 2146 1641 WRITE_STAT("ROBUST_STDEV", PS_STAT_ROBUST_STDEV); 2147 1642 WRITE_STAT("ROBUST_QUARTILE", PS_STAT_ROBUST_QUARTILE); 2148 WRITE_STAT("FITTED_MEAN", PS_STAT_FITTED_MEAN); 2149 WRITE_STAT("FITTED_STDEV", PS_STAT_ROBUST_STDEV); 2150 WRITE_STAT("CLIPPED_MEAN", PS_STAT_CLIPPED_MEAN); 2151 WRITE_STAT("CLIPPED_STDEV", PS_STAT_CLIPPED_STDEV); 1643 WRITE_STAT("FITTED_MEAN", PS_STAT_FITTED_MEAN); 1644 WRITE_STAT("FITTED_STDEV", PS_STAT_FITTED_STDEV); 1645 WRITE_STAT("FITTED_MEAN_V2", PS_STAT_FITTED_MEAN_V2); 1646 WRITE_STAT("FITTED_STDEV_V2", PS_STAT_FITTED_STDEV_V2); 1647 WRITE_STAT("CLIPPED_MEAN", PS_STAT_CLIPPED_MEAN); 1648 WRITE_STAT("CLIPPED_STDEV", PS_STAT_CLIPPED_STDEV); 2152 1649 2153 1650 return string; … … 2197 1694 case PS_STAT_FITTED_MEAN: 2198 1695 case PS_STAT_FITTED_STDEV: 1696 case PS_STAT_FITTED_MEAN_V2: 1697 case PS_STAT_FITTED_STDEV_V2: 2199 1698 case PS_STAT_CLIPPED_MEAN: 2200 1699 case PS_STAT_CLIPPED_STDEV: … … 2227 1726 case PS_STAT_FITTED_STDEV: 2228 1727 return stats->fittedStdev; 1728 case PS_STAT_FITTED_MEAN_V2: 1729 return stats->fittedMean; 1730 case PS_STAT_FITTED_STDEV_V2: 1731 return stats->fittedStdev; 2229 1732 case PS_STAT_CLIPPED_MEAN: 2230 1733 return stats->clippedMean; … … 2246 1749 return NAN; 2247 1750 } 1751 1752 // other private functions used above 1753 1754 static psF32 QuadraticInverse(psF32 a, 1755 psF32 b, 1756 psF32 c, 1757 psF32 y, 1758 psF32 xLo, 1759 psF32 xHi 1760 ) 1761 { 1762 psF64 tmp = sqrt((y - c)/a + (b*b)/(4.0 * a * a)); 1763 1764 psF64 x1 = -b/(2.0*a) + tmp; 1765 psF64 x2 = -b/(2.0*a) - tmp; 1766 1767 if (xLo <= x1 && x1 <= xHi) { 1768 return x1; 1769 } 1770 if (xLo <= x2 && x2 <= xHi) { 1771 return x2; 1772 } 1773 return 0.5 * (xLo + xHi); 1774 } 1775 1776 1777 /****************************************************************************** 1778 fitQuadraticSearchForYThenReturnX(*xVec, *yVec, binNum, yVal): A general 1779 routine which fits a quadratic to three points and returns the x-value 1780 corresponding to the input y-value. This routine takes psVectors of x/y pairs 1781 as input, and fits a quadratic to the 3 points surrounding element binNum in 1782 the vectors (the midpoint between element i and i+1 is used for x[i]). It 1783 then determines for what value x does that quadratic f(x) = yVal (the input 1784 parameter). 1785 1786 *****************************************************************************/ 1787 static psF32 fitQuadraticSearchForYThenReturnX(const psVector *xVec, 1788 psVector *yVec, 1789 psS32 binNum, 1790 psF32 yVal 1791 ) 1792 { 1793 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__); 1794 psTrace("psLib.math", 5, "binNum, yVal is (%d, %f)\n", binNum, yVal); 1795 if (psTraceGetLevel("psLib.math") >= 8) { 1796 PS_VECTOR_PRINT_F32(xVec); 1797 PS_VECTOR_PRINT_F32(yVec); 1798 } 1799 1800 PS_ASSERT_VECTOR_NON_NULL(xVec, NAN); 1801 PS_ASSERT_VECTOR_NON_NULL(yVec, NAN); 1802 PS_ASSERT_VECTOR_TYPE(xVec, PS_TYPE_F32, NAN); 1803 PS_ASSERT_VECTOR_TYPE(yVec, PS_TYPE_F32, NAN); 1804 // PS_ASSERT_VECTORS_SIZE_EQUAL(xVec, yVec, NAN); 1805 PS_ASSERT_INT_WITHIN_RANGE(binNum, 0, (int)(xVec->n - 1), NAN); 1806 PS_ASSERT_INT_WITHIN_RANGE(binNum, 0, (int)(yVec->n - 1), NAN); 1807 1808 psVector *x = psVectorAlloc(3, PS_TYPE_F64); 1809 psVector *y = psVectorAlloc(3, PS_TYPE_F64); 1810 psF32 tmpFloat = 0.0f; 1811 1812 if ((binNum >= 1) && (binNum < (yVec->n - 2)) && (binNum < (xVec->n - 2))) { 1813 // The general case. We have all three points. 1814 x->data.F64[0] = (psF64) (0.5 * (xVec->data.F32[binNum - 1] + xVec->data.F32[binNum])); 1815 x->data.F64[1] = (psF64) (0.5 * (xVec->data.F32[binNum] + xVec->data.F32[binNum+1])); 1816 x->data.F64[2] = (psF64) (0.5 * (xVec->data.F32[binNum+1] + xVec->data.F32[binNum+2])); 1817 y->data.F64[0] = yVec->data.F32[binNum - 1]; 1818 y->data.F64[1] = yVec->data.F32[binNum]; 1819 y->data.F64[2] = yVec->data.F32[binNum + 1]; 1820 psTrace("psLib.math", 6, "x vec (orig) is (%f %f %f %f)\n", xVec->data.F32[binNum - 1], 1821 xVec->data.F32[binNum], xVec->data.F32[binNum+1], xVec->data.F32[binNum+2]); 1822 psTrace("psLib.math", 6, "x data is (%f %f %f)\n", x->data.F64[0], x->data.F64[1], x->data.F64[2]); 1823 psTrace("psLib.math", 6, "y data is (%f %f %f)\n", y->data.F64[0], y->data.F64[1], y->data.F64[2]); 1824 1825 // 1826 // Ensure that the y value lies within range of the y values. 1827 // 1828 if (! (((y->data.F64[0] <= yVal) && (yVal <= y->data.F64[2])) || 1829 ((y->data.F64[2] <= yVal) && (yVal <= y->data.F64[0]))) ) { 1830 psError(PS_ERR_BAD_PARAMETER_VALUE, true, 1831 _("Specified yVal, %g, is not within y-range, %g to %g."), 1832 (psF64)yVal, y->data.F64[0], y->data.F64[2]); 1833 } 1834 1835 // 1836 // Ensure that the y values are monotonic. 1837 // 1838 if (((y->data.F64[0] < y->data.F64[1]) && !(y->data.F64[1] <= y->data.F64[2])) || 1839 ((y->data.F64[0] > y->data.F64[1]) && !(y->data.F64[1] >= y->data.F64[2]))) { 1840 psError(PS_ERR_UNKNOWN, true, 1841 "This routine must be called with monotonically increasing or decreasing data points.\n"); 1842 psFree(x); 1843 psFree(y); 1844 psTrace("psLib.math", 5, "---- %s() end ----\n", __func__); 1845 return NAN; 1846 } 1847 1848 // Determine the coefficients of the polynomial. 1849 psPolynomial1D *myPoly = psPolynomial1DAlloc(PS_POLYNOMIAL_ORD, 2); 1850 myPoly = psVectorFitPolynomial1D(myPoly, NULL, 0, y, NULL, x); 1851 if (myPoly == NULL) { 1852 psError(PS_ERR_UNEXPECTED_NULL, false, 1853 _("Failed to fit a 1-dimensional polynomial to the three specified data points. Returning NAN.")); 1854 psFree(x); 1855 psFree(y); 1856 psTrace("psLib.math", 5, "---- %s(NAN) end ----\n", __func__); 1857 return NAN; 1858 } 1859 psTrace("psLib.math", 6, "myPoly->coeff[0] is %f\n", myPoly->coeff[0]); 1860 psTrace("psLib.math", 6, "myPoly->coeff[1] is %f\n", myPoly->coeff[1]); 1861 psTrace("psLib.math", 6, "myPoly->coeff[2] is %f\n", myPoly->coeff[2]); 1862 psTrace("psLib.math", 6, "Fitted y vec is (%f %f %f)\n", 1863 (psF32) psPolynomial1DEval(myPoly, (psF64) x->data.F64[0]), 1864 (psF32) psPolynomial1DEval(myPoly, (psF64) x->data.F64[1]), 1865 (psF32) psPolynomial1DEval(myPoly, (psF64) x->data.F64[2])); 1866 1867 psTrace("psLib.math", 6, "We fit the polynomial, now find x such that f(x) equals %f\n", yVal); 1868 tmpFloat = QuadraticInverse(myPoly->coeff[2], myPoly->coeff[1], myPoly->coeff[0], yVal, 1869 x->data.F64[0], x->data.F64[2]); 1870 psFree(myPoly); 1871 1872 if (isnan(tmpFloat)) { 1873 psError(PS_ERR_UNEXPECTED_NULL, 1874 false, _("Failed to determine the median of the fitted polynomial. Returning NAN.")); 1875 psFree(x); 1876 psFree(y); 1877 psTrace("psLib.math", 5, "---- %s(NAN) end ----\n", __func__); 1878 return(NAN); 1879 } 1880 } else { 1881 // These are special cases where the bin is at the beginning or end of the vector. 1882 if (binNum == 0) { 1883 // We have two points only at the beginning of the vectors x and y. 1884 tmpFloat = 0.5 * (xVec->data.F32[binNum] + 1885 xVec->data.F32[binNum + 1]); 1886 } else if (binNum == (xVec->n - 1)) { 1887 // The special case where we have two points only at the end of 1888 // the vectors x and y. 1889 // XXX: Is this right? 1890 tmpFloat = xVec->data.F32[binNum]; 1891 } else if (binNum == (xVec->n - 2)) { 1892 // XXX: Is this right? 1893 tmpFloat = 0.5 * (xVec->data.F32[binNum] + xVec->data.F32[binNum + 1]); 1894 } 1895 } 1896 1897 psTrace("psLib.math", 6, "FIT: return %f\n", tmpFloat); 1898 psFree(x); 1899 psFree(y); 1900 1901 psTrace("psLib.math", 5, "---- %s(%f) end ----\n", __func__, tmpFloat); 1902 return tmpFloat; 1903 } 1904 1905 /****************************************************************************** 1906 NOTE: We assume unnormalized gaussians. 1907 *****************************************************************************/ 1908 static psF32 minimizeLMChi2Gauss1D(psVector *deriv, 1909 const psVector *params, 1910 const psVector *coords 1911 ) 1912 { 1913 psTrace("psLib.math", 4, "---- %s() begin ----\n", __func__); 1914 PS_ASSERT_VECTOR_NON_NULL(params, NAN); 1915 PS_ASSERT_VECTOR_SIZE(params, (long)2, NAN); 1916 PS_ASSERT_VECTOR_TYPE(params, PS_TYPE_F32, NAN); 1917 PS_ASSERT_VECTOR_NON_NULL(coords, NAN); 1918 PS_ASSERT_VECTOR_SIZE(coords, (long)1, NAN); 1919 PS_ASSERT_VECTOR_TYPE(coords, PS_TYPE_F32, NAN); 1920 1921 psF32 x = coords->data.F32[0]; 1922 psF32 mean = params->data.F32[0]; 1923 psF32 var = params->data.F32[1]; 1924 psF32 dx = (x - mean); 1925 1926 psF32 gauss = exp (-0.5*PS_SQR(dx)/var); 1927 if (deriv) { 1928 PS_ASSERT_VECTOR_SIZE(deriv, (long)2, NAN); 1929 PS_ASSERT_VECTOR_TYPE(deriv, PS_TYPE_F32, NAN); 1930 psF32 tmp = dx * gauss; 1931 deriv->data.F32[0] = tmp / var; 1932 deriv->data.F32[1] = tmp * dx / (var * var); 1933 } 1934 1935 1936 psTrace("psLib.math", 4, "---- %s() end ----\n", __func__); 1937 return gauss; 1938 }
Note:
See TracChangeset
for help on using the changeset viewer.
