IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 42382 for trunk/ppImage


Ignore:
Timestamp:
Feb 8, 2023, 11:57:40 AM (3 years ago)
Author:
eugene
Message:

merge from eam_branches/ipp-20220316. add code for PatternDeadCells, fix organization of DetrendPattern, handle psphotModelBackground failure

Location:
trunk/ppImage
Files:
13 edited

Legend:

Unmodified
Added
Removed

  • trunk/ppImage

  • trunk/ppImage/src/Makefile.am

    r42340 r42382  
     1
    12bin_PROGRAMS = ppImage
    23

  • trunk/ppImage/src/ppImage.h

    r42340 r42382  
    4747    bool doPatternContinuity;           // Cell continuity correction
    4848    bool doBackgroundContinuity;        // Do mosaic continuity correction
     49    bool doPatternDeadCells;            // match dead cell background patterns
    4950    bool doFringe;                      // Fringe subtraction
    5051    bool doPhotom;                      // Source identification and photometry

  • trunk/ppImage/src/ppImageArguments.c

    r42340 r42382  
    119119    pmConfigFileSetsMD (config->arguments, &argc, argv, "NEWNONLIN", "-newnonlin", "-newnonlinlist");
    120120    pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.ROW.AMP", "-pattern-row-amplitude", "-not-defined");
     121    pmConfigFileSetsMD (config->arguments, &argc, argv, "PATTERN.DEAD.CELLS", "-pattern-dead-cells", "-not-defined");
    121122
    122123    if ((argnum = psArgumentGet(argc, argv, "-burntool"))) {

  • trunk/ppImage/src/ppImageDetrendFringe.c

    r37406 r42382  
    327327    pmCell *cell = NULL;
    328328
    329     // psTimerStart("apply.fringe");
    330 
    331     assert (options->doFringe); // do not call if not needed
     329    if (!options->doFringe) return true;
     330
    332331    assert (inputView->chip != -1);
    333332    assert (inputView->cell == -1);

  • trunk/ppImage/src/ppImageDetrendPattern.c

    r41894 r42382  
    55#include "ppImage.h"
    66
    7 #define ESCAPE(STATUS,...) {                    \
     7#define ESCAPE(STATUS,...) {                            \
    88        psError(PS_ERR_UNKNOWN, STATUS, __VA_ARGS__);   \
    99        psFree(view);                                   \
    … …  
    1313static bool doPatternForView (bool *doit, const pmConfig *config, const pmChip *chip, const pmFPAview *view, const char *recipename, const char *recipevalue);
    1414
    15 bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView,
    16                                 ppImageOptions *options)
     15bool ppImageDetrendPatternRowApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options);
     16bool ppImageDetrendPatternContinuityApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options);
     17bool ppImageDetrendPatternCellApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options);
     18bool ppImageDetrendPatternDeadCellsApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options);
     19
     20// Apply the desired pattern corrections (if any).  This function is passed the chip for the
     21// image being processed, but may interact with pmFPAfiles containing the pattern reference information
     22bool ppImageDetrendPatternApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options)
    1723{
    18     pmCell *cell = NULL;
    19 
    20     assert(options->doPatternRow || options->doPatternCell || options->doPatternContinuity); // do not call if not needed
     24    assert(inputView->chip != -1);
     25    assert(inputView->cell == -1);
     26    assert(inputView->readout == -1);
     27
     28    if (!ppImageDetrendPatternRowApply (config, chip, inputView, options)) {
     29        return false;
     30    }
     31    if (!ppImageDetrendPatternContinuityApply (config, chip, inputView, options)) {
     32        return false;
     33    }
     34    if (!ppImageDetrendPatternCellApply (config, chip, inputView, options)) {
     35        return false;
     36    }
     37    if (!ppImageDetrendPatternDeadCellsApply (config, chip, inputView, options)) {
     38        return false;
     39    }
     40    return(true);
     41}
     42
     43bool ppImageDetrendPatternRowApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options)
     44{
     45    bool status;
     46
     47    if (!options->doPatternRow) return true;
     48   
    2149    assert(inputView->chip != -1);
    2250    assert(inputView->cell == -1);
    … …  
    3563    // tables, one per chip.  Each table lists the typical bias-drift amplitude for cell,
    3664    // measured from a collection of dark images.  These values are passed (via
    37     // cell->analysis) to pmPatternRow which choosed to apply the correction based on the
     65    // cell->analysis) to pmPatternRow which chooses to apply the correction based on the
    3866    // ratio of the poisson noise due to the median background (+ read noise) and the
    3967    // typical amplitude.  Tests show that if the amplitude / noise < 1/6, then the
    … …  
    4472    // chip.
    4573
    46     // psTimerStart("apply.pattern");
    47     if (options->doPatternRow) {
    48 
    49         //psLogMsg("ppImage", PS_LOG_INFO, "Log: entering pattern.row routine");
    50         //psWarning ("Entering pattern.row routine");
    51 
    52         bool status;
    53         pmHDU *hdu = pmHDUFromChip(chip);
    54         if (psMetadataLookupBool(NULL,hdu->header,"PTRN_ROW")) {
    55           psLogMsg("ppImage", PS_LOG_INFO, "Not performing row pattern correction as it has already been done.");
    56           goto pattern_continuity;
    57         }
    58 
    59         const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME");
    60         psLogMsg("ppImage", PS_LOG_INFO, "Performing row pattern correction for %s\n", chipName);
    61 
    62         pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.INPUT");
    63 
    64         // grab the PATTERN.ROW.AMP file
    65         pmFPAfile *PRAfile = psMetadataLookupPtr (NULL, config->files, "PPIMAGE.PATTERN.ROW.AMP");
    66         if (!PRAfile) {
    67           psLogMsg("ppImage", PS_LOG_INFO, "Pattern Row Amplitude file not found, applying to all (with limits from ROW.SUBSET) ");
    68         }
    69        
    70         pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
    71         *view = *inputView;
    72         while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) {
    73             //const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME");
    74             //const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME");
    75             //psWarning ("Looping through %s, %s\n", chipName, cellName);
    76 
    77             if (!cell->process || !cell->file_exists) {
    78                 continue;
    79             }
    80             if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
    81                 ESCAPE(false, "load failure for Cell");
    82             }
     74    pmHDU *hdu = pmHDUFromChip(chip);
     75    if (psMetadataLookupBool(&status, hdu->header, "PTRN_ROW")) {
     76        psLogMsg("ppImage", PS_LOG_INFO, "Not performing row pattern correction as it has already been done.");
     77        return true;
     78    }
     79
     80    const char *chipName = psMetadataLookupStr(&status, chip->concepts, "CHIP.NAME");
     81    psLogMsg("ppImage", PS_LOG_INFO, "Performing row pattern correction for %s\n", chipName);
     82
     83    // XXX we use input->fpa below, but could we just use chip->parent?
     84    pmFPAfile *input = psMetadataLookupPtr(&status, config->files, "PPIMAGE.INPUT");
     85
     86    // grab the PATTERN.ROW.AMP file
     87    pmFPAfile *PRAfile = psMetadataLookupPtr (&status, config->files, "PPIMAGE.PATTERN.ROW.AMP");
     88    if (!PRAfile) {
     89        psLogMsg("ppImage", PS_LOG_INFO, "Pattern Row Amplitude file not found, applying to all (with limits from ROW.SUBSET) ");
     90    }
     91       
     92    pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
     93    *view = *inputView;
     94
     95    pmCell *cell = NULL;
     96    while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) {
     97        // const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME");
     98        // const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME");
     99        // psWarning ("Looping through %s, %s\n", chipName, cellName);
     100
     101        if (!cell->process || !cell->file_exists) {
     102            continue;
     103        }
     104
     105        // this forces pmFPAfileRead of the PATTERN.ROW.AMP file (XXX but is this needed?)
     106        if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
     107            ESCAPE(false, "load failure for Cell");
     108        }
    83109           
    84             if (!cell->data_exists) {
    85                 continue;
    86             }
    87 
    88             if (cell->readouts->n > 1) {
    89                 psWarning ("Skipping Video Cell for ppImageDetrendPatternApply");
    90                 continue;
    91             }
    92 
    93             // grab the corresponding cell
    94             if (PRAfile) {
    95               pmCell *PRAcell = pmFPAviewThisCell (view, PRAfile->fpa);
    96               psAssert (PRAcell, "found Pattern Row Amplitude file, but not cell?");
    97 
    98               // find the nominal signal amplitude (check the ghost and/or crosstalk recipe file)
    99               float amplitude = psMetadataLookupF32 (&status, PRAcell->analysis, "PTN.ROW.AMP");
    100               if (!status) amplitude = NAN;
     110        if (!cell->data_exists) {
     111            continue;
     112        }
     113
     114        if (cell->readouts->n > 1) {
     115            psWarning ("Skipping Video Cell for ppImageDetrendPatternApply");
     116            continue;
     117        }
     118
     119        // grab the corresponding cell
     120        if (PRAfile) {
     121            pmCell *PRAcell = pmFPAviewThisCell (view, PRAfile->fpa);
     122            psAssert (PRAcell, "found Pattern Row Amplitude file, but not cell?");
     123
     124            // find the nominal signal amplitude (check the ghost and/or crosstalk recipe file)
     125            float amplitude = psMetadataLookupF32 (&status, PRAcell->analysis, "PTN.ROW.AMP");
     126            if (!status) amplitude = NAN;
    101127           
    102               // put the value on the science cell
    103               psMetadataAddF32 (cell->analysis, PS_LIST_TAIL, "PTN.ROW.AMP", PS_META_REPLACE, "", amplitude);
     128            // put the value on the science cell
     129            psMetadataAddF32 (cell->analysis, PS_LIST_TAIL, "PTN.ROW.AMP", PS_META_REPLACE, "", amplitude);
     130        }
     131
     132        bool doPattern = false;
     133        if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.ROW.SUBSET")) {
     134            ESCAPE(false, "Unable to determine whether row pattern matching should be applied.");
     135        }
     136        if (!doPattern) continue;
     137
     138        // switch to test threaded version
     139        if (true) {
     140            // I need to allocate a view here to be freed by the
     141            // called function below. 
     142            pmFPAview *myView = pmFPAviewAlloc(0); // View for local processing
     143            *myView = *view;
     144
     145            // allocate a job, construct the arguments for this job
     146            psThreadJob *job = psThreadJobAlloc("PPIMAGE_PATTERN_ROW_CELL");
     147            psArrayAdd(job->args, 1, config);
     148            psArrayAdd(job->args, 1, input->fpa);
     149            psArrayAdd(job->args, 1, chip);
     150            psArrayAdd(job->args, 1, cell);
     151            psArrayAdd(job->args, 1, myView);
     152            psArrayAdd(job->args, 1, options);
     153            if (!psThreadJobAddPending(job)) {
     154                return false;
    104155            }
    105 
    106             bool doPattern = false;
    107             if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.ROW.SUBSET")) {
    108                 ESCAPE(false, "Unable to determine whether row pattern matching should be applied.");
    109             }
    110             if (!doPattern) continue;
    111 
    112             // switch to test threaded version
    113             if (true) {
    114                 // I need to allocate a view here to be freed by the
    115                 // called function below. 
    116                 pmFPAview *myView = pmFPAviewAlloc(0); // View for local processing
    117                 *myView = *view;
    118 
    119                 // allocate a job, construct the arguments for this job
    120                 psThreadJob *job = psThreadJobAlloc("PPIMAGE_PATTERN_ROW_CELL");
    121                 psArrayAdd(job->args, 1, config);
    122                 psArrayAdd(job->args, 1, input->fpa);
    123                 psArrayAdd(job->args, 1, chip);
    124                 psArrayAdd(job->args, 1, cell);
    125                 psArrayAdd(job->args, 1, myView);
    126                 psArrayAdd(job->args, 1, options);
    127                 if (!psThreadJobAddPending(job)) {
    128                     return false;
    129                 }
    130             } else {
    131                 // bump the counter since it must be freed by the function below. 
    132                 psMemIncrRefCounter (view); // View for local processing
    133                 if (!ppImageDetrendPatternApplyCell (config, input->fpa, chip, cell, view, options)) {
    134                     return false;
    135                 }
     156        } else {
     157            // bump the counter since it must be freed by the function below. 
     158            psMemIncrRefCounter (view); // View for local processing
     159            if (!ppImageDetrendPatternApplyCell (config, input->fpa, chip, cell, view, options)) {
     160                return false;
    136161            }
    137 
    138         }
    139 
    140         // wait here for the threaded jobs to finish
    141         // if no threads are allocated, this
    142         if (!psThreadPoolWait(true, true)) {
    143             psError(PS_ERR_UNKNOWN, false, "Unable to apply bias correction.");
    144             return false;
    145         }
    146 
    147         psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_ROW",PS_META_REPLACE,"PATTERN.ROW correction applied",true);
    148         psFree(view);
    149     }
    150 
    151  pattern_continuity:
     162        }
     163    }
     164
     165    // wait here for the threaded jobs to finish
     166    // if no threads are allocated, this
     167    if (!psThreadPoolWait(true, true)) {
     168        psError(PS_ERR_UNKNOWN, false, "Unable to apply bias correction.");
     169        return false;
     170    }
     171
     172    psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_ROW",PS_META_REPLACE,"PATTERN.ROW correction applied",true);
     173    psFree(view);
     174   
     175    return true;
     176}
     177
     178bool ppImageDetrendPatternContinuityApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options)
     179{
     180    bool status;
     181
     182    if (!options->doPatternContinuity) return true;
     183
     184    assert(inputView->chip != -1);
     185    assert(inputView->cell == -1);
     186    assert(inputView->readout == -1);
    152187
    153188    // see the comment for PATTERN.ROW; the same rules apply for PATTERN.CELL
    154189
    155     if (options->doPatternContinuity) {
    156         int numCells = chip->cells->n;       // Number of cells
    157         psVector *tweak = psVectorAlloc(numCells, PS_TYPE_U8); // Tweak cell?
    158         pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
    159         *view = *inputView;
    160 
    161         pmHDU *hdu = pmHDUFromChip(chip);
    162         if (psMetadataLookupBool(NULL,hdu->header,"PTRN_CON")) {
    163           psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell continuity correction as it has already been done.");
    164           goto pattern_cell;
    165         }
    166 
    167         const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME");
    168         psLogMsg("ppImage", PS_LOG_INFO, "Performing pattern continuity correction for %s\n", chipName);
    169 
    170         for (int i = 0; i < chip->cells->n; i++) {
    171             view->cell = i;
    172 
    173             pmCell *cell = chip->cells->data[i]; // Cell of interest
    174 
    175             if (cell->readouts->n > 1) {
    176                 psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell continuity correction on video cell.");
    177                 continue;
    178             }
    179 
    180             bool doPattern = false;
    181             if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.CONTINUITY.SUBSET")) {
    182                 ESCAPE(false, "Unable to determine whether row pattern matching should be applied.");
    183             }
    184 
    185             if (doPattern) {
    186                 tweak->data.U8[i] = 0xFF;
    187             }
    188         }
    189 
    190         // Tweak the cells
    191         if (!pmPatternContinuity(chip, tweak, options->patternCellBG, options->patternCellMean,
    192                                  options->maskValue, options->darkMask,options->patternContinuityEdgeWidth)) {
    193             psFree(tweak);
    194             psFree(view);
    195             return false;
    196         }
     190    int numCells = chip->cells->n;       // Number of cells
     191    psVector *tweak = psVectorAlloc(numCells, PS_TYPE_U8); // Tweak cell?
     192    pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
     193    *view = *inputView;
     194
     195    pmHDU *hdu = pmHDUFromChip(chip);
     196    if (psMetadataLookupBool(&status, hdu->header, "PTRN_CON")) {
     197        psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell continuity correction as it has already been done.");
     198        return true;
     199    }
     200
     201    const char *chipName = psMetadataLookupStr(&status, chip->concepts, "CHIP.NAME");
     202    psLogMsg("ppImage", PS_LOG_INFO, "Performing pattern continuity correction for %s\n", chipName);
     203
     204    for (int i = 0; i < chip->cells->n; i++) {
     205        view->cell = i;
     206
     207        pmCell *cell = chip->cells->data[i]; // Cell of interest
     208
     209        if (cell->readouts->n > 1) {
     210            psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell continuity correction on video cell.");
     211            continue;
     212        }
     213
     214        bool doPattern = false;
     215        if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.CONTINUITY.SUBSET")) {
     216            ESCAPE(false, "Unable to determine whether row pattern matching should be applied.");
     217        }
     218
     219        if (doPattern) {
     220            tweak->data.U8[i] = 0xFF;
     221        }
     222    }
     223
     224    // Tweak the cells
     225    if (!pmPatternContinuity(chip, tweak, options->patternCellBG, options->patternCellMean,
     226                             options->maskValue, options->darkMask,options->patternContinuityEdgeWidth)) {
    197227        psFree(tweak);
    198228        psFree(view);
    199 
    200         psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_CON",PS_META_REPLACE,"PATTERN.CONTINUITY correction applied",true);
    201     }
    202 
    203  pattern_cell:
    204    
    205     if (options->doPatternCell) {
    206         int numCells = chip->cells->n;       // Number of cells
    207         psVector *tweak = psVectorAlloc(numCells, PS_TYPE_U8); // Tweak cell?
    208         pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
    209         *view = *inputView;
    210 
    211         pmHDU *hdu = pmHDUFromChip(chip);
    212         if (psMetadataLookupBool(NULL,hdu->header,"PTRN_CEL")) {
    213           psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell pattern correction as it has already been done.");
    214           goto pattern_done;
    215         }
    216 
    217         const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME");
    218         psLogMsg("ppImage", PS_LOG_INFO, "Performing cell pattern correction for %s\n", chipName);
    219 
    220         for (int i = 0; i < chip->cells->n; i++) {
    221             view->cell = i;
    222 
    223             pmCell *cell = chip->cells->data[i]; // Cell of interest
    224 
    225             if (cell->readouts->n > 1) {
    226                 psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell pattern correction on video cell.");
    227                 continue;
    228             }
    229 
    230             bool doPattern = false;
    231             if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.CELL.SUBSET")) {
    232                 ESCAPE(false, "Unable to determine whether row pattern matching should be applied.");
    233             }
    234             if (doPattern) {
    235                 tweak->data.U8[i] = 0xFF;
    236             }
    237         }
    238 
    239         // Tweak the cells
    240         if (!pmPatternCell(chip, tweak, options->patternCellBG, options->patternCellMean,
    241                            options->maskValue, options->darkMask)) {
    242             psFree(tweak);
    243             psFree(view);
    244             return false;
    245         }
     229        return false;
     230    }
     231    psFree(tweak);
     232    psFree(view);
     233
     234    psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_CON",PS_META_REPLACE,"PATTERN.CONTINUITY correction applied",true);
     235    return true;
     236}
     237
     238bool ppImageDetrendPatternCellApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options)
     239{
     240    bool status;
     241
     242    if (!options->doPatternCell) return true;
     243
     244    assert(inputView->chip != -1);
     245    assert(inputView->cell == -1);
     246    assert(inputView->readout == -1);
     247
     248    int numCells = chip->cells->n;       // Number of cells
     249    psVector *tweak = psVectorAlloc(numCells, PS_TYPE_U8); // Tweak cell?
     250    pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
     251    *view = *inputView;
     252
     253    pmHDU *hdu = pmHDUFromChip(chip);
     254    if (psMetadataLookupBool(&status, hdu->header, "PTRN_CEL")) {
     255        psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell pattern correction as it has already been done.");
     256        return true;
     257    }
     258
     259    const char *chipName = psMetadataLookupStr(&status, chip->concepts, "CHIP.NAME");
     260    psLogMsg("ppImage", PS_LOG_INFO, "Performing cell pattern correction for %s\n", chipName);
     261
     262    for (int i = 0; i < chip->cells->n; i++) {
     263        view->cell = i;
     264
     265        pmCell *cell = chip->cells->data[i]; // Cell of interest
     266
     267        if (cell->readouts->n > 1) {
     268            psLogMsg("ppImage", PS_LOG_INFO, "Not performing cell pattern correction on video cell.");
     269            continue;
     270        }
     271
     272        bool doPattern = false;
     273        if (!doPatternForView(&doPattern, config, chip, view, RECIPE_NAME, "PATTERN.CELL.SUBSET")) {
     274            ESCAPE(false, "Unable to determine whether row pattern matching should be applied.");
     275        }
     276        if (doPattern) {
     277            tweak->data.U8[i] = 0xFF;
     278        }
     279    }
     280
     281    // Tweak the cells
     282    if (!pmPatternCell(chip, tweak, options->patternCellBG, options->patternCellMean,
     283                       options->maskValue, options->darkMask)) {
    246284        psFree(tweak);
    247285        psFree(view);
    248 
    249         psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_CEL",PS_META_REPLACE,"PATTERN.CELL correction applied",true);
    250     }
    251 
    252  pattern_done:
    253 
    254     // psLogMsg ("ppImage", 5, "apply pattern: %f sec\n", psTimerMark ("apply.pattern"));
    255 
    256     return(true);
     286        return false;
     287    }
     288    psFree(tweak);
     289    psFree(view);
     290
     291    psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_CEL",PS_META_REPLACE,"PATTERN.CELL correction applied",true);
     292    return true;
     293}
     294
     295bool ppImageDetrendPatternDeadCellsMask (pmChip *chip, psImageMaskType maskVal) {
     296
     297    int numCells = chip->cells->n;       // Number of cells
     298
     299    // now mask bad cells
     300    for (int i = 0; i < numCells; i++) {
     301        pmCell *cell = chip->cells->data[i]; // Cell of interest
     302        pmReadout *ro = cell->readouts->data[0]; // Readout of interest
     303
     304        psImage *mask = ro->mask; // mask of interest
     305        int numCols = mask->numCols, numRows = mask->numRows; // Size of image
     306       
     307        for (int y = 0; y < numRows; y++) {
     308            for (int x = 0; x < numCols; x++) {
     309                mask->data.PS_TYPE_IMAGE_MASK_DATA[y][x] |= maskVal;
     310            }
     311        }
     312    }
     313    return true;
     314}
     315
     316bool ppImageDetrendPatternDeadCellsXY42 (pmChip *chip, psImageMaskType maskVal) {
     317
     318    bool status;
     319
     320    // extract the MEDIAN_CELL_BACKGROUND values to check for problems
     321    int numCells = chip->cells->n;       // Number of cells
     322    psVector *cellBackground = psVectorAllocEmpty (numCells, PS_DATA_F32);
     323
     324    for (int i = 0; i < numCells; i++) {
     325        pmCell *cell = chip->cells->data[i];;
     326
     327        if (!cell->process || !cell->file_exists || !cell->data_exists) continue;
     328       
     329        // select the HDU for this cell
     330        pmHDU *cellHDU = pmHDUFromCell(cell);  // HDU of interest
     331       
     332        psF32 BackValue = psMetadataLookupF32(&status, cellHDU->header, "BACK_VAL");
     333        if (!status) continue;
     334
     335        psF32 BackError = psMetadataLookupF32(&status, cellHDU->header, "BACK_ERR");
     336        if (!status) continue;
     337
     338        if (BackValue == 0.0) continue;
     339        if (BackError == 0.0) continue;
     340
     341        psVectorAppend (cellBackground, BackValue / BackError);
     342    }
     343
     344    if (cellBackground->n < 0.375*numCells) {
     345        // Chip is bad, mask the whole thing
     346        ppImageDetrendPatternDeadCellsMask (chip, maskVal);
     347        psFree (cellBackground);
     348        return true;
     349    }
     350
     351    // Second, calculate the median
     352    psVectorSortInPlace (cellBackground);
     353    int midPt = cellBackground->n / 2.0;
     354    float median = cellBackground->n % 2 ? cellBackground->data.F32[midPt] : 0.5*(cellBackground->data.F32[midPt] + cellBackground->data.F32[midPt-1]);
     355    psFree (cellBackground);
     356
     357    // XXX hardwired number
     358    if (median < 3.0) {
     359        // Chip is bad, mask the whole thing
     360        ppImageDetrendPatternDeadCellsMask (chip, maskVal);
     361    }
     362    psFree (cellBackground);
     363    return true;
     364}
     365
     366// to apply the dead cell pattern, we need to transfer the pattern for this chip from the
     367// pmFPAfile for the pattern to the one for the image being processed. 
     368bool ppImageDetrendPatternDeadCellsApply(pmConfig *config, pmChip *chip, const pmFPAview *inputView, ppImageOptions *options)
     369{
     370    bool status;
     371
     372    if (!options->doPatternDeadCells) return true;
     373
     374    assert(inputView->chip != -1);
     375    assert(inputView->cell == -1);
     376    assert(inputView->readout == -1);
     377
     378    const char *chipName = psMetadataLookupStr(&status, chip->concepts, "CHIP.NAME");
     379    psLogMsg("ppImage", PS_LOG_INFO, "Performing cell pattern correction for %s\n", chipName);
     380
     381    pmHDU *hdu = pmHDUFromChip(chip);
     382    if (psMetadataLookupBool(&status, hdu->header, "PTRN_DED")) {
     383        psLogMsg("ppImage", PS_LOG_INFO, "Not performing dead cell pattern correction as it has already been done.");
     384        return true;
     385    }
     386
     387    if (!strcmp (chipName, "XY40") || !strcmp (chipName, "XY42")) {
     388        // special case : check for BACK_VAL / BACK_ERR > 3 or < 3
     389        psLogMsg("ppImage", PS_LOG_INFO, "Using special case for XY40 and XY42");
     390        ppImageDetrendPatternDeadCellsXY42 (chip, options->blankMask);
     391        return true;
     392    }
     393
     394    pmFPAfile *pattern = psMetadataLookupPtr(&status, config->files, "PPIMAGE.PATTERN.DEAD.CELLS");
     395    if (!pattern) {
     396        psLogMsg("ppImage", PS_LOG_INFO, "Pattern Dead Cells file not found, skipping");
     397        return true;
     398    }
     399
     400    pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
     401    *view = *inputView;
     402
     403    pmChip *patternChip = pmFPAviewThisChip (view, pattern->fpa);
     404
     405    // grab the pattern from the input pattern file chip
     406    psImage *deadCellPattern = (psImage *) psMetadataLookupPtr (&status, patternChip->analysis, "PTN.DEAD.CELL");
     407    if (!deadCellPattern) {
     408        psLogMsg("psModules.detrend", PS_LOG_DETAIL, "No DEAD CELL pattern for chip, skipping\n");
     409        psFree (view);
     410        return true;
     411    }
     412
     413    // copy the pattern pointer to the chip of the image being processed
     414    psMetadataAddImage (chip->analysis, PS_LIST_TAIL, "PTN.DEAD.CELL", PS_META_REPLACE, "", deadCellPattern);
     415
     416    // extract the MEDIAN_CELL_BACKGROUND values to check for problems
     417    int numCells = chip->cells->n;       // Number of cells
     418    psVector *cellBackground = psVectorAllocEmpty (numCells, PS_DATA_F32);
     419
     420    pmCell *cell = NULL;
     421    while ((cell = pmFPAviewNextCell(view, chip->parent, 1)) != NULL) {
     422        if (!cell->process || !cell->file_exists || !cell->data_exists) {
     423            psVectorAppend (cellBackground, NAN);
     424            continue;
     425        }
     426       
     427        // select the HDU for this cell
     428        pmHDU *cellHDU = pmHDUFromCell(cell);  // HDU of interest
     429       
     430        psF32 value = psMetadataLookupF32(&status, cellHDU->header, "BACK_VAL");
     431        if (!status) {
     432            psVectorAppend (cellBackground, NAN);
     433            continue;
     434        }
     435        psVectorAppend (cellBackground, value);
     436    }
     437
     438    // match cellBackground pattern to registered patterns and mask as needed
     439    if (!pmPatternDeadCells(chip, cellBackground, options->blankMask)) {
     440        psFree(cellBackground);
     441        psFree(view);
     442        return false;
     443    }
     444    psFree(cellBackground);
     445    psFree(view);
     446
     447    psMetadataAddBool(hdu->header, PS_LIST_TAIL, "PTRN_DED", PS_META_REPLACE, "PATTERN.DEADCELLS applied", true);
     448    return true;
    257449}
    258450
    … …  
    327519}
    328520
     521psVector *ppImageDetrendPatternCellFailures(pmFPAfile *input, const pmFPAview *inputView) {
     522   
     523    bool status = false;
     524    pmCell *cell = NULL;
     525   
     526    pmFPAview *view = pmFPAviewAlloc(0); // View for local processing
     527    *view = *inputView;
     528
     529    // extract the MEDIAN_CELL_BACKGROUND values to check for problems
     530    psVector *cellBackground = psVectorAllocEmpty (64, PS_DATA_F32);
     531    while ((cell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) {
     532        if (!cell->process || !cell->file_exists || !cell->data_exists) {
     533            psVectorAppend (cellBackground, NAN);
     534            continue;
     535        }
     536       
     537        // select the HDU for this cell
     538        pmHDU *hdu = pmHDUFromCell(cell);  // HDU of interest
     539       
     540        psF32 value = psMetadataLookupF32(&status, hdu->header, "BACK_VAL");
     541        psVectorAppend (cellBackground, value);
     542    }
     543    return cellBackground;
     544}

  • trunk/ppImage/src/ppImageDetrendReadout.c

    r42340 r42382  
    6363    // psLogMsg ("ppImage", 6, "check video: %f sec\n", psTimerMark ("detrend.readout"));
    6464
     65    // XXX for GPC2, there are some bizzare cell levels that need to be filtered out
     66    // Here I measure the median background before overscan subtraction, but an alternative
     67    // is to use the overscan-subtracted image.
     68    // XX if (0) {
     69    // XX       psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN);
     70    // XX       psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
     71    // XX       psImageBackground (stats, NULL, input->image, NULL, 0xffff, rng);
     72    // XX       readoutBackground = stats->robustMedian;
     73    // XX }
     74
    6575    // Masking on the basis of pixel value needs to be done before anything else, so the values are pristine.
    6676    if (options->doMaskBuild) {
    … …  
    113123      }
    114124      // psLogMsg ("ppImage", 6, "subtract overscan: %f sec\n", psTimerMark ("detrend.readout"));
     125    }
     126
     127    // measure the overscan-subtracted readoutBackground here (or subtract the overscan value?)
     128    // XXX this is the measurements and should be independent of the pattern masking
     129    if (options->doPatternDeadCells) {
     130        psStats *stats = psStatsAlloc (PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV);
     131        psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS);
     132        psImageBackground (stats, NULL, input->image, NULL, 0xffff, rng);
     133
     134        // save this value somewhere
     135        pmHDU *hdu = pmHDUFromReadout(input);  // HDU of interest
     136        psMetadataAddF32(hdu->header, PS_LIST_TAIL, "BACK_VAL", PS_META_REPLACE, "Median cell background", stats->robustMedian);
     137        psMetadataAddF32(hdu->header, PS_LIST_TAIL, "BACK_ERR", PS_META_REPLACE, "Stdev of cell background", stats->robustStdev);
     138        psFree (stats);
     139        psFree (rng);
    115140    }
    116141

  • trunk/ppImage/src/ppImageLoop.c

    r39501 r42382  
    172172
    173173        // Apply the fringe correction
    174         if (options->doFringe) {
    175             if (!ppImageDetrendFringeApply(config, chip, view, options)) {
    176                 ESCAPE("Unable to defringe");
    177             }
     174        if (!ppImageDetrendFringeApply(config, chip, view, options)) {
     175            ESCAPE("Unable to defringe");
    178176        }
    179177
    … …  
    183181        }
    184182
    185         // Apply the pattern correction
    186         if (options->doPatternRow || options->doPatternCell || options->doPatternContinuity) {
    187           if (!ppImageDetrendPatternApply(config,chip,view,options)) {
    188             ESCAPE("Unable to apply pattern corrections");
    189           }
     183        // Apply the pattern correction (only done if requested)
     184        if (!ppImageDetrendPatternApply(config,chip,view,options)) {
     185            ESCAPE("Problem applying pattern corrections");
    190186        }
    191187

  • trunk/ppImage/src/ppImageOptions.c

    r42340 r42382  
    226226        }
    227227
     228        bool mdok;
    228229        int boxcar = psMetadataLookupS32(NULL, recipe, "OVERSCAN.BOXCAR");
    229230        float gauss = psMetadataLookupF32(NULL, recipe, "OVERSCAN.GAUSS");
     231        float minValid = psMetadataLookupF32(&mdok, recipe, "OVERSCAN.MIN.VALID");
     232        if (!mdok) { minValid = 0.0; }
     233
     234        float maxValid = psMetadataLookupF32(&mdok, recipe, "OVERSCAN.MAX.VALID");
     235        if (!mdok) { maxValid = (float) 0x10000; }
    230236
    231237        // Fill in the options
    … …  
    235241        options->overscan->constant = psMetadataLookupBool(NULL, recipe, "OVERSCAN.CONSTANT");
    236242        options->overscan->value = psMetadataLookupF32(NULL, recipe, "OVERSCAN.VALUE");
     243        options->overscan->minValid = minValid;
     244        options->overscan->maxValid = maxValid;
     245        options->overscan->maskVal  = 0x0001;
    237246
    238247        psFree(overscanStats);
    … …  
    290299    options->doPatternCell = psMetadataLookupBool(NULL, recipe, "PATTERN.CELL");
    291300    options->doPatternContinuity = psMetadataLookupBool(NULL, recipe, "PATTERN.CONTINUITY");
     301    options->doPatternDeadCells = psMetadataLookupBool(NULL, recipe, "PATTERN.DEAD.CELLS");
    292302
    293303    options->doMaskStats = psMetadataLookupBool(NULL, recipe, "MASK.STATS");

  • trunk/ppImage/src/ppImageParseCamera.c

    r42340 r42382  
    201201            // an empty or invalid file may have been generated -- we want to skip, not raise an error
    202202            pmFPAfile *PRAfile = psMetadataLookupPtr (NULL, config->files, "PPIMAGE.PATTERN.ROW.AMP");
     203            if (PRAfile) {
     204              PRAfile->state |= PM_FPA_STATE_INACTIVE;
     205            }
     206        }
     207    }
     208    if (options->doPatternDeadCells) {
     209        if (!ppImageDefineFile(config, input->fpa, "PPIMAGE.PATTERN.DEAD.CELLS", "PATTERN.DEAD.CELLS",
     210                               PM_FPA_FILE_PATTERN_DEAD_CELLS, PM_DETREND_TYPE_PATTERN_DEAD_CELLS)) {
     211            psWarning ("Can't find a pattern dead cells source");
     212            // an empty or invalid file may have been generated -- we want to skip, not raise an error
     213            pmFPAfile *PRAfile = psMetadataLookupPtr (NULL, config->files, "PPIMAGE.PATTERN.DEAD.CELLS");
    203214            if (PRAfile) {
    204215              PRAfile->state |= PM_FPA_STATE_INACTIVE;

  • trunk/ppImage/src/ppImageRebinReadout.c

    r37406 r42382  
    3535
    3636            // run the rebin code
     37            // XXX EAM 2022.04.21 : this function rebins the signal image and makes an attempt to
     38            // generate a mask only with bits raised in masked pixels that have > 50% of input pixels masked.
     39            // this step is very expensive because it must count the input mask bits for each pixel.
     40            // What is particularly silly is that the mask is not even used in the jpeg image.
    3741            if (!pmReadoutRebin(outReadout, inReadout, options->maskValue, outFile->xBin, outFile->yBin)) {
    3842                psError(PS_ERR_UNKNOWN, false, "Unable to rebin readout.");

  • trunk/ppImage/src/ppImageReplaceBackground.c

    r41382 r42382  
    8181    if (!modelFile || !modelRO) {
    8282        if (!psphotModelBackgroundReadoutFileIndex(config, &roView, "PPIMAGE.CHIP", 0)) {
    83             psError(PS_ERR_UNKNOWN, false, "Unable to model background");
    84             return false;
     83            int lastError = psErrorCodeLast();
     84            if (lastError == PSPHOT_ERR_DATA) {
     85              // a data error in psphotModelBackground* means an empty or bad image: skip background subtraction
     86              psErrorStackPrint(stderr, "Unable to model background");
     87              psErrorClear();
     88              return true;
     89            } else {
     90              psError(PS_ERR_UNKNOWN, false, "Unable to model background");
     91              return false;
     92            }
    8593        }
    8694        // the model file should now at least be defined

  • trunk/ppImage/src/ppImageSetMaskBits.c

    r42293 r42382  
    7979    options->blankMask = pmConfigMaskGet("BLANK", config);
    8080    psAssert (options->blankMask, "blank mask not set");
     81    if (options->overscan) {
     82        options->overscan->maskVal  = options->blankMask;
     83    }
    8184
    8285    // mask for saturated data  (default to RANGE if not defined)
Note: See TracChangeset for help on using the changeset viewer.