IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 23688 for trunk/ppSub/src


Ignore:
Timestamp:
Apr 2, 2009, 2:51:37 PM (17 years ago)
Author:
Paul Price
Message:

Merging in branches/pap/ from r23685. Some conflicts, most notably in ippTools, but these resolved fairly simply. Only question mark is on regtool reversion, but I think I got it right. Everything builds fine.

Location:
trunk
Files:
11 edited
1 copied

Legend:

Unmodified
Added
Removed

  • trunk

  • trunk/ppSub

  • trunk/ppSub/src/Makefile.am

    r23229 r23688  
    1919        ppSubBackground.c        \
    2020        ppSubCamera.c            \
     21        ppSubData.c              \
    2122        ppSubLoop.c              \
    2223        ppSubReadout.c           \

  • trunk/ppSub/src/ppSub.h

    r23195 r23688  
    2222#define PPSUB_RECIPE "PPSUB"            /// Name of the recipe to use
    2323
     24// Output files, for activation/deactivation
     25typedef enum {
     26    PPSUB_FILES_IMAGE = 0x01,           // Image files
     27    PPSUB_FILES_PHOT  = 0x02,           // Photometry files
     28    PPSUB_FILES_ALL   = 0xFF,           // All files
     29} ppSubFiles;
     30
     31/// Data for processing
     32typedef struct {
     33    psErrorCode quality;                /// Quality code; 0 for no problem
     34    psMetadata *stats;                  /// Statistics
     35} ppSubData;
     36
     37/// Constructor
     38ppSubData *ppSubDataAlloc(void);
     39
    2440/// Setup the arguments parsing
    2541bool ppSubArgumentsSetup(int argc, char *argv[], ///< Command-line arguments
    … …  
    4157/// Perform PSF-matched image subtraction on the readout
    4258bool ppSubReadout(pmConfig *config,     ///< Configuration
    43                   psMetadata *stats,    ///< Statistics, for output
     59                  ppSubData *data,      ///< Processing data
    4460                  const pmFPAview *view ///< View of readout to subtract
    4561    );
    … …  
    5369/// looking up the parameters in the recipe and supplying them to the function pmSubtractionMatch()
    5470bool ppSubMatchPSFs(pmConfig *config,    ///< Configuration
     71                    ppSubData *data,    ///< Processing data
    5572                    const pmFPAview *view ///< View of active readout
    5673    );
    … …  
    6380/// Photometry stage 1: measure the PSF from the minuend image
    6481bool ppSubMakePSF(pmConfig *config,       ///< Configuration
     82                  ppSubData *data,    ///< Processing data
    6583                  const pmFPAview *view ///< View of active readout
    6684    );
    … …  
    7492/// Photometry stage 2: find and measure sources on the subtracted image
    7593bool ppSubReadoutPhotometry(pmConfig *config,     ///< Configuration
    76                             psMetadata *stats,    ///< Statistics, for output
     94                            ppSubData *data,    ///< Processing data
    7795                            const pmFPAview *view ///< View of active readout
    7896    );
    … …  
    8098/// Renormalize, update headers and generate JPEGs
    8199bool ppSubReadoutUpdate(pmConfig *config, ///< Configuration
    82                         psMetadata *stats, ///< Statistics for output, or NULL
     100                        ppSubData *data,    ///< Processing data
    83101                        const pmFPAview *view ///< View of active readout
    84102    );
    … …  
    96114void ppSubVersionPrint(void);
    97115
     116/// Mark the data quality as bad and prepare to suspend processing
     117void ppSubDataQuality(pmConfig *config, ///< Configuration
     118                      ppSubData *data,  ///< Processing data
     119                      psErrorCode error,///< Error code
     120                      ppSubFiles files  ///< Files to deactivate
     121    );
    98122
    99123// Copy every instance of a single keyword from one metadata to another

  • trunk/ppSub/src/ppSubCamera.c

    r23449 r23688  
    285285    return true;
    286286}
     287
     288

  • trunk/ppSub/src/ppSubLoop.c

    r23315 r23688  
    2828    pmConfigRecipesCull(config, "PPSUB,PPSTATS,PSPHOT,MASKS,JPEG");
    2929
     30    ppSubData *data = ppSubDataAlloc(); // Processing data
     31
    3032    bool mdok;                          // Status of MD lookup
    3133    const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics
    32     psMetadata *stats = NULL;           // Container for statistics
    3334    FILE *statsFile = NULL;             // File stream for statistics
    3435    if (statsName && strlen(statsName) > 0) {
    … …  
    3839            psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved);
    3940            psFree(resolved);
    40             return false;
    41         } else {
    42             stats = psMetadataAlloc();
     41            goto ERROR;
    4342        }
    4443        psFree(resolved);
    … …  
    4847    if (!input) {
    4948        psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find input data!\n");
    50         return false;
     49        goto ERROR;
    5150    }
    5251
    … …  
    5453    if (!reference) {
    5554        psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find reference data!\n");
    56         return false;
     55        goto ERROR;
    5756    }
    5857
    … …  
    6059    if (!output) {
    6160        psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find output data!\n");
    62         return false;
     61        goto ERROR;
    6362    }
    6463
    … …  
    6766    // Iterate over the FPA hierarchy
    6867    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
    69         return false;
     68        goto ERROR;
    7069    }
    7170
    … …  
    7776            psError(PS_ERR_BAD_PARAMETER_VALUE, true, "FPA format discrepency between input and reference");
    7877            psFree(view);
    79             return false;
     78            goto ERROR;
    8079        }
    8180
    … …  
    8584
    8685        if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
    87             return false;
     86            goto ERROR;
    8887        }
    8988
    … …  
    9695                        "FPA format discrepency between input and reference");
    9796                psFree(view);
    98                 return false;
     97                goto ERROR;
    9998            }
    10099            if (!inCell->file_exists) {
    … …  
    102101            }
    103102            if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
    104                 return false;
     103                goto ERROR;
    105104            }
    106105
    … …  
    108107            while ((inRO = pmFPAviewNextReadout(view, input->fpa, 1))) {
    109108                if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
    110                     return false;
     109                    goto ERROR;
    111110                }
    112111                pmReadout *refRO = pmFPAviewThisReadout(view, reference->fpa);// Reference readout of interest
    … …  
    116115                            "FPA format discrepency between input and reference");
    117116                    psFree(view);
    118                     return false;
     117                    goto ERROR;
    119118                }
    120119                if (!inRO->data_exists) {
    … …  
    123122
    124123                // Perform the analysis
    125                 if (!ppSubReadout(config, stats, view)) {
     124                if (!ppSubReadout(config, data, view)) {
    126125                    psError(PS_ERR_UNKNOWN, false, "Unable to subtract images.\n");
    127                     return false;
     126                    goto ERROR;
    128127                }
    129128
    130129                if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
    131                     return false;
     130                    goto ERROR;
    132131                }
    133132            }
    134133
    135134            // Perform statistics on the cell
    136             if (stats) {
     135            if (statsFile) {
    137136                pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSUB.OUTPUT"); // Output file
    138137                if (!output) {
    139138                    psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find file PPSUB.OUTPUT.\n");
    140                     return false;
     139                    goto ERROR;
    141140                }
    142141                psImageMaskType maskValue = pmConfigMaskGet("MASK.VALUE", config);
    143                 ppStatsFPA(stats, output->fpa, view, maskValue, config);
     142                ppStatsFPA(data->stats, output->fpa, view, maskValue, config);
    144143            }
    145144
    146145            if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
    147                 return false;
     146                goto ERROR;
    148147            }
    149148        }
    150149
    151150        if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
    152             return false;
     151            goto ERROR;
    153152        }
    154153    }
    155154
    156155    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
    157         return false;
     156        goto ERROR;
    158157    }
    159158
    … …  
    161160
    162161    // Write out summary statistics
    163     if (stats) {
    164         psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_SUB", 0, "Time for subtraction completion",
     162    if (statsFile) {
     163        psMetadataAddF32(data->stats, PS_LIST_TAIL, "TIME_SUB", 0, "Time for subtraction completion",
    165164                         psTimerMark("ppSub"));
    166165
    167         const char *statsMDC = psMetadataConfigFormat(stats);
     166        const char *statsMDC = psMetadataConfigFormat(data->stats);
    168167        if (!statsMDC || strlen(statsMDC) == 0) {
    169168            psWarning("Unable to generate statistics MDC file.\n");
    … …  
    173172        psFree((void *)statsMDC);
    174173        fclose(statsFile);
    175 
    176         psFree(stats);
    177174    }
    178175
    179176    psString dump_file = psMetadataLookupStr(&mdok, config->arguments, "-dumpconfig");
    180177    if (dump_file) {
    181 
    182178        pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); // Input file
    183179        pmConfigDump(config, input->fpa, dump_file);
    184180    }
    185181
     182    psFree(data);
    186183    return true;
     184
     185ERROR:
     186    psFree(data);
     187    return false;
    187188}

  • trunk/ppSub/src/ppSubMakePSF.c

    r23235 r23688  
    2222#include "ppSub.h"
    2323
    24 bool ppSubMakePSF(pmConfig *config, const pmFPAview *view)
     24bool ppSubMakePSF(pmConfig *config, ppSubData *data, const pmFPAview *view)
    2525{
    2626    psAssert(config, "Require configuration");
    … …  
    8383    psArray *sources = psMetadataLookupPtr(&mdok, minuend->analysis, "PSPHOT.SOURCES");
    8484    if (!psphotReadoutFindPSF(config, view, sources)) {
    85         psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry on subtracted image.");
    86         return false;
     85        // This is likely a data quality issue
     86        // XXX Split into multiple cases using error codes?
     87        psErrorStackPrint(stderr, "Unable to determine PSF");
     88        psWarning("Unable to determine PSF --- suspect bad data quality.");
     89        ppSubDataQuality(config, data, PSPHOT_ERR_PSF, PPSUB_FILES_PHOT);
     90        return true;
    8791    }
    8892
    … …  
    98102    return true;
    99103}
    100 
    101 // XXX we used to need this, is it still needed?
    102 
    103 // pmCell *photCell = pmFPAfileThisCell(config->files, view, "PSPHOT.INPUT");
    104 // pmCellFreeReadouts(photCell);
    105 
    106 // if (!pmFPAfileDropInternal(config->files, "PSPHOT.BACKMDL") ||
    107 //     !pmFPAfileDropInternal (config->files, "PSPHOT.BACKMDL.STDEV") ||
    108 //     !pmFPAfileDropInternal (config->files, "PSPHOT.BACKGND")) {
    109 //     psError(PS_ERR_UNKNOWN, false, "Unable to drop PSPHOT internal files.");
    110 //     return false;
    111 // }
    112 
    113 // Blow away the sources psphot found --- they're irrelevant for the subtraction
    114 // XXX is this still needed?  These are now probably not being set
    115 // pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout with sources
    116 // psMetadataRemoveKey(photRO->analysis, "PSPHOT.SOURCES");
    117 // psMetadataRemoveKey(photRO->analysis, "PSPHOT.HEADER");
    118 

  • trunk/ppSub/src/ppSubMatchPSFs.c

    r23505 r23688  
    2222#include "ppSub.h"
    2323
    24 bool ppSubMatchPSFs(pmConfig *config, const pmFPAview *view)
     24bool ppSubMatchPSFs(pmConfig *config, ppSubData *data, const pmFPAview *view)
    2525{
    2626    psAssert(config, "Require configuration");
    … …  
    136136
    137137    // Match the PSFs
     138    bool success = false;               // Operation was successful?
    138139    if (kernelRO) {
    139         if (!pmSubtractionMatchPrecalc(inConv, refConv, inRO, refRO, kernelRO->analysis,
    140                                        stride, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac)) {
    141             psError(PS_ERR_UNKNOWN, false, "Unable to convolve images.");
    142             return false;
    143         }
     140        success = pmSubtractionMatchPrecalc(inConv, refConv, inRO, refRO, kernelRO->analysis,
     141                                            stride, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac);
    144142    } else {
    145         if (!pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, stride, regionSize, spacing,
    146                                 threshold, sources, stampsName, type, size, order, widths, orders, inner,
    147                                 ringsOrder, binning, penalty, optimum, optWidths, optOrder, optThresh, iter,
    148                                 rej, sys, maskVal, maskBad, maskPoor, poorFrac, badFrac, subMode)) {
     143        success = pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, stride, regionSize,
     144                                     spacing, threshold, sources, stampsName, type, size, order,
     145                                     widths, orders, inner, ringsOrder, binning, penalty, optimum,
     146                                     optWidths, optOrder, optThresh, iter, rej, sys, maskVal,
     147                                     maskBad, maskPoor, poorFrac, badFrac, subMode);
     148    }
     149
     150    psFree(optWidths);
     151    pmSubtractionThreadsFinalize(inRO, refRO);
     152
     153    if (!success) {
     154        psErrorCode error = psErrorCodeLast(); // Error code
     155        if (error == PM_ERR_STAMPS) {
     156            psErrorStackPrint(stderr, "Unable to find stamps");
     157            psWarning("Unable to find stamps --- suspect bad data quality.");
     158            ppSubDataQuality(config, data, error, PPSUB_FILES_ALL);
     159            return true;
     160        } else {
    149161            psError(PS_ERR_UNKNOWN, false, "Unable to match images.");
    150162            return false;
    … …  
    152164    }
    153165
    154     psFree(optWidths);
    155 
    156     pmSubtractionThreadsFinalize(inRO, refRO);
    157 
    158166    psImageCovarianceTransfer(inConv->variance, inConv->covariance);
    159167    psImageCovarianceTransfer(refConv->variance, refConv->covariance);
    160168
    161     // XXX drop the pixels associated with inRO and refRO (now that we have inConv and refConf)
    162 #ifdef TESTING
    163     psphotSaveImage (NULL, inRO->image, "inRO.fits");
    164     psphotSaveImage (NULL, refRO->image, "refRO.fits");
    165     psphotSaveImage (NULL, inConv->image, "inConv.fits");
    166     psphotSaveImage (NULL, refConv->image, "refConv.fits");
    167 #endif
    168 
    169169    return true;
    170170}

  • trunk/ppSub/src/ppSubReadout.c

    r21524 r23688  
    2121#include "ppSub.h"
    2222
    23 bool ppSubReadout(pmConfig *config, psMetadata *stats, const pmFPAview *view)
     23bool ppSubReadout(pmConfig *config, ppSubData *data, const pmFPAview *view)
    2424{
    2525    psTimerStart("PPSUB_MATCH");
    … …  
    3030    }
    3131
    32     if (!ppSubMatchPSFs(config, view)) {
     32    if (!ppSubMatchPSFs(config, data, view)) {
    3333        psError(PS_ERR_UNKNOWN, false, "Unable to match PSFs.");
    3434        return false;
     35    } else if (data->quality) {
     36        // Can't do anything at all
     37        return true;
    3538    }
    3639
    … …  
    4043    }
    4144
    42     if (!ppSubMakePSF(config, view)) {
     45    if (!data->quality && !ppSubMakePSF(config, data, view)) {
    4346        psError(PS_ERR_UNKNOWN, false, "Unable to generate PSF.");
    4447        return false;
    … …  
    5659    }
    5760
    58     if (!ppSubReadoutPhotometry(config, stats, view)) {
     61    if (!data->quality && !ppSubReadoutPhotometry(config, data, view)) {
    5962        psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry.");
    6063        return false;
    6164    }
    6265
    63     if (!ppSubReadoutUpdate(config, stats, view)) {
     66    if (!ppSubReadoutUpdate(config, data, view)) {
    6467        psError(PS_ERR_UNKNOWN, false, "Unable to update.");
    6568        return false;

  • trunk/ppSub/src/ppSubReadoutPhotometry.c

    r21524 r23688  
    2222#include "ppSub.h"
    2323
    24 bool ppSubReadoutPhotometry (pmConfig *config, psMetadata *stats, const pmFPAview *view)
     24bool ppSubReadoutPhotometry (pmConfig *config, ppSubData *data, const pmFPAview *view)
    2525{
    2626    psAssert(config, "Require configuration");
    … …  
    4545    pmPSF *psf = psMetadataLookupPtr(NULL, psfInputChip->analysis, "PSPHOT.PSF"); // PSF for photometry
    4646    if (!psf) {
    47         psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find PSF from psphot");
    48         return false;
     47        psErrorStackPrint(stderr, "No PSF available");
     48        psWarning("No PSF available --- suspect bad data quality.");
     49        ppSubDataQuality(config, data, psErrorCodeLast(), PPSUB_FILES_PHOT);
     50        return true;
    4951    }
    5052    psMetadataAddPtr(psfLoadChip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN | PS_META_REPLACE,
    … …  
    8082    }
    8183
    82 #if 0
    83     psMetadataAddPtr(config->files, PS_LIST_TAIL, "PSPHOT.INPUT", PS_DATA_UNKNOWN | PS_META_REPLACE,
    84                      "psphot input: view on another pmFPAfile", photFile);
    85 #endif
     84    if (!psphotReadoutMinimal(config, view)) {
     85        // This is likely a data quality issue
     86        // XXX Split into multiple cases using error codes?
     87        psErrorStackPrint(stderr, "Unable to perform photometry on image");
     88        psWarning("Unable to perform photometry on image --- suspect bad data quality.");
     89        ppSubDataQuality(config, data, psErrorCodeLast(), PPSUB_FILES_PHOT);
     90    }
    8691
    87     if (!psphotReadoutMinimal(config, view)) {
    88         psWarning("Unable to perform photometry on subtracted image.");
    89         psErrorStackPrint(stderr, "Error stack from photometry:");
    90         psErrorClear();
    91     }
    92 #if 1
    9392    photRO->data_exists = true;
    9493    photRO->parent->data_exists = true;
    9594    photRO->parent->parent->data_exists = true;
    96 #endif
    9795
    98     if (stats) {
     96    if (data->stats) {
    9997        psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources
    100         psMetadataAddS32(stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected",
     98        psMetadataAddS32(data->stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected",
    10199                         sources ? sources->n : 0);
    102         psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_PHOT", 0, "Time to do photometry",
     100        psMetadataAddF32(data->stats, PS_LIST_TAIL, "TIME_PHOT", 0, "Time to do photometry",
    103101                         psTimerClear("PPSUB_PHOT"));
    104102    }

  • trunk/ppSub/src/ppSubReadoutUpdate.c

    r23173 r23688  
    2121#include "ppSub.h"
    2222
    23 bool ppSubReadoutUpdate(pmConfig *config, psMetadata *stats, const pmFPAview *view)
     23bool ppSubReadoutUpdate(pmConfig *config, ppSubData *data, const pmFPAview *view)
    2424{
    2525    psAssert(config, "Require configuration");
    … …  
    4747
    4848    // Statistics on the matching
    49     if (stats) {
    50         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MODE);
    51         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS);
    52         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_MEAN);
    53         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_RMS);
    54         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_NORM);
    55         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_BGDIFF);
    56         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MX);
    57         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MY);
    58         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXX);
    59         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXY);
    60         psMetadataCopySingle(stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MYY);
     49    if (data->stats) {
     50        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MODE);
     51        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS);
     52        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_MEAN);
     53        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_DEV_RMS);
     54        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_NORM);
     55        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_BGDIFF);
     56        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MX);
     57        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MY);
     58        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXX);
     59        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MXY);
     60        psMetadataCopySingle(data->stats, outRO->analysis, PM_SUBTRACTION_ANALYSIS_MYY);
    6161
    62         psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_MATCH", 0, "Time to match PSFs",
     62        psMetadataAddF32(data->stats, PS_LIST_TAIL, "TIME_MATCH", 0, "Time to match PSFs",
    6363                         psTimerClear("PPSUB_MATCH"));
    6464    }
Note: See TracChangeset for help on using the changeset viewer.