IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 30193


Ignore:
Timestamp:
Dec 28, 2010, 10:40:28 AM (15 years ago)
Author:
rhenders
Message:

Was not correctly handling removeList, for detections with bogus detection IDs. Fixed and cleaned up.

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippToPsps/src/DetectionBatch.c

    r30189 r30193  
    11#include "DetectionBatch.h"
    22#include "DetectionBatchEnums.h"
     3
     4/**
     5  Source for all methods in the DetectionBatch class
     6  */
    37
    48/**
    … …  
    1418
    1519/**
    16   Does the work.
     20  Does the work. Loops through the provided smf files, tallies the contents with the DVO database already connected to and generates a new FITS file of the fomat required by PSPS (i.e. a binary table per PSPS database table)
    1721  */
    1822static int run(DetectionBatch* this) {
    … …  
    8286
    8387    // stuff for detections table
    84     uint32_t s,d, invalidDvoRows, unmatched, totalDetections = 0;
     88    uint32_t s,d, invalidDvoRows, nChipDetectionsOut = 0;
    8589
    8690    long longnull = -999;
    … …  
    122126    short nOta = 0;
    123127    long i;
    124     bool isDuplicate;
    125128    uint32_t numOfDuplicates, numInvalidFlux, numOfInvalidIppIDet;
    126     long numDetectionsOut, totalDetectionsOut = 0, numPhotoRef, totalNumPhotoRef = 0;
     129    long totalDetectionsOut = 0, numPhotoRef, totalNumPhotoRef = 0;
    127130    long* removeList = (long*)calloc(this->MAXDETECT, sizeof(long));
    128131    long thisObjId;
    … …  
    220223            nOta++;
    221224
    222             long nDet = 0;
    223             if (fits_get_num_rows(fitsIn, &nDet, &status)) {
     225            long nChipDetectionsIn = 0;
     226            if (fits_get_num_rows(fitsIn, &nChipDetectionsIn, &status)) {
    224227                fits_report_error(stderr, status);
    225228            }
    226229
    227230            // loop round detections to populate some extra stuff
    228             s = d = totalDetections = invalidDvoRows = unmatched = 0;
     231            s = d = nChipDetectionsOut = invalidDvoRows = 0;
    229232
    230233            // determine column numbers for certain IPP detection columns
    … …  
    244247
    245248            anynull = 0;
    246             fits_read_col(fitsIn, TLONG, ippIDetNum, 1, 1, nDet, &longnull, ippIDet, &anynull, &status);
    247             fits_read_col(fitsIn, TFLOAT, instMagNum, 1, 1, nDet, &floatnull, instMag, &anynull, &status);
    248             fits_read_col(fitsIn, TFLOAT, instMagErrNum, 1, 1, nDet, &floatnull, instMagErr, &anynull, &status);
    249             fits_read_col(fitsIn, TFLOAT, peakMagNum, 1, 1, nDet, &floatnull, peakMag, &anynull, &status);
     249            fits_read_col(fitsIn, TLONG, ippIDetNum, 1, 1, nChipDetectionsIn, &longnull, ippIDet, &anynull, &status);
     250            fits_read_col(fitsIn, TFLOAT, instMagNum, 1, 1, nChipDetectionsIn, &floatnull, instMag, &anynull, &status);
     251            fits_read_col(fitsIn, TFLOAT, instMagErrNum, 1, 1, nChipDetectionsIn, &floatnull, instMagErr, &anynull, &status);
     252            fits_read_col(fitsIn, TFLOAT, peakMagNum, 1, 1, nChipDetectionsIn, &floatnull, peakMag, &anynull, &status);
    250253
    251254            // DVO detections are ordered by IPP_IDET, which increments from 0 in SMF table
    … …  
    254257            numInvalidFlux = 0;
    255258            numOfInvalidIppIDet = 0;
    256             for (long s = 0; s<nDet; s++) {
    257 
    258                 isDuplicate = false;
    259 
    260                 // catch crazy large or negative IPP_IDET
     259           
     260            // loop through all detections in smf file extension for this chip.
     261            // there are three ways for a detection to be ommitted from the final FITS output:
     262            // 1. it has a duplicate obj ID to a previous detection
     263            // 2. it has an invalid detection ID (< 0 or > max DVO det ID)
     264            // 3. it has an invalid flux value
     265            for (long s = 0; s<nChipDetectionsIn; s++) {
     266
     267                // catch crazy large or negative IPP_IDET, and skip them
    261268                if (ippIDet[s] < 0 || ippIDet[s] > maxDvoDetId) {
    262269
    263270                    removeList[numOfDuplicates+numInvalidFlux+numOfInvalidIppIDet] = s+1;
    264271                    numOfInvalidIppIDet++;
     272                    continue;
    265273                }
    266                 else {
    267                     while (ippIDet[s] > dvoDetections[d].meas.detID && dvoDetections[d].meas.detID <= maxDvoDetId) {
    268 
    269                         d++;
    270                         if (!dvoDetections[d].valid) invalidDvoRows++;
    271                     }
     274
     275                // loop forward through DVO detections until we find corresponding detection ID
     276                while (ippIDet[s] > dvoDetections[d].meas.detID && dvoDetections[d].meas.detID <= maxDvoDetId) {
     277
     278                    d++;
     279                    if (!dvoDetections[d].valid) invalidDvoRows++;
    272280                }
     281
     282                // if we have found matching detection IDs then continue to populate other columns
    273283                if (ippIDet[s] == dvoDetections[d].meas.detID ) {
    274284
    275                     // TODO HACK duplicates bug - don't include any duplicates
     285                    // check if we have already seen this obj ID, and if we have, skip this detection
    276286                    thisObjId = dvoDetections[d].ave.extID;
    277                     for (i=0; i<totalDetections; i++) {
     287                    bool isDuplicate = false;
     288                    for (i=0; i<s; i++) {
    278289                        if (thisObjId == objID[i]) {
    279290                            removeList[numOfDuplicates+numInvalidFlux+numOfInvalidIppIDet] = s+1;
    … …  
    283294                        }
    284295                    }
     296                    if (isDuplicate) continue;
     297
    285298                    // populate some arrays to be shoved in table in bulk later
    286299                    objID[s] = thisObjId;
    … …  
    298311                    instFluxOk = getFlux(exposureTime, instMag[s], &instFlux[s], instMagErr[s], &instFluxErr[s]);
    299312
    300                     // check for invalid flux values (if not already labelled as a duplicate)
    301                     if ( (!peakFluxOk || !instFluxOk) && !isDuplicate) {
     313                    // check for invalid flux values, and skip them
     314                    if ( !peakFluxOk || !instFluxOk) {
    302315                        removeList[numOfDuplicates+numInvalidFlux+numOfInvalidIppIDet] = s+1;
    303316                        numInvalidFlux++;
     317                        continue;
    304318                    }
    305                     else {
    306 
    307                         // store max/min objID
    308                         if (objID[s] > maxObjID) maxObjID = objID[s];
    309                         if (objID[s] < minObjID) minObjID = objID[s];
    310                     }
    311                     totalDetections++;
     319
     320                    // store max/min objID
     321                    if (objID[s] > maxObjID) maxObjID = objID[s];
     322                    if (objID[s] < minObjID) minObjID = objID[s];
     323
     324                    nChipDetectionsOut++;
    312325                }
    313                 // if we get here then we cannot find this smf detection in DVO
    314                 else {
    315 
    316                     unmatched++;
    317                     continue;
    318                 }
    319 
    320                 if (dvoDetections[d].meas.detID == maxDvoDetId) break;
    321             }
     326            } // end of loop round detections
    322327
    323328            // write number of rows (detections) to ImageMeta
    324             numDetectionsOut = totalDetections - numOfDuplicates - numInvalidFlux;
    325             fits_write_col(this->base.fitsOut, TLONG, IMAGEMETA_NDETECT, 1, 1, 1, &numDetectionsOut, &status);
     329            fits_write_col(this->base.fitsOut, TLONG, IMAGEMETA_NDETECT, 1, 1, 1, &nChipDetectionsOut, &status);
     330
     331            int totalRemove = numOfDuplicates+numInvalidFlux+numOfInvalidIppIDet;
    326332
    327333            // avoid writing out an empty table of detections
    328             if (numDetectionsOut > 0) {
     334            if (nChipDetectionsOut > 0) {
    329335
    330336                // detections
    331                 ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nDet, "Detection", false);
    332                 fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_OBJID, 1, 1, nDet, objID, &status);
    333                 fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_DETECTID, 1, 1, nDet, detectID, &status);
    334                 fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IPPOBJID, 1, 1, nDet, ippObjID, &status);
    335                 fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IPPDETECTID, 1, 1, nDet, ippDetectID, &status);
    336                 fits_write_col(this->base.fitsOut, TBYTE, DETECTION_FILTERID, 1, 1, nDet, filterIDs, &status);
    337                 fits_write_col(this->base.fitsOut, TBYTE, DETECTION_SURVEYID, 1, 1, nDet, surveyIDs, &status);
    338                 fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IMAGEID, 1, 1, nDet, imageID, &status);
    339                 fits_write_col(this->base.fitsOut, TDOUBLE, DETECTION_OBSTIME, 1, 1, nDet, obsTimes, &status);
    340                 fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_INSTFLUX, 1, 1, nDet, instFlux, &status);
    341                 fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_INSTFLUXERR, 1, 1, nDet, instFluxErr, &status);
    342                 fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_PEAKADU, 1, 1, nDet, peakFlux, &status);
    343                 fits_write_col(this->base.fitsOut, TSTRING, DETECTION_ASSOCDATE, 1, 1, nDet, assocDate, &status);
    344                 fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_INFOFLAG, 1, 1, nDet, flags, &status);
    345                 if (numOfDuplicates||numInvalidFlux) fits_delete_rowlist(this->base.fitsOut, removeList, numOfDuplicates+numInvalidFlux, &status);
     337                ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nChipDetectionsIn, "Detection", false);
     338                fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_OBJID, 1, 1, nChipDetectionsIn, objID, &status);
     339                fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_DETECTID, 1, 1, nChipDetectionsIn, detectID, &status);
     340                fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IPPOBJID, 1, 1, nChipDetectionsIn, ippObjID, &status);
     341                fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IPPDETECTID, 1, 1, nChipDetectionsIn, ippDetectID, &status);
     342                fits_write_col(this->base.fitsOut, TBYTE, DETECTION_FILTERID, 1, 1, nChipDetectionsIn, filterIDs, &status);
     343                fits_write_col(this->base.fitsOut, TBYTE, DETECTION_SURVEYID, 1, 1, nChipDetectionsIn, surveyIDs, &status);
     344                fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_IMAGEID, 1, 1, nChipDetectionsIn, imageID, &status);
     345                fits_write_col(this->base.fitsOut, TDOUBLE, DETECTION_OBSTIME, 1, 1, nChipDetectionsIn, obsTimes, &status);
     346                fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_INSTFLUX, 1, 1, nChipDetectionsIn, instFlux, &status);
     347                fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_INSTFLUXERR, 1, 1, nChipDetectionsIn, instFluxErr, &status);
     348                fits_write_col(this->base.fitsOut, TFLOAT, DETECTION_PEAKADU, 1, 1, nChipDetectionsIn, peakFlux, &status);
     349                fits_write_col(this->base.fitsOut, TSTRING, DETECTION_ASSOCDATE, 1, 1, nChipDetectionsIn, assocDate, &status);
     350                fits_write_col(this->base.fitsOut, TLONGLONG, DETECTION_INFOFLAG, 1, 1, nChipDetectionsIn, flags, &status);
     351                if (numOfDuplicates||numInvalidFlux||numOfInvalidIppIDet)
     352                    fits_delete_rowlist(this->base.fitsOut, removeList, totalRemove, &status);
    346353
    347354                // skinny object
    348                 ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nDet, "SkinnyObject", false);
    349                 fits_write_col(this->base.fitsOut, TLONGLONG, SKINNYOBJECT_OBJID, 1, 1, nDet, objID, &status);
    350                 fits_write_col(this->base.fitsOut, TLONGLONG, SKINNYOBJECT_IPPOBJID, 1, 1, nDet, ippObjID, &status);
    351                 fits_write_col(this->base.fitsOut, TBYTE, SKINNYOBJECT_SURVEYID, 1, 1, nDet, surveyIDs, &status);
    352                 if (numOfDuplicates||numInvalidFlux) fits_delete_rowlist(this->base.fitsOut, removeList, numOfDuplicates+numInvalidFlux, &status);
     355                ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nChipDetectionsIn, "SkinnyObject", false);
     356                fits_write_col(this->base.fitsOut, TLONGLONG, SKINNYOBJECT_OBJID, 1, 1, nChipDetectionsIn, objID, &status);
     357                fits_write_col(this->base.fitsOut, TLONGLONG, SKINNYOBJECT_IPPOBJID, 1, 1, nChipDetectionsIn, ippObjID, &status);
     358                fits_write_col(this->base.fitsOut, TBYTE, SKINNYOBJECT_SURVEYID, 1, 1, nChipDetectionsIn, surveyIDs, &status);
     359                if (numOfDuplicates||numInvalidFlux||numOfInvalidIppIDet)
     360                    fits_delete_rowlist(this->base.fitsOut, removeList, totalRemove, &status);
    353361
    354362                // object calibration color
    355                 ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nDet, "ObjectCalColor", false);
    356                 fits_write_col(this->base.fitsOut, TLONGLONG, OBJECTCALCOLOR_OBJID, 1, 1, nDet, objID, &status);
    357                 fits_write_col(this->base.fitsOut, TLONGLONG, OBJECTCALCOLOR_IPPOBJID, 1, 1, nDet, ippObjID, &status);
    358                 fits_write_col(this->base.fitsOut, TBYTE, OBJECTCALCOLOR_FILTERID, 1, 1, nDet, filterIDs, &status);
    359                 if (numOfDuplicates||numInvalidFlux) fits_delete_rowlist(this->base.fitsOut, removeList, numOfDuplicates+numInvalidFlux, &status);
     363                ippToPspsConfig_writeTable(this->base.config, fitsIn, this->base.fitsOut, nChipDetectionsIn, "ObjectCalColor", false);
     364                fits_write_col(this->base.fitsOut, TLONGLONG, OBJECTCALCOLOR_OBJID, 1, 1, nChipDetectionsIn, objID, &status);
     365                fits_write_col(this->base.fitsOut, TLONGLONG, OBJECTCALCOLOR_IPPOBJID, 1, 1, nChipDetectionsIn, ippObjID, &status);
     366                fits_write_col(this->base.fitsOut, TBYTE, OBJECTCALCOLOR_FILTERID, 1, 1, nChipDetectionsIn, filterIDs, &status);
     367                if (numOfDuplicates||numInvalidFlux||numOfInvalidIppIDet)
     368                    fits_delete_rowlist(this->base.fitsOut, removeList, totalRemove, &status);
    360369            }
    361370
    362371            psLogMsg("ippToPsps", PS_LOG_INFO,
    363                     "+-----------+---------+----------+------------------+---------------+--------------+\n"
    364                     "| Extension | Rows in | Rows out | Missing from DVO | Duplicate IDs | Invalid Flux |\n"
    365                     "|  %s |  %5ld  |   %5ld  |      %5d       |    %5d      |    %5d     |\n",
    366                     extensionName, nDet, numDetectionsOut, unmatched, numOfDuplicates, numInvalidFlux);
    367 
    368             totalDetectionsOut += numDetectionsOut;
     372                    "+-----------+---------+----------+----------------+--------------+--------------+\n"
     373                    "| Extension | Rows in | Rows out |  Duplicate IDs | Invalid Flux | Bogus det ID |\n"
     374                    "|  %s |  %5ld  |   %5d  |     %5d      |    %5d     |    %5d     |\n",
     375                    extensionName, nChipDetectionsIn, nChipDetectionsOut, numOfDuplicates, numInvalidFlux, numOfInvalidIppIDet);
     376
     377            totalDetectionsOut += nChipDetectionsOut;
    369378
    370379            if (dvoDetections!= NULL) dvoFree(dvoDetections);
    … …  
    492501
    493502/**
    494   Constructor
     503  Constructor. Returns a new DetectionBatch object.
    495504  */
    496505DetectionBatch* new_DetectionBatch(int *argc, char **argv) {
Note: See TracChangeset for help on using the changeset viewer.