IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 30447


Ignore:
Timestamp:
Jan 31, 2011, 2:52:18 PM (15 years ago)
Author:
eugene
Message:

merging changes from trunk

Location:
branches/eam_branches/ipp-20101205
Files:
10 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20101205/Nebulous/bin/neb-cull

    r25121 r30447  
    1717
    1818my ($min_copies, $server, $volume, $one_only);
    19 
     19my ($soft_volume);
    2020$server = $ENV{'NEB_SERVER'} unless $server;
    2121
    … …  
    2525    'volume|v=s'    => \$volume,
    2626    'one_only|o'    => \$one_only,
     27    'soft_volume'   => \$soft_volume,
    2728) || pod2usage( 2 );
    2829
    … …  
    4243    unless defined $neb;
    4344
     45if ($volume) {
     46    unless (($soft_volume)) {
     47        $volume = '~' . $volume;
     48    }
     49}
     50
    4451$volume ||= "any";
    4552$min_copies ||= 2;
    … …  
    4855push @cull_args, $key;
    4956push @cull_args, $volume;
     57
     58# print "$key $volume\n";
     59# print "@cull_args\n";
     60
    5061
    5162if ($one_only) {
    … …  
    95106
    96107Symbolic name of the volume to create the new instance on.
     108
     109Optional.
     110
     111=item * --soft_volume
     112
     113Treat the specified volume name as soft, and allow an instance to be culled
     114from another volume if the specified volume does not have an instance.
    97115
    98116Optional.

  • branches/eam_branches/ipp-20101205/PS-IPP-Config/lib/PS/IPP/Config.pm

    r30103 r30447  
    10481048}
    10491049
     1050sub magic_filename
     1051{
     1052    my $file = shift;
     1053    my $prefix = shift;
     1054    my $dirname = dirname($file);
     1055    my $base = basename($file);
     1056
     1057    return "$dirname/$prefix$base";
     1058}
     1059
     1060sub destreaked_filename
     1061{
     1062    my $self  = shift;
     1063    my $file  = shift;
     1064
     1065    return magic_filename($file, "SR_");
     1066}
     1067sub recovery_filename
     1068{
     1069    my $self  = shift;
     1070    my $file  = shift;
     1071
     1072    return magic_filename($file, "REC_");
     1073}
     1074
    10501075# Return an EXTNAME From the EXTNAME.RULE table in the camera configuration
    10511076sub extname_rule

  • branches/eam_branches/ipp-20101205/magic/remove/src/streaksVersion.c

    r28043 r30447  
    5858    psString source = streaksSource();   // Software source
    5959
    60     psMetadataAddStr(header, PS_LIST_TAIL, "STREAK_V", 0, NULL, source);
     60    psMetadataAddStr(header, PS_LIST_TAIL, "STREAK_V", PS_META_REPLACE, NULL, source);
    6161   
    6262    psStringPrepend(&version, "%s version: ", streaksProgram);

  • branches/eam_branches/ipp-20101205/magic/remove/src/streakscompare.c

    r26477 r30447  
    1818    ippStage stage = psMetadataLookupS32(&status, config->arguments, "STAGE");
    1919
    20     sFile *file1 = sFileOpen(config, stage, "INPUT1", NULL, true);
    21     sFile *file2 = sFileOpen(config, stage, "INPUT2", NULL, true);
     20    sFile *file1 = sFileOpen(config, stage, "INPUT1", NULL, true, false);
     21    sFile *file2 = sFileOpen(config, stage, "INPUT2", NULL, true, false);
    2222
    2323    int ncomponents;

  • branches/eam_branches/ipp-20101205/magic/remove/src/streaksio.c

    r29392 r30447  
    4242    sf->class_id = psMetadataLookupStr(&status, config->arguments, "CLASS_ID");
    4343
    44     sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true);
     44    sf->inImage = sFileOpen(config, stage, "INPUT", NULL, true, false);
    4545    sf->nHDU = sf->inImage->nHDU;
    4646
    … …  
    4848    // The names of the temporary and recovery files are taken from the input
    4949    char *inputBasename = basename(sf->inImage->name);
    50     sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
     50    sf->outImage = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
    5151
    5252    if (remove) {
    5353        // XXX: the recovery file should be required if stage is raw and the replace flag is set
    5454        // that is if the input raw image is to be destroyed
    55         sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
    56     } else {
    57         sf->recImage = sFileOpen(config, stage, "RECOVERY.IMAGE", NULL, true);
    58     }
    59 
    60     sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false);
     55        sf->recImage = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false);
     56    } else {
     57        sf->recImage = sFileOpen(config, stage, "RECOVERY.IMAGE", NULL, true, false);
     58    }
     59
     60    sf->inMask = sFileOpen(config, stage, "INPUT.MASK", NULL, false, false);
    6161    if (sf->inMask && (sf->stage != IPP_STAGE_RAW)) {
    6262        inputBasename = basename(sf->inMask->name);
    63         sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
     63        sf->outMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
    6464        if (remove) {
    65             sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
     65            sf->recMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false);
    6666        } else {
    67             sf->recMask = sFileOpen(config, stage, "RECOVERY.MASK", NULL, true);
     67            sf->recMask = sFileOpen(config, stage, "RECOVERY.MASK", NULL, true, false);
    6868        }
    6969    }
    … …  
    7373    // If it doesn't exist, we didn't have a camera mask
    7474    if (remove && sf->inMask && (stage == IPP_STAGE_CHIP)) {
    75         sf->inChMask = sFileOpen(config, stage, "INPUT.CHMASK", NULL, false);
     75        sf->inChMask = sFileOpen(config, stage, "INPUT.CHMASK", NULL, false, false);
    7676        if (sf->inChMask) {
    7777            inputBasename = basename(sf->inChMask->name);
    78             sf->outChMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
    79             sf->recChMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
    80         }
    81     }
    82 
    83     sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false);
     78            sf->outChMask = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
     79            sf->recChMask = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false);
     80        }
     81    }
     82
     83    sf->inWeight = sFileOpen(config, stage, "INPUT.WEIGHT", NULL, false, false);
    8484    if (sf->inWeight) {
    8585        inputBasename = basename(sf->inWeight->name);
    86         sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
     86        sf->outWeight = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
    8787        if (remove) {
    88             sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, false);
     88            sf->recWeight = sFileOpen(config, stage, "RECOVERY", inputBasename, false, false);
    8989        } else {
    90             sf->recWeight = sFileOpen(config, stage, "RECOVERY.WEIGHT", NULL, true);
     90            sf->recWeight = sFileOpen(config, stage, "RECOVERY.WEIGHT", NULL, true, false);
    9191        }
    9292    }
    9393    if (remove) {
    94         sf->inSources = sFileOpen(config, stage, "SOURCES", NULL, false);
     94        sf->inSources = sFileOpen(config, stage, "SOURCES", NULL, false, false);
    9595        if (sf->inSources) {
    9696            inputBasename = basename(sf->inSources->name);
    97             sf->outSources = sFileOpen(config, stage, "OUTPUT", inputBasename, true);
     97            sf->outSources = sFileOpen(config, stage, "OUTPUT", inputBasename, true, true);
    9898        }
    9999    }
    … …  
    121121    sf->transparentStreaks = psMetadataLookupF64(&status, config->arguments, "TRANSPARENT_STREAKS");
    122122
    123     sf->stats = psMetadataAlloc();
    124     psString statsFileName= psMetadataLookupStr(&status, config->arguments, "STATS");
    125     if (statsFileName) {
    126         sf->statsFile = fopen(statsFileName, "w");
    127         if (!sf->statsFile) {
    128             psError(PS_ERR_IO, true, "failed to open stats file %s", statsFileName);
    129             streaksExit("", PS_EXIT_CONFIG_ERROR);
    130         }
    131     }
    132 
    133123    return sf;
    134124}
    … …  
    138128{
    139129    freeImages(sf);
     130    psFree(sf->stats);
    140131    psFree(sf->diffedPixels);
    141132    psFree(sf->tiles);
    … …  
    221212
    222213static psString
    223 resolveFilename(pmConfig *config, sFile *sfile, bool create)
     214resolveFilename(pmConfig *config, sFile *sfile, bool create, bool checkIfDestreaked)
    224215{
    225216    sfile->inNebulous = IN_NEBULOUS(sfile->name);
    … …  
    232223            // instance. It will get created below in pmConfigConvertFilename
    233224            if ((sfile->resolved_name = nebFind(server, sfile->name)) != NULL) {
    234                 if (!nebFileIsDestreaked(sfile)) {
     225                if (checkIfDestreaked && !nebFileIsDestreaked(sfile)) {
    235226                    psError(PS_ERR_IO, false, "attempting to delete file that has not been destreaked %s", sfile->name);
    236227                    return NULL;
    … …  
    247238
    248239sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect,
    249     psString outputFilename, bool required)
     240    psString outputFilename, bool required, bool checkIfDestreaked)
    250241{
    251242    bool status;
    … …  
    322313    // and the file is to be opened for writing
    323314    if (outputFilename) {
     315        sfile->write = true;
    324316        psStringAppend(&sfile->name, "%s%s", name, outputFilename);
    325         sfile->resolved_name = resolveFilename(config, sfile, true);
     317        sfile->resolved_name = resolveFilename(config, sfile, true, checkIfDestreaked);
    326318        if (!sfile->resolved_name) {
    327319            psError(PS_ERR_IO, false, "Failed to resolve filename for %s", sfile->name);
    … …  
    335327    } else {
    336328        sfile->name = psStringCopy(name);
    337         sfile->resolved_name = resolveFilename(config, sfile, false);
     329        sfile->resolved_name = resolveFilename(config, sfile, false, false);
    338330        if (!sfile->resolved_name) {
    339331            psError(PS_ERR_IO, false, "Failed to resolve name for %s", sfile->name);
    … …  
    402394
    403395void
    404 addDestreakKeyword(psMetadata *header)
     396addDestreakKeyword(psMetadata *header, bool value)
    405397{
    406398    psMetadataAddBool(header, PS_LIST_TAIL, "PSDESTRK", PS_META_REPLACE,
    407         "Have streaks been removed from image?", true);
     399        "Have streaks been removed from image?", value);
    408400}
    409401
    … …  
    416408
    417409void
    418 copyPHU(streakFiles *sfiles, bool remove)
     410copyPHU(streakFiles *sfiles, bool remove, bool destreak)
    419411{
    420412    psAssert(sfiles->stage == IPP_STAGE_RAW, "copyPHU should only be used for raw stage");
    … …  
    433425
    434426    // add keyword indicating that streaks have been removed
    435     addDestreakKeyword(imageHeader);
     427    addDestreakKeyword(imageHeader, destreak);
    436428
    437429    if (!psFitsWriteBlank(sfiles->outImage->fits, imageHeader, NULL)) {
    … …  
    463455        }
    464456        // add keyword indicating that streaks have been removed
    465         addDestreakKeyword(maskHeader);
     457        addDestreakKeyword(maskHeader, destreak);
    466458        if (!psFitsWriteBlank(sfiles->outMask->fits, maskHeader, NULL)) {
    467459            psError(PS_ERR_IO, false, "failed to write primary header to %s",
    … …  
    493485
    494486        // add keyword indicating that streaks have been removed
    495         addDestreakKeyword(weightHeader);
     487        addDestreakKeyword(weightHeader, destreak);
    496488        if (!psFitsWriteBlank(sfiles->outWeight->fits, weightHeader, NULL)) {
    497489            psError(PS_ERR_IO, false, "failed to write primary header to %s",
    … …  
    683675    psVector *tiles)
    684676{
    685     if (!sfile) {
     677    if (!sfile || !sfile->write) {
    686678        return;
    687679    }
    … …  
    861853        streaksExit("", PS_EXIT_DATA_ERROR);
    862854    }
     855    sfile->fits = NULL;
    863856    psFree(sfile->header);
    864857    sfile->header = NULL;
    … …  
    11981191                                weight->data.F32[y][x] = NAN;
    11991192                            }
     1193                        } else if (weightType == PS_TYPE_F64) {
     1194                            double weightVal = weight->data.F64[y][x];
     1195                            if (!isnan(weightVal)) {
     1196                                ++nandWeights;
     1197                                weight->data.F64[y][x] = NAN;
     1198                            }
    12001199                        } else if(weightType == PS_TYPE_S16) {
    12011200                            double weightVal = weight->data.S16[y][x];
    … …  
    12491248        psMetadataAddU16(in->header, PS_LIST_TAIL, "ZBLANK", 0, "", 32767);
    12501249    } else if (in->image->type.type == PS_TYPE_F32) {
     1250        in->exciseValue = NAN;
     1251    } else if (in->image->type.type == PS_TYPE_F64) {
    12511252        in->exciseValue = NAN;
    12521253    } else {

  • branches/eam_branches/ipp-20101205/magic/remove/src/streaksio.h

    r26408 r30447  
    55
    66sFile *sFileOpen(pmConfig *config, ippStage stage, psString fileSelect,
    7     psString outputFilename, bool required);
     7    psString outputFilename, bool required, bool checkIfDestreaked);
    88
    99void closeImages(streakFiles *sfiles);
    … …  
    1212
    1313void readImage(sFile *sfile, int extnum, ippStage stage, bool isMask);
    14 void copyPHU(streakFiles *sfiles, bool remove);
     14void copyPHU(streakFiles *sfiles, bool remove, bool destreaked);
    1515void copyTable(sFile *out, sFile *in, int extnum);
    1616void copyFitsOptions(sFile *out, sFile *rec, sFile *in, psVector *tiles);
    … …  
    2424void readImageFrom_pmFile(streakFiles *sf);
    2525
    26 void addDestreakKeyword(psMetadata *);
     26void addDestreakKeyword(psMetadata *, bool value);
    2727void addRecoveryKeyword(psMetadata *);
    2828

  • branches/eam_branches/ipp-20101205/magic/remove/src/streaksrelease.c

    r27468 r30447  
    2424    if (sfiles->stage == IPP_STAGE_RAW) {
    2525        // copy PHU to output files
    26         copyPHU(sfiles, true);
     26        copyPHU(sfiles, false, true);
    2727
    2828        // advance to the first image extension

  • branches/eam_branches/ipp-20101205/magic/remove/src/streaksremove.c

    r29806 r30447  
    9393        // Raw files have a phu and multiple extensions, one per chip
    9494        // Since this is a raw file, copy it's PHU to output files
    95         copyPHU(sfiles, true);
     95        copyPHU(sfiles, true, true);
    9696
    9797        // advance to the first image extension
    … …  
    275275    psLogMsg("streaksremove", PS_LOG_INFO, "time to run streaksremove: %f\n", total_time);
    276276
    277     if (sfiles->statsFile) {
    278         const char *statsMDC = psMetadataConfigFormat(sfiles->stats);
    279         if (!statsMDC || strlen(statsMDC) == 0) {
    280             psError(PS_ERR_IO, false, "Unable to get statistics MDC file.\n");
    281         } else {
    282             fprintf(sfiles->statsFile, "%s", statsMDC);
    283             psFree(statsMDC);
    284             fclose(sfiles->statsFile);
    285             sfiles->statsFile = NULL;
    286             psFree(sfiles->stats);
    287             sfiles->stats = NULL;
    288         }
    289     }
     277    psString statsFileName= psMetadataLookupStr(&status, config->arguments, "STATS");
     278
     279    if (statsFileName) {
     280    // Write out
     281        psString resolved = pmConfigConvertFilename(statsFileName, config, true, true); // Resolved filename
     282        if (!resolved) {
     283            psError(psErrorCodeLast(), false, "Unable to resolve statistics file name");
     284            return false;
     285        }
     286        if (!psMetadataConfigWrite(sfiles->stats, resolved, NULL)) {
     287            psError(psErrorCodeLast(), false, "Unable to serialize stats metadata.\n");
     288            psFree(resolved);
     289            return false;
     290        }
     291        psFree(resolved);
     292    }
     293
    290294    // all done. Clean up to look for memory leaks.
    291295
    … …  
    613617        addRecoveryKeyword(sf->recImage->header);
    614618    }
    615     addDestreakKeyword(sf->outImage->header);
     619    addDestreakKeyword(sf->outImage->header, true);
    616620
    617621    if (!SFILE_IS_IMAGE(sf->inImage)) {
    … …  
    639643                addRecoveryKeyword(sf->recMask->header);
    640644            }
    641             addDestreakKeyword(sf->outMask->header);
     645            addDestreakKeyword(sf->outMask->header, true);
    642646            // Note: we don't excise the mask pixels even if exciseAll is true.
    643647            setupImageRefs(sf->outMask, sf->recMask, sf->inMask, sf->extnum, false);
    … …  
    671675            addRecoveryKeyword(sf->recWeight->header);
    672676        }
    673         addDestreakKeyword(sf->outWeight->header);
     677        addDestreakKeyword(sf->outWeight->header, true);
    674678        setupImageRefs(sf->outWeight, sf->recWeight, sf->inWeight, sf->extnum, exciseAll);
    675679
    … …  
    767771            }
    768772        }
    769     } else {
     773    } else if (sfiles->inImage->image->type.type == PS_TYPE_F32) {
    770774        float imageValue  = sfiles->inImage->image->data.F32[y][x];
    771775        if (sfiles->recImage && !isExciseValue(imageValue, sfiles->inImage->exciseValue) ) {
    … …  
    783787            }
    784788        }
     789    } else {
     790        // We could handle F64 but I don't think we ever get them.
     791        // Should catch this earlier
     792        psError(PS_ERR_IO, true, "unexpected image type %x found", sfiles->inImage->image->type.type );
     793        streaksExit("", PS_EXIT_PROG_ERROR);
    785794    }
    786795
    787796    if (sfiles->outWeight) {
    788         if (sfiles->recWeight) {
    789             sfiles->recWeight->image->data.F32[y][x] = sfiles->inWeight->image->data.F32[y][x];
    790         }
    791797        // Assume that weight images are always a floating point type
    792         sfiles->outWeight->image->data.F32[y][x] = NAN;
     798        if (sfiles->inWeight->image->type.type == PS_TYPE_F32) {
     799            if (sfiles->recWeight) {
     800                sfiles->recWeight->image->data.F32[y][x] = sfiles->inWeight->image->data.F32[y][x];
     801            }
     802            sfiles->outWeight->image->data.F32[y][x] = NAN;
     803        } else if (sfiles->inWeight->image->type.type == PS_TYPE_F64) {
     804            if (sfiles->recWeight) {
     805                sfiles->recWeight->image->data.F64[y][x] = sfiles->inWeight->image->data.F64[y][x];
     806            }
     807            sfiles->outWeight->image->data.F64[y][x] = NAN;
     808        } else {
     809            // Should catch this earlier
     810            psError(PS_ERR_IO, true, "unexpected weight image type %x found", sfiles->inWeight->image->type.type );
     811            streaksExit("", PS_EXIT_PROG_ERROR);
     812        }
    793813    }
    794814    if (sfiles->outMask) {
    … …  
    912932            streaksExit("", PS_EXIT_DATA_ERROR);
    913933        }
    914         addDestreakKeyword(header);
     934        addDestreakKeyword(header, true);
    915935
    916936        if (!psFitsWriteBlank(out->fits, header, extname)) {
    … …  
    973993        psArrayRealloc(outTable, j);
    974994
    975         addDestreakKeyword(header);
     995        addDestreakKeyword(header, true);
    976996        if (psArrayLength(outTable) > 0) {
    977997            printf("Censored %d sources\n", numCensored);

  • branches/eam_branches/ipp-20101205/magic/remove/src/streaksremove.h

    r27752 r30447  
    1818    psString    name;
    1919    bool        inNebulous;
     20    bool        write;
    2021    psFits      *fits;
    2122    int         nHDU;

  • branches/eam_branches/ipp-20101205/magic/remove/src/streaksreplace.c

    r26477 r30447  
    3636    if (sfiles->stage == IPP_STAGE_RAW) {
    3737        // copy PHU to output files
    38         copyPHU(sfiles, false);
     38        copyPHU(sfiles, false, false);
    3939
    4040        // advance to the first image extension
    … …  
    296296
    297297    sf->outImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header);
     298    addDestreakKeyword(sf->outImage->header, false);
    298299
    299300    if (!SFILE_IS_IMAGE(sf->inImage)) {
    … …  
    317318        readImage(sf->recMask, sf->extnum, sf->stage, true);
    318319        sf->outMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
     320       
     321        addDestreakKeyword(sf->outMask->header, false);
    319322
    320323        // XXX: TODO
    … …  
    329332
    330333        sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header);
     334        addDestreakKeyword(sf->outWeight->header, false);
    331335        setupImageRefs(sf->outMask, NULL, sf->inMask, sf->extnum, false);
    332336
Note: See TracChangeset for help on using the changeset viewer.