IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 26446


Ignore:
Timestamp:
Dec 16, 2009, 3:57:48 PM (16 years ago)
Author:
bills
Message:

Add some stats to magicDSFile.
On the receive side, make it possible to disable a source

Location:
trunk
Files:
10 edited

Legend:

Unmodified
Added
Removed

  • trunk/dbconfig/changes.txt

    r26378 r26446  
    14341434UPDATE dbversion set schema_version = '1.1.57',  updated= CURRENT_TIMESTAMP();
    14351435
     1436ALTER TABLE receiveSource add column state varchar (64) after workdir;
     1437UPDATE receiveSource SET state = 'enabled';
     1438
    14361439-- Vesion 1.1.58
    14371440-- UPDATE dbversion set schema_version = '1.1.58',  updated= CURRENT_TIMESTAMP();
    … …  
    14791482
    14801483UPDATE dbversion set schema_version = '1.1.61',  updated= CURRENT_TIMESTAMP();
     1484
     1485ALTER TABLE magicDSFile ADD COLUMN streak_frac FLOAT AFTER recovery_path_base;
     1486ALTER TABLE magicDSFile ADD COLUMN total_time FLOAT AFTER streak_frac;
     1487
     1488ALTER TABLE receiveSource ADD COLUMN state VARCHAR(64) AFTER workdir;
     1489
     1490UPDATE dbversion set schema_version = '1.1.62',  updated= CURRENT_TIMESTAMP();

  • trunk/dbconfig/magic.md

    r25835 r26446  
    6969    backup_path_base  STR         255
    7070    recovery_path_base STR        255
     71    streak_frac F32         0
     72    run_time    F32         0
    7173    fault       S16         0
    7274    data_state  STR        64     # Key

  • trunk/dbconfig/receive.md

    r24125 r26446  
    66        product         STR     64      # Key
    77        workdir         STR     255
     8        state           STR     64
    89        comment         STR     255     # Key
    910        fileset_last    STR     128

  • trunk/ippTools/share/pxadmin_create_tables.sql

    r26015 r26446  
    11811181    backup_path_base VARCHAR(255),
    11821182    recovery_path_base VARCHAR(255),
     1183    streak_frac FLOAT,
     1184    run_time FLOAT,
    11831185    fault SMALLINT,
    11841186    data_state VARCHAR(64),
    … …  
    14281430    product VARCHAR(64) NOT NULL, -- product of interest
    14291431    workdir VARCHAR(255) NOT NULL, -- where to extract
     1432    state VARCHAR(64) NOT NULL, -- state 'enabled' or 'disabled'
    14301433    comment VARCHAR(255),       -- for human memory
    14311434    fileset_last VARCHAR(128),  -- last fileset seen

  • trunk/ippTools/share/receivetool_list.sql

    r23880 r26446  
    11SELECT *
    22FROM receiveSource
     3WHERE receiveSource.state = 'enabled'

  • trunk/ippTools/share/receivetool_pendingfile.sql

    r24788 r26446  
    1818    ON receiveResult.file_id = receiveFile.file_id
    1919WHERE
    20     ((receiveFileset.state = 'new')
     20    receiveSource.state = 'enabled'
     21    AND ((receiveFileset.state = 'new')
    2122    OR (receiveFileset.state = 'listed' AND receiveFile.component = 'dirinfo' AND receiveFileset.dirinfo IS NULL))
    2223    AND receiveFileset.fault = 0

  • trunk/ippTools/share/receivetool_pendingfileset.sql

    r24788 r26446  
    1 SELECT *
     1SELECT receiveFileset.*,
     2    receiveSource.source,
     3    receiveSource.product,
     4    receiveSource.workdir
    25FROM receiveFileset
    36JOIN receiveSource USING(source_id)
    47LEFT JOIN receiveFile USING(fileset_id)
    5 WHERE receiveFileset.state = 'reg'
     8WHERE receiveSource.state = 'enabled'
     9    AND receiveFileset.state = 'reg'
    610    AND receiveFileset.fault = 0
    711    AND receiveFile.fileset_id IS NULL

  • trunk/ippTools/share/receivetool_toadvance.sql

    r24125 r26446  
    1010    JOIN receiveFile USING(fileset_id)
    1111    LEFT JOIN receiveResult USING(file_id)
    12 WHERE receiveFileset.state = 'new'
     12WHERE receiveSource.state = 'enabled'
     13    AND receiveFileset.state = 'new'
    1314    AND receiveFileset.fault = 0
    1415GROUP BY fileset_id

  • trunk/ippconfig/recipes/ppStatsFromMetadata.config

    r26443 r26446  
    403403  ENTRY MULTI
    404404  TYPE   VAL  KEYWORD             TYPE STATISTIC         FLAG
    405   ENTRY  VAL  MASKED_FRACTION     F64  CONSTANT          -masked_frac # fraction of pixels masked by streaks
    406   ENTRY  VAL  TOTAL_TIME          F64  CONSTANT          -run_time    # total run time for streaksremove
     405  ENTRY  VAL  STREAK_FRACTION     F32  CONSTANT          -streak_frac # fraction of pixels masked by streaks
     406  ENTRY  VAL  TOTAL_TIME          F32  CONSTANT          -run_time    # total run time for streaksremove
    407407
    408408  # The following timers are included in the stats not saved in the datbase
    409 # ENTRY  VAL  GET_STREAK_PIXELS   F64  CONSTANT          -dtime_compute_masked  # time to compute list of streak pixels
    410 # ENTRY  VAL  REMOVE_STREAKS      F64  CONSTANT          -dtime_remove # time to mask the pixels
    411 # ENTRY  VAL  WRITE_IMAGES        F64  CONSTANT          -dtime_write  # time to write out the results
    412 # ENTRY  VAL  REPLICATE_OUTPUTS   F64  CONSTANT          -dtime_replicate  # time to replicate the outputs
    413 END
     409# ENTRY  VAL  GET_STREAK_PIXELS   F32  CONSTANT          -dtime_compute_masked  # time to compute list of streak pixels
     410# ENTRY  VAL  REMOVE_STREAKS      F32  CONSTANT          -dtime_remove # time to mask the pixels
     411# ENTRY  VAL  WRITE_IMAGES        F32  CONSTANT          -dtime_write  # time to write out the results
     412# ENTRY  VAL  REPLICATE_OUTPUTS   F32  CONSTANT          -dtime_replicate  # time to replicate the outputs
     413END

  • trunk/magic/remove/src/streaksremove.c

    r26438 r26446  
    8383        }
    8484        psF64 cdp_t = psTimerClear("COMPUTE_DIFFED_PIXELS");
    85         psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "COMPUTE_UNDIFFED_PIXELS", PS_META_REPLACE, "time to compute non-diffedpixels", cdp_t);
     85        psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "COMPUTE_UNDIFFED_PIXELS", PS_META_REPLACE, "time to compute non-diffedpixels", cdp_t);
    8686        psLogMsg("streaksremove", PS_LOG_INFO, "time to compute diffed pixels: %f\n", cdp_t);
    8787    }
    … …  
    139139                                                        sfiles->inImage->numRows);
    140140            gsp_t +=  psTimerClear("GET_STREAK_PIXELS");
    141             psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "GET_STREAK_PIXELS", PS_META_REPLACE, "", gsp_t);
     141            psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "GET_STREAK_PIXELS", PS_META_REPLACE, "", gsp_t);
    142142            psLogMsg("streaksremove", PS_LOG_INFO, "time to get streak pixels: %f\n", psTimerClear("GET_STREAK_PIXELS"));
    143143
    … …  
    153153
    154154                    enw_t +=  psTimerClear("EXCISE_NON_DIFFED");
    155                     psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "EXCISE_NON_DIFFED", PS_META_REPLACE, "", enw_t);
     155                    psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "EXCISE_NON_DIFFED", PS_META_REPLACE, "", enw_t);
    156156                    psLogMsg("streaksremove", PS_LOG_INFO, "time to excise non diffed pixels: %f\n", enw_t);
    157157                }
    … …  
    162162
    163163                rms_t += psTimerClear("REMOVE_STREAKS");
    164                 psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "REMOVE_STREAKS", PS_META_REPLACE, "", enw_t);
     164                psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "REMOVE_STREAKS", PS_META_REPLACE, "", enw_t);
    165165                psLogMsg("streaksremove", PS_LOG_INFO, "time to remove streak pixels: %f\n", rms_t);
    166166
    … …  
    188188        censorSources(sfiles, sfiles->maskStreak);
    189189        cs_t += psTimerClear("CENSOR_SOURCES");
    190         psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "CENSOR_SOURCES", PS_META_REPLACE, "", cs_t);
     190        psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "CENSOR_SOURCES", PS_META_REPLACE, "", cs_t);
    191191
    192192
    … …  
    195195        writeImages(sfiles, exciseImageCube);
    196196        wi_t += psTimerClear("WRITE_IMAGES");
    197         psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "WRITE_IMAGES", PS_META_REPLACE, "", wi_t);
     197        psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "WRITE_IMAGES", PS_META_REPLACE, "", wi_t);
    198198
    199199        psLogMsg("streaksremove", PS_LOG_INFO, "time to process component %d: %f\n", sfiles->extnum, psTimerClear("PROCESS_COMPONENT"));
    … …  
    207207    }
    208208
    209     psF64 maskedFraction = totalStreakPixels / totalPixels;
    210     psLogMsg("streaksremove", PS_LOG_INFO, "pixels: %ld streak pixels: %ld %4.2f%%\n", totalPixels, totalStreakPixels, maskedFraction * 100);
    211     psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "MASKED_FRACTION", PS_META_REPLACE, "", maskedFraction);
     209    psF64 streakFraction = totalStreakPixels / totalPixels;
     210    psLogMsg("streaksremove", PS_LOG_INFO, "pixels: %ld streak pixels: %ld %4.2f%%\n", totalPixels, totalStreakPixels, streakFraction * 100);
     211    psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "STREAK_FRACTION", PS_META_REPLACE, "", streakFraction);
    212212
    213213    // check the weight and mask files for extra extensions that might be in files
    … …  
    220220    closeImages(sfiles);
    221221    psF64 ci_t = psTimerClear("CLOSE_IMAGES");
    222     psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "CLOSE_IMAGES", PS_META_REPLACE, "", ci_t);
     222    psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "CLOSE_IMAGES", PS_META_REPLACE, "", ci_t);
    223223
    224224    psLogMsg("streaksremove", PS_LOG_INFO, "time to close images: %f\n", ci_t);
    … …  
    231231    }
    232232    psF64 ro_t = psTimerClear("REPLICATE_OUTPUTS");
    233     psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "REPLICATE_OUTPUTS", PS_META_REPLACE, "", ro_t);
     233    psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "REPLICATE_OUTPUTS", PS_META_REPLACE, "", ro_t);
    234234
    235235    // NOTE: from here on we can't just quit if something goes wrong.
    … …  
    253253        }
    254254        psF64 si_t = psTimerClear("SWAP_INSTANCES");
    255         psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "SWAP_INSTANCES", PS_META_REPLACE, "", si_t);
     255        psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "SWAP_INSTANCES", PS_META_REPLACE, "", si_t);
    256256    }
    257257
    258258    psF64 total_time = psTimerClear("TOTAL_TIME");
    259     psMetadataAddF64(sfiles->stats, PS_LIST_TAIL, "TOTAL_TIME", PS_META_REPLACE, "", total_time);
     259    psMetadataAddF32(sfiles->stats, PS_LIST_TAIL, "TOTAL_TIME", PS_META_REPLACE, "", total_time);
    260260    psLogMsg("streaksremove", PS_LOG_INFO, "time to run streaksremove: %f\n", total_time);
    261261
    … …  
    424424        psArgumentRemove(argnum, &argc, argv);
    425425        double transparentStreaks = atof(argv[argnum]);
    426         psMetadataAddF64(config->arguments, PS_LIST_TAIL, "TRANSPARENT_STREAKS", 0,
     426        psMetadataAddF32(config->arguments, PS_LIST_TAIL, "TRANSPARENT_STREAKS", 0,
    427427            "value to adjust excised pixels", transparentStreaks);
    428428        psArgumentRemove(argnum, &argc, argv);
Note: See TracChangeset for help on using the changeset viewer.