IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 23208


Ignore:
Timestamp:
Mar 6, 2009, 11:22:21 AM (17 years ago)
Author:
eugene
Message:

modify the detrend sequence to generate the detResidImfiles before generating the detNormalizedStat

Location:
branches/eam_branches/eam_branch_20090303/ippTools
Files:
9 edited

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_processedimfile.sql

    r18561 r23208  
     1-- is this DISTINCT needed?
    12SELECT DISTINCT
    23  detRun.det_type,

  • branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_residimfile.sql

    r19632 r23208  
    22   detRun.det_type,
    33   detRun.mode,
    4    detResidImfile.*,
    5    rawExp.exp_time
    6  FROM detResidImfile
    7  JOIN detRun
    8    USING(det_id, iteration)
    9  JOIN rawExp
    10    USING(exp_id)
     4   rawExp.exp_time,
     5   detResidImfile.*
     6FROM detResidImfile
     7JOIN detRun
     8  USING(det_id, iteration)
     9JOIN detInputExp
     10  ON detRun.det_id = detInputExp.det_id
     11  AND detRun.iteration = detInputExp.iteration
     12  AND detResidImfile.exp_id = detInputExp.exp_id
     13JOIN rawExp
     14  ON rawExp.exp_id = detResidImfile.exp_id

  • branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_tonormalizedstat.sql

    r18118 r23208  
    1 -- select detRun.det_id (det_id)
    2 -- select detRun.iteration
    3 -- by:
    4 -- find the current iteration bassed on det_id
    5 -- find all exp_ids in the current det_id/iteration from detInputExp
    6 -- sort to detInputExp.imfiles to find the largest value per det_id/iter
    7 -- compare imfiles to the number of detStackedImfiles by class_id
    8 -- and:
    9 -- ???
    10 -- det_id is not in detStackedImfile;
    11 -- iteration is not in detStackedImfile;
    12 
    13 SELECT
    14     det_id,
    15     det_type,
    16     iteration,
    17     camera,
    18     workdir,
    19     class_id
    20 FROM
    21     (SELECT DISTINCT
    22         detRun.det_id,
    23         detRun.det_type,
    24         detRun.iteration,
    25         detRun.workdir,
    26         rawExp.camera,
    27         detStackedImfile.class_id,
    28         rawImfile.class_id as rawimfile_class_id
    29     FROM detRun
    30     JOIN detInputExp
    31         ON detRun.det_id = detInputExp.det_id
    32         AND detRun.iteration = detInputExp.iteration
    33     JOIN rawExp
    34         ON detInputExp.exp_id = rawExp.exp_id
    35     JOIN rawImfile
    36         ON rawExp.exp_id = rawImfile.exp_id
    37     LEFT JOIN detStackedImfile
    38         ON detInputExp.det_id = detStackedImfile.det_id
    39         AND detInputExp.iteration = detStackedImfile.iteration
    40         AND rawImfile.class_id = detStackedImfile.class_id
    41     LEFT JOIN detNormalizedStatImfile
    42         ON detStackedImfile.det_id = detNormalizedStatImfile.det_id
    43         AND detStackedImfile.iteration = detNormalizedStatImfile.iteration
    44         AND detStackedImfile.class_id = detNormalizedStatImfile.class_id
    45     WHERE
    46         detRun.state = 'run'
    47         AND detRun.mode = 'master'
    48         AND detNormalizedStatImfile.det_id IS NULL
    49         AND detNormalizedStatImfile.iteration IS NULL
    50         AND detNormalizedStatImfile.class_id IS NULL
    51     GROUP BY
    52 --        rawExp.exp_id,
    53         detRun.iteration,
    54         detRun.det_id
    55     HAVING
    56         COUNT(rawImfile.class_id) = COUNT(detStackedImfile.class_id)
    57         AND SUM(detStackedImfile.fault) = 0
    58     ) as tonormalizedstat
     1-- a det_run + iteration is ready for normstat when: all detResidImfile entries for that iteration corresponding to all detProcessedImfile entries are available
     2SELECT
     3  detRun.det_id,
     4  detRun.det_type,
     5  detRun.iteration,
     6  detRun.camera,
     7  detRun.workdir,
     8  detProcessedImfile.class_id,
     9  COUNT(detProcessedImfile.class_id),
     10  COUNT(detResidImfile.class_id)
     11FROM detRun
     12JOIN detProcessedImfile
     13    ON  detProcessedImfile.det_id    = detRun.det_id
     14LEFT JOIN detResidImfile
     15    ON  detResidImfile.det_id    = detProcessedImfile.det_id
     16    AND detResidImfile.class_id  = detProcessedImfile.class_id
     17    AND detResidImfile.iteration = detRun.iteration
     18LEFT JOIN detNormalizedStatImfile
     19    ON  detNormalizedStatImfile.det_id    = detRun.det_id   
     20    AND detNormalizedStatImfile.iteration = detRun.iteration
     21    AND detNormalizedStatImfile.class_id  = detProcessedImfile.class_id
     22WHERE
     23    detRun.state = 'run'
     24    AND detRun.mode = 'master'
     25    AND detNormalizedStatImfile.det_id IS NULL
     26    AND detNormalizedStatImfile.iteration IS NULL
     27    AND detNormalizedStatImfile.class_id IS NULL
     28GROUP BY
     29    detRun.iteration,
     30    detRun.det_id
     31HAVING
     32    COUNT(detProcessedImfile.class_id) = COUNT(detResidImfile.class_id)
     33    AND SUM(detResidImfile.fault) = 0

  • branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_toresidexp.sql

    r19815 r23208  
    55-- id, detrend type, and whether the exposure was included in the stack for
    66-- this iteration.
     7
     8-- require the corresponding detNormalizedExp to complete before starting
    79
    810-- select detRun.det_id
    … …  
    4143        detResidImfile.class_id
    4244    FROM detRun
     45    JOIN detNormalizedExp
     46        USING(det_id, iteration)
    4347    JOIN detInputExp
    4448        USING(det_id, iteration)

  • branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_toresidimfile.sql

    r19816 r23208  
    1313    detProcessedImfile.class_id,
    1414    detProcessedImfile.uri,
    15     detNormalizedImfile.uri AS det_uri,
     15--  detNormalizedImfile.uri AS det_uri,
     16    detStackedImfile.uri AS det_uri,
    1617    detRun.det_id AS ref_det_id,
    1718    detRun.iteration AS ref_iter,
    … …  
    2627    ON detRun.det_id = detProcessedImfile.det_id
    2728    AND detInputExp.exp_id = detProcessedImfile.exp_id
    28 JOIN detNormalizedImfile
    29     ON detRun.det_id = detNormalizedImfile.det_id
    30     AND detRun.iteration = detNormalizedImfile.iteration
    31     AND detProcessedImfile.class_id = detNormalizedImfile.class_id
    32 JOIN detNormalizedExp
    33     ON detRun.det_id = detNormalizedExp.det_id
    34     AND detRun.iteration = detNormalizedExp.iteration
     29JOIN detStackedImfile
     30    ON detRun.det_id = detStackedImfile.det_id
     31    AND detRun.iteration = detStackedImfile.iteration
     32    AND detProcessedImfile.class_id = detStackedImfile.class_id
     33-- EAM : replacing detNormalizedImfile with detStackedImfile to change the sequencing
     34-- JOIN detNormalizedImfile
     35--     ON detRun.det_id = detNormalizedImfile.det_id
     36--     AND detRun.iteration = detNormalizedImfile.iteration
     37--     AND detProcessedImfile.class_id = detNormalizedImfile.class_id
     38-- EAM : we there is no reason to wait for all stacks to complete before continuing
     39-- JOIN detNormalizedExp
     40--     ON detRun.det_id = detNormalizedExp.det_id
     41--     AND detRun.iteration = detNormalizedExp.iteration
    3542LEFT JOIN detResidImfile
    3643    ON detRun.det_id = detResidImfile.det_id
    … …  
    4148    detRun.state = 'run'
    4249    AND detRun.mode = 'master'
    43     AND detNormalizedImfile.fault = 0
    44     AND detNormalizedExp.fault = 0
     50    AND detStackedImfile.fault = 0
     51--  AND detNormalizedImfile.fault = 0
     52--  AND detNormalizedExp.fault = 0
    4553    AND detResidImfile.det_id IS NULL
    4654    AND detResidImfile.iteration IS NULL

  • branches/eam_branches/eam_branch_20090303/ippTools/share/pxadmin_create_tables.sql

    r21308 r23208  
    737737    REFERENCES  detInputExp(det_id, iteration, exp_id),
    738738    FOREIGN KEY (det_id, exp_id, class_id)
    739     REFERENCES  detProcessedImfile(det_id, exp_id, class_id),
    740     FOREIGN KEY (ref_det_id, ref_iter)
    741     REFERENCES  detNormalizedExp(det_id, iteration)
     739    REFERENCES  detProcessedImfile(det_id, exp_id, class_id)
    742740) ENGINE=innodb DEFAULT CHARSET=latin1;
    743741

  • branches/eam_branches/eam_branch_20090303/ippTools/src/dettoolConfig.c

    r23154 r23208  
    640640    // -residimfile
    641641    psMetadata *residimfileArgs = psMetadataAlloc();
    642     psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-det_id",  0,            "search for detrend ID", 0);
    643     psMetadataAddS32(residimfileArgs, PS_LIST_TAIL, "-iteration",  0,            "search for iteration number", 0);
    644     psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by detrend ID", 0);
    645     psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-class_id",  0,            "search for class ID", NULL);
    646     psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-recip",  0,            "search for recipe", NULL);
    647     psMetadataAddU64(residimfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     642    psMetadataAddS64(residimfileArgs,  PS_LIST_TAIL, "-det_id",  0,            "search for detrend ID", 0);
     643    psMetadataAddS32(residimfileArgs,  PS_LIST_TAIL, "-iteration",  0,            "search for iteration number", 0);
     644    psMetadataAddS64(residimfileArgs,  PS_LIST_TAIL, "-exp_id",  0,            "search by detrend ID", 0);
     645    psMetadataAddStr(residimfileArgs,  PS_LIST_TAIL, "-class_id",  0,            "search for class ID", NULL);
     646    psMetadataAddStr(residimfileArgs,  PS_LIST_TAIL, "-recip",  0,            "search for recipe", NULL);
     647    psMetadataAddU64(residimfileArgs,  PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
    648648    psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-faulted",  0,            "only return imfiles with a fault status set", false);
    649649    psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
    650     psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-select_state",  0,            "search for state", NULL);
     650    psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-included",  0,            "restrict results to exposures 'includeded' in the current iteration", false);
     651    psMetadataAddStr(residimfileArgs,  PS_LIST_TAIL, "-select_state",  0,            "search for state", NULL);
    651652
    652653    // -revertresidimfile

  • branches/eam_branches/eam_branch_20090303/ippTools/src/dettool_processedimfile.c

    r19621 r23208  
    165165    PS_ASSERT_PTR_NON_NULL(config, false);
    166166
    167     bool hasWhere = false;
    168 
    169     PXOPT_LOOKUP_BOOL(included, config->args, "-included", false);
    170     PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
    171     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    172     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    173 
    174167    psMetadata *where = psMetadataAlloc();
    175168    PXOPT_COPY_S64(config->args, where, "-det_id", "detProcessedImfile.det_id", "==");
    … …  
    179172    PXOPT_COPY_STR(config->args, where, "-select_mode", "detRun.mode", "==");
    180173
     174    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     175    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     176
     177    PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
     178    PXOPT_LOOKUP_BOOL(included, config->args, "-included", false);
     179
    181180    psString query = pxDataGet("dettool_processedimfile.sql");
    182181    if (!query) {
    … …  
    186185    }
    187186
     187    bool hasWhere = false;
    188188    if (psListLength(where->list)) {
    189189        psString whereClause = psDBGenerateWhereSQL(where, NULL);
    … …  
    211211
    212212    if (faulted) {
     213        // list only faulted rows
    213214        psStringAppend(&query, " %s", " detProcessedImfile.fault != 0");
    214215    } else {

  • branches/eam_branches/eam_branch_20090303/ippTools/src/dettool_residimfile.c

    r19632 r23208  
    159159    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    160160    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     161
    161162    PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
     163    PXOPT_LOOKUP_BOOL(included, config->args, "-included", false);
    162164
    163165    psString query = pxDataGet("dettool_residimfile.sql");
    … …  
    175177    }
    176178    psFree(where);
     179
     180    // restrict search to included imfiles
     181    if (included) {
     182        if (hasWhere) {
     183            psStringAppend(&query, " AND detInputExp.include = 1");
     184        } else {
     185            psStringAppend(&query, " WHERE detInputExp.include = 1");
     186        }
     187        hasWhere = true;
     188    }
    177189
    178190    if (hasWhere) {
Note: See TracChangeset for help on using the changeset viewer.