Index: /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_processedimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_processedimfile.sql (revision 23207) +++ /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_processedimfile.sql (revision 23208) @@ -1,2 +1,3 @@ +-- is this DISTINCT needed? SELECT DISTINCT detRun.det_type, Index: /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_residimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_residimfile.sql (revision 23207) +++ /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_residimfile.sql (revision 23208) @@ -2,9 +2,13 @@ detRun.det_type, detRun.mode, - detResidImfile.*, - rawExp.exp_time - FROM detResidImfile - JOIN detRun - USING(det_id, iteration) - JOIN rawExp - USING(exp_id) + rawExp.exp_time, + detResidImfile.* +FROM detResidImfile +JOIN detRun + USING(det_id, iteration) +JOIN detInputExp + ON detRun.det_id = detInputExp.det_id + AND detRun.iteration = detInputExp.iteration + AND detResidImfile.exp_id = detInputExp.exp_id +JOIN rawExp + ON rawExp.exp_id = detResidImfile.exp_id Index: /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_tonormalizedstat.sql =================================================================== --- /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_tonormalizedstat.sql (revision 23207) +++ /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_tonormalizedstat.sql (revision 23208) @@ -1,58 +1,33 @@ --- select detRun.det_id (det_id) --- select detRun.iteration --- by: --- find the current iteration bassed on det_id --- find all exp_ids in the current det_id/iteration from detInputExp --- sort to detInputExp.imfiles to find the largest value per det_id/iter --- compare imfiles to the number of detStackedImfiles by class_id --- and: --- ??? --- det_id is not in detStackedImfile; --- iteration is not in detStackedImfile; - -SELECT - det_id, - det_type, - iteration, - camera, - workdir, - class_id -FROM - (SELECT DISTINCT - detRun.det_id, - detRun.det_type, - detRun.iteration, - detRun.workdir, - rawExp.camera, - detStackedImfile.class_id, - rawImfile.class_id as rawimfile_class_id - FROM detRun - JOIN detInputExp - ON detRun.det_id = detInputExp.det_id - AND detRun.iteration = detInputExp.iteration - JOIN rawExp - ON detInputExp.exp_id = rawExp.exp_id - JOIN rawImfile - ON rawExp.exp_id = rawImfile.exp_id - LEFT JOIN detStackedImfile - ON detInputExp.det_id = detStackedImfile.det_id - AND detInputExp.iteration = detStackedImfile.iteration - AND rawImfile.class_id = detStackedImfile.class_id - LEFT JOIN detNormalizedStatImfile - ON detStackedImfile.det_id = detNormalizedStatImfile.det_id - AND detStackedImfile.iteration = detNormalizedStatImfile.iteration - AND detStackedImfile.class_id = detNormalizedStatImfile.class_id - WHERE - detRun.state = 'run' - AND detRun.mode = 'master' - AND detNormalizedStatImfile.det_id IS NULL - AND detNormalizedStatImfile.iteration IS NULL - AND detNormalizedStatImfile.class_id IS NULL - GROUP BY --- rawExp.exp_id, - detRun.iteration, - detRun.det_id - HAVING - COUNT(rawImfile.class_id) = COUNT(detStackedImfile.class_id) - AND SUM(detStackedImfile.fault) = 0 - ) as tonormalizedstat +-- a det_run + iteration is ready for normstat when: all detResidImfile entries for that iteration corresponding to all detProcessedImfile entries are available +SELECT + detRun.det_id, + detRun.det_type, + detRun.iteration, + detRun.camera, + detRun.workdir, + detProcessedImfile.class_id, + COUNT(detProcessedImfile.class_id), + COUNT(detResidImfile.class_id) +FROM detRun +JOIN detProcessedImfile + ON detProcessedImfile.det_id = detRun.det_id +LEFT JOIN detResidImfile + ON detResidImfile.det_id = detProcessedImfile.det_id + AND detResidImfile.class_id = detProcessedImfile.class_id + AND detResidImfile.iteration = detRun.iteration +LEFT JOIN detNormalizedStatImfile + ON detNormalizedStatImfile.det_id = detRun.det_id + AND detNormalizedStatImfile.iteration = detRun.iteration + AND detNormalizedStatImfile.class_id = detProcessedImfile.class_id +WHERE + detRun.state = 'run' + AND detRun.mode = 'master' + AND detNormalizedStatImfile.det_id IS NULL + AND detNormalizedStatImfile.iteration IS NULL + AND detNormalizedStatImfile.class_id IS NULL +GROUP BY + detRun.iteration, + detRun.det_id +HAVING + COUNT(detProcessedImfile.class_id) = COUNT(detResidImfile.class_id) + AND SUM(detResidImfile.fault) = 0 Index: /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_toresidexp.sql =================================================================== --- /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_toresidexp.sql (revision 23207) +++ /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_toresidexp.sql (revision 23208) @@ -5,4 +5,6 @@ -- id, detrend type, and whether the exposure was included in the stack for -- this iteration. + +-- require the corresponding detNormalizedExp to complete before starting -- select detRun.det_id @@ -41,4 +43,6 @@ detResidImfile.class_id FROM detRun + JOIN detNormalizedExp + USING(det_id, iteration) JOIN detInputExp USING(det_id, iteration) Index: /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_toresidimfile.sql =================================================================== --- /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_toresidimfile.sql (revision 23207) +++ /branches/eam_branches/eam_branch_20090303/ippTools/share/dettool_toresidimfile.sql (revision 23208) @@ -13,5 +13,6 @@ detProcessedImfile.class_id, detProcessedImfile.uri, - detNormalizedImfile.uri AS det_uri, +-- detNormalizedImfile.uri AS det_uri, + detStackedImfile.uri AS det_uri, detRun.det_id AS ref_det_id, detRun.iteration AS ref_iter, @@ -26,11 +27,17 @@ ON detRun.det_id = detProcessedImfile.det_id AND detInputExp.exp_id = detProcessedImfile.exp_id -JOIN detNormalizedImfile - ON detRun.det_id = detNormalizedImfile.det_id - AND detRun.iteration = detNormalizedImfile.iteration - AND detProcessedImfile.class_id = detNormalizedImfile.class_id -JOIN detNormalizedExp - ON detRun.det_id = detNormalizedExp.det_id - AND detRun.iteration = detNormalizedExp.iteration +JOIN detStackedImfile + ON detRun.det_id = detStackedImfile.det_id + AND detRun.iteration = detStackedImfile.iteration + AND detProcessedImfile.class_id = detStackedImfile.class_id +-- EAM : replacing detNormalizedImfile with detStackedImfile to change the sequencing +-- JOIN detNormalizedImfile +-- ON detRun.det_id = detNormalizedImfile.det_id +-- AND detRun.iteration = detNormalizedImfile.iteration +-- AND detProcessedImfile.class_id = detNormalizedImfile.class_id +-- EAM : we there is no reason to wait for all stacks to complete before continuing +-- JOIN detNormalizedExp +-- ON detRun.det_id = detNormalizedExp.det_id +-- AND detRun.iteration = detNormalizedExp.iteration LEFT JOIN detResidImfile ON detRun.det_id = detResidImfile.det_id @@ -41,6 +48,7 @@ detRun.state = 'run' AND detRun.mode = 'master' - AND detNormalizedImfile.fault = 0 - AND detNormalizedExp.fault = 0 + AND detStackedImfile.fault = 0 +-- AND detNormalizedImfile.fault = 0 +-- AND detNormalizedExp.fault = 0 AND detResidImfile.det_id IS NULL AND detResidImfile.iteration IS NULL Index: /branches/eam_branches/eam_branch_20090303/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branches/eam_branch_20090303/ippTools/share/pxadmin_create_tables.sql (revision 23207) +++ /branches/eam_branches/eam_branch_20090303/ippTools/share/pxadmin_create_tables.sql (revision 23208) @@ -737,7 +737,5 @@ REFERENCES detInputExp(det_id, iteration, exp_id), FOREIGN KEY (det_id, exp_id, class_id) - REFERENCES detProcessedImfile(det_id, exp_id, class_id), - FOREIGN KEY (ref_det_id, ref_iter) - REFERENCES detNormalizedExp(det_id, iteration) + REFERENCES detProcessedImfile(det_id, exp_id, class_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/eam_branches/eam_branch_20090303/ippTools/src/dettoolConfig.c =================================================================== --- /branches/eam_branches/eam_branch_20090303/ippTools/src/dettoolConfig.c (revision 23207) +++ /branches/eam_branches/eam_branch_20090303/ippTools/src/dettoolConfig.c (revision 23208) @@ -640,13 +640,14 @@ // -residimfile psMetadata *residimfileArgs = psMetadataAlloc(); - psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); - psMetadataAddS32(residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); - psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); - psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); - psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); - psMetadataAddU64(residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-det_id", 0, "search for detrend ID", 0); + psMetadataAddS32(residimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search for iteration number", 0); + psMetadataAddS64(residimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by detrend ID", 0); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID", NULL); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-recip", 0, "search for recipe", NULL); + psMetadataAddU64(residimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-select_state", 0, "search for state", NULL); + psMetadataAddBool(residimfileArgs, PS_LIST_TAIL, "-included", 0, "restrict results to exposures 'includeded' in the current iteration", false); + psMetadataAddStr(residimfileArgs, PS_LIST_TAIL, "-select_state", 0, "search for state", NULL); // -revertresidimfile Index: /branches/eam_branches/eam_branch_20090303/ippTools/src/dettool_processedimfile.c =================================================================== --- /branches/eam_branches/eam_branch_20090303/ippTools/src/dettool_processedimfile.c (revision 23207) +++ /branches/eam_branches/eam_branch_20090303/ippTools/src/dettool_processedimfile.c (revision 23208) @@ -165,11 +165,4 @@ PS_ASSERT_PTR_NON_NULL(config, false); - bool hasWhere = false; - - PXOPT_LOOKUP_BOOL(included, config->args, "-included", false); - PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-det_id", "detProcessedImfile.det_id", "=="); @@ -179,4 +172,10 @@ PXOPT_COPY_STR(config->args, where, "-select_mode", "detRun.mode", "=="); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(included, config->args, "-included", false); + psString query = pxDataGet("dettool_processedimfile.sql"); if (!query) { @@ -186,4 +185,5 @@ } + bool hasWhere = false; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereSQL(where, NULL); @@ -211,4 +211,5 @@ if (faulted) { + // list only faulted rows psStringAppend(&query, " %s", " detProcessedImfile.fault != 0"); } else { Index: /branches/eam_branches/eam_branch_20090303/ippTools/src/dettool_residimfile.c =================================================================== --- /branches/eam_branches/eam_branch_20090303/ippTools/src/dettool_residimfile.c (revision 23207) +++ /branches/eam_branches/eam_branch_20090303/ippTools/src/dettool_residimfile.c (revision 23208) @@ -159,5 +159,7 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + PXOPT_LOOKUP_BOOL(included, config->args, "-included", false); psString query = pxDataGet("dettool_residimfile.sql"); @@ -175,4 +177,14 @@ } psFree(where); + + // restrict search to included imfiles + if (included) { + if (hasWhere) { + psStringAppend(&query, " AND detInputExp.include = 1"); + } else { + psStringAppend(&query, " WHERE detInputExp.include = 1"); + } + hasWhere = true; + } if (hasWhere) {