Index: branches/eam_branches/ipp-20130419/ippTools/share/chiptool_pendingcleanupimfile.sql =================================================================== --- branches/eam_branches/ipp-20130419/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 35436) +++ branches/eam_branches/ipp-20130419/ippTools/share/chiptool_pendingcleanupimfile.sql (revision 35546) @@ -13,6 +13,9 @@ USING(chip_id) WHERE - ((chipRun.state = 'goto_cleaned' AND (chipProcessedImfile.data_state = 'full' - OR chipProcessedImfile.data_state = 'update') AND chipProcessedImfile.quality = 0) + ((chipRun.state = 'goto_cleaned' + AND (chipProcessedImfile.data_state = 'full' + OR chipProcessedImfile.data_state = 'update' + OR chipProcessedImfile.data_state = 'error_cleaned') + AND chipProcessedImfile.quality = 0) OR (chipRun.state = 'goto_scrubbed' AND chipProcessedImfile.data_state != 'scrubbed') Index: branches/eam_branches/ipp-20130419/ippTools/share/pxadmin_create_tables.sql =================================================================== --- branches/eam_branches/ipp-20130419/ippTools/share/pxadmin_create_tables.sql (revision 35436) +++ branches/eam_branches/ipp-20130419/ippTools/share/pxadmin_create_tables.sql (revision 35546) @@ -1,3 +1,3 @@ -CREATE TABLE dbversion ( +eREATE TABLE dbversion ( schema_version VARCHAR(64), updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP @@ -2172,4 +2172,5 @@ priority INT, PRIMARY KEY(rel_id), + UNIQUE KEY(surveyID, release_name), KEY(release_name), KEY(release_state), @@ -2196,4 +2197,5 @@ PRIMARY KEY (relexp_id), FOREIGN KEY(rel_id) REFERENCES ippRelease(rel_id), + UNIQUE KEY(rel_id, exp_id), KEY (state), KEY (fault), @@ -2224,6 +2226,6 @@ time_stamp DATETIME, PRIMARY KEY (relstack_id), + UNIQUE KEY (rel_id, tess_id, skycell_id, filter, stack_type, mjd_obs), KEY (tess_id, skycell_id), - KEY (rel_id, tess_id, skycell_id, filter, mjd_obs), KEY (stack_type), KEY (mjd_obs), Index: branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack.sql =================================================================== --- branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack.sql (revision 35436) +++ branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack.sql (revision 35546) @@ -6,5 +6,5 @@ stackRun.tess_id, stackRun.filter, - 0 AS mjd_obs, + stackSumSkyfile.mjd_obs, 0 AS zpt_obs, 0 AS zpt_stdev @@ -12,5 +12,13 @@ FROM ippRelease JOIN stackRun -LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter) - +JOIN stackSumSkyfile using(stack_id) +LEFT JOIN relStack AS previousRelStack + ON previousRelStack.rel_id = ippRelease.rel_id + AND previousRelStack.tess_id = stackRun.tess_id + AND previousRelStack.skycell_id = stackRun.skycell_id + AND previousRelStack.filter = stackRun.filter + -- JOIN hook %s WHERE previousRelStack.relstack_id IS NULL + AND stackRun.state ='full' + AND stackSumSkyfile.quality = 0 + AND stackSumSkyfile.fault = 0 Index: branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack_with_skycal.sql =================================================================== --- branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack_with_skycal.sql (revision 35436) +++ branches/eam_branches/ipp-20130419/ippTools/share/releasetool_definerelstack_with_skycal.sql (revision 35546) @@ -13,7 +13,14 @@ JOIN stackRun JOIN stackSumSkyfile USING(stack_id) -LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter) JOIN skycalRun ON skycalRun.stack_id = stackRun.stack_id JOIN skycalResult ON skycalRun.skycal_id = skycalResult.skycal_id +-- LEFT JOIN relStack AS previousRelStack USING(rel_id, tess_id, skycell_id, filter) +LEFT JOIN relStack AS previousRelStack + ON ippRelease.rel_id = previousRelStack.rel_id + AND stackRun.tess_id = previousRelStack.tess_id + AND stackRun.skycell_id = previousRelStack.skycell_id + AND stackRun.filter = previousRelStack.filter +-- JOIN Hook %s WHERE previousRelStack.relstack_id IS NULL + AND skycalRun.state = 'full' Index: branches/eam_branches/ipp-20130419/ippTools/share/releasetool_listrelstack.sql =================================================================== --- branches/eam_branches/ipp-20130419/ippTools/share/releasetool_listrelstack.sql (revision 35436) +++ branches/eam_branches/ipp-20130419/ippTools/share/releasetool_listrelstack.sql (revision 35546) @@ -7,4 +7,5 @@ stackRun.state AS stack_state, stackRun.data_group AS stack_data_group, + skycalResult.fwhm_major, skycalResult.path_base AS skycal_path_base, skycalRun.data_group AS skycal_data_group, Index: branches/eam_branches/ipp-20130419/ippTools/share/warptool_pendingcleanupskyfile.sql =================================================================== --- branches/eam_branches/ipp-20130419/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 35436) +++ branches/eam_branches/ipp-20130419/ippTools/share/warptool_pendingcleanupskyfile.sql (revision 35546) @@ -10,7 +10,9 @@ USING(warp_id) WHERE - ((warpRun.state = 'goto_cleaned' AND (warpSkyfile.data_state = 'full' - OR warpSkyfile.data_state = 'update') - AND warpSkyfile.quality = 0) + ((warpRun.state = 'goto_cleaned' + AND (warpSkyfile.data_state = 'full' + OR warpSkyfile.data_state = 'error_cleaned' + OR warpSkyfile.data_state = 'update') + AND warpSkyfile.quality = 0) OR (warpRun.state = 'goto_scrubbed' AND warpSkyfile.data_state != 'scrubbed') Index: branches/eam_branches/ipp-20130419/ippTools/src/releasetool.c =================================================================== --- branches/eam_branches/ipp-20130419/ippTools/src/releasetool.c (revision 35436) +++ branches/eam_branches/ipp-20130419/ippTools/src/releasetool.c (revision 35546) @@ -702,4 +702,8 @@ PXOPT_COPY_STR(config->args, where, "-cam_data_group", "camRun.data_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-warp_data_group", "warpRun.data_group", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-tess_id", "warpRun.tess_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); + + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); PXOPT_COPY_STR(config->args, where, "-surveyName", "survey.surveyName", "LIKE"); @@ -715,4 +719,8 @@ psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); return false; + } + + if (skycell_id) { + psStringAppend(&query, "\nJOIN warpSkyfile ON warpRun.warp_id = warpSkyfile.warp_id AND warpRun.tess_id = warpSkyfile.tess_id"); } @@ -741,5 +749,5 @@ if (priority_order) { - psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority"); + psStringAppend(&query, "\nAND priority > 0 order by exp_id, priority DESC"); } @@ -787,7 +795,7 @@ bool includeSkycal = false; + PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE"); PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycalRun.skycal_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycal_label", "skycalRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "=="); if (psListLength(where->list)) { includeSkycal = true; @@ -795,5 +803,5 @@ PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE"); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackRun.stack_id", "=="); @@ -848,7 +856,24 @@ } + psString joinConditions = psStringCopy(""); + // Finish up the selectors for the JOIN to previousRelStack + // join to rows with our stack_type + psStringAppend(&joinConditions, "\nAND previousRelStack.stack_type = '%s'", stack_type); + + if (!strcmp(stack_type, "nightly")) { + // detect nightly stack entries that already exist for this day + psStringAppend(&joinConditions, "\nAND previousRelStack.mjd_obs = floor(stackSumSkyfile.mjd_obs)"); + } + + // Add in the where conditions psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&query, "\nAND %s", whereClause); psFree(whereClause); + + if (!strcmp(stack_type, "nightly")) { + // avoid stacks with NAN mjd_obs. These are old skycells that were lost due to system failure + // prior to the time that the mjd_obs was extracted from the headers and added to stackSumSkyfile + psStringAppend(&query, "\nAND stackSumSkyfile.mjd_obs IS NOT NULL"); + } psFree(where); @@ -860,10 +885,12 @@ } - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, joinConditions)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); + psFree(joinConditions); return false; } psFree(query); + psFree(joinConditions); psArray *output = p_psDBFetchResult(config->dbh); @@ -985,8 +1012,7 @@ PXOPT_COPY_F32(config->args, where, "-mjd_min", "stackSumSkyfile.mjd_obs", ">="); PXOPT_COPY_F32(config->args, where, "-mjd_max", "stackSumSkyfile.mjd_obs", "<="); - PXOPT_COPY_F32(config->args, where, "-fwhm_min", "skycalResult.fwhm_minor", ">="); - PXOPT_COPY_F32(config->args, where, "-fwhm_max", "skycalresult.fwhm_major", "<="); PXOPT_COPY_STR(config->args, where, "-tess_id", "relStack.tess_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "relStack.skycell_id", "LIKE"); +// PXOPT_COPY_STR(config->args, where, "-stack_type", "relStack.stack_type", "=="); PXOPT_COPY_STR(config->args, where, "-stack_data_group", "stackRun.data_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-skycal_data_group", "skycalRun.data_group", "LIKE"); @@ -996,8 +1022,13 @@ pxskycellAddWhere(config, where); + PXOPT_COPY_F32(config->args, where, "-fwhm_min", "IFNULL(skycalResult.fwhm_major, 999)", ">="); + PXOPT_COPY_F32(config->args, where, "-fwhm_max", "IFNULL(skycalResult.fwhm_major, 0)", "<="); + PXOPT_LOOKUP_BOOL(priority_order, config->args, "-priority_order", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + pxAddLabelSearchArgs (config, where, "-stack_type", "relStack.stack_type", "=="); psString query = pxDataGet("releasetool_listrelstack.sql"); @@ -1018,5 +1049,5 @@ if (priority_order) { - psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority"); + psStringAppend(&query, "\nAND priority > 0 order by stack_id, priority DESC"); } @@ -1487,5 +1518,5 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); PXOPT_COPY_F32(config->args, where, "-fwhm_min", "skycalResult.fwhm_major", ">="); - PXOPT_COPY_F32(config->args, where, "-fwhm_max", "skycalresult.fwhm_major", "<="); + PXOPT_COPY_F32(config->args, where, "-fwhm_max", "skycalResult.fwhm_major", "<="); PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_ame", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "relExp.exp_id", "=="); Index: branches/eam_branches/ipp-20130419/ippTools/src/releasetoolConfig.c =================================================================== --- branches/eam_branches/ipp-20130419/ippTools/src/releasetoolConfig.c (revision 35436) +++ branches/eam_branches/ipp-20130419/ippTools/src/releasetoolConfig.c (revision 35546) @@ -164,4 +164,6 @@ psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-cam_data_group", 0, "camRun.data_group (LIKE comparison)", NULL); psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-warp_data_group", 0, "warpRun.data_group (LIKE comparison)", NULL); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-skycell_id", 0, "select by skycell", NULL); + psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-tess_id", 0, "warpRun.tess_id", NULL); psMetadataAddStr(listrelexpArgs, PS_LIST_TAIL, "-surveyName", 0, "select by survey name (LIKE comparision)", NULL); @@ -220,6 +222,10 @@ psMetadataAddF32(listrelstackArgs, PS_LIST_TAIL, "-fwhm_max", 0, "search by seeing (<=)", NAN); + psMetadataAddF32(listrelstackArgs, PS_LIST_TAIL, "-mjd_min", 0, "search by MJD seeing (>=)", NAN); + psMetadataAddF32(listrelstackArgs, PS_LIST_TAIL, "-mjd_max", 0, "search by MJD (<=)", NAN); + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-tess_id", 0, "select by tess_id", NULL); psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-skycell_id", 0, "select by skycell_id (LIKE comparision)", NULL); + psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-stack_type", PS_META_DUPLICATE_OK, "select by stack_type", NULL); psMetadataAddStr(listrelstackArgs, PS_LIST_TAIL, "-stack_data_group", 0, "select by stackRun.data_group (LIKE comparison)", NULL);