Index: branches/czw_branch/20120906/ippTools/share/Makefile.am =================================================================== --- branches/czw_branch/20120906/ippTools/share/Makefile.am (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/Makefile.am (revision 34786) @@ -88,4 +88,5 @@ chiptool_listrun.sql \ chiptool_pendingcleanupimfile.sql \ + chiptool_pendingcleanupimfile_all.sql \ chiptool_pendingcleanuprun.sql \ chiptool_pendingimfile.sql \ @@ -172,4 +173,5 @@ difftool_pendingcleanuprun.sql \ difftool_pendingcleanupskyfile.sql \ + difftool_pendingcleanupskyfile_all.sql \ difftool_revertdiffskyfile_delete.sql \ difftool_revertdiffskyfile_updated.sql \ @@ -424,4 +426,5 @@ warptool_pendingcleanuprun.sql \ warptool_pendingcleanupskyfile.sql \ + warptool_pendingcleanupskyfile_all.sql \ warptool_revertoverlap.sql \ warptool_revertwarped_delete.sql \ Index: branches/czw_branch/20120906/ippTools/share/camtool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/camtool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/camtool_pendingcleanuprun.sql (revision 34786) @@ -2,4 +2,6 @@ camRun.cam_id, rawExp.camera, + rawExp.exp_tag, + camRun.workdir, camRun.state FROM camRun Index: branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanupimfile_all.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanupimfile_all.sql (revision 34786) +++ branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanupimfile_all.sql (revision 34786) @@ -0,0 +1,16 @@ +-- list of chipProcessedImfiles for runs in a goto_cleaned like state without regard to +-- data_state or quality. Used for confirming that a run is really in the cleaned state +SELECT + chipProcessedImfile.*, + chipRun.state, + chipRun.workdir, + chipRun.label, + chipRun.reduction, + chipRun.expgroup, + chipRun.dvodb, + chipRun.tess_id, + chipRun.end_stage +FROM chipRun +JOIN chipProcessedImfile + USING(chip_id) +WHERE ( chipRun.state = 'goto_cleaned' OR chipRun.state = 'goto_scrubbed' OR chipRun.state = 'goto_purged' ) Index: branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/chiptool_pendingcleanuprun.sql (revision 34786) @@ -1,6 +1,8 @@ -SELECT +SELECT DISTINCT chipRun.chip_id, rawExp.camera, - chipRun.state + rawExp.exp_tag, + chipRun.state, + chipRun.workdir FROM chipRun JOIN rawExp Index: branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanuprun.sql (revision 34786) @@ -6,4 +6,6 @@ rawExp.camera, diffRun.state, + diffRun.workdir, + diffRun.tess_id, diffRun.label FROM diffRun @@ -35,4 +37,6 @@ rawExp.camera, diffRun.state, + diffRun.workdir, + diffRun.tess_id, diffRun.label FROM diffRun @@ -64,4 +68,6 @@ rawExp.camera, diffRun.state, + diffRun.workdir, + diffRun.tess_id, diffRun.label FROM diffRun @@ -93,4 +99,6 @@ rawExp.camera, diffRun.state, + diffRun.workdir, + diffRun.tess_id, diffRun.label FROM diffRun Index: branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanupskyfile_all.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanupskyfile_all.sql (revision 34786) +++ branches/czw_branch/20120906/ippTools/share/difftool_pendingcleanupskyfile_all.sql (revision 34786) @@ -0,0 +1,10 @@ +SELECT + diffSkyfile.*, + diffRun.state, + diffRun.workdir, + diffRun.dvodb, + diffRun.tess_id +FROM diffRun +JOIN diffSkyfile + USING(diff_id) +WHERE ( diffRun.state = 'goto_cleaned' OR diffRun.state = 'goto_scrubbed' OR diffRun.state = 'goto_purged') Index: branches/czw_branch/20120906/ippTools/share/pstamptool_revertdependent.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/pstamptool_revertdependent.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/pstamptool_revertdependent.sql (revision 34786) @@ -4,4 +4,6 @@ SET pstampDependent.fault = 0 -- fault count hook %s -WHERE pstampDependent.state = 'new' +WHERE pstampRequest.state ='run' + AND pstampJob.state ='run' + AND pstampDependent.state = 'new' AND (pstampDependent.fault > 0) Index: branches/czw_branch/20120906/ippTools/share/stacktool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/stacktool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/stacktool_pendingcleanuprun.sql (revision 34786) @@ -8,6 +8,8 @@ stackRun.stack_id, rawExp.camera, - stackRun.state + stackRun.state, + stackSumSkyfile.path_base FROM stackRun +JOIN stackSumSkyfile using(stack_id) JOIN stackInputSkyfile USING(stack_id) Index: branches/czw_branch/20120906/ippTools/share/staticskytool_defineskycalrun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/staticskytool_defineskycalrun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/staticskytool_defineskycalrun.sql (revision 34786) @@ -2,4 +2,6 @@ sky_id, stack_id, + skycell_id, + filter, staticskyRun.workdir, staticskyRun.label, @@ -11,4 +13,5 @@ JOIN stackRun USING(stack_id) JOIN stackSumSkyfile USING(stack_id) + JOIN skycell USING(tess_id, skycell_id) LEFT JOIN skycalRun USING(sky_id, stack_id) WHERE staticskyRun.state = 'full' Index: branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanuprun.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanuprun.sql (revision 34785) +++ branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanuprun.sql (revision 34786) @@ -1,5 +1,7 @@ SELECT warpRun.warp_id, + warpRun.workdir, rawExp.camera, + rawExp.exp_tag, warpRun.state FROM warpRun Index: branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanupskyfile_all.sql =================================================================== --- branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanupskyfile_all.sql (revision 34786) +++ branches/czw_branch/20120906/ippTools/share/warptool_pendingcleanupskyfile_all.sql (revision 34786) @@ -0,0 +1,11 @@ +SELECT + warpSkyfile.*, + warpRun.state, + warpRun.workdir, + warpRun.label, + warpRun.dvodb, + warpRun.end_stage +FROM warpRun +JOIN warpSkyfile + USING(warp_id) +WHERE (warpRun.state = 'goto_cleaned' OR warpRun.state = 'goto_scrubbed' OR warpRun.state = 'goto_purged') Index: branches/czw_branch/20120906/ippTools/src/chiptool.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/chiptool.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/chiptool.c (revision 34786) @@ -1311,4 +1311,5 @@ PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", false, false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -1319,5 +1320,6 @@ pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - psString query = pxDataGet("chiptool_pendingcleanupimfile.sql"); + char *sql_file = all ? "chiptool_pendingcleanupimfile_all.sql" : "chiptool_pendingcleanupimfile.sql"; + psString query = pxDataGet(sql_file); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); Index: branches/czw_branch/20120906/ippTools/src/chiptoolConfig.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/chiptoolConfig.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/chiptoolConfig.c (revision 34786) @@ -296,4 +296,5 @@ psMetadataAddS64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(pendingcleanupimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", NULL); + psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-all", 0, "list all components regardless of data_state", false); psMetadataAddBool(pendingcleanupimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanupimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); Index: branches/czw_branch/20120906/ippTools/src/difftool.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/difftool.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/difftool.c (revision 34786) @@ -2561,4 +2561,5 @@ PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -2570,5 +2571,6 @@ pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); - psString query = pxDataGet("difftool_pendingcleanupskyfile.sql"); + char * sql_file = all ? "difftool_pendingcleanupskyfile_all.sql": "difftool_pendingcleanupskyfile.sql" ; + psString query = pxDataGet(sql_file); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -2740,17 +2742,35 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); - - if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) { - psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!state) { + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + PXOPT_LOOKUP_S16(quality, config->args, "-set_quality", false, false); + + if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, fault, quality)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } psFree (where); - return false; - } - psFree (where); + } else { + if (strcmp(state,"error_cleaned") == 0) { + change_skyfile_data_state(config,"error_cleaned", "goto_cleaned"); + } + else if (strcmp(state, "error_scrubbed") == 0) { + change_skyfile_data_state(config,"error_scrubbed", "goto_scrubbed"); + } + else if (strcmp(state, "error_purged") == 0) { + change_skyfile_data_state(config,"error_purged", "goto_purged"); + } + else { + psError(PS_ERR_UNKNOWN, false, "unhandled state given"); + return(false); + } + } return true; Index: branches/czw_branch/20120906/ippTools/src/difftoolConfig.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/difftoolConfig.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/difftoolConfig.c (revision 34786) @@ -395,4 +395,5 @@ psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all", 0, "list all skycells regardless of data_state", false); psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -411,4 +412,5 @@ psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -updatediffskyfile psMetadata *updatediffskyfileArgs = psMetadataAlloc(); @@ -416,5 +418,6 @@ psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); - psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); + psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-set_quality", 0, "set quality", 0); + psMetadataAddStr(updatediffskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "set data_state state", NULL); // -tocleanedskyfile Index: branches/czw_branch/20120906/ippTools/src/staticskytool.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/staticskytool.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/staticskytool.c (revision 34786) @@ -132,14 +132,4 @@ pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "stackRun.data_group", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); -#ifdef notdef - PXOPT_COPY_F32(config->args, whereMD, "-select_glat_min", "skycell.glat", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_glat_max", "skycell.glat", "<="); - PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_min", "skycell.radeg/15", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_rahours_max", "skycell.radeg/15", "<="); - PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_min", "skycell.radeg", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_radeg_max", "skycell.radeg", "<="); - PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_min", "skycell.decdeg", ">="); - PXOPT_COPY_F32(config->args, whereMD, "-select_decdeg_max", "skycell.decdeg", "<="); -#endif if (!pxskycellAddWhere(config, whereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); @@ -686,5 +676,5 @@ psStringAppend(&query, "\nGROUP BY sky_id"); if (num_filters) { - psStringAppend(&query, "\nHAVING COUNT(filter) >= %d", num_filters); + psStringAppend(&query, "\nHAVING COUNT(filter) = %d", num_filters); } @@ -1049,4 +1039,8 @@ pxAddLabelSearchArgs(config, whereMD, "-select_data_group", "staticskyRun.data_group", "LIKE"); pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, whereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + return false; + } PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); @@ -1128,4 +1122,5 @@ psString sky_label = psMetadataLookupStr(&status, row, "label"); psString sky_data_group = psMetadataLookupStr(&status, row, "data_group"); + psString sky_dist_group = psMetadataLookupStr(&status, row, "dist_group"); // create a staticskyRun @@ -1138,5 +1133,5 @@ label ? label : sky_label, data_group ? data_group : sky_data_group, - dist_group, + dist_group ? dist_group : sky_dist_group, reduction, registered, Index: branches/czw_branch/20120906/ippTools/src/staticskytoolConfig.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/staticskytoolConfig.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/staticskytoolConfig.c (revision 34786) @@ -126,5 +126,5 @@ psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL); - psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs (>=)", 0); + psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-num_filters", 0, "search by number of filters in the inputs", 0); pxskycellAddArguments(resultArgs); psMetadataAddS16(resultArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); @@ -166,4 +166,5 @@ psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_data_group", PS_META_DUPLICATE_OK, "search by stackRun data_group (LIKE comparison, multiple OK)", NULL); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL); + pxskycellAddArguments(defineskycalrunArgs); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); psMetadataAddStr(defineskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); @@ -183,4 +184,5 @@ psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + pxskycellAddArguments(updateskycalrunArgs); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", NULL); psMetadataAddStr(updateskycalrunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); Index: branches/czw_branch/20120906/ippTools/src/warptool.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/warptool.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/warptool.c (revision 34786) @@ -1838,4 +1838,5 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); psMetadata *where = psMetadataAlloc(); @@ -1843,5 +1844,6 @@ pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); - psString query = pxDataGet("warptool_pendingcleanupskyfile.sql"); + char * sql_file = all ? "warptool_pendingcleanupskyfile_all.sql" : "warptool_pendingcleanupskyfile.sql"; + psString query = pxDataGet(sql_file); if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); Index: branches/czw_branch/20120906/ippTools/src/warptoolConfig.c =================================================================== --- branches/czw_branch/20120906/ippTools/src/warptoolConfig.c (revision 34785) +++ branches/czw_branch/20120906/ippTools/src/warptoolConfig.c (revision 34786) @@ -364,4 +364,5 @@ psMetadataAddStr(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); psMetadataAddS64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID (required)", 0); + psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-all", 0, "list all skycells regardless of data_state", false); psMetadataAddBool(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);