Index: branches/eam_branches/ipp-20111122/ippTools/src/addtool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/addtool.c (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/addtool.c (revision 32851) @@ -143,5 +143,11 @@ pxAddLabelSearchArgs (config, where, "-filter", "stackRun.filter", "=="); // define using camRun label } - + if (strcmp(stage, "staticsky_multi")== 0) { + + pxAddLabelSearchArgs (config, where, "-label", "staticskyRun.label", "=="); // define using camRun label + pxAddLabelSearchArgs (config, where, "-data_group","staticskyRun.data_group", "=="); // define using camRun label + PXOPT_COPY_STR(config->args, where, "-reduction", "staticskyyRun.reduction", "=="); + //no filter here + } if (!psListLength(where->list)) { psFree(where); @@ -237,5 +243,19 @@ } - + if (strcmp(stage,"staticsky_multi") == 0) { + if (dvodb ) { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument found (%s) using addtool_find_sky_id_multi_dvo.sql\n%s\n", dvodb,stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id_multi_dvo.sql"); + // user supplied dvodb + psStringAppend(&dvodb_string, "addRun.dvodb = '%s'", dvodb); + } else { + psTrace("addtool.c", PS_LOG_INFO, "dvodb argument not found using addtool_find_sky_id_multi.sql\n%s\n",stage); + // find the cam_id of all the exposures that we want to queue up. + bare_query = pxDataGet("addtool_find_sky_id_multi.sql"); + // inherit dvodb from camRun, avoid matching NULL + psStringAppend(&dvodb_string, "(staticskyRun.dvodb IS NOT NULL AND previous_dvodb = staticskyRun.dvodb)"); + } + } if (!bare_query) { @@ -292,5 +312,7 @@ psStringAppend(&query, " GROUP BY stack_id"); } - + if (strcmp(stage,"staticsky_multi") == 0) { + psStringAppend(&query, " GROUP BY sky_id"); //some reason it needs this + } @@ -405,4 +427,43 @@ } } + if (strcmp(stage,"staticsky_multi") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + bool status = false; + psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs"); + return false; + } + if (num_inputs < 0) { + psError(PS_ERR_UNKNOWN, true, "invalid value for num_inputs"); + return false; + } + + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb) { //there's no staticsky.dvodb + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, sky_id %" PRId64, row->label, row->sky_id); + psFree(output); + return false; + } + + psFree(row); + } + } + + + // start a transaction so we don't end up with an exp without any associted @@ -438,4 +499,5 @@ stage, stage_id, + 0, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -479,4 +541,5 @@ stage, stage_id, + 0, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -520,4 +583,5 @@ stage, stage_id, + 0, workdir ? workdir : row->workdir, reduction ? reduction : row->reduction, @@ -544,4 +608,57 @@ } + if (strcmp(stage,"staticsky_multi") == 0) { + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + psS64 stage_id =0; + bool status = false; + psS32 num_inputs = psMetadataLookupS32(&status, md, "num_inputs"); + if (!status) { + psError(PS_ERR_UNKNOWN, true, "failed to lookup value for item num_inputs"); + return false; + } + staticskyRunRow *row = staticskyRunObjectFromMetadata(md); + stage_id = row->sky_id; + + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + // queue the exp + for (int filter_id = 0; filter_id < num_inputs; filter_id++) { + if (!pxaddQueueByCamID(config, + stage, + stage_id, + filter_id, + workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, + label ? label : row->label, + data_group ? data_group : (row->data_group ? row->data_group : (label ? label : row->label)), + dvodb ? dvodb : NULL, + note ? note : NULL, + image_only, + minidvodb, + minidvodb_group, + minidvodb_name + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error sfg"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue stage %s %" PRId64,stage, stage_id); + psFree(row); + psFree(output); + return false; + } + } + psFree(row); + } + } + + + + psFree(output); @@ -585,4 +702,7 @@ } if (strcmp(stage, "staticsky")==0) { + query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); + } + if (strcmp(stage, "staticsky_multi")==0) { query = psStringCopy("UPDATE addRun JOIN staticskyRun on sky_id = stage_id"); } @@ -626,5 +746,8 @@ query = pxDataGet("addtool_find_pendingexp_staticsky.sql"); } - + if (strcmp(stage, "staticsky_multi")==0) { + query = pxDataGet("addtool_find_pendingexp_staticsky_multi.sql"); + } + if (!query) { @@ -648,5 +771,8 @@ psStringAppend(&query, " GROUP BY %s", "sky_id"); } - + if (strcmp(stage, "staticsky_multi") == 0) { + //this group by is needed to join against all the warps (to get camera) + psStringAppend(&query, " GROUP BY %s", "sky_id, stage_extra1"); + } // treat limit == 0 as "no limit" if (limit) { @@ -838,4 +964,8 @@ query = pxDataGet("addtool_find_processedexp_staticsky.sql"); } + if (strcmp (stage,"staticsky_multi") == 0) { + query = pxDataGet("addtool_find_processedexp_staticsky_multi.sql"); + } + if (!query) { @@ -948,4 +1078,10 @@ query = pxDataGet("addtool_revertprocessedexp_staticsky.sql"); } + if (strcmp(stage, "staticsky_multi") == 0) { + query = pxDataGet("addtool_revertprocessedexp_staticsky_multi.sql"); + } + + + if (!query) { // rollback Index: branches/eam_branches/ipp-20111122/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/disttool.c (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/disttool.c (revision 32851) @@ -419,6 +419,10 @@ } - if (!strcmp(stage, "sky") && single) { - psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) = 1"); + if (!strcmp(stage, "sky")) { + if (single) { + psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) = 1"); + } else { + psStringAppend(&query, "\nGROUP BY sky_id HAVING count(stack_id) > 1"); + } } Index: branches/eam_branches/ipp-20111122/ippTools/src/flatcorr.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/flatcorr.c (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/flatcorr.c (revision 32851) @@ -666,4 +666,5 @@ "cam", row->cam_id, + 0, row->workdir, row->reduction, Index: branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.c (revision 32851) @@ -56,4 +56,7 @@ static bool revertdependentMode(pxConfig *config); static bool getwebrequestnumMode(pxConfig *config); +static bool addfileMode(pxConfig *config); +static bool listfileMode(pxConfig *config); +static bool deletefileMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -103,4 +106,7 @@ MODECASE(PSTAMPTOOL_MODE_REVERTDEPENDENT, revertdependentMode); MODECASE(PSTAMPTOOL_MODE_GETWEBREQUESTNUM, getwebrequestnumMode); + MODECASE(PSTAMPTOOL_MODE_ADDFILE, addfileMode); + MODECASE(PSTAMPTOOL_MODE_LISTFILE, listfileMode); + MODECASE(PSTAMPTOOL_MODE_DELETEFILE, deletefileMode); default: psAbort("invalid option (this should not happen)"); @@ -1476,2 +1482,122 @@ return true; } + +static bool addfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + PXOPT_LOOKUP_STR(path, config->args, "-path", true, false); + + if (!pstampFileInsert(config->dbh, + 0, // file_id + job_id, + path + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, + "should have affected one row but %" PRIu64 " rows were modified", + affected); + return false; + } + + psS64 file_id = psDBLastInsertID(config->dbh); + printf("%" PRId64 "\n", file_id); + + return true; +} + +static bool listfileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-file_id", "file_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (!psListLength(where->list)) { + fprintf(stderr, "search arguments are required\n"); + exit (1); + } + + psString query = pxDataGet("pstamptool_listfile.sql"); + if (!query) { + psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampFile", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool deletefileMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", true, false); + + psString query = NULL; + psStringAppend(&query, "DELETE FROM pstampFile WHERE job_id = %" PRId64, job_id); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psU64 affected = psDBAffectedRows(config->dbh); + psLogMsg("pstamptool", PS_LOG_INFO, "Deleted %" PRIu64 " rows from pstampFile", affected); + + return true; +} Index: branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.h =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.h (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/pstamptool.h (revision 32851) @@ -50,4 +50,7 @@ PSTAMPTOOL_MODE_REVERTDEPENDENT, PSTAMPTOOL_MODE_GETWEBREQUESTNUM, + PSTAMPTOOL_MODE_ADDFILE, + PSTAMPTOOL_MODE_LISTFILE, + PSTAMPTOOL_MODE_DELETEFILE, } pstamptoolMode; Index: branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/pstamptoolConfig.c (revision 32851) @@ -274,4 +274,21 @@ psMetadata *modes = psMetadataAlloc(); + // -addfile + psMetadata *addfileArgs = psMetadataAlloc(); + psMetadataAddS64(addfileArgs, PS_LIST_TAIL, "-job_id", 0, "define job ID for file (required)", 0); + psMetadataAddStr(addfileArgs, PS_LIST_TAIL, "-path", 0, "define path for file (required)", NULL); + + // -listfile + psMetadata *listfileArgs = psMetadataAlloc(); + psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-file_id", 0, "select by file ID", 0); + psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID", 0); + psMetadataAddS64(listfileArgs, PS_LIST_TAIL, "-req_id", 0, "select by request ID", 0); + psMetadataAddU64(listfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -deletefile + psMetadata *deletefileArgs = psMetadataAlloc(); + psMetadataAddS64(deletefileArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID (required)", 0); + PXOPT_ADD_MODE("-addreq", "", PSTAMPTOOL_MODE_ADDREQ, addreqArgs); PXOPT_ADD_MODE("-pendingreq", "", PSTAMPTOOL_MODE_PENDINGREQ, pendingreqArgs); @@ -302,4 +319,7 @@ PXOPT_ADD_MODE("-project", "", PSTAMPTOOL_MODE_PROJECT, projectArgs); PXOPT_ADD_MODE("-getwebrequestnum","", PSTAMPTOOL_MODE_GETWEBREQUESTNUM, getwebrequestnumArgs); + PXOPT_ADD_MODE("-addfile", "", PSTAMPTOOL_MODE_ADDFILE, addfileArgs); + PXOPT_ADD_MODE("-listfile", "", PSTAMPTOOL_MODE_LISTFILE, listfileArgs); + PXOPT_ADD_MODE("-deletefile", "", PSTAMPTOOL_MODE_DELETEFILE, deletefileArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/pxadd.c (revision 32851) @@ -131,4 +131,5 @@ char *stage, psS64 stage_id, + psS32 stage_extra1, char *workdir, char *reduction, @@ -159,4 +160,8 @@ psMemSetPersistent(query, true); } + if (strcmp(stage,"staticsky_multi") == 0) { + query = pxDataGet("addtool_queue_sky_id_multi.sql"); + psMemSetPersistent(query, true); + } if (!query) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); @@ -170,4 +175,5 @@ // on 32/64 if (!p_psDBRunQueryF(config->dbh, query, + stage_extra1 ? stage_extra1 : 0, "new", // state workdir ? workdir : "NULL", Index: branches/eam_branches/ipp-20111122/ippTools/src/pxadd.h =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/pxadd.h (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/pxadd.h (revision 32851) @@ -33,4 +33,5 @@ char *stage, psS64 stage_id, + psS32 stage_extra1, char *workdir, char *reduction, Index: branches/eam_branches/ipp-20111122/ippTools/src/vptool.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/vptool.c (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/vptool.c (revision 32851) @@ -178,13 +178,4 @@ PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); - psMetadata *where = psMetadataAlloc(); - pxchipGetSearchArgs (config, where); // rawExp only - pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE"); // define using newExp label - - if (!psListLength(where->list)) { - psFree(where); - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - return false; - } PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false); @@ -195,7 +186,17 @@ // default - PXOPT_LOOKUP_BOOL(unique, config->args, "-unique", false); + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // rawExp only + pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "="); // define using newExp label + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } // find the exp_id of all the exposures that we want to queue up. @@ -213,6 +214,6 @@ psFree(where); - if (unique) { - psStringAppend(&query, "AND vp_id IS NULL"); + if (!rerun) { + psStringAppend(&query, "\n AND vp_id IS NULL\n AND (vpRun.label = '%s' OR vpRun.label IS NULL)", label); } Index: branches/eam_branches/ipp-20111122/ippTools/src/vptoolConfig.c =================================================================== --- branches/eam_branches/ipp-20111122/ippTools/src/vptoolConfig.c (revision 32758) +++ branches/eam_branches/ipp-20111122/ippTools/src/vptoolConfig.c (revision 32851) @@ -50,5 +50,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by newExp label (LIKE comparison)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label (required)", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); @@ -56,5 +56,5 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-set_dest_id", 0, "define destination", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-unique", 0, "only queue exposures that have no previous vpRun", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun", 0, "queue exposures even if a previous vpRun exists", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -pendingrun