Index: branches/eam_branches/20090820/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/20090820/ippTools/src/Makefile.am (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/Makefile.am (revision 25870) @@ -34,4 +34,5 @@ pxmagic.h \ pxregister.h \ + pxspace.h \ pxtag.h \ pxtools.h \ @@ -78,4 +79,5 @@ pxmagic.c \ pxregister.c \ + pxspace.c \ pxtag.c \ pxtools.c \ Index: branches/eam_branches/20090820/ippTools/src/addtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/addtool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/addtool.c (revision 25870) @@ -40,9 +40,4 @@ static bool maskedMode(pxConfig *config); static bool unblockMode(pxConfig *config); -static bool pendingcleanuprunMode(pxConfig *config); -static bool pendingcleanupexpMode(pxConfig *config); -static bool donecleanupMode(pxConfig *config); -static bool exportrunMode(pxConfig *config); -static bool importrunMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -74,9 +69,4 @@ MODECASE(ADDTOOL_MODE_MASKED, maskedMode); MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); - MODECASE(ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); - MODECASE(ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpMode); - MODECASE(ADDTOOL_MODE_DONECLEANUP, donecleanupMode); - MODECASE(ADDTOOL_MODE_EXPORTRUN, exportrunMode); - MODECASE(ADDTOOL_MODE_IMPORTRUN, importrunMode); default: psAbort("invalid option (this should not happen)"); @@ -106,7 +96,8 @@ psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); - pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + pxcamGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "camRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); if (!psListLength(where->list) && @@ -117,8 +108,13 @@ } - PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_BOOL(image_only, config->args, "-image_only", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // find the cam_id of all the exposures that we want to queue up. @@ -130,5 +126,5 @@ } - // use psDBGenerateWhereSQL because the SQL yields an intermediate table + // use psDBGenerateWhereConditionSQL because the SQL ends in a WHERE if (where && psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); @@ -136,5 +132,4 @@ psFree(whereClause); } - psFree(where); @@ -155,4 +150,40 @@ psFree(output); return true; + } + + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + return true; + } + + // loop over our list of camRun rows to check the supplied and selected dvodb and workdir values: + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + camRunRow *row = camRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); + psFree(output); + return false; + } + + if (!dvodb && !row->dvodb) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined dvodb: label: %s, cam_id %" PRId64, row->label, row->cam_id); + psFree(output); + return false; + } + if (!workdir && !row->workdir) { + psError(PS_ERR_UNKNOWN, false, "cannot queue addstar run without a defined workdir: label: %s, cam_id %" PRId64, row->label, row->cam_id); + psFree(output); + return false; + } + + psFree(row); } @@ -170,12 +201,11 @@ // old values in place (i.e., passing the values through). - - // loop over our list of addRun rows + // loop over our list of camRun rows for (long i = 0; i < psArrayLength(output); i++) { psMetadata *md = output->data[i]; - addRunRow *row = addRunObjectFromMetadata(md); + camRunRow *row = camRunObjectFromMetadata(md); if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into addRun"); + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into camRun"); psFree(output); return false; @@ -186,8 +216,10 @@ row->cam_id, workdir ? workdir : row->workdir, + reduction ? reduction : row->reduction, label ? label : row->label, - "RECIPE", - dvodb ? dvodb : row->dvodb - + data_group ? data_group : (label ? label : row->label), + dvodb ? dvodb : row->dvodb, + note ? note : NULL, + image_only )) { if (!psDBRollback(config->dbh)) { @@ -218,12 +250,12 @@ psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); // most search arguments based on camera PXOPT_COPY_STR(config->args, where, "-label", "addRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + if (!psListLength(where->list)) { psFree(where); psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); @@ -231,4 +263,16 @@ } + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and runs the update query + bool result = pxUpdateRun(config, where, &query, "addRun", false); + + psFree(query); + psFree(where); + + return result; + + +#ifdef notdef PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); @@ -259,4 +303,5 @@ return true; +#endif } @@ -267,9 +312,9 @@ psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -325,62 +370,4 @@ } -/* static bool pendingimfileMode(pxConfig *config) */ -/* { */ -/* PS_ASSERT_PTR_NON_NULL(config, false); */ - -/* psMetadata *where = psMetadataAlloc(); */ -/* pxaddGetSearchArgs (config, where); */ -/* PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); */ -/* pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); */ -/* PXOPT_COPY_S64(config->args, where, "-chip_id", "addRun.chip_id", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-class_id", "addProcessedExp.class_id", "=="); */ - -/* PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); */ - -/* psString query = pxDataGet("addtool_find_pendingimfile.sql"); */ -/* if (!query) { */ -/* psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); */ -/* return false; */ -/* } */ - -/* // use psDBGenerateWhereSQL because the SQL yields an intermediate table */ -/* if (psListLength(where->list)) { */ -/* psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); */ -/* psStringAppend(&query, " AND %s", whereClause); */ -/* psFree(whereClause); */ -/* } */ -/* psFree(where); */ - -/* if (!p_psDBRunQuery(config->dbh, query)) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* psFree(query); */ -/* return false; */ -/* } */ -/* psFree(query); */ - -/* psArray *output = p_psDBFetchResult(config->dbh); */ -/* if (!output) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* return false; */ -/* } */ -/* if (!psArrayLength(output)) { */ -/* psTrace("addtool", PS_LOG_INFO, "no rows found"); */ -/* psFree(output); */ -/* return true; */ -/* } */ - -/* // negate simple so the default is true */ -/* if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { */ -/* psError(PS_ERR_UNKNOWN, false, "failed to print array"); */ -/* psFree(output); */ -/* return false; */ -/* } */ - -/* psFree(output); */ - -/* return true; */ -/* } */ - static bool addprocessedexpMode(pxConfig *config) { @@ -391,11 +378,7 @@ // optional - PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); - - PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); - - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); - - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); // generate restrictions @@ -446,7 +429,6 @@ pendingRow->add_id, dtime_addstar, - n_stars, path_base, - 0 + fault ); @@ -463,5 +445,5 @@ // since there is only one exp per 'new' set addRun.state = 'full' - if (!pxaddRunSetState(config, row->add_id, "full", magicked)) { + if (!pxaddRunSetState(config, row->add_id, "full")) { psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id); psFree(row); @@ -469,38 +451,5 @@ return false; } - - // NULL for end_stage means go as far as possible - // EAM : skip here if fault != 0 - // Also, we can run fake even if tess_id is not defined -/* if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "add"))) { */ -/* psFree(row); */ -/* psFree(pendingRow); */ -/* if (!psDBCommit(config->dbh)) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* return false; */ -/* } */ -/* return true; */ -/* } */ psFree(row); - // else continue on... - -/* if (!pxfakeQueueByAddID(config, */ -/* pendingRow->add_id, */ -/* pendingRow->workdir, */ -/* pendingRow->label, */ -/* pendingRow->reduction, */ -/* pendingRow->expgroup, */ -/* pendingRow->dvodb, */ -/* pendingRow->tess_id, */ -/* pendingRow->end_stage */ -/* )) { */ -/* // rollback */ -/* if (!psDBRollback(config->dbh)) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* } */ -/* psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); */ -/* psFree(pendingRow); */ -/* return false; */ -/* } */ psFree(pendingRow); @@ -517,15 +466,16 @@ { PS_ASSERT_PTR_NON_NULL(config, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); - - // generate restrictions - psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); - PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); if (!psListLength(where->list) && @@ -613,9 +563,9 @@ psMetadata *where = psMetadataAlloc(); - pxaddGetSearchArgs (config, where); PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); + pxcamGetSearchArgs (config, where); pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); -/* PXOPT_COPY_S16(config->args, where, "-fault", "addProcessedExp.fault", "=="); */ if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { @@ -705,11 +655,8 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); -/* PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); */ + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) { @@ -777,5 +724,4 @@ } - static bool unblockMode(pxConfig *config) { @@ -793,324 +739,2 @@ return true; } - -static bool pendingcleanuprunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); - - psString query = pxDataGet("addtool_pendingcleanuprun.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupRun", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - - -static bool pendingcleanupexpMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", false, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - if (add_id) { - PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); - } - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - psString query = pxDataGet("addtool_pendingcleanupexp.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("chiptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupExp", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - - -static bool donecleanupMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - - psString query = pxDataGet("addtool_donecleanup.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("addtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "addDoneCleanup", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - -bool exportrunMode(pxConfig *config) -{ - typedef struct ExportTable { - char tableName[80]; - char sqlFilename[80]; - } ExportTable; - - int numExportTables = 2; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(det_id, config->args, "-add_id", true, false); - PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); - - FILE *f = fopen (outfile, "w"); - if (f == NULL) { - psError(PS_ERR_UNKNOWN, false, "failed to open output file"); - return false; - } - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); - - ExportTable tables [] = { - {"addRun", "addtool_export_run.sql"}, - {"addProcessedExp", "addtool_export_processed_exp.sql"}, - }; - - for (int i=0; i < numExportTables; i++) { - psString query = pxDataGet(tables[i].sqlFilename); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psError(PS_ERR_UNKNOWN, true, "no rows found"); - psFree(output); - return false; - } - - if (clean) { - if (!strcmp(tables[i].tableName, "addRun")) { - if (!pxSetStateCleaned("addRun", "state", output)) { - psFree(output); - psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); - return false; - } - } - } - - // we must write the export table in non-simple (true) format - if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - psFree(output); - } - - fclose (f); - - return true; -} - -bool importrunMode(pxConfig *config) -{ - unsigned int nFail; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); - - psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - - fprintf (stdout, "---- input ----\n"); - psMetadataPrint (stderr, input, 1); - - psMetadataItem *item = psMetadataLookup (input, "addRun"); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - psMetadataItem *entry = psListGet (item->data.list, 0); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - addRunRow *addRun = addRunObjectFromMetadata (entry->data.md); - addRunInsertObject (config->dbh, addRun); - - // fprintf (stdout, "---- add run ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - - item = psMetadataLookup (input, "addProcessedExp"); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - for (int i = 0; i < item->data.list->n; i++) { - psMetadataItem *entry = psListGet (item->data.list, i); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - addProcessedExpRow *addProcessedExp = addProcessedExpObjectFromMetadata (entry->data.md); - addProcessedExpInsertObject (config->dbh, addProcessedExp); - - // fprintf (stdout, "---- row %d ----\n", i); - // psMetadataPrint (stderr, entry->data.md, 1); - } - - return true; -} Index: branches/eam_branches/20090820/ippTools/src/addtoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/addtoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/addtoolConfig.c (revision 25870) @@ -48,10 +48,9 @@ // -definebyquery - // XXX need to allow multiple chip_ids - // XXX need to allow multiple exp_ids psMetadata *definebyqueryArgs = psMetadataAlloc(); + psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); pxcamSetSearchArgs(definebyqueryArgs); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); @@ -59,60 +58,62 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-image_only", 0, "addstar image metadata but not detections", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); // -updaterun - // XXX need to allow multiple add_ids - // XXX need to allow multiple chip_ids - // XXX need to allow multiple exp_ids psMetadata *updaterunArgs = psMetadataAlloc(); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); pxcamSetSearchArgs(updaterunArgs); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by addRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by addRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by addRun reduction class", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); - pxcamSetSearchArgs(pendingexpArgs); psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); - psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); - psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by addRun reduction class", NULL); psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // XXX is this used? psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); // -addprocessedexp psMetadata *addprocessedexpArgs = psMetadataAlloc(); psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); + psMetadataAddS16(addprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); - psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); - psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); - - psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); - psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); // -processedexp psMetadata *processedexpArgs = psMetadataAlloc(); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); pxcamSetSearchArgs(processedexpArgs); - psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all", 0, "list everything without restriction", false); - psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); -/* psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); */ // -revertprocessedexp - // XXX need to allow multiple add_ids - // XXX need to allow multiple chip_ids - // XXX need to allow multiple exp_ids psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); pxcamSetSearchArgs(revertprocessedexpArgs); - psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); @@ -121,8 +122,8 @@ // -updateprocessedexp - // XXX allow full search options? psMetadata *updateprocessedexpArgs = psMetadataAlloc(); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addtool ID", 0); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + psMetadataAddS16(updateprocessedexpArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); // -block @@ -137,37 +138,4 @@ psMetadata *unblockArgs = psMetadataAlloc(); psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); - - // -pendingcleanuprun - // XXX allow full search options? - psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); - psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); - psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -pendingcleanupexp - // XXX allow full search options? - psMetadata *pendingcleanupexpArgs = psMetadataAlloc(); - psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); - psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addstar ID", 0); - psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -donecleanup - psMetadata *donecleanupArgs = psMetadataAlloc(); - psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); - psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - - // -exportrun - psMetadata *exportrunArgs = psMetadataAlloc(); - psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-add_id", 0, "export this addstar ID (required)", 0); - psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); - psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export tables as cleaned", false); - - // -importrun - psMetadata *importrunArgs = psMetadataAlloc(); - psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); - psMetadata *argSets = psMetadataAlloc(); @@ -184,9 +152,4 @@ PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); - PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); - PXOPT_ADD_MODE("-pendingcleanupexp", "show exps for cleanup runs", ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs); - PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", ADDTOOL_MODE_DONECLEANUP, donecleanupArgs); - PXOPT_ADD_MODE("-exportrun", "export run for import on other database", ADDTOOL_MODE_EXPORTRUN, exportrunArgs); - PXOPT_ADD_MODE("-importrun", "import run from metadata file", ADDTOOL_MODE_IMPORTRUN, importrunArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090820/ippTools/src/camtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/camtool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/camtool.c (revision 25870) @@ -121,4 +121,6 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false); @@ -126,4 +128,5 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // find the exp_id of all the exposures that we want to queue up. @@ -212,9 +215,12 @@ workdir ? workdir : row->workdir, label ? label : row->label, + data_group ? data_group: row->data_group, + dist_group ? dist_group: row->dist_group, reduction ? reduction : row->reduction, expgroup ? expgroup : row->expgroup, dvodb ? dvodb : row->dvodb, tess_id ? tess_id : row->tess_id, - end_stage ? end_stage : row->end_stage + end_stage ? end_stage : row->end_stage, + note )) { if (!psDBRollback(config->dbh)) { @@ -251,39 +257,21 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "camRun.reduction", "=="); - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + if (!psListLength(where->list)) { psFree(where); psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); return false; } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set camRun.state to state - if (!pxcamRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set camRun.label to label - if (!pxcamRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "camRun", true); + if (!result) { + psError(PXTOOLS_ERR_DATA, false, "pxUpdateRun failed"); + } + + psFree(query); psFree(where); - return true; + return result; } @@ -642,9 +630,12 @@ pendingRow->workdir, pendingRow->label, + pendingRow->data_group, + pendingRow->dist_group, pendingRow->reduction, pendingRow->expgroup, pendingRow->dvodb, pendingRow->tess_id, - pendingRow->end_stage + pendingRow->end_stage, + NULL // note does not propagate )) { // rollback @@ -656,20 +647,4 @@ return false; } - -/* if (!pxaddQueueByCamID(config, */ -/* pendingRow->cam_id, */ -/* pendingRow->workdir, */ -/* pendingRow->label, */ -/* pendingRow->reduction, */ -/* pendingRow->dvodb */ -/* )) { */ -/* // rollback */ -/* if (!psDBRollback(config->dbh)) { */ -/* psError(PS_ERR_UNKNOWN, false, "database error"); */ -/* } */ -/* psError(PS_ERR_UNKNOWN, false, "failed to queue new addRun"); */ -/* psFree(pendingRow); */ -/* return false; */ -/* } */ psFree(pendingRow); @@ -1172,4 +1147,9 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); @@ -1252,6 +1232,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "camRun"); Index: branches/eam_branches/20090820/ippTools/src/camtoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/camtoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/camtoolConfig.c (revision 25870) @@ -62,5 +62,7 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tess ID", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); // -updaterun @@ -77,5 +79,7 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -pendingexp psMetadata *pendingexpArgs = psMetadataAlloc(); Index: branches/eam_branches/20090820/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/chiptool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/chiptool.c (revision 25870) @@ -57,4 +57,5 @@ static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool runstateMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -97,4 +98,5 @@ MODECASE(CHIPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(CHIPTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(CHIPTOOL_MODE_RUNSTATE, runstateMode); default: psAbort("invalid option (this should not happen)"); @@ -144,4 +146,7 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -263,4 +268,6 @@ workdir ? workdir : raw_workdir, label ? label : raw_label, + data_group ? data_group : (label ? label : raw_label), + dist_group, reduction ? reduction : raw_reduction, // expgroup ? expgroup : raw_expgroup, @@ -269,5 +276,6 @@ dvodb ? dvodb : raw_dvodb, tess_id ? tess_id : raw_tess_id, - end_stage ? end_stage : raw_end_stage + end_stage ? end_stage : raw_end_stage, + note )) { if (!psDBRollback(config->dbh)) { @@ -301,41 +309,24 @@ PXOPT_COPY_STR(config->args, where, "-label", "chipRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "=="); - - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group", "chipRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "chipRun.dist_group", "=="); + + if (!psListLength(where->list)) { psFree(where); - where = NULL; psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); return false; } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set chipRun.state to state - if (!pxchipRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set chipRun.label to label - if (!pxchipRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "chipRun", true); + if (!result) { + psError(PXTOOLS_ERR_DATA, false, "pxUpdateRun failed"); + } + + psFree(query); psFree(where); - return true; + return result; } @@ -605,12 +596,9 @@ PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "=="); - PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); - PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); - PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); - - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { - psFree(where); - psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "chipProcessedImfile"); + // add cuts on ra and decl if supplied + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); return false; } @@ -622,8 +610,14 @@ } - if (where && psListLength(where->list)) { + if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (psMetadataLookupBool(NULL, config->args, "-all") || (faulted || where2)) { + psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : ""); + } else { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + return false; } psFree(where); @@ -635,18 +629,4 @@ // don't list faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); - } - if (not_destreaked) { - if (destreaked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); - return false; - } - if (magicked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); - return false; - } - psStringAppend(&query, " AND chipProcessedImfile.magicked = 0"); - } - if (destreaked) { - psStringAppend(&query, " AND chipProcessedImfile.magicked != 0"); } @@ -739,12 +719,23 @@ PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); - - if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) { + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL); + + if (!state) { + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) { psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); return false; - } - psFree(where); - + } + psFree(where); + } + else { + if (!pxchipProcessedImfileSetStateByQuery(config,where,state)) { + psError(PS_ERR_UNKNOWN, false, "failed to set chipProcessedImfile state"); + return(false); + } + } + + return true; } @@ -1114,4 +1105,7 @@ psMetadata *where = psMetadataAlloc(); + pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); @@ -1253,9 +1247,12 @@ chipRun->workdir, chipRun->label, + chipRun->data_group, + chipRun->dist_group, chipRun->reduction, chipRun->expgroup, chipRun->dvodb, chipRun->tess_id, - chipRun->end_stage + chipRun->end_stage, + NULL // note does not propagate )) { if (!psDBRollback(config->dbh)) { @@ -1380,4 +1377,9 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "=="); @@ -1471,4 +1473,14 @@ psVector *identifiers = psVectorAllocEmpty(16, PS_TYPE_U64); // Identifiers inserted + psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL); // Iterator + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } + // first item is the dbversion, skip it + psMetadataItem *dbversion = psMetadataGetAndIncrement(iter); + (void) dbversion; + if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1476,5 +1488,4 @@ } - psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL); // Iterator psMetadataItem *item; // Item from iteration while ((item = psMetadataGetAndIncrement(iter))) { @@ -1505,2 +1516,82 @@ return true; } + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("chiptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/chiptool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/chiptool.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/chiptool.h (revision 25870) @@ -47,5 +47,6 @@ CHIPTOOL_MODE_TOSCRUBBEDIMFILE, CHIPTOOL_MODE_EXPORTRUN, - CHIPTOOL_MODE_IMPORTRUN + CHIPTOOL_MODE_IMPORTRUN, + CHIPTOOL_MODE_RUNSTATE } chiptoolMode; Index: branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c (revision 25870) @@ -58,7 +58,9 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define distribution group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); // -updaterun @@ -67,8 +69,12 @@ psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by dist_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "set data group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "set dist group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "set note", NULL); // -pendingimfile @@ -158,12 +164,12 @@ psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); - - psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); - psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); - psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); + pxmagicAddArguments(processedimfileArgs); + psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + pxspaceAddArguments(processedimfileArgs); // -revertprocessedimfile @@ -182,5 +188,5 @@ psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); psMetadataAddS16(updateprocessedimfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); - + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); // -block psMetadata *blockArgs = psMetadataAlloc(); @@ -229,4 +235,7 @@ psMetadataAddU64(runArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddStr(runArgs, PS_LIST_TAIL, "-state", 0, "search by state (required)", NULL); + pxchipSetSearchArgs(runArgs); + psMetadataAddS64(runArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddStr(runArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); // -advanceexp @@ -268,4 +277,15 @@ psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-chip_id", 0, "search by chip ID", 0); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -278,4 +298,5 @@ PXOPT_ADD_MODE("-updateprocessedimfile","change procesed imfile properties", CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,updateprocessedimfileArgs); PXOPT_ADD_MODE("-revertprocessedimfile","undo a processed imfile", CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,revertprocessedimfileArgs); + PXOPT_ADD_MODE("-runstate", "list the states of chip run", CHIPTOOL_MODE_RUNSTATE, runstateArgs); PXOPT_ADD_MODE("-advanceexp", "advance completed exposures", CHIPTOOL_MODE_ADVANCEEXP, advanceexpArgs); PXOPT_ADD_MODE("-block", "set a label block", CHIPTOOL_MODE_BLOCK, blockArgs); Index: branches/eam_branches/20090820/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/difftool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/difftool.c (revision 25870) @@ -51,4 +51,10 @@ static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked); +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state); +static bool tocleanedskyfileMode(pxConfig *config); +static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); + + # define MODECASE(caseName, func) \ @@ -88,4 +94,8 @@ MODECASE(DIFFTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(DIFFTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); + default: psAbort("invalid option (this should not happen)"); @@ -115,10 +125,13 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false); PXOPT_LOOKUP_BOOL(exposure, config->args, "-exposure", false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-rset_eduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -131,4 +144,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -137,5 +152,6 @@ bothways, exposure, - false + false, + note ); if (!run) { @@ -168,14 +184,45 @@ PS_ASSERT_PTR_NON_NULL(config, false); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-diff_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE diffRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "diffRun", true); + + psFree(query); + psFree(where); + + return result; + +#ifdef notdef // required options - PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false); PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + // Copy of my hacky work around from stacktool.c + if ((state)&&(diff_id)) { // set detRun.state to state return setdiffRunState(config, diff_id, state, false); } - return true; + if ((state)&&(label)) { + return setdiffRunStateByLabel(config, label, state); + } + + psError(PS_ERR_UNKNOWN, false, "Required options not found."); + + return false; +#endif } @@ -692,8 +739,8 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); - PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); - PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); - + + psString where2 = NULL; + pxmagicAddWhere(config, &where2, "diffSkyfile"); + pxspaceAddWhere(config, &where2, template ? "rawTemplate" : "rawInput"); psString query = pxDataGet("difftool_skyfile.sql"); if (!query) { @@ -706,24 +753,11 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (where2) { + psStringAppend(&query, " WHERE diffRun.diff_id is not null %s", where2); } else if (!all) { psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; } psFree(where); - - if (not_destreaked) { - if (destreaked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); - return false; - } - if (magicked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); - return false; - } - psStringAppend(&query, " AND diffSkyfile.magicked = 0"); - } - if (destreaked) { - psStringAppend(&query, " AND diffSkyfile.magicked != 0"); - } - // treat limit == 0 as "no limit" @@ -878,14 +912,47 @@ } - char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { + if (magicked) { + char *query = "UPDATE diffRun SET state = '%s', magicked = %" PRId64 " WHERE diff_id = %"PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, diff_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for diff_id %"PRId64, diff_id); return false; - } - + } + } + else { + char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = %"PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, diff_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for diff_id %"PRId64, diff_id); + return false; + } + } + return true; } + + +#ifdef notdef +static bool setdiffRunStateByLabel(pxConfig *config, const char *label, const char *state) { + PS_ASSERT_PTR_NON_NULL(state,false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid diffRun state: %s", state); + return false; + } + + char *query = "UPDATE diffRun SET state = '%s' WHERE label = '%s'"; + if (!p_psDBRunQueryF(config->dbh,query,state,label)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for label %s", label); + return(false); + } + + return true; +} +#endif // Generate a single populated run @@ -895,5 +962,8 @@ const char *tess_id, // Tessellation identifier const char *label, // label + const char *data_group, // data_group + const char *dist_group, // dist_group const char *reduction, // reduction + const char *note, // note psS64 input_warp_id, // Warp identifier for input image, PS_MAX_S64 for none psS64 input_stack_id, // Stack identifier for input image, PS_MAX_S64 for none @@ -926,4 +996,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -932,5 +1004,6 @@ false, false, - 0 // magicked + 0, // magicked + note ); @@ -1006,4 +1079,6 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required options PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); PXOPT_LOOKUP_S64(template_warp_id, config->args, "-template_warp_id", false, false); @@ -1012,4 +1087,5 @@ PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); if (template_stack_id && template_warp_id) { @@ -1035,5 +1111,5 @@ psArray *list = psArrayAllocEmpty(16); // List of runs, to print - if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction, + if (!populatedrun(list, workdir, skycell_id, tess_id, label, data_group ? data_group : label, dist_group, reduction, note, input_warp_id ? input_warp_id : PS_MAX_S64, input_stack_id ? input_stack_id : PS_MAX_S64, @@ -1074,8 +1150,12 @@ PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "=="); - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false); @@ -1313,4 +1393,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, NULL, // dvodb @@ -1319,5 +1401,6 @@ false, // bothways true, // exposure - 0 // magicked + 0, // magicked + note ); @@ -1388,5 +1471,5 @@ } - if (!diffRunPrintObjects(stdout, list, !simple)) { + if (numGood && !diffRunPrintObjects(stdout, list, !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print object"); psFree(list); @@ -1469,8 +1552,12 @@ // Settings to apply to defined run - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required options + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); // option + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); // option + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // option + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -1656,6 +1743,6 @@ } - diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, reduction, NULL, registered, - tess_id, true, true, false); // Run to insert + diffRunRow *run = diffRunRowAlloc(0, "reg", workdir, label, data_group ? data_group : label, dist_group, reduction, NULL, registered, + tess_id, true, true, false, note); // Run to insert if (!diffRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1738,5 +1825,5 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); psString query = pxDataGet("difftool_pendingcleanuprun.sql"); @@ -1760,4 +1847,5 @@ } + // fprintf(stderr,"%s",query); if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -1931,4 +2019,59 @@ return true; +} + +static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) { + PS_ASSERT_PTR_NON_NULL(config, false); + + // diff_id, skycell_id are required + PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); + PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); + + psString query = pxDataGet("difftool_change_skyfile_data_state.sql"); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + // Uses the unconstrained (diffRun.state [NEED NOT EQUAL] run_state) version from warptool.c + + if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + psFree(query); + + query = pxDataGet("difftool_change_run_state.sql"); + if (!p_psDBRunQueryF(config->dbh, query, data_state, diff_id, data_state)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + + return(true); +} + +static bool tocleanedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "cleaned","goto_cleaned"); +} +static bool topurgedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "purged", "goto_purged"); +} +static bool toscrubbedskyfileMode(pxConfig *config) { + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); } @@ -1956,4 +2099,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); @@ -2047,7 +2194,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "diffRun"); psAssert (item, "entry not in input?"); Index: branches/eam_branches/20090820/ippTools/src/difftool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/difftool.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/difftool.h (revision 25870) @@ -42,5 +42,9 @@ DIFFTOOL_MODE_UPDATEDIFFSKYFILE, DIFFTOOL_MODE_EXPORTRUN, - DIFFTOOL_MODE_IMPORTRUN + DIFFTOOL_MODE_IMPORTRUN, + DIFFTOOL_MODE_TOCLEANEDSKYFILE, + DIFFTOOL_MODE_TOPURGEDSKYFILE, + DIFFTOOL_MODE_TOSCRUBBEDSKYFILE + } difftoolMode; Index: branches/eam_branches/20090820/ippTools/src/difftoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/difftoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/difftoolConfig.c (revision 25870) @@ -47,17 +47,26 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways", 0, "do the subtraction both ways?", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure", 0, "subtraction for entire exposure?", false); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-bothways", 0, "do the subtraction both ways?", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-exposure", 0, "subtraction for entire exposure?", false); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-diff_id", 0, "define diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define by label instead of diff ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -addinputskyfile @@ -134,9 +143,8 @@ psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); - psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); - psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that are not destreaked", false); psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + pxmagicAddArguments(diffskyfileArgs); + pxspaceAddArguments(diffskyfileArgs); psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); @@ -175,8 +183,11 @@ psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-warp_label", 0, "search by warp label", NULL); psMetadataAddF32(definewarpstackArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); - psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definewarpstackArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definewarpstackArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-new-templates", 0, "also search for diffs with new template", false); psMetadataAddBool(definewarpstackArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); @@ -198,8 +209,11 @@ psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-template_label", 0, "search by warp label for template", NULL); psMetadataAddF32(definewarpwarpArgs, PS_LIST_TAIL, "-good_frac", 0, "minimum good fraction of skycell", NAN); - psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction class", NULL); - psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddTime(definewarpwarpArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definewarpwarpArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-rerun", 0, "define new run even if one exists", false); psMetadataAddBool(definewarpwarpArgs, PS_LIST_TAIL, "-available", 0, "define new run even if warpRun has some faults", false); @@ -259,4 +273,19 @@ psMetadataAddS16(updatediffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); + // -tocleanedskyfile + psMetadata *tocleanedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(tocleanedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -topurgedskyfile + psMetadata *topurgedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(topurgedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -toscrubbedskyfile + psMetadata *toscrubbedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "difftool ID to update", 0); + psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -296,4 +325,8 @@ PXOPT_ADD_MODE("-importrun", "import run from metadata file", DIFFTOOL_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", DIFFTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); + PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", DIFFTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", DIFFTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { psError(PS_ERR_UNKNOWN, true, "option parsing failed"); Index: branches/eam_branches/20090820/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/disttool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/disttool.c (revision 25870) @@ -32,5 +32,4 @@ static bool definebyqueryMode(pxConfig *config); -static bool definerunMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool revertrunMode(pxConfig *config); @@ -79,5 +78,4 @@ switch (config->mode) { MODECASE(DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); - MODECASE(DISTTOOL_MODE_DEFINERUN, definerunMode); MODECASE(DISTTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(DISTTOOL_MODE_REVERTRUN, revertrunMode); @@ -123,46 +121,4 @@ } -static bool definerunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); - PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); - PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false); - PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", true, false); - - // optional - PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); - PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); - PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); - - // TODO: check that stage has an expected value - // XXX: all of the following concerns will be managed properly by definebyquery - - // TODO: should we check that stage_id actually exists for stage - // in magicdstool we queue off of a magic_id so the stage_id, exp_id, and cam_id get looked up - // when the run is queued - - if (!distRunInsert(config->dbh, - 0, // dist_id - target_id, - stage, - stage_id, - 0, - set_label, - outroot, - clean, - no_magic, - "new", - NULL, // time_stamp - 0 // fault - )) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} static bool definebyqueryMode(pxConfig *config) @@ -178,4 +134,6 @@ PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); @@ -191,4 +149,5 @@ PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); psString query = NULL; @@ -209,4 +168,7 @@ psStringAppend(&query, " AND (chipRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (chipRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "chip")) { magicRunType = "chipRun"; @@ -222,4 +184,7 @@ psStringAppend(&query, " AND (chipRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (chipRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "camera")) { magicRunType = "camRun"; // This is used below to set the magicked business @@ -235,4 +200,7 @@ psStringAppend(&query, " AND (camRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (camRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "fake")) { magicRunType = "fakeRun"; @@ -246,4 +214,7 @@ if (label) { psStringAppend(&query, " AND (fakeRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (fakeRun.dist_group = '%s')", dist_group); } // fake stage doesn't require magic @@ -262,4 +233,7 @@ psStringAppend(&query, " AND (warpRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (warpRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "diff")) { @@ -276,4 +250,7 @@ psStringAppend(&query, " AND (diffRun.label = '%s')", label); } + if (dist_group) { + psStringAppend(&query, " AND (diffRun.dist_group = '%s')", dist_group); + } } else if (!strcmp(stage, "stack")) { @@ -288,4 +265,7 @@ if (label) { psStringAppend(&query, " AND (stackRun.label = '%s')", label); + } + if (dist_group) { + psStringAppend(&query, " AND (stackRun.dist_group = '%s')", dist_group); } // stack stage doesn't require magic @@ -371,5 +351,5 @@ psS64 magic_ds_id = psMetadataLookupS64(NULL, md, "magicked"); psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); - psString target_label = psMetadataLookupStr(NULL, md, "label"); + psString stage_label = psMetadataLookupStr(NULL, md, "label"); bool clean = psMetadataLookupBool(NULL, md, "clean"); @@ -381,5 +361,5 @@ new_label = set_label; } else { - new_label = target_label; + new_label = stage_label; } distRunRow *row = distRunRowAlloc( @@ -395,5 +375,6 @@ "new", NULL, // time_stamp - 0 // fault + 0, // fault + set_note // note does not propagate ); @@ -1239,5 +1220,5 @@ // required - PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", true, false); PXOPT_LOOKUP_STR(filter, config->args, "-filter", true, false); PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); @@ -1250,5 +1231,5 @@ distTargetRow *row = distTargetRowAlloc( 0, // target_id - label, + dist_group, filter, stage, @@ -1287,5 +1268,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "=="); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); @@ -1323,5 +1304,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); @@ -1518,5 +1499,5 @@ PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", false, false); PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); @@ -1528,6 +1509,6 @@ error = true; } - if (!label) { - psError(PS_ERR_UNKNOWN, !error, "label is required if target_id is not supplied"); + if (!dist_group) { + psError(PS_ERR_UNKNOWN, !error, "dist_group is required if target_id is not supplied"); error = true; } @@ -1539,5 +1520,5 @@ // optional PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); if (state) { if (strcmp(state, "enabled") && strcmp(state, "disabled")) { @@ -1555,5 +1536,5 @@ PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "=="); PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); @@ -1658,5 +1639,5 @@ PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "dist_group", "LIKE"); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); Index: branches/eam_branches/20090820/ippTools/src/disttool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/disttool.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/disttool.h (revision 25870) @@ -26,5 +26,4 @@ DISTTOOL_MODE_NONE = 0x0, DISTTOOL_MODE_DEFINEBYQUERY, - DISTTOOL_MODE_DEFINERUN, DISTTOOL_MODE_UPDATERUN, DISTTOOL_MODE_REVERTRUN, Index: branches/eam_branches/20090820/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/disttoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/disttoolConfig.c (revision 25870) @@ -50,4 +50,5 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "select by dist_group", NULL); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be selected", false); @@ -58,18 +59,9 @@ psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id", 0, "define chip_id", 0); psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define chip_id", 0); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "select by run label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dist_group", 0, "select by dist_group", NULL); psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -definerun - psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define stage for bundle (required)", NULL); - psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id (required)", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output destination (required)", NULL); - psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-clean", 0, "build clean distribution bundle", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not needed", false); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label for run", NULL); // -updaterun @@ -139,4 +131,5 @@ psMetadataAddS64(pendingfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); + psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-dist_group", PS_META_DUPLICATE_OK, "limit results to dist_group", NULL); psMetadataAddStr(pendingfilesetArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); psMetadataAddU64(pendingfilesetArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -219,5 +212,5 @@ // -definetarget psMetadata *definetargetArgs = psMetadataAlloc(); - psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-label", 0, "define label (required)", NULL); + psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-dist_group", 0, "define dist_group (required)", NULL); psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter (required)", NULL); psMetadataAddStr(definetargetArgs, PS_LIST_TAIL, "-stage", 0, "define stage (required)", NULL); @@ -230,12 +223,13 @@ psMetadataAddS64(updatetargetArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); - psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); - psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-dist_group", 0, "define dist_group", NULL); psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); + + psMetadataAddStr(updatetargetArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); // -listtarget psMetadata *listtargetArgs = psMetadataAlloc(); psMetadataAddS64(listtargetArgs, PS_LIST_TAIL, "-target_id", 0, "list target with target_id", 0); - psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-label", 0, "list targets for label", NULL); + psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-dist_group", 0, "list targets for dist_group", NULL); psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-filter", 0, "define filter", NULL); psMetadataAddStr(listtargetArgs, PS_LIST_TAIL, "-stage", 0, "list targets for stage", NULL); @@ -252,5 +246,5 @@ psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); - psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-dist_group", 0, "define dist_group", NULL); psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-filter", 0, "define filter (LIKE comparison)", NULL); psMetadataAddBool(defineinterestArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); @@ -271,5 +265,5 @@ psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-int_id", 0, "list interests with int_id", 0); psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dest_name", 0, "list interests for destinationn name)", NULL); - psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-label", 0, "list interests for label (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dist_group", 0, "list interests for dist_group (LIKE comparison)", NULL); psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-filter", 0, "list interests by filter (LIKE comparison)", NULL); psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-stage", 0, "list interests for stage", NULL); @@ -284,5 +278,4 @@ PXOPT_ADD_MODE("-definebyquery", "", DISTTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-definerun", "", DISTTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", DISTTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-revertrun", "", DISTTOOL_MODE_REVERTRUN, revertrunArgs); Index: branches/eam_branches/20090820/ippTools/src/faketool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/faketool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/faketool.c (revision 25870) @@ -177,4 +177,7 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); // default @@ -265,5 +268,5 @@ // queue the exp - if (!pxfakeQueueByCamID(config, cam_id, workdir, label, reduction, expgroup, dvodb, tess_id, end_stage)) { + if (!pxfakeQueueByCamID(config, cam_id, workdir, label, data_group ? data_group : label, dist_group, reduction, expgroup, dvodb, tess_id, end_stage, note)) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -330,7 +333,8 @@ PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "sun_angle", "<"); PXOPT_COPY_STR(config->args, where, "-label", "fakeRun.label", "=="); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group", "fakeRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group", "fakeRun.dist_group", "=="); + + if (!psListLength(where->list)) { psFree(where); where = NULL; @@ -339,32 +343,16 @@ } - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters are required"); - psFree(where); - return false; - } - - if (state) { - // set fakeRun.state to state - if (!pxfakeRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set fakeRun.label to label - if (!pxfakeRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "fakeRun", true); + if (!result) { + psError(PXTOOLS_ERR_DATA, false, "pxUpdateRun failed"); + } + + psFree(query); psFree(where); - return true; + return result; } @@ -1162,8 +1150,11 @@ fakeRun->workdir, fakeRun->label, + fakeRun->data_group, + fakeRun->dist_group, fakeRun->dvodb, fakeRun->tess_id, fakeRun->reduction, - fakeRun->end_stage + fakeRun->end_stage, + NULL // note does not propagate )) { psError(PS_ERR_UNKNOWN, false, "failed to queue warpRun"); @@ -1279,5 +1270,8 @@ return false; } - + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); @@ -1360,7 +1354,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "fakeRun"); psAssert (item, "entry not in input?"); Index: branches/eam_branches/20090820/ippTools/src/faketoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/faketoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/faketoolConfig.c (revision 25870) @@ -93,4 +93,7 @@ psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_tess_id", 0, "define tessellation identifier", NULL); psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-pretend", 0, "do not actually modify the database", false); psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -139,9 +142,14 @@ psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF32(updaterunArgs, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by data_group", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -pendingexp Index: branches/eam_branches/20090820/ippTools/src/flatcorr.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/flatcorr.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/flatcorr.c (revision 25870) @@ -33,4 +33,6 @@ #include "pxtools.h" #include "pxchip.h" +#include "pxcam.h" +#include "pxadd.h" #include "flatcorr.h" @@ -38,7 +40,9 @@ static bool definerunMode(pxConfig *config); static bool addchipMode(pxConfig *config); +static bool dropchipMode(pxConfig *config); static bool addcameraMode(pxConfig *config); -static bool dropchipMode(pxConfig *config); static bool dropcameraMode(pxConfig *config); +static bool advancecameraMode(pxConfig *config); +static bool advanceaddstarMode(pxConfig *config); static bool pendingprocessMode(pxConfig *config); static bool addprocessMode(pxConfig *config); @@ -46,6 +50,4 @@ static bool inputexpMode(pxConfig *config); static bool inputimfileMode(pxConfig *config); -static bool exportrunMode(pxConfig *config); -static bool importrunMode(pxConfig *config); static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state); @@ -72,7 +74,9 @@ MODECASE(FLATCORR_MODE_DEFINERUN, definerunMode); MODECASE(FLATCORR_MODE_ADDCHIP, addchipMode); + MODECASE(FLATCORR_MODE_DROPCHIP, dropchipMode); MODECASE(FLATCORR_MODE_ADDCAMERA, addcameraMode); - MODECASE(FLATCORR_MODE_DROPCHIP, dropchipMode); MODECASE(FLATCORR_MODE_DROPCAMERA, dropcameraMode); + MODECASE(FLATCORR_MODE_ADVANCECAMERA, advancecameraMode); + MODECASE(FLATCORR_MODE_ADVANCEADDSTAR, advanceaddstarMode); MODECASE(FLATCORR_MODE_PENDINGPROCESS, pendingprocessMode); MODECASE(FLATCORR_MODE_ADDPROCESS, addprocessMode); @@ -80,6 +84,4 @@ MODECASE(FLATCORR_MODE_INPUTEXP, inputexpMode); MODECASE(FLATCORR_MODE_INPUTIMFILE, inputimfileMode); - MODECASE(FLATCORR_MODE_EXPORTRUN, exportrunMode); - MODECASE(FLATCORR_MODE_IMPORTRUN, importrunMode); default: psAbort("invalid option (this should not happen)"); @@ -131,4 +133,5 @@ PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(region, config->args, "-set_region", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); @@ -233,5 +236,5 @@ // queue the exp : force this to stop at the chip stage - psS64 chip_id = pxchipQueueByExpTag(config, exp_id, workdir, label, reduction, expgroup, dvodb, tess_id, "chip"); + psS64 chip_id = pxchipQueueByExpTag(config, exp_id, workdir, label, label, NULL, reduction, expgroup, dvodb, tess_id, "chip", note); if (!chip_id) { if (!psDBRollback(config->dbh)) { @@ -297,8 +300,8 @@ // start a transaction so we don't end up with an exp without any associted // imfiles - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + // if (!psDBTransaction(config->dbh)) { + // psError(PS_ERR_UNKNOWN, false, "database error"); + // return false; + // } // create a new flatcorrRun @@ -333,4 +336,9 @@ row->corr_id = corr_id; + // if (!psDBCommit(config->dbh)) { + // psError(PS_ERR_UNKNOWN, false, "database error"); + // return false; + // } + flatcorrRunPrintObject (stdout, row, !simple); return true; @@ -378,7 +386,50 @@ } +static bool addcameraMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + + // add a flatcorrCamLink (initial state has include = TRUE) + // XXX should add checks that the chip_id and corr_id are in ChipLink + if (!flatcorrCamLinkInsert(config->dbh, corr_id, chip_id, cam_id, 1)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool dropcameraMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); + + // UPDATE flatcorrCamLink set include = 0 where corr_id = %lld AND cam_id = %lld + psString query = pxDataGet("flatcorr_dropcamera.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, (long long) corr_id, (long long) cam_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + return true; +} + // select the flatcorr chip runs that have completed and for which there is no camera entry // queue a new camera run for them -static bool addcameraMode(pxConfig *config) +static bool advancecameraMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -387,4 +438,7 @@ PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "flatcorrRun.label", "=="); psString query = pxDataGet("flatcorr_chiprundone.sql"); @@ -393,4 +447,11 @@ return false; } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); // treat limit == 0 as "no limit" @@ -463,9 +524,13 @@ row->workdir, row->label, + row->data_group, + row->dist_group, row->reduction, row->expgroup, row->dvodb, row->tess_id, - "camera")) { + "camera", + NULL // note does not propragate + )) { if (!psDBRollback(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -502,31 +567,7 @@ } -static bool dropcameraMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_S64(cam_id, config->args, "-cam_id", true, false); - - // UPDATE flatcorrCamLink set include = 0 where corr_id = %lld AND cam_id = %lld - psString query = pxDataGet("flatcorr_dropcamera.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, (long long) corr_id, (long long) cam_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - - return true; -} - -// select the flatcorr chip runs that have completed and for which there is no camera entry -// queue a new camera run for them -static bool pendingprocessMode(pxConfig *config) +// Select the flatcorr camera runs that have completed and for which there is no addstar +// entry. Queue a new addstar run for them +static bool advanceaddstarMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -534,9 +575,10 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - psString query = pxDataGet("flatcorr_pendingprocess.sql"); + pxAddLabelSearchArgs (config, where, "-label", "flatcorrRun.label", "=="); + + psString query = pxDataGet("flatcorr_camerarundone.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -575,8 +617,177 @@ } + if (pretend) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "flatcorr_addcamera", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // loop over our list of chipRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + bool status; + psS64 corr_id = psMetadataLookupS64(&status, md, "corr_id"); + if (!status) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to lookup value for corr_id"); + psFree(output); + return false; + } + + camRunRow *row = camRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun"); + psFree(output); + return false; + } + + // queue the exp : force image_only to be false (flatcorr is meaningless with just image info) + if (!pxaddQueueByCamID( + config, + row->cam_id, + row->workdir, + row->reduction, + row->label, + row->data_group, + row->dvodb, + NULL, // note is not propagated + 0)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue cam_id: %" PRId64, row->cam_id); + psFree(row); + psFree(output); + return false; + } + + // figure out the ID of the flatcorrRun we just created + psS64 add_id = psDBLastInsertID(config->dbh); + + // add the addRun entry to the flatcorrAddstarLink table (include is TRUE) + if (!flatcorrAddstarLinkInsert(config->dbh, corr_id, row->cam_id, add_id, 1)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return false; +} + +// select the flatcorr chip runs that have completed and for which there is no camera entry +// queue a new camera run for them +static bool pendingprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(limit, config->args, "-limit", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("flatcorr_pendingprocess.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("flatcorr", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "flatcorrPending", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); return false; + } + + return true; +} + +// XXX need a fault state +static bool addprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, hostname, code, corr_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); + return false; + } + + return true; +} + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + + if (!setflatcorrRunState(config, corr_id, state)) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "failed to set run state"); + return false; } @@ -688,41 +899,4 @@ } -// XXX need a fault state -static bool addprocessMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - - char *query = "UPDATE flatcorrRun SET state = 'full', hostname = '%s', fault = '%hd' WHERE corr_id = %" PRId64; - - if (!p_psDBRunQueryF(config->dbh, query, hostname, code, corr_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to change state for corr_id %" PRId64, corr_id); - return false; - } - - return true; -} - -static bool updaterunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_S64(corr_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (!setflatcorrRunState(config, corr_id, state)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "failed to set run state"); - return false; - } - - return true; -} - static bool setflatcorrRunState(pxConfig *config, psS64 corr_id, const char *state) { @@ -748,144 +922,2 @@ return true; } - -bool exportrunMode(pxConfig *config) -{ - typedef struct ExportTable { - char tableName[80]; - char sqlFilename[80]; - } ExportTable; - - int numExportTables = 3; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(det_id, config->args, "-corr_id", true, false); - PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - - FILE *f = fopen (outfile, "w"); - if (f == NULL) { - psError(PS_ERR_UNKNOWN, false, "failed to open output file"); - return false; - } - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-corr_id", "corr_id", "=="); - - ExportTable tables [] = { - {"flatcorrRun", "flatcorr_export_run.sql"}, - {"flatcorrCamLink", "flatcorr_export_cam_link.sql"}, - {"flatcorrChipLink", "flatcorr_export_chip_link.sql"}, - }; - - for (int i=0; i < numExportTables; i++) { - psString query = pxDataGet(tables[i].sqlFilename); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("regtool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // we must write the export table in non-simple (true) format - if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - psFree(output); - } - - fclose (f); - - return true; -} - -bool importrunMode(pxConfig *config) -{ - unsigned int nFail; - psMetadataItem *item, *entry; - - int numImportTables = 3; - - char tables[3] [80] = {"flatcorrRun", "flatcorrCamLink", "flatcorrChipLink"}; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); - - psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - - fprintf (stdout, "---- input ----\n"); - psMetadataPrint (stderr, input, 1); - - for (int i = 0; i < numImportTables; i++) { - item = psMetadataLookup (input, tables[i]); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - entry = psListGet (item->data.list, 0); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - - switch (i) { - case 0: - { - flatcorrRunRow *flatcorrRun = flatcorrRunObjectFromMetadata (entry->data.md); - flatcorrRunInsertObject (config->dbh, flatcorrRun); - - // fprintf (stdout, "---- flatcorr run ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - break; - } - case 1: - { - flatcorrCamLinkRow *flatcorrCamLink = flatcorrCamLinkObjectFromMetadata (entry->data.md); - flatcorrCamLinkInsertObject (config->dbh, flatcorrCamLink); - - // fprintf (stdout, "---- flatcorr cam link ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - break; - } - case 2: - { - flatcorrChipLinkRow *flatcorrChipLink = flatcorrChipLinkObjectFromMetadata (entry->data.md); - flatcorrChipLinkInsertObject (config->dbh, flatcorrChipLink); - - // fprintf (stdout, "---- flatcorr chip link ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - break; - } - } - } - return true; -} Index: branches/eam_branches/20090820/ippTools/src/flatcorr.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/flatcorr.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/flatcorr.h (revision 25870) @@ -28,7 +28,9 @@ FLATCORR_MODE_DEFINERUN, FLATCORR_MODE_ADDCHIP, + FLATCORR_MODE_DROPCHIP, FLATCORR_MODE_ADDCAMERA, - FLATCORR_MODE_DROPCHIP, FLATCORR_MODE_DROPCAMERA, + FLATCORR_MODE_ADVANCECAMERA, + FLATCORR_MODE_ADVANCEADDSTAR, FLATCORR_MODE_PENDINGPROCESS, FLATCORR_MODE_ADDPROCESS, @@ -36,6 +38,4 @@ FLATCORR_MODE_INPUTEXP, FLATCORR_MODE_INPUTIMFILE, - FLATCORR_MODE_EXPORTRUN, - FLATCORR_MODE_IMPORTRUN } flatcorrMode; Index: branches/eam_branches/20090820/ippTools/src/flatcorrConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/flatcorrConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/flatcorrConfig.c (revision 25870) @@ -65,4 +65,6 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-inst", 0, "define camera", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-det_type", 0, "define detrend type to be generated", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); @@ -88,7 +90,7 @@ // -addcamera psMetadata *addcameraArgs = psMetadataAlloc(); - psMetadataAddU64 (addcameraArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddBool(addcameraArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); + psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-chip_id", 0, "define Chip ID (required)", 0); + psMetadataAddS64(addcameraArgs, PS_LIST_TAIL, "-cam_id", 0, "define Camera ID (required)", 0); // -dropcamera @@ -96,4 +98,18 @@ psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-corr_id", 0, "define Flat Correction ID (required)", 0); psMetadataAddS64(dropcameraArgs, PS_LIST_TAIL, "-cam_id", 0, "define Camera ID (required)", 0); + + // -advancecamera + psMetadata *advancecameraArgs = psMetadataAlloc(); + psMetadataAddU64 (advancecameraArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(advancecameraArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(advancecameraArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + psMetadataAddStr(advancecameraArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); + + // -advanceaddstar + psMetadata *advanceaddstarArgs = psMetadataAlloc(); + psMetadataAddU64 (advanceaddstarArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(advanceaddstarArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(advanceaddstarArgs, PS_LIST_TAIL, "-pretend", 0, "use the simple output format", false); + psMetadataAddStr(advanceaddstarArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by label", NULL); // -pendingprocess @@ -142,17 +158,17 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "create a new, populated flat correction run", FLATCORR_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-definerun", "create a new, empty flat correction run", FLATCORR_MODE_DEFINERUN, definerunArgs); - PXOPT_ADD_MODE("-addchip", "add a chip to a flat correction run", FLATCORR_MODE_ADDCHIP, addchipArgs); - PXOPT_ADD_MODE("-addcamera", "migrate completed chips to camera stage analysis", FLATCORR_MODE_ADDCAMERA, addcameraArgs); - PXOPT_ADD_MODE("-dropchip", "drop a chip from a flat correction run", FLATCORR_MODE_DROPCHIP, dropchipArgs); - PXOPT_ADD_MODE("-dropcamera", "drop an exposure (camera stage analysis)", FLATCORR_MODE_DROPCAMERA, dropcameraArgs); - PXOPT_ADD_MODE("-pendingprocess", "show flat correction runs needing to be processed", FLATCORR_MODE_PENDINGPROCESS, pendingprocessArgs); - PXOPT_ADD_MODE("-addprocess", "report completed flat correction analysis", FLATCORR_MODE_ADDPROCESS, addprocessArgs); - PXOPT_ADD_MODE("-updaterun", "change a flat calibration run's state", FLATCORR_MODE_UPDATERUN, updaterunArgs); - PXOPT_ADD_MODE("-inputexp", "list exposures for a correction run", FLATCORR_MODE_INPUTEXP, inputexpArgs); - PXOPT_ADD_MODE("-inputimfile", "list imfiles for a chip run", FLATCORR_MODE_INPUTIMFILE, inputimfileArgs); - PXOPT_ADD_MODE("-exportrun", "export run for import on other database", FLATCORR_MODE_EXPORTRUN, exportrunArgs); - PXOPT_ADD_MODE("-importrun", "import run from metadata file", FLATCORR_MODE_IMPORTRUN, importrunArgs); + PXOPT_ADD_MODE("-definebyquery", "create a new, populated flat correction run", FLATCORR_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-definerun", "create a new, empty flat correction run", FLATCORR_MODE_DEFINERUN, definerunArgs); + PXOPT_ADD_MODE("-addchip", "add an existing chip run to a flat correction run", FLATCORR_MODE_ADDCHIP, addchipArgs); + PXOPT_ADD_MODE("-addcamera", "add an existing camera run to a flat correction run", FLATCORR_MODE_ADDCAMERA, addcameraArgs); + PXOPT_ADD_MODE("-advancecamera", "migrate completed chips to camera stage analysis", FLATCORR_MODE_ADVANCECAMERA, advancecameraArgs); + PXOPT_ADD_MODE("-advanceaddstar", "migrate completed exposures to addstar processing", FLATCORR_MODE_ADVANCEADDSTAR, advanceaddstarArgs); + PXOPT_ADD_MODE("-dropchip", "drop a chip from a flat correction run", FLATCORR_MODE_DROPCHIP, dropchipArgs); + PXOPT_ADD_MODE("-dropcamera", "drop an exposure (camera stage analysis)", FLATCORR_MODE_DROPCAMERA, dropcameraArgs); + PXOPT_ADD_MODE("-pendingprocess", "show flat correction runs needing to be processed", FLATCORR_MODE_PENDINGPROCESS, pendingprocessArgs); + PXOPT_ADD_MODE("-addprocess", "report completed flat correction analysis", FLATCORR_MODE_ADDPROCESS, addprocessArgs); + PXOPT_ADD_MODE("-updaterun", "change a flat calibration run's state", FLATCORR_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-inputexp", "list exposures for a correction run", FLATCORR_MODE_INPUTEXP, inputexpArgs); + PXOPT_ADD_MODE("-inputimfile", "list imfiles for a chip run", FLATCORR_MODE_INPUTIMFILE, inputimfileArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090820/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magicdstool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/magicdstool.c (revision 25870) @@ -33,5 +33,4 @@ static bool definebyqueryMode(pxConfig *config); -static psS64 definerunMode(pxConfig *config); static bool updaterunMode(pxConfig *config); static bool todestreakMode(pxConfig *config); @@ -43,7 +42,7 @@ static bool torevertMode(pxConfig *config); static bool completedrevertMode(pxConfig *config); - -static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state); -static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id); +static bool tocleanupMode(pxConfig *config); + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psMetadata *where, const char *state); # define MODECASE(caseName, func) \ @@ -66,5 +65,4 @@ switch (config->mode) { MODECASE(MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); - MODECASE(MAGICDSTOOL_MODE_DEFINERUN, definerunMode); MODECASE(MAGICDSTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(MAGICDSTOOL_MODE_TODESTREAK, todestreakMode); @@ -76,4 +74,5 @@ MODECASE(MAGICDSTOOL_MODE_TOREVERT, torevertMode); MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); + MODECASE(MAGICDSTOOL_MODE_TOCLEANUP, tocleanupMode); default: psAbort("invalid option (this should not happen)"); @@ -108,4 +107,6 @@ PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(set_data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false); @@ -246,4 +247,5 @@ psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id"); psString magicRunLabel = psMetadataLookupStr(NULL, row, "label"); + psString magicRunDataGroup = psMetadataLookupStr(NULL, row, "data_group"); psString magicRunWorkdir = psMetadataLookupStr(NULL, row, "workdir"); @@ -267,8 +269,11 @@ cam_id, set_label ? set_label : magicRunLabel, + set_data_group ? set_data_group : magicRunDataGroup, outroot, recoveryroot, noreplace ? 0 :1, // re_place - 0); // remove + 0, // remove + 0, // fault + note); // remove psFree(outroot); @@ -313,81 +318,43 @@ } -static psS64 definerunMode(pxConfig *config) + +static bool updaterunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); // required - PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); - PXOPT_LOOKUP_STR(outroot, config->args, "-outroot", true, false); - - // optional - PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); - PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); - PXOPT_LOOKUP_BOOL(remove, config->args, "-remove", false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - - psS64 stage_id = 0, cam_id = 0; - - if (!magicDSGetIDs(config, stage, magic_id, &stage_id, &cam_id)) { - psError(PS_ERR_UNKNOWN, false, "failed to get ids"); - return false; - } - - magicDSRunRow *run = magicDSRunRowAlloc( - 0, // ID - magic_id, - 0, // inv_magic_id - "new", // state - stage, - stage_id, - cam_id, - label, - outroot, - recoveryroot, - re_place, - remove - ); - - if (!run) { - psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object"); - return false; - } - if (!magicDSRunInsertObject(config->dbh, run)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(run); - return false; - } - - psS64 magic_ds_id = psDBLastInsertID(config->dbh); - run->magic_ds_id = magic_ds_id; - - if (!magicDSRunPrintObject(stdout, run, !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(run); - return false; - } - - psFree(run); - - return magic_id; -} - - -static bool updaterunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); - PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { - // set detRun.state to state - return setmagicDSRunState(config, magic_ds_id, state); - } - - return true; + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", false, false); + if (magic_ds_id) { + + return setmagicDSRunState(config, magic_ds_id, NULL, state); + + } else if (!strcmp(state, "full")) { + psError(PS_ERR_UNKNOWN, true, "magic_ds_id is required to update run state to full"); + return false; + } + // we can transition by query as well + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + + + if (psListLength(where->list) < 2) { + psError(PS_ERR_UNKNOWN, true, "at least 2 search arguments are required"); + return false; + } + + + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); + if (!noreplace) { + psMetadataAddS32(where, PS_LIST_TAIL, "re_place", 0, ">", 0); + } + bool result = setmagicDSRunState(config, magic_ds_id, where, state); + psFree(where); + + return result; } @@ -670,68 +637,4 @@ } -static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(stage, false); - PS_ASSERT_PTR_NON_NULL(stage_id, false); - PS_ASSERT_PTR_NON_NULL(cam_id, false); - - int stageNum = ippStringToStage(stage);; - if (stageNum == IPP_STAGE_NONE) { - psError(PXTOOLS_ERR_DATA, false, "%s is not a valid value for stage", stage); - return false; - } - - psString query = pxDataGet("magicdstool_getrunids.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQueryF(config->dbh, query, magic_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("magicdstool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - if (psArrayLength(output) > 1) { - psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found %ld for magic_id %" PRId64, - psArrayLength(output), magic_id); - return false; - } - psMetadata *row = output->data[0]; - - *cam_id = psMetadataLookupS64(NULL, row, "cam_id"); - switch (stageNum) { - case IPP_STAGE_RAW: - *stage_id = psMetadataLookupS64(NULL, row, "exp_id"); - break; - case IPP_STAGE_CHIP: - *stage_id = psMetadataLookupS64(NULL, row, "chip_id"); - break; - case IPP_STAGE_CAMERA: - *stage_id = *cam_id; - return true; - case IPP_STAGE_WARP: - *stage_id = psMetadataLookupS64(NULL, row, "warp_id"); - break; - case IPP_STAGE_DIFF: - *stage_id = psMetadataLookupS64(NULL, row, "diff_id"); - break; - } - - return true; -} static bool advancerunMode(pxConfig *config) @@ -796,5 +699,5 @@ // set magicDSRun.state to 'full' - if (!setmagicDSRunState(config, magic_ds_id, "full")) { + if (!setmagicDSRunState(config, magic_ds_id, NULL, "full")) { psError(PS_ERR_UNKNOWN, false, "failed to change magicDSRun.state for magic_ds_id: %" PRId64, magic_ds_id); @@ -994,18 +897,26 @@ } -static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value +static bool validDSRunState(const char *state) +{ if (!((strcmp(state, "new") == 0) || (strcmp(state, "full") == 0) || (strcmp(state, "restored") == 0) || (strcmp(state, "censored") == 0) || - (strcmp(state, "purged") == 0) || + (strcmp(state, "cleaned") == 0) || (strcmp(state, "goto_restored") == 0) || (strcmp(state, "goto_censored") == 0) || - (strcmp(state, "goto_purged") == 0)) + (strcmp(state, "goto_cleaned") == 0)) ) { + return false; + } else { + return true; + } +} + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + if (!validDSRunState(state)) { psError(PS_ERR_UNKNOWN, false, "invalid magicDSRun state: %s", state); @@ -1013,6 +924,17 @@ } - char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { + psString query = psStringCopy("UPDATE magicDSRun SET state = '%s'"); + if (magic_ds_id) { + psStringAppend(&query, " WHERE magic_ds_id = %" PRId64, magic_ds_id); + } else if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search arugments are required"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for magic_id %" PRId64, magic_ds_id); @@ -1178,2 +1100,76 @@ +static bool tocleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magicdstool_tocleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magicdstool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "tocleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/magicdstool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/magicdstool.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/magicdstool.h (revision 25870) @@ -26,5 +26,4 @@ MAGICDSTOOL_MODE_NONE = 0x0, MAGICDSTOOL_MODE_DEFINEBYQUERY, - MAGICDSTOOL_MODE_DEFINERUN, MAGICDSTOOL_MODE_UPDATERUN, MAGICDSTOOL_MODE_TODESTREAK, @@ -36,4 +35,5 @@ MAGICDSTOOL_MODE_TOREVERT, MAGICDSTOOL_MODE_COMPLETEDREVERT, + MAGICDSTOOL_MODE_TOCLEANUP, } MAGICDStoolMode; Index: branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c (revision 25870) @@ -54,4 +54,6 @@ psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data_group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by magicRun.label", NULL); @@ -69,19 +71,12 @@ psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - // -definerun - psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magic_id (required)", 0); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define stage for run (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace the input wit with the destreaked versions", false); -// psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-noreplace", 0, "only update runs with replace not set", false); // -addinputskyfile @@ -153,4 +148,12 @@ psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + // -tocleanup + psMetadata *tocleanupArgs = psMetadataAlloc(); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-stage", 0, "define output directory", NULL); + psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -160,6 +163,4 @@ PXOPT_ADD_MODE("-definebyquery", "create magic de-streak runs from magic runs", MAGICDSTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); - PXOPT_ADD_MODE("-definerun", "create one magic de-streak run for one magic run", - MAGICDSTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "update state of magic de-streak run", MAGICDSTOOL_MODE_UPDATERUN, updaterunArgs); @@ -181,4 +182,6 @@ PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting", MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs); + PXOPT_ADD_MODE("-tocleanup", "destreak runs to clean up", + MAGICDSTOOL_MODE_TOCLEANUP, tocleanupArgs); Index: branches/eam_branches/20090820/ippTools/src/magictool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magictool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/magictool.c (revision 25870) @@ -117,4 +117,6 @@ // Optional PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-note", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -236,6 +238,17 @@ // create a new magicRun for this group - magicRunRow *run = magicRunRowAlloc(0, exp_id, diff_id, inverse, "new", workdir, "dirty", label, - dvodb, registered, 0); + magicRunRow *run = magicRunRowAlloc(0, + exp_id, + diff_id, + inverse, + "new", // state + workdir, + "dirty", // workdir_state + label, + data_group ? data_group : label, + dvodb, + registered, + 0, // fault + note); if (!run) { psAbort("failed to alloc magicRun object"); @@ -331,7 +344,9 @@ "dirty", // workdir_state label, + NULL, // data_group dvodb, registered, - 0 + 0, // fault + NULL ); @@ -1449,22 +1464,22 @@ // Now queue any destreaked files to be re-verted - // note: on failure magicRestoreStage issues the rollback - if (!magicRestoreStage(config, "raw", whereClause, "goto_censored")) { + // note: on failure pxmagicRestoreStage issues the rollback + if (!pxmagicRestoreStage(config, "raw", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!magicRestoreStage(config, "chip", whereClause, "goto_censored")) { + if (!pxmagicRestoreStage(config, "chip", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!magicRestoreStage(config, "camera", whereClause, "goto_censored")) { + if (!pxmagicRestoreStage(config, "camera", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!magicRestoreStage(config, "warp", whereClause, "goto_censored")) { + if (!pxmagicRestoreStage(config, "warp", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!magicRestoreStage(config, "diff", whereClause, "goto_censored")) { + if (!pxmagicRestoreStage(config, "diff", whereClause, "goto_censored")) { psFree(whereClause); return false; Index: branches/eam_branches/20090820/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pstamptool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pstamptool.c (revision 25870) @@ -48,4 +48,7 @@ static bool projectMode(pxConfig *config); static bool modprojectMode(pxConfig *config); +static bool getdependentMode(pxConfig *config); +static bool pendingdependentMode(pxConfig *config); +static bool updatedependentMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -85,4 +88,7 @@ MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); MODECASE(PSTAMPTOOL_MODE_PROJECT, projectMode); + MODECASE(PSTAMPTOOL_MODE_GETDEPENDENT, getdependentMode); + MODECASE(PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentMode); + MODECASE(PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentMode); default: psAbort("invalid option (this should not happen)"); @@ -545,4 +551,5 @@ PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", false, false); // unless the job is being inserted with stop state require outputBase @@ -570,5 +577,6 @@ exp_id, outputBase, - options + options, + dep_id )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -896,2 +904,150 @@ } +static bool getdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + PXOPT_LOOKUP_S64(stage_id, config->args, "-stage_id", true, false); + PXOPT_LOOKUP_STR(imagedb, config->args, "-imagedb", true, false); + PXOPT_LOOKUP_STR(rlabel, config->args, "-rlabel", false, false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + PXOPT_LOOKUP_BOOL(no_create, config->args, "-no_create", false); + + psString query = pxDataGet("pstamptool_pendingdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (psArrayLength(output)) { + psMetadata *dep = output->data[0]; + psS64 dep_id = psMetadataLookupS64(NULL, dep, "dep_id"); + if (!dep_id) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + printf("%" PRId64 "\n", dep_id); + return true; + } + if (no_create) { + return true; + } + // no existing dependent that matches, insert one + + if (!pstampDependentInsert( + config->dbh, + 0, // dep_id + "new", // state + stage, + stage_id, + imagedb, + rlabel, + no_magic + )) { + psError(PS_ERR_UNKNOWN, false, "failed to insert pstampDependent"); + return false; + } + + psS64 dep_id = psDBLastInsertID(config->dbh); + + printf("%" PRId64 "\n", dep_id); + + return true; +} + +static bool pendingdependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("pstamptool_pendingdependent.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampDependent"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("pstamptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pstampDependent", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} +static bool updatedependentMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(dep_id, config->args, "-dep_id", true, false); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false); + + char *query ="UPDATE pstampDependent" + " SET state = '%s'" + " WHERE dep_id = %" PRId64; + + if (!p_psDBRunQueryF(config->dbh, query, state, dep_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + psU64 affected = psDBAffectedRows(config->dbh); + if (affected != 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" + PRIu64 " rows were modified", affected); + return false; + } + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/pstamptool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pstamptool.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pstamptool.h (revision 25870) @@ -43,4 +43,7 @@ PSTAMPTOOL_MODE_MODPROJECT, PSTAMPTOOL_MODE_PROJECT, + PSTAMPTOOL_MODE_GETDEPENDENT, + PSTAMPTOOL_MODE_PENDINGDEPENDENT, + PSTAMPTOOL_MODE_UPDATEDEPENDENT, } pstamptoolMode; Index: branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c (revision 25870) @@ -112,4 +112,5 @@ psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure id", 0); psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-dep_id", 0, "define job dep_id", 0); psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); @@ -143,4 +144,29 @@ psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted jobs", false); psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -getdependent + psMetadata *getdependentArgs = psMetadataAlloc(); + psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run (required)", 0); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run (required)", NULL); + psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); + psMetadataAddBool(getdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + + // -updatedependent + psMetadata *updatedependentArgs = psMetadataAlloc(); + psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent run (required)", 0); + psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state (required)", NULL); + + // -pendingdependent + psMetadata *pendingdependentArgs = psMetadataAlloc(); + psMetadataAddS64(pendingdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define id for dependent run", 0); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-stage", 0, "define stage for dependent run", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-imagedb", 0, "define imagedb for depenent run", NULL); + psMetadataAddStr(pendingdependentArgs, PS_LIST_TAIL, "-rlabel", 0, "define label for dependent run", NULL); + psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_magic", 0, "define no_magic", false); + psMetadataAddBool(pendingdependentArgs,PS_LIST_TAIL, "-no_create", 0, "if no run exists do not create one", false); + psMetadataAddU64(pendingdependentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingdependentArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -addproject @@ -189,4 +215,8 @@ PXOPT_ADD_MODE("-moddatastore", "", PSTAMPTOOL_MODE_MODDATASTORE, moddatastoreArgs); + PXOPT_ADD_MODE("-getdependent", "", PSTAMPTOOL_MODE_GETDEPENDENT, getdependentArgs); + PXOPT_ADD_MODE("-updatedependent", "", PSTAMPTOOL_MODE_UPDATEDEPENDENT, updatedependentArgs); + PXOPT_ADD_MODE("-pendingdependent","", PSTAMPTOOL_MODE_PENDINGDEPENDENT, pendingdependentArgs); + PXOPT_ADD_MODE("-addproject", "", PSTAMPTOOL_MODE_ADDPROJECT, addprojectArgs); PXOPT_ADD_MODE("-modproject", "", PSTAMPTOOL_MODE_MODPROJECT, modprojectArgs); Index: branches/eam_branches/20090820/ippTools/src/pxadd.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxadd.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxadd.c (revision 25870) @@ -29,98 +29,5 @@ #include "pxadd.h" -bool pxaddSetSearchArgs (psMetadata *md) { - - psMetadataAddS64(md, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); - psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); - -/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); */ -/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); */ -/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); */ -/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); */ -/* psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); */ -/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); */ - - return true; -} - -bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where) { - - PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); -/* PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); */ -/* PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); */ -/* PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); */ -/* PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); */ -/* PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); */ -/* PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); */ -/* PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); */ -/* PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); */ -/* PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); */ -/* PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); */ -/* PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); */ -/* PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); */ -/* PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); */ -/* PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); */ -/* PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); */ -/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); */ -/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); */ -/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); */ -/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); */ -/* PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); */ -/* PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); */ -/* PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); */ -/* PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); */ -/* PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); */ - - return true; -} - -bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked) +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -134,8 +41,7 @@ } - char *query = "UPDATE addRun SET state = '%s', magicked = %" PRId64 " WHERE add_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magicked, add_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for add_id %" PRId64, add_id); + char *query = "UPDATE addRun SET state = '%s' WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, add_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to change state for add_id %" PRId64, add_id); return false; } @@ -216,11 +122,13 @@ } -// Need to think more about this to see what we want it to do. BROKEN bool pxaddQueueByCamID(pxConfig *config, psS64 cam_id, char *workdir, + char *reduction, char *label, - char *recipe, - char *dvodb) + char *data_group, + char *dvodb, + char *note, + bool image_only) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -236,15 +144,19 @@ } } - // fprintf(stderr,"%s",query); + // queue the exp - // XXX chip_id is being cast here work around psS64 have a different type - // different on 32/64 + // Note: cam_id is being cast here work around psS64 have a different type different + // on 32/64 if (!p_psDBRunQueryF(config->dbh, query, "new", // state - workdir ? workdir : "NULL", + workdir ? workdir : "NULL", "dirty", //workdir_state - label ? label : "NULL", - dvodb ? dvodb : "NULL", - (long long)cam_id + reduction? reduction : "NULL", + label ? label : "NULL", + data_group ? data_group : "NULL", + dvodb ? dvodb : "NULL", + note ? note : "NULL", + image_only, + (long long) cam_id )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: branches/eam_branches/20090820/ippTools/src/pxadd.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxadd.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxadd.h (revision 25870) @@ -25,20 +25,18 @@ #include "pxtools.h" -bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked); +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state); bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label); bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); -bool pxaddSetSearchArgs (psMetadata *md); -bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where); - -// Likely BROKEN bool pxaddQueueByCamID(pxConfig *config, - psS64 cam_id, - char *workdir, - char *label, - char *recipe, - char *dvodb); - + psS64 cam_id, + char *workdir, + char *reduction, + char *label, + char *data_group, + char *dvodb, + char *note, + bool image_only); #endif // PXADD_H Index: branches/eam_branches/20090820/ippTools/src/pxadmin.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxadmin.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxadmin.c (revision 25870) @@ -2,5 +2,5 @@ * pxadmin.c * - * Copyright (C) 2006-2008 Joshua Hoblitt + * Copyright (C) 2006-2009 Joshua Hoblitt * * This program is free software; you can redistribute it and/or modify it @@ -31,4 +31,5 @@ bool createMirrorMode(pxConfig *config); bool deleteMode(pxConfig *config); +static bool insert_dbversion(pxConfig * config, const char *versionString); static bool runMultipleStatments(pxConfig *config, const char *query); @@ -113,4 +114,9 @@ psFree(query); + if (!insert_dbversion(config, PACKAGE_VERSION)) { + psError(PS_ERR_UNKNOWN, false, "failed to set database version"); + return false; + } + // COMMIT if (!psDBCommit(config->dbh)) { @@ -148,4 +154,9 @@ } psFree(query); + + if (!insert_dbversion(config, PACKAGE_VERSION)) { + psError(PS_ERR_UNKNOWN, false, "failed to set database version"); + return false; + } // COMMIT @@ -262,2 +273,35 @@ return true; } + +#ifdef notdef +static bool update_dbversion(pxConfig * config, const char *versionString) +{ + psString query = pxDataGet("pxadmin_update_version.sql"); + if (!query) { + psError(PS_ERR_UNKNOWN, false, "failed to retrieve SQL statement"); + psFree(query); + return false; + } + if (!p_psDBRunQueryF(config->dbh, query, versionString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + return true; +} +#endif +static bool insert_dbversion(pxConfig * config, const char *versionString) +{ + psString query = "INSERT INTO dbversion VALUES('%s', CURRENT_TIMESTAMP())"; + if (!query) { + psError(PS_ERR_UNKNOWN, false, "failed to retrieve SQL statement"); + psFree(query); + return false; + } + if (!p_psDBRunQueryF(config->dbh, query, versionString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + return true; +} Index: branches/eam_branches/20090820/ippTools/src/pxcam.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxcam.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxcam.c (revision 25870) @@ -141,86 +141,16 @@ } - -bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, - "invalid chipRun state: %s", state); - return false; - } - - psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.state = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - -bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE camRun SET camRun.label = '%s' WHERE cam_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, cam_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for cam_id %" PRId64, cam_id); - return false; - } - - return true; -} - -bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE camRun JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET camRun.label = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - bool pxcamQueueByChipID(pxConfig *config, psS64 chip_id, char *workdir, char *label, + char *data_group, + char *dist_group, char *recipe, char *expgroup, char *dvodb, char *tess_id, - char *end_stage) + char *end_stage, + char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -245,4 +175,6 @@ "dirty", //workdir_state label ? label : "NULL", + data_group ? data_group : "NULL", + dist_group ? dist_group : "NULL", recipe ? recipe : "NULL", expgroup ? expgroup : "NULL", @@ -250,4 +182,5 @@ tess_id ? tess_id : "NULL", end_stage ? end_stage : "NULL", + note ? note : "NULL", (long long)chip_id )) { Index: branches/eam_branches/20090820/ippTools/src/pxcam.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxcam.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxcam.h (revision 25870) @@ -26,8 +26,4 @@ bool pxcamRunSetState(pxConfig *config, psS64 cam_id, const char *state, psS64 magicked); -bool pxcamRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxcamRunSetLabel(pxConfig *config, psS64 cam_id, const char *label); -bool pxcamRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); - bool pxcamSetSearchArgs (psMetadata *md); bool pxcamGetSearchArgs (pxConfig *config, psMetadata *where); @@ -37,9 +33,12 @@ char *workdir, char *label, + char *data_group, + char *dist_group, char *recipe, char *expgroup, char *dvodb, char *tess_id, - char *end_stage); + char *end_stage, + char *note); #endif // PXCAM_H Index: branches/eam_branches/20090820/ippTools/src/pxchip.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxchip.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxchip.c (revision 25870) @@ -192,27 +192,26 @@ } - -bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label) +bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) { PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE chipRun SET label = '%s' WHERE chip_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, chip_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for chip_id %" PRId64, chip_id); - return false; - } - - return true; -} - - -bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) SET label = '%s'"); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid chipProcessedImfile state: %s", state); + return false; + } + +/* if (!strcmp(state, "full")) { */ +/* // There are states that need to be met for a run to be set to full that we don't */ +/* // check here. */ +/* // for example all of the run's Imfiles must have chipProcessedImfile.data_state == "full" */ +/* // chipRun.magicked = (SUM(!chipProcessedImfile.magicked) = 0) */ +/* // so don't do allow setting the state to full */ +/* psError(PS_ERR_UNKNOWN, true, "cannot use -updaterun so set chipRun state to full"); */ +/* return false; */ +/* } */ + + psString query = psStringCopy("UPDATE chipProcessedImfile JOIN chipRun USING(chip_id) JOIN rawExp ON chipRun.exp_id = rawExp.exp_id SET state = '%s'"); if (where && psListLength(where->list) > 0) { @@ -222,5 +221,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, label)) { + if (!p_psDBRunQueryF(config->dbh, query, state)) { psFree(query); psError(PS_ERR_UNKNOWN, false, "database error"); @@ -232,5 +231,4 @@ return true; } - psS64 pxchipQueueByExpTag(pxConfig *config, @@ -238,9 +236,12 @@ const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *reduction, const char *expgroup, const char *dvodb, const char *tess_id, - const char *end_stage) + const char *end_stage, + const char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -260,4 +261,6 @@ "dirty", // workdir_state label, + data_group, + dist_group, reduction, expgroup, @@ -265,5 +268,6 @@ tess_id, end_stage, - 0 // magicked + 0, // magicked + note ) ) { Index: branches/eam_branches/20090820/ippTools/src/pxchip.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxchip.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxchip.h (revision 25870) @@ -25,8 +25,8 @@ #include "pxtools.h" + + bool pxchipRunSetState(pxConfig *config, psS64 chip_id, const char *state, const bool magicked); -bool pxchipRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxchipRunSetLabel(pxConfig *config, psS64 chip_id, const char *label); -bool pxchipRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); +bool pxchipProcessedImfileSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); psS64 pxchipQueueByExpTag(pxConfig *config, @@ -34,9 +34,12 @@ const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *reduction, const char *expgroup, const char *dvodb, const char *tess_id, - const char *end_stage); + const char *end_stage, + const char *note); Index: branches/eam_branches/20090820/ippTools/src/pxfake.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxfake.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxfake.c (revision 25870) @@ -51,87 +51,16 @@ } - -bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, - "invalid fakeRun state: %s", state); - return false; - } - - psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.state = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - -bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE fakeRun SET fakeRun.label = '%s' WHERE fake_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, fake_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for fake_id %" PRId64, fake_id); - return false; - } - - return true; -} - - -bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE fakeRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET fakeRun.label = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - psS64 pxfakeQueueByCamID(pxConfig *config, psS64 cam_id, char *workdir, char *label, + char *data_group, + char *dist_group, char *reduction, char *expgroup, char *dvodb, char *tess_id, - char *end_stage) + char *end_stage, + char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -157,4 +86,6 @@ workdir ? workdir : "NULL", label ? label : "NULL", + data_group ? data_group : "NULL", + dist_group ? dist_group : "NULL", reduction? reduction: "NULL", expgroup ? expgroup : "NULL", @@ -162,4 +93,5 @@ tess_id ? tess_id : "NULL", end_stage ? end_stage : "NULL", + note ? note : "NULL", (long long)cam_id )) { Index: branches/eam_branches/20090820/ippTools/src/pxfake.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxfake.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxfake.h (revision 25870) @@ -26,8 +26,4 @@ bool pxfakeRunSetState(pxConfig *config, psS64 fake_id, const char *state); -bool pxfakeRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxfakeRunSetLabel(pxConfig *config, psS64 fake_id, const char *label); -bool pxfakeRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); - psS64 pxfakeQueueByCamID(pxConfig *config, @@ -35,9 +31,12 @@ char *workdir, char *label, + char *data_group, + char *dist_group, char *reduction, char *expgroup, char *dvodb, char *tess_id, - char *end_stage); + char *end_stage, + char *note); Index: branches/eam_branches/20090820/ippTools/src/pxmagic.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxmagic.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxmagic.c (revision 25870) @@ -29,5 +29,5 @@ #include "pxmagic.h" -bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState) +bool pxmagicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState) { psString queryFile = NULL; @@ -66,2 +66,32 @@ return true; } + +bool pxmagicAddWhere(pxConfig *config, psString *out, psString table) +{ + PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); + + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(out, " AND %s.magicked = 0", table); + } + if (destreaked) { + psStringAppend(out, " AND %s.magicked != 0", table); + } + return true; +} + +void pxmagicAddArguments(psMetadata *md) +{ + psMetadataAddBool(md, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); + psMetadataAddBool(md, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); + psMetadataAddS64(md, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); +} Index: branches/eam_branches/20090820/ippTools/src/pxmagic.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxmagic.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxmagic.h (revision 25870) @@ -25,5 +25,8 @@ #include "pxtools.h" -extern bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState); +extern bool pxmagicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState); + +extern bool pxmagicAddWhere(pxConfig *config, psString *string, psString table); +extern void pxmagicAddArguments(psMetadata *md); #endif // PXMAGIC_H Index: branches/eam_branches/20090820/ippTools/src/pxspace.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxspace.c (revision 25870) +++ branches/eam_branches/20090820/ippTools/src/pxspace.c (revision 25870) @@ -0,0 +1,57 @@ +/* + * pxmagic.c + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +// #include +// #include +// #include + +#include "pxtools.h" + +bool pxspaceAddWhere(pxConfig *config, psString *pQuery, psString table) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(table, NULL); + + PXOPT_LOOKUP_F64(radius, config->args, "-radius", false, false); + + if (radius > 0) { + PXOPT_LOOKUP_F64(ra, config->args, "-ra", false, false); + PXOPT_LOOKUP_F64(decl, config->args, "-decl", false, false); + + ra *= PS_RAD_DEG; + decl *= PS_RAD_DEG; + radius *= PS_RAD_DEG; + + psStringAppend(pQuery, " AND (ACOS((cos(%s.decl) * cos(%lf) * cos(%s.ra - %lf)) +(sin(%s.decl) * sin(%lf))) < %lf)", + table, decl, table, ra, table, decl, radius); + } + + return true; +} + +void pxspaceAddArguments(psMetadata *md) +{ + psMetadataAddF64(md, PS_LIST_TAIL, "-radius", 0, "search for exposures within radius RA DEC (degrees)", false); + psMetadataAddF64(md, PS_LIST_TAIL, "-ra", 0, "RA value for radius search (degrees)", false); + psMetadataAddF64(md, PS_LIST_TAIL, "-decl", 0, "DEC value for radius search (degrees)", false); +} Index: branches/eam_branches/20090820/ippTools/src/pxspace.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxspace.h (revision 25870) +++ branches/eam_branches/20090820/ippTools/src/pxspace.h (revision 25870) @@ -0,0 +1,26 @@ +/* + * pxmagic.h + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXSPACE_H +#define PXSPACE_H 1 + +extern bool pxspaceAddWhere(pxConfig *config, psString *pQuery, psString table); +extern void pxspaceAddArguments(psMetadata *md); + +#endif // PXSPACE_H Index: branches/eam_branches/20090820/ippTools/src/pxtools.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxtools.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxtools.c (revision 25870) @@ -108,2 +108,185 @@ return true; } + +// shared code for updating the various strings for a Run +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString table, bool has_dist_group) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(where, false); + PS_ASSERT_PTR_NON_NULL(pQuery, false); + PS_ASSERT_PTR_NON_NULL(*pQuery, false); + + // make sure that -state is not the only selection parameter + PXOPT_LOOKUP_STR(where_state, config->args, "-state", false, false); + if (where_state && (psListLength(where->list) < 2)) { + psError(PXTOOLS_ERR_DATA, true, "selection by -state alone is not allowed"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + + psString dist_group = NULL; + if (has_dist_group) { + PXOPT_LOOKUP_STR(tmp_dist_group, config->args, "-set_dist_group", false, false); + dist_group = tmp_dist_group; + } + + if ((!state) && (!label) && (!data_group) && (has_dist_group && !dist_group) && !(note)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + return false; + } + + if (state && ! pxIsValidState(state)) { + psError(PXTOOLS_ERR_DATA, false, "pxIsValidState failed"); + return false; + } + + // first paramter is added with "SET param = 'value'" + // subseqent ones with ", param = 'value'" + char *separator = " SET "; + char *comma = ","; + +# define addColumn(_tab, _val) \ + do { \ + if (_val) { \ + psStringAppend(pQuery, "%s %s.%s = '%s'", separator, _tab, #_val, _val); \ + separator = comma; \ + } \ + } while (0) + + addColumn(table, state); + addColumn(table, data_group); + if (has_dist_group) { + addColumn(table, dist_group); + } + addColumn(table, note); + addColumn(table, label); + + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(pQuery, " %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, *pQuery)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +bool pxLookupVersion(pxConfig *config, psArray **pArray) +{ + const char *query = "SELECT * FROM dbversion"; + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, true, "no rows in dbversion"); + return false; + } + if (psArrayLength(output) > 1) { + psError(PS_ERR_UNKNOWN, true, "unexpected number of rows found in dbversion: %" PRId64, + psArrayLength(output)); + return false; + } + *pArray = output; + + return true; +} + +psString pxGetDBVersion(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psArray *array = NULL; + if (!pxLookupVersion(config, &array)) { + psError(PS_ERR_UNKNOWN, false, "pxLookupVersion failed"); + return NULL; + } + psMetadata *md = array->data[0]; + if (!md) { + psError(PS_ERR_UNKNOWN, true, "output of pxLookupVersion is null"); + return NULL; + } + + psString version = psMetadataLookupStr(NULL, md, "schema_version"); + + return version; +} + +bool pxExportVersion(pxConfig *config, FILE *file) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(file, NULL); + + psArray *array = NULL; + if (!pxLookupVersion(config, &array) || !array) { + psError(PS_ERR_UNKNOWN, false, "pxLookupVersion failed"); + return false; + } + if (!ippdbPrintMetadatas(file, array, "dbversion", true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(array); + return false; + } + return true; +} + +bool pxCheckImportVersion(pxConfig *config, psMetadata *input) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + PS_ASSERT_PTR_NON_NULL(input, NULL); + + // This code was adapted from the way camtool parses the structures. + // Is this really the way to do it? + psMetadataItem *multi_item = psMetadataLookup(input, "dbversion"); + if (!multi_item || (multi_item->type != PS_DATA_METADATA_MULTI)) { + psError(PS_ERR_UNKNOWN, true, "dbversion multi not found in input"); + return false; + } + + psMetadataItem *dbversion = psListGet(multi_item->data.list, 0); + if (!dbversion) { + psError(PS_ERR_UNKNOWN, true, "dbversion not found in input"); + return false; + } + + if (!strcmp(dbversion->name, "dbversion")) { + // horray + psMetadata *md = dbversion->data.md; + psString schema_version = pxGetDBVersion(config); + if (!schema_version) { + psError(PS_ERR_UNKNOWN, false, "pxGetDBVersion failed"); + return false; + } + + psString import_version = psMetadataLookupStr(NULL, md, "schema_version"); + if (import_version && strcmp(import_version, schema_version)) { + psError(PS_ERR_UNKNOWN, true, "input file schema_version: %s does not match data base: %s", + import_version, schema_version); + return false; + } else if (!import_version) { + psError(PS_ERR_UNKNOWN, true, "input file schema_version is NULL"); + return false; + } else { + // YIPPEE this file is the same version + } + } else { + psError(PS_ERR_UNKNOWN, true, "Unexpected config dump format"); + return false; + } + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/pxtools.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxtools.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxtools.h (revision 25870) @@ -47,4 +47,5 @@ #include "pxtree.h" #include "pxmagic.h" +#include "pxspace.h" # define MAX_ROWS 10e9 @@ -56,4 +57,6 @@ bool pxSetFaultCode(psDB *dbh, const char *tableName, psMetadata *where, psS16 code); +bool pxExportVersion(pxConfig *config, FILE *f); +bool pxCheckImportVersion(pxConfig *config, psMetadata *md); psExit pxerrorGetExitStatus(void); @@ -61,4 +64,6 @@ void pxUsage(FILE *stream, int argc, char **argv, const char *modeName, psMetadata *argSet); bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets); + +bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString table, bool has_dist_group); #define PXOPT_ADD_MODE(option, comment, modeval, argset) \ Index: branches/eam_branches/20090820/ippTools/src/pxwarp.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxwarp.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxwarp.c (revision 25870) @@ -29,108 +29,15 @@ #include "pxwarp.h" -bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, "invalid warpRun state: %s", state); - return false; - } - - char *query = "UPDATE warpRun SET state = '%s' WHERE warp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, warp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for warp_id %" PRId64, warp_id); - return false; - } - - return true; -} - - -bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!pxIsValidState(state)) { - psError(PS_ERR_UNKNOWN, false, "invalid warpRun state: %s", state); - return false; - } - - psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.state = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - if (whereClause && strlen(whereClause) > 0) { - psStringAppend(&query, " %s", whereClause); - } - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, state)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - -bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - char *query = "UPDATE warpRun SET warpRun.label = '%s' WHERE warp_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, label, warp_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change label for warp_id %" PRId64, warp_id); - return false; - } - - return true; -} - - -bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - // note label == NULL should be explicitly allowed - - psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.label = '%s'"); - - if (where) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psFree(query); - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(query); - - return true; -} - - bool pxwarpQueueByFakeID(pxConfig *config, psS64 fake_id, const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *dvodb, const char *tess_id, const char *reduction, - const char *end_stage) + const char *end_stage, + const char *note) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -145,10 +52,13 @@ "dirty", // workdir_state label, + data_group, + dist_group, dvodb, tess_id, - reduction, + reduction, end_stage, NULL, // registered - 0 // magicked zero when created will get updated when warpRun goes to 'full' + 0, // magicked set to zero when created may get updated when warpRun goes to 'full' + NULL // note )) { psError(PS_ERR_UNKNOWN, false, "database error"); Index: branches/eam_branches/20090820/ippTools/src/pxwarp.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxwarp.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxwarp.h (revision 25870) @@ -25,17 +25,15 @@ #include "pxtools.h" -bool pxwarpRunSetState(pxConfig *config, psS64 warp_id, const char *state); -bool pxwarpRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); -bool pxwarpRunSetLabel(pxConfig *config, psS64 warp_id, const char *label); -bool pxwarpRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); - bool pxwarpQueueByFakeID(pxConfig *config, psS64 fake_id, const char *workdir, const char *label, + const char *data_group, + const char *dist_group, const char *dvodb, const char *tess_id, const char *reduction, - const char *end_stage); + const char *end_stage, + const char *note); #endif // PXWARP_H Index: branches/eam_branches/20090820/ippTools/src/regtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/regtool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/regtool.c (revision 25870) @@ -334,14 +334,24 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "obs_mode", "=="); PXOPT_COPY_S64(config->args, where, "-magicked", "magicked", "=="); - - PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); - PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); - PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_BOOL(ordered_by_date, config->args, "-ordered_by_date", false); + + // build where string for some parameters that don't fit PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "rawImfile")) { + psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawImfile")) { + psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + return false; + } psString query = pxDataGet("regtool_processedimfile.sql"); @@ -356,6 +366,13 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else if (!all && !where2) { + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; } psFree(where); + if (where2) { + psStringAppend(&query, " %s", where2); + psFree(where2); + } if (faulted) { @@ -366,20 +383,4 @@ psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); } - - if (not_destreaked) { - if (destreaked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); - return false; - } - if (magicked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); - return false; - } - psStringAppend(&query, " AND rawImfile.magicked = 0"); - } - if (destreaked) { - psStringAppend(&query, " AND rawImfile.magicked != 0"); - } - // add the ORDER BY statement if desired @@ -884,9 +885,12 @@ workdir, label, + NULL, // data_group + NULL, // dist_group reduction, - NULL, // expgroup + NULL, // expgroup dvodb, tess_id, - end_stage + end_stage, + NULL // note )) { // rollback Index: branches/eam_branches/20090820/ippTools/src/regtoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/regtoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/regtoolConfig.c (revision 25870) @@ -137,13 +137,15 @@ ADD_OPT(Str, processedimfileArgs, "-class_id", "search by class ID", NULL); ADD_OPT(Str, processedimfileArgs, "-filter", "search by filter", NULL); + ADD_OPT(Str, processedimfileArgs, "-obs_mode", "search by obs_mod", NULL); + ADD_OPT(Str, processedimfileArgs, "-exp_type", "search by exposure type", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL); - ADD_OPT(S64, processedimfileArgs, "-magicked", "search by magicked value", 0); - ADD_OPT(Bool, processedimfileArgs, "-destreaked", "only return imfiles that have been destreaked", false); - ADD_OPT(Bool, processedimfileArgs, "-not_destreaked", "only return imfiles that have not been destreaked", false); ADD_OPT(U64, processedimfileArgs, "-limit", "limit result set to N items", 0); ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); + ADD_OPT(Bool, processedimfileArgs, "-all", "list without search arguments", false); ADD_OPT(Bool, processedimfileArgs, "-simple", "use the simple output format", false); ADD_OPT(Bool, processedimfileArgs, "-ordered_by_date", "force output to be in DATE order", false); + pxspaceAddArguments(processedimfileArgs); + pxmagicAddArguments(processedimfileArgs); // -revertprocessedimfile Index: branches/eam_branches/20090820/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/stacktool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/stacktool.c (revision 25870) @@ -109,8 +109,14 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); psMetadata *where = psMetadataAlloc(); @@ -160,8 +166,6 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); - - if (!psListLength(where->list) && - !psMetadataLookupBool(NULL, config->args, "-all")) { + + if (!psListLength(where->list)) { psFree(where); psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); @@ -345,4 +349,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, dvodb, @@ -350,5 +356,6 @@ skycell_id, tess_id, - filter); + filter, + note); if (!stackRunInsertObject(config->dbh, run)) { @@ -468,5 +475,5 @@ // required options - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); @@ -475,10 +482,13 @@ // default PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false); // options - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); - PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); // we have to support multipe exp_ids @@ -495,4 +505,6 @@ workdir, label, + data_group ? data_group : label, + dist_group, reduction, dvodb, @@ -500,5 +512,6 @@ skycell_id, tess_id, - filter); + filter, + note); if (!run) { @@ -569,13 +582,44 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false); +#ifdef notdef + PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false); PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); - - if (state) { + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); +#endif + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE stackRun"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "stackRun", true); + + psFree(query); + psFree(where); + + return result; + +#ifdef notdef + // Hack-y work around to make stacktool more like the other tools, without breaking other stuff (hopefully). + + if ((state)&&(stack_id)) { // set detRun.state to state return setstackRunState(config, stack_id, state); } - return true; + if ((state)&&(label)) { + return setstackRunStateByLabel(config, label, state); + } + + psError(PS_ERR_UNKNOWN, false, "Required options not found."); + return false; +#endif } @@ -1006,4 +1050,26 @@ } +#ifdef notdef +static bool setstackRunStateByLabel(pxConfig *config, const char *label, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, "invalid stackRun state: %s", state); + return false; + } + + char *query = "UPDATE stackRun SET state = '%s' WHERE label = '%s'"; + if (!p_psDBRunQueryF(config->dbh, query, state, label)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for label %s", label); + return false; + } + + return true; +} +#endif + static bool pendingcleanuprunMode(pxConfig *config) { @@ -1231,4 +1297,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "=="); @@ -1317,7 +1387,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); - +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "stackRun"); psAssert (item, "entry not in input?"); Index: branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c (revision 25870) @@ -47,9 +47,12 @@ // -definebyquery psMetadata *definebyqueryArgs = psMetadataAlloc(); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); - psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-select_skycell_id", 0, "search for skycell_id", NULL); psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0); @@ -84,5 +87,4 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_max", 0, "define max iq_m4", NAN); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_num", 0, "minimum number of inputs", 0); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-min_new", 0, "minimum number of new inputs", 0); @@ -93,9 +95,9 @@ // -definerun psMetadata *definerunArgs = psMetadataAlloc(); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir (required)", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-reduction", 0, "define reduction", NULL); - psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-dvodb", 0, "define dvodb", NULL); - psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define dvodb", NULL); + psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-set_registered", 0, "time detrend run was registered", now); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID (required)", NULL); @@ -106,7 +108,13 @@ // -updaterun psMetadata *updaterunArgs = psMetadataAlloc(); - psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL); - + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); + // -addinputskyfile psMetadata *addinputskyfileArgs = psMetadataAlloc(); Index: branches/eam_branches/20090820/ippTools/src/warptool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/warptool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/warptool.c (revision 25870) @@ -58,4 +58,5 @@ static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); +static bool runstateMode(pxConfig *config); static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile); @@ -108,4 +109,5 @@ MODECASE(WARPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(WARPTOOL_MODE_IMPORTRUN, importrunMode); + MODECASE(WARPTOOL_MODE_RUNSTATE, runstateMode); default: @@ -139,4 +141,7 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-dist_group", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-note", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required (no default TESS) @@ -160,4 +165,6 @@ "dirty", // workdir_state label, + data_group ? data_group : label, + dist_group, dvodb, tess_id, @@ -165,5 +172,6 @@ end_stage, registered, - 0 // magicked + 0, // magicked + note ); if (!warpRun) { @@ -251,8 +259,11 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false); PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); @@ -340,8 +351,11 @@ workdir ? workdir : row->workdir, label ? label : row->label, + data_group ? data_group: row->data_group, + dist_group ? dist_group: row->dist_group, dvodb ? dvodb : row->dvodb, tess_id ? tess_id : row->tess_id, reduction ? reduction : row->reduction, - end_stage ? end_stage : row->end_stage)) + end_stage ? end_stage : row->end_stage, + note)) { psError(PS_ERR_UNKNOWN, false, "failed to trying to queue fake_id: %" PRId64, row->fake_id); @@ -371,43 +385,23 @@ PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "=="); PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { + PXOPT_COPY_STR(config->args, where, "-data_group","warpRun.data_group", "=="); + PXOPT_COPY_STR(config->args, where, "-dist_group","warpRun.dist_group", "=="); + + if (!psListLength(where->list)) { psFree(where); - where = NULL; psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); return false; } - - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - if ((!state) && (!label)) { - psError(PXTOOLS_ERR_DATA, false, "parameters (-state or -label) are required"); - psFree(where); - return false; - } - - if (state) { - // set warpRun.state to state - if (!pxwarpRunSetStateByQuery(config, where, state)) { - psFree(where); - return false; - } - } - - if (label) { - // set chipRun.label to label - if (!pxwarpRunSetLabelByQuery(config, where, label)) { - psFree(where); - return false; - } - } - + + psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)"); + + // pxUpdateRun gets parameters from config->args and updates + bool result = pxUpdateRun(config, where, &query, "warpRun", true); + + psFree(query); psFree(where); - return true; -} - + return result; +} static bool expMode(pxConfig *config) @@ -1204,8 +1198,4 @@ pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); - PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); - PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); - PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); - PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); @@ -1220,9 +1210,21 @@ } + // generate where strings for arguments that require extra processing + // beyond PXOPT_COPY* + psString where2 = NULL; + if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) { + psError(PXTOOLS_ERR_DATA, false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &where2, "rawExp")) { + psError(PXTOOLS_ERR_DATA, false, "pxSpaceAddWhere failed"); + return false; + } + if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); - } else if (!all) { + } else if (!all && !where2) { psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); return false; @@ -1230,17 +1232,6 @@ psFree(where); - if (not_destreaked) { - if (destreaked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); - return false; - } - if (magicked) { - psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); - return false; - } - psStringAppend(&query, " AND warpSkyfile.magicked = 0"); - } - if (destreaked) { - psStringAppend(&query, " AND warpSkyfile.magicked != 0"); + if (where2) { + psStringAppend(&query, " %s", where2); } @@ -1712,12 +1703,30 @@ PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - - psString query = pxDataGet("warptool_updateskyfile.sql"); - - if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + + if (!state) { + psString query = pxDataGet("warptool_updateskyfile.sql"); + + if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + } + else { + if (strcmp(state,"error_cleaned") == 0) { + change_skyfile_data_state(config,"error_cleaned","goto_cleaned"); + } + else if (strcmp(state, "error_scrubbed") == 0) { + change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed"); + } + else if (strcmp(state, "error_purged") == 0) { + change_skyfile_data_state(config,"error_purged","goto_purged"); + } + else { + psError(PS_ERR_UNKNOWN, false, "unhandled state given"); + return(false); + } + } return true; @@ -1744,4 +1753,8 @@ } + if (!pxExportVersion(config, f)) { + psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); + return false; + } psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); @@ -1836,6 +1849,13 @@ psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - fprintf (stdout, "---- input ----\n"); +#ifdef notdef + fprintf (stderr, "---- input ----\n"); psMetadataPrint (stderr, input, 1); +#endif + + if (!pxCheckImportVersion(config, input)) { + psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); + return false; + } psMetadataItem *item = psMetadataLookup (input, "warpRun"); @@ -1900,2 +1920,82 @@ return true; } + +static bool runstateMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + +// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("warptool_runstate.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } else { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("warptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "warpRunState", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/warptool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/warptool.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/warptool.h (revision 25870) @@ -53,5 +53,6 @@ WARPTOOL_MODE_UPDATESKYFILE, WARPTOOL_MODE_EXPORTRUN, - WARPTOOL_MODE_IMPORTRUN + WARPTOOL_MODE_IMPORTRUN, + WARPTOOL_MODE_RUNSTATE, } warptoolMode; Index: branches/eam_branches/20090820/ippTools/src/warptoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/warptoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/warptoolConfig.c (revision 25870) @@ -100,5 +100,7 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); @@ -117,4 +119,8 @@ psMetadataAddTime(definerunArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_end_stage", 0, "define end stage", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define data group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define dist group", NULL); + psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-set_note", 0, "define note", NULL); // -updaterun @@ -128,8 +134,12 @@ psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by warpRun state", NULL); psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by warpRun label", NULL); - psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); - psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); - + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-data_group", 0, "search by warpRun data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-dist_group", 0, "search by warpRun dist_group", NULL); + + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "define new state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "define new value for label", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_data_group", 0, "define new data_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0, "define new dist_group", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0, "define new note", NULL); // -exp psMetadata *expArgs = psMetadataAlloc(); @@ -218,9 +228,10 @@ psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); - psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); - psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); - psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); psMetadataAddS16(warpedArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + // add magic related arguments + pxmagicAddArguments(warpedArgs); + // add arguments for spatial search + pxspaceAddArguments(warpedArgs); psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); @@ -304,5 +315,6 @@ psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); psMetadataAddS16(updateskyfileArgs, PS_LIST_TAIL, "-fault", 0, "new fault code", 0); - + psMetadataAddStr(updateskyfileArgs, PS_LIST_TAIL, "-set_state", 0, "set state", 0); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -315,4 +327,16 @@ psMetadata *importrunArgs = psMetadataAlloc(); psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + // -runstate + psMetadata *runstateArgs = psMetadataAlloc(); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warptool ID", 0); +// psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddS64(runstateArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); + psMetadataAddStr(runstateArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-no_magic", 0, "magic is not necessary for result", false); + + psMetadataAddU64(runstateArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(runstateArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -334,4 +358,5 @@ PXOPT_ADD_MODE("-advancerun", "", WARPTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-warped", "", WARPTOOL_MODE_WARPED, warpedArgs); + PXOPT_ADD_MODE("-runstate", "", WARPTOOL_MODE_RUNSTATE, runstateArgs); PXOPT_ADD_MODE("-revertwarped", "", WARPTOOL_MODE_REVERTWARPED, revertwarpedArgs); PXOPT_ADD_MODE("-block", "set a label block", WARPTOOL_MODE_BLOCK, blockArgs);