Index: branches/eam_branches/20090820/ippTools/src/Makefile.am =================================================================== --- branches/eam_branches/20090820/ippTools/src/Makefile.am (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/Makefile.am (revision 25766) @@ -1,3 +1,4 @@ bin_PROGRAMS = \ + addtool \ caltool \ camtool \ @@ -23,9 +24,6 @@ pubtool - -bin_SCRIPTS = \ - fakemagic - pkginclude_HEADERS = \ + pxadd.h \ pxadmin.h \ pxcam.h \ @@ -34,4 +32,5 @@ pxdata.h \ pxfake.h \ + pxmagic.h \ pxregister.h \ pxtag.h \ @@ -42,4 +41,5 @@ noinst_HEADERS = \ + addtool.h \ caltool.h \ camtool.h \ @@ -68,4 +68,5 @@ libpxtools_la_LDFLAGS = -release $(PACKAGE_VERSION) libpxtools_la_SOURCES = \ + pxadd.c \ pxcam.c \ pxchip.c \ @@ -75,4 +76,5 @@ pxfake.c \ pxfault.c \ + pxmagic.c \ pxregister.c \ pxtag.c \ @@ -96,4 +98,10 @@ pstamptool.c \ pstamptoolConfig.c + +addtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) +addtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la +addtool_SOURCES = \ + addtool.c \ + addtoolConfig.c caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS) Index: branches/eam_branches/20090820/ippTools/src/addtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/addtool.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/addtool.c (revision 25766) @@ -0,0 +1,1116 @@ +/* + * addtool.c + * + * Copyright (C) 2006 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +static bool definebyqueryMode(pxConfig *config); +static bool updaterunMode(pxConfig *config); +static bool pendingexpMode(pxConfig *config); +static bool addprocessedexpMode(pxConfig *config); +static bool processedexpMode(pxConfig *config); +static bool revertprocessedexpMode(pxConfig *config); +static bool updateprocessedexpMode(pxConfig *config); +static bool blockMode(pxConfig *config); +static bool maskedMode(pxConfig *config); +static bool unblockMode(pxConfig *config); +static bool pendingcleanuprunMode(pxConfig *config); +static bool pendingcleanupexpMode(pxConfig *config); +static bool donecleanupMode(pxConfig *config); +static bool exportrunMode(pxConfig *config); +static bool importrunMode(pxConfig *config); + +# define MODECASE(caseName, func) \ + case caseName: \ + if (!func(config)) { \ + goto FAIL; \ + } \ + break; + +int main(int argc, char **argv) +{ + psLibInit(NULL); + + pxConfig *config = addtoolConfig(NULL, argc, argv); + if (!config) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to configure"); + goto FAIL; + } + + switch (config->mode) { + MODECASE(ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(ADDTOOL_MODE_UPDATERUN, updaterunMode); + MODECASE(ADDTOOL_MODE_PENDINGEXP, pendingexpMode); + MODECASE(ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpMode); + MODECASE(ADDTOOL_MODE_PROCESSEDEXP, processedexpMode); + MODECASE(ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpMode); + MODECASE(ADDTOOL_MODE_UPDATEPROCESSEDEXP, updateprocessedexpMode); + MODECASE(ADDTOOL_MODE_BLOCK, blockMode); + MODECASE(ADDTOOL_MODE_MASKED, maskedMode); + MODECASE(ADDTOOL_MODE_UNBLOCK, unblockMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); + MODECASE(ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpMode); + MODECASE(ADDTOOL_MODE_DONECLEANUP, donecleanupMode); + MODECASE(ADDTOOL_MODE_EXPORTRUN, exportrunMode); + MODECASE(ADDTOOL_MODE_IMPORTRUN, importrunMode); + default: + psAbort("invalid option (this should not happen)"); + } + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(EXIT_SUCCESS); + +FAIL: + psErrorStackPrint(stderr, "\n"); + int exit_status = pxerrorGetExitStatus(); + + psFree(config); + pmConfigDone(); + psLibFinalize(); + + exit(exit_status); +} + + +static bool definebyqueryMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false); + + // find the cam_id of all the exposures that we want to queue up. + psString query = pxDataGet("addtool_find_cam_id.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // start a transaction so we don't end up with an exp without any associted + // imfiles + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + // would could do this "all in the database" if we didn't want the option + // of changing the label/reduction/expgroup/dvodb/etc. So we're pulling the + // data out so we have the option of changing these values or leaving the + // old values in place (i.e., passing the values through). + + + // loop over our list of addRun rows + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + addRunRow *row = addRunObjectFromMetadata(md); + if (!row) { + psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into addRun"); + psFree(output); + return false; + } + + // queue the exp + if (!pxaddQueueByCamID(config, + row->cam_id, + workdir ? workdir : row->workdir, + label ? label : row->label, + "RECIPE", + dvodb ? dvodb : row->dvodb + + )) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, + "failed to trying to queue chip_id: %" PRId64, row->cam_id); + psFree(row); + psFree(output); + return false; + } + psFree(row); + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updaterunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "addRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) + && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + + if ((!state) && (!label)) { + psError(PXTOOLS_ERR_DATA, false, "parameters are required"); + psFree(where); + return false; + } + + if (state) { + // set addRun.state to state + if (!pxaddRunSetStateByQuery(config, where, state)) { + psFree(where); + return false; + } + } + + if (label) { + // set addRun.label to label + if (!pxaddRunSetLabelByQuery(config, where, label)) { + psFree(where); + return false; + } + } + + psFree(where); + + return true; +} + + +static bool pendingexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +/* static bool pendingimfileMode(pxConfig *config) */ +/* { */ +/* PS_ASSERT_PTR_NON_NULL(config, false); */ + +/* psMetadata *where = psMetadataAlloc(); */ +/* pxaddGetSearchArgs (config, where); */ +/* PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); */ +/* pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "addRun.chip_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "addProcessedExp.class_id", "=="); */ + +/* PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); */ + +/* psString query = pxDataGet("addtool_find_pendingimfile.sql"); */ +/* if (!query) { */ +/* psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); */ +/* return false; */ +/* } */ + +/* // use psDBGenerateWhereSQL because the SQL yields an intermediate table */ +/* if (psListLength(where->list)) { */ +/* psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); */ +/* psStringAppend(&query, " AND %s", whereClause); */ +/* psFree(whereClause); */ +/* } */ +/* psFree(where); */ + +/* if (!p_psDBRunQuery(config->dbh, query)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* psFree(query); */ +/* return false; */ +/* } */ +/* psFree(query); */ + +/* psArray *output = p_psDBFetchResult(config->dbh); */ +/* if (!output) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* if (!psArrayLength(output)) { */ +/* psTrace("addtool", PS_LOG_INFO, "no rows found"); */ +/* psFree(output); */ +/* return true; */ +/* } */ + +/* // negate simple so the default is true */ +/* if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { */ +/* psError(PS_ERR_UNKNOWN, false, "failed to print array"); */ +/* psFree(output); */ +/* return false; */ +/* } */ + +/* psFree(output); */ + +/* return true; */ +/* } */ + +static bool addprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", true, false); + + // optional + PXOPT_LOOKUP_F32(dtime_addstar, config->args, "-dtime_addstar", false, false); + + PXOPT_LOOKUP_S32(n_stars, config->args, "-n_stars", false, false); + + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + + psString query = pxDataGet("addtool_find_pendingexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClaus = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClaus); + psFree(whereClaus); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(output); + return false; + } + + addRunRow *pendingRow = addRunObjectFromMetadata(output->data[0]); + psFree(output); + addProcessedExpRow *row = addProcessedExpRowAlloc( + pendingRow->add_id, + dtime_addstar, + n_stars, + path_base, + 0 + ); + + if (!addProcessedExpInsertObject(config->dbh, row)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(row); + psFree(pendingRow); + return false; + } + + // since there is only one exp per 'new' set addRun.state = 'full' + if (!pxaddRunSetState(config, row->add_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id); + psFree(row); + psFree(pendingRow); + return false; + } + + // NULL for end_stage means go as far as possible + // EAM : skip here if fault != 0 + // Also, we can run fake even if tess_id is not defined +/* if (fault || (pendingRow->end_stage && psStrcasestr(pendingRow->end_stage, "add"))) { */ +/* psFree(row); */ +/* psFree(pendingRow); */ +/* if (!psDBCommit(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* return false; */ +/* } */ +/* return true; */ +/* } */ + psFree(row); + // else continue on... + +/* if (!pxfakeQueueByAddID(config, */ +/* pendingRow->add_id, */ +/* pendingRow->workdir, */ +/* pendingRow->label, */ +/* pendingRow->reduction, */ +/* pendingRow->expgroup, */ +/* pendingRow->dvodb, */ +/* pendingRow->tess_id, */ +/* pendingRow->end_stage */ +/* )) { */ +/* // rollback */ +/* if (!psDBRollback(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* } */ +/* psError(PS_ERR_UNKNOWN, false, "failed to queue new fakeRun"); */ +/* psFree(pendingRow); */ +/* return false; */ +/* } */ + psFree(pendingRow); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool processedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); + + // generate restrictions + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); + + if (!psListLength(where->list) && + !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required"); + return false; + } + + psString query = pxDataGet("addtool_find_processedexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + // we either add AND (condition) or WHERE (condition): + if (where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault != 0"); + } + if (where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " AND addProcessedExp.fault = 0"); + } + if (!where->list && faulted) { + // list only faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault != 0"); + } + if (!where->list && !faulted) { + // don't list faulted rows + psStringAppend(&query, " %s", " WHERE addProcessedExp.fault = 0"); + } + psFree(where); + + // order by add_id so that the postage stamp parser can easliy find the 'latest' astrometry + psStringAppend(&query, " ORDER BY add_id"); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negate simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addProcessedExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool revertprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + pxaddGetSearchArgs (config, where); + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + PXOPT_COPY_STR(config->args, where, "-reduction", "addRun.reduction", "=="); +/* PXOPT_COPY_S16(config->args, where, "-fault", "addProcessedExp.fault", "=="); */ + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(where); + return false; + } + + { + psString query = pxDataGet("addtool_reset_faulted_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + + { + psString query = pxDataGet("addtool_revertprocessedexp.sql"); + if (!query) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + // use psDBGenerateWhereConditionalSQL with AND ... because the SQL ends in a WHERE + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(where); + return false; + } + psFree(query); + } + psFree(where); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool updateprocessedexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + PXOPT_COPY_S64(config->args, where, "-cam_id", "cam_id", "=="); +/* PXOPT_COPY_STR(config->args, where, "-class", "class", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); */ + + if (!pxSetFaultCode(config->dbh, "addProcessedExp", where, fault)) { + psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag"); + psFree (where); + return false; + } + psFree (where); + + return true; +} + + +static bool blockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + if (!addMaskInsert(config->dbh, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskedMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = psStringCopy("SELECT * FROM addMask"); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addMask", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool unblockMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); + + char *query = "DELETE FROM addMask WHERE label = '%s'"; + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + +static bool pendingcleanuprunMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs (config, where, "-label", "addRun.label", "=="); + + psString query = pxDataGet("addtool_pendingcleanuprun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool pendingcleanupexpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(add_id, config->args, "-add_id", false, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + if (add_id) { + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + } + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_pendingcleanupexp.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("chiptool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addPendingCleanupExp", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool donecleanupMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + psString query = pxDataGet("addtool_donecleanup.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("addtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "addDoneCleanup", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +bool exportrunMode(pxConfig *config) +{ + typedef struct ExportTable { + char tableName[80]; + char sqlFilename[80]; + } ExportTable; + + int numExportTables = 2; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_S64(det_id, config->args, "-add_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + FILE *f = fopen (outfile, "w"); + if (f == NULL) { + psError(PS_ERR_UNKNOWN, false, "failed to open output file"); + return false; + } + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-add_id", "add_id", "=="); + + ExportTable tables [] = { + {"addRun", "addtool_export_run.sql"}, + {"addProcessedExp", "addtool_export_processed_exp.sql"}, + }; + + for (int i=0; i < numExportTables; i++) { + psString query = pxDataGet(tables[i].sqlFilename); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PS_ERR_UNKNOWN, true, "no rows found"); + psFree(output); + return false; + } + + if (clean) { + if (!strcmp(tables[i].tableName, "addRun")) { + if (!pxSetStateCleaned("addRun", "state", output)) { + psFree(output); + psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); + return false; + } + } + } + + // we must write the export table in non-simple (true) format + if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + + fclose (f); + + return true; +} + +bool importrunMode(pxConfig *config) +{ + unsigned int nFail; + + PS_ASSERT_PTR_NON_NULL(config, NULL); + + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); + + fprintf (stdout, "---- input ----\n"); + psMetadataPrint (stderr, input, 1); + + psMetadataItem *item = psMetadataLookup (input, "addRun"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + psMetadataItem *entry = psListGet (item->data.list, 0); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addRunRow *addRun = addRunObjectFromMetadata (entry->data.md); + addRunInsertObject (config->dbh, addRun); + + // fprintf (stdout, "---- add run ----\n"); + // psMetadataPrint (stderr, entry->data.md, 1); + + item = psMetadataLookup (input, "addProcessedExp"); + psAssert (item, "entry not in input?"); + psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); + + for (int i = 0; i < item->data.list->n; i++) { + psMetadataItem *entry = psListGet (item->data.list, i); + assert (entry); + assert (entry->type == PS_DATA_METADATA); + addProcessedExpRow *addProcessedExp = addProcessedExpObjectFromMetadata (entry->data.md); + addProcessedExpInsertObject (config->dbh, addProcessedExp); + + // fprintf (stdout, "---- row %d ----\n", i); + // psMetadataPrint (stderr, entry->data.md, 1); + } + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/addtool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/addtool.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/addtool.h (revision 25766) @@ -0,0 +1,46 @@ +/* + * addtool.h + * + * Copyright (C) 2006 Joshua Hoblitt, Christopher Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef ADDTOOL_H +#define ADDTOOL_H 1 + +#include "pxtools.h" + +typedef enum { + ADDTOOL_MODE_NONE = 0x0, + ADDTOOL_MODE_DEFINEBYQUERY, + ADDTOOL_MODE_UPDATERUN, + ADDTOOL_MODE_PENDINGEXP, + ADDTOOL_MODE_ADDPROCESSEDEXP, + ADDTOOL_MODE_PROCESSEDEXP, + ADDTOOL_MODE_REVERTPROCESSEDEXP, + ADDTOOL_MODE_UPDATEPROCESSEDEXP, + ADDTOOL_MODE_BLOCK, + ADDTOOL_MODE_MASKED, + ADDTOOL_MODE_UNBLOCK, + ADDTOOL_MODE_PENDINGCLEANUPRUN, + ADDTOOL_MODE_PENDINGCLEANUPEXP, + ADDTOOL_MODE_DONECLEANUP, + ADDTOOL_MODE_EXPORTRUN, + ADDTOOL_MODE_IMPORTRUN +} addtoolMode; + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv); + +#endif // ADDTOOL_H Index: branches/eam_branches/20090820/ippTools/src/addtoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/addtoolConfig.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/addtoolConfig.c (revision 25766) @@ -0,0 +1,212 @@ +/* + * addtoolConfig.c + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include + +#include "pxtools.h" +#include "pxadd.h" +#include "addtool.h" + +pxConfig *addtoolConfig(pxConfig *config, int argc, char **argv) +{ + if (!config) { + config = pxConfigAlloc(); + } + + pmConfigReadParamsSet(false); + + // setup site config + config->modules = pmConfigRead(&argc, argv, NULL); + if (!config->modules) { + psError(PS_ERR_UNKNOWN, false, "Can't find site configuration"); + psFree(config); + return NULL; + } + + // -definebyquery + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *definebyqueryArgs = psMetadataAlloc(); + pxcamSetSearchArgs(definebyqueryArgs); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-reduction", 0, "search by chipRun reduction class", NULL); + + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_workdir", 0, "define workdir", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_reduction", 0, "define reduction class", NULL); + psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dvodb", 0, "define DVO db", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updaterun + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *updaterunArgs = psMetadataAlloc(); + pxcamSetSearchArgs(updaterunArgs); + psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-label", 0, "search by camRun label", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "search by camRun state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state", NULL); + psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_label", 0, "set label", NULL); + + // -pendingexp + psMetadata *pendingexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(pendingexpArgs); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(pendingexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by camRun label", NULL); + psMetadataAddStr(pendingexpArgs, PS_LIST_TAIL, "-reduction", 0, "search by camRun reduction class", NULL); + psMetadataAddU64(pendingexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // XXX is this used? psMetadataAddStr(pendingimfileArgs, PS_LIST_TAIL, "-class", 0, "search by class", NULL); + + // -addprocessedexp + psMetadata *addprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "define addtool ID (required)", 0); + + psMetadataAddF32(addprocessedexpArgs, PS_LIST_TAIL, "-dtime_addstar", 0, "define elapsed time for DVO insertion (seconds)", NAN); + psMetadataAddS32(addprocessedexpArgs, PS_LIST_TAIL, "-n_stars", 0, "define number of stars", 0); + + psMetadataAddStr(addprocessedexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); + psMetadataAddS64(addprocessedexpArgs, PS_LIST_TAIL, "-magicked", 0, "set magicked", 0); + // -processedexp + psMetadata *processedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(processedexpArgs); + psMetadataAddS64(processedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(processedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddU64(processedexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-all", 0, "list everything without restriction", false); + psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); +/* psMetadataAddBool(processedexpArgs, PS_LIST_TAIL, "-faulted", 0, "only return imfiles with a fault status set", false); */ + + // -revertprocessedexp + // XXX need to allow multiple add_ids + // XXX need to allow multiple chip_ids + // XXX need to allow multiple exp_ids + psMetadata *revertprocessedexpArgs = psMetadataAlloc(); + pxcamSetSearchArgs(revertprocessedexpArgs); + psMetadataAddS64(revertprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by addRun label", NULL); + psMetadataAddStr(revertprocessedexpArgs, PS_LIST_TAIL, "-reduction",0, "search by addRun reduction class", NULL); + + psMetadataAddBool(revertprocessedexpArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); + + // -updateprocessedexp + // XXX allow full search options? + psMetadata *updateprocessedexpArgs = psMetadataAlloc(); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addtool ID", 0); + psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-cam_id", 0, "search by camtool ID", 0); + + // -block + psMetadata *blockArgs = psMetadataAlloc(); + psMetadataAddStr(blockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to mask out (required)", NULL); + + // -masked + psMetadata *maskedArgs = psMetadataAlloc(); + psMetadataAddBool(maskedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -unblock + psMetadata *unblockArgs = psMetadataAlloc(); + psMetadataAddStr(unblockArgs, PS_LIST_TAIL, "-label", 0, "name of a label to unmask (required)", NULL); + + // -pendingcleanuprun + // XXX allow full search options? + psMetadata *pendingcleanuprunArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanuprunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddBool(pendingcleanuprunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanuprunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -pendingcleanupexp + // XXX allow full search options? + psMetadata *pendingcleanupexpArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcleanupexpArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "list blocks for specified label", NULL); + psMetadataAddS64(pendingcleanupexpArgs, PS_LIST_TAIL, "-add_id", 0, "search by addstar ID", 0); + psMetadataAddBool(pendingcleanupexpArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(pendingcleanupexpArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -donecleanup + psMetadata *donecleanupArgs = psMetadataAlloc(); + psMetadataAddStr(donecleanupArgs, PS_LIST_TAIL, "-label", 0, "list blocks for specified label", NULL); + psMetadataAddBool(donecleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -exportrun + psMetadata *exportrunArgs = psMetadataAlloc(); + psMetadataAddS64(exportrunArgs, PS_LIST_TAIL, "-add_id", 0, "export this addstar ID (required)", 0); + psMetadataAddStr(exportrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL); + psMetadataAddU64(exportrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(exportrunArgs, PS_LIST_TAIL, "-clean", 0, "export tables as cleaned", false); + + // -importrun + psMetadata *importrunArgs = psMetadataAlloc(); + psMetadataAddStr(importrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL); + + + psMetadata *argSets = psMetadataAlloc(); + psMetadata *modes = psMetadataAlloc(); + + PXOPT_ADD_MODE("-definebyquery", "create runs from cam stage", ADDTOOL_MODE_DEFINEBYQUERY, definebyqueryArgs); + PXOPT_ADD_MODE("-updaterun", "change add run properties", ADDTOOL_MODE_UPDATERUN, updaterunArgs); + PXOPT_ADD_MODE("-pendingexp", "show pending exps", ADDTOOL_MODE_PENDINGEXP, pendingexpArgs); + PXOPT_ADD_MODE("-addprocessedexp", "add a processed exp", ADDTOOL_MODE_ADDPROCESSEDEXP, addprocessedexpArgs); + PXOPT_ADD_MODE("-processedexp", "show processed exps", ADDTOOL_MODE_PROCESSEDEXP, processedexpArgs); + PXOPT_ADD_MODE("-revertprocessedexp", "change procesed exp properties", ADDTOOL_MODE_REVERTPROCESSEDEXP, revertprocessedexpArgs); + PXOPT_ADD_MODE("-updateprocessedexp", "undo a processed exp", ADDTOOL_MODE_UPDATEPROCESSEDEXP,updateprocessedexpArgs); + PXOPT_ADD_MODE("-block", "set a label block", ADDTOOL_MODE_BLOCK, blockArgs); + PXOPT_ADD_MODE("-masked", "show blocked labels", ADDTOOL_MODE_MASKED, maskedArgs); + PXOPT_ADD_MODE("-unblock", "remove a label block", ADDTOOL_MODE_UNBLOCK, unblockArgs); + PXOPT_ADD_MODE("-pendingcleanuprun", "show runs that need to be cleaned up", ADDTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunArgs); + PXOPT_ADD_MODE("-pendingcleanupexp", "show exps for cleanup runs", ADDTOOL_MODE_PENDINGCLEANUPEXP, pendingcleanupexpArgs); + PXOPT_ADD_MODE("-donecleanup", "show runs that have been cleaned", ADDTOOL_MODE_DONECLEANUP, donecleanupArgs); + PXOPT_ADD_MODE("-exportrun", "export run for import on other database", ADDTOOL_MODE_EXPORTRUN, exportrunArgs); + PXOPT_ADD_MODE("-importrun", "import run from metadata file", ADDTOOL_MODE_IMPORTRUN, importrunArgs); + + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { + psError(PS_ERR_UNKNOWN, false, "option parsing failed"); + psFree(argSets); + psFree(modes); + psFree(config); + return NULL; + } + + psFree(argSets); + psFree(modes); + + // define Database handle, if used + config->dbh = psMemIncrRefCounter(pmConfigDB(config->modules)); + if (!config->dbh) { + psError(PS_ERR_UNKNOWN, false, "Can't configure database"); + psFree(config); + return NULL; + } + + return config; +} Index: branches/eam_branches/20090820/ippTools/src/camtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/camtool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/camtool.c (revision 25766) @@ -656,4 +656,21 @@ return false; } + +/* if (!pxaddQueueByCamID(config, */ +/* pendingRow->cam_id, */ +/* pendingRow->workdir, */ +/* pendingRow->label, */ +/* pendingRow->reduction, */ +/* pendingRow->dvodb */ +/* )) { */ +/* // rollback */ +/* if (!psDBRollback(config->dbh)) { */ +/* psError(PS_ERR_UNKNOWN, false, "database error"); */ +/* } */ +/* psError(PS_ERR_UNKNOWN, false, "failed to queue new addRun"); */ +/* psFree(pendingRow); */ +/* return false; */ +/* } */ + psFree(pendingRow); Index: branches/eam_branches/20090820/ippTools/src/chiptool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/chiptool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/chiptool.c (revision 25766) @@ -54,4 +54,5 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -93,4 +94,5 @@ MODECASE(CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(CHIPTOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(CHIPTOOL_MODE_IMPORTRUN, importrunMode); @@ -601,5 +603,9 @@ PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "=="); pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE"); - PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "=="); + + PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); if (!psListLength(where->list) && @@ -629,4 +635,18 @@ // don't list faulted rows psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0"); + } + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(&query, " AND chipProcessedImfile.magicked = 0"); + } + if (destreaked) { + psStringAppend(&query, " AND chipProcessedImfile.magicked != 0"); } @@ -1333,5 +1353,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } - +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) { Index: branches/eam_branches/20090820/ippTools/src/chiptool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/chiptool.h (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/chiptool.h (revision 25766) @@ -45,4 +45,5 @@ CHIPTOOL_MODE_TOFULLIMFILE, CHIPTOOL_MODE_TOPURGEDIMFILE, + CHIPTOOL_MODE_TOSCRUBBEDIMFILE, CHIPTOOL_MODE_EXPORTRUN, CHIPTOOL_MODE_IMPORTRUN Index: branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c (revision 25766) @@ -158,5 +158,8 @@ psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-reduction", 0, "search by reduction class", NULL); psMetadataAddStr(processedimfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL); - psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked status", false); + + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); + psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); + psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); psMetadataAddU64(processedimfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all", 0, "list everything without search terms", false); @@ -247,4 +250,9 @@ psMetadataAddS64(topurgedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-chip_id", 0, "chip ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); // -exportrun @@ -282,4 +290,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", CHIPTOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", CHIPTOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", CHIPTOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", CHIPTOOL_MODE_EXPORTRUN, exportrunArgs); Index: branches/eam_branches/20090820/ippTools/src/dettool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/dettool.c (revision 25766) @@ -150,4 +150,5 @@ MODECASE(DETTOOL_MODE_UPDATEDETRUN, updatedetrunMode); MODECASE(DETTOOL_MODE_RERUN, rerunMode); + MODECASE(DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryMode); // register MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode); @@ -1827,4 +1828,15 @@ if (!strcmp(data_state, "drop")) return true; if (!strcmp(data_state, "register")) return true; + // These are valid data states, and are necessary for the cleanup to work correctly. + if (!strcmp(data_state, "full")) return true; + if (!strcmp(data_state, "goto_cleaned")) return true; + if (!strcmp(data_state, "goto_scrubbed")) return true; + if (!strcmp(data_state, "goto_purged")) return true; + if (!strcmp(data_state, "cleaned")) return true; + if (!strcmp(data_state, "scrubbed")) return true; + if (!strcmp(data_state, "purged")) return true; + if (!strcmp(data_state, "error_cleaned")) return true; + if (!strcmp(data_state, "error_scrubbed")) return true; + if (!strcmp(data_state, "error_purged")) return true; psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state); @@ -1853,5 +1865,5 @@ if (!isValidDataState (data_state)) return false; - + char *query = "UPDATE detProcessedImfile SET data_state = '%s'" " WHERE det_id = %" PRId64 @@ -1900,5 +1912,5 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, @@ -1922,6 +1934,7 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; - if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration)) { + " AND class_id = '%s'"; +/* fprintf(stderr,"DETTOOL SAYS: %s\n",query); */ + if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration,class_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change state for det_id %" PRId64 ", iteration %" PRId32, @@ -1944,5 +1957,5 @@ " WHERE det_id = %" PRId64 " AND iteration = %" PRId32 - " AND class_id = %s"; + " AND class_id = '%s'"; if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) { psError(PS_ERR_UNKNOWN, false, Index: branches/eam_branches/20090820/ippTools/src/dettool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettool.h (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/dettool.h (revision 25766) @@ -111,4 +111,5 @@ DETTOOL_MODE_REVERTDETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUNSUMMARY, + DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, DETTOOL_MODE_UPDATEDETRUN, DETTOOL_MODE_RERUN, @@ -209,4 +210,5 @@ bool revertdetrunsummaryMode(pxConfig *config); bool updatedetrunsummaryMode(pxConfig *config); +bool pendingcleanup_detrunsummaryMode(pxConfig *config); // other utilities Index: branches/eam_branches/20090820/ippTools/src/dettoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/dettoolConfig.c (revision 25766) @@ -41,11 +41,11 @@ // XXX EAM : is this used? does it make sense? psMetadata *pendingArgs = psMetadataAlloc(); - psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); - psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); + psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type", 0, "search by exposure type", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst", 0, "search by camera", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope", 0, "search by telescope", NULL); + psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter", 0, "search by filter", NULL); psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri", 0, "search by URL", NULL); - psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -definebytag @@ -125,4 +125,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min", 0, "define min rotator position angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max", 0, "define max rotator position angle", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min", 0, "define min solar angle", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max", 0, "define max solar angle", NAN); @@ -137,4 +138,5 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max", 0, "define max moon phase", NAN); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment", 0, "search by comment field (LIKE comparison)", NULL); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend", 0, "print the exposures that would be included in the detrend run and exit", false); @@ -207,7 +209,7 @@ // -addcorrectimfile psMetadata *addcorrectimfileArgs = psMetadataAlloc(); - psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); - psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); + psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id", 0, "define detrend ID (required)", 0); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search for class ID (required)", NULL); + psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri", 0, "define resid file URI", NULL); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg", 0, "define exposure background", NAN); psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev", 0, "define exposure background stdev", NAN); @@ -306,5 +308,5 @@ psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedimfile @@ -366,5 +368,5 @@ psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); - psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_processedexp @@ -426,5 +428,5 @@ psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id", 0, "search by class ID", NULL); - psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_stacked @@ -478,5 +480,5 @@ psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedstat @@ -540,5 +542,5 @@ psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedimfile @@ -598,5 +600,5 @@ psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id", 0, "search by chip ID", 0); psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); - psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_normalizedexp @@ -678,5 +680,5 @@ psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exp_id", 0); psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id", 0, "search by exp_name", NULL); - psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residimfile @@ -759,5 +761,5 @@ psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", NULL); psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject", 0, "exposure is not to be stacked in the next iteration", false); - psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "search for telescope", NULL); + psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state", 0, "set state", NULL); // -pendingcleanup_residexp @@ -813,4 +815,13 @@ psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-accept", 0, "declare that this detrun iteration is accepted as a master", false); psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-reject", 0, "reject this detrun iteration as a master", false); + psMetadataAddStr(updatedetrunsummaryArgs, PS_LIST_TAIL, "-data_state", 0, "set the data state", NULL); + psMetadataAddS32(updatedetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + + // -pendingcleanup_detrunsummary + psMetadata *pendingcleanup_detrunsummaryArgs = psMetadataAlloc(); + psMetadataAddS64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0, "search by detrend ID (required)", 0); + psMetadataAddS32(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0, "search by iteration number", 0); + psMetadataAddU64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -updatedetrun @@ -968,4 +979,6 @@ PXOPT_ADD_MODE("-updatedetrunsummary", "", DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryArgs); + PXOPT_ADD_MODE("-pendingcleanup_detrunsummary", "", DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryArgs); + PXOPT_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs); PXOPT_ADD_MODE("-rerun", "", DETTOOL_MODE_RERUN, rerunArgs); Index: branches/eam_branches/20090820/ippTools/src/dettool_detrunsummary.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettool_detrunsummary.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/dettool_detrunsummary.c (revision 25766) @@ -336,5 +336,65 @@ } -// XXX need to add -data_state here +bool pendingcleanup_detrunsummaryMode(pxConfig *config) { + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); + PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "=="); + + psString query = pxDataGet("dettool_pendingcleanup_detrunsummary.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return(false); + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + // fprintf(stderr,">>>%s<<<\n",query); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return(false); + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return(false); + } + if (!psArrayLength(output)) { + psTrace("dettool", PS_LOG_INFO, "no rows found"); + psFree(output); + return(true); + } + + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "pendingCleanupDetRun", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return(false); + } + psFree(output); + + return(true); +} + + +// preliminary code now. bool updatedetrunsummaryMode(pxConfig *config) { @@ -344,4 +404,7 @@ PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false); PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false); + PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", ((accept == 0)&&(reject == 0)), false); + PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", ((accept == 0)&&(reject == 0)), false); + if (accept && reject) { @@ -350,13 +413,109 @@ } - if (!(accept || reject)) { - psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required"); - return false; - } - - char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; - if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; + if (!(accept || reject || (data_state != NULL))) { + psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required if -data_state is not supplied"); + return false; + } + + if (accept || reject) { + char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64; + if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + } + else { + PS_ASSERT_PTR_NON_NULL(data_state,false); + +/* if (!isValidDataState(data_state)) return false; */ + + + + char *query_detRunSummary = "UPDATE detRunSummary SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detRunSummary, data_state, det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return false; + } + + /* This check allows the one update to flag everything for cleanup. The check for full is only temporary while I test for bugs. */ + if ((!strncmp(data_state,"goto_",5) + // || (!strcmp(data_state,"full")) + // || (!strcmp(data_state,"cleaned")) + )) { + char *query_detProcessedImfile = "UPDATE detProcessedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedImfile,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detProcessedExp = "UPDATE detProcessedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query_detProcessedExp,data_state,det_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detProcessedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedImfile = "UPDATE detNormalizedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedStatImfile = "UPDATE detNormalizedStatImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedStatImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedStatImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detNormalizedExp = "UPDATE detNormalizedExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detNormalizedExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detNormalizedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidImfile = "UPDATE detResidImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detResidExp = "UPDATE detResidExp SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detResidExp,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detResidExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + + char *query_detStackedImfile = "UPDATE detStackedImfile SET data_state = '%s'" + " WHERE det_id = %" PRId64 + " AND iteration = %" PRId32; + if (!p_psDBRunQueryF(config->dbh, query_detStackedImfile,data_state,det_id,iteration)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for detStackedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration); + return(false); + } + } + /* End if */ + } Index: branches/eam_branches/20090820/ippTools/src/dettool_processedimfile.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/dettool_processedimfile.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/dettool_processedimfile.c (revision 25766) @@ -314,5 +314,5 @@ PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - +/* int i; */ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "=="); @@ -345,4 +345,5 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",query); */ psFree(query); @@ -350,8 +351,10 @@ if (!output) { psError(PS_ERR_UNKNOWN, false, "database error"); +/* fprintf(stderr,"WTF !output?\n"); */ return false; } if (!psArrayLength(output)) { psTrace("dettool", PS_LOG_INFO, "no rows found"); +/* fprintf(stderr,"WTF no rows??\n"); */ psFree(output); return true; @@ -359,4 +362,6 @@ // negative simple so the default is true +/* i = (int) ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple); */ +/* fprintf(stderr,">>%d<<\n",i); */ if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); @@ -364,4 +369,6 @@ return false; } +/* fprintf(stderr,"DETTOOL:procimfile: %s\n",output); */ +/* psFree(output); */ psFree(output); Index: branches/eam_branches/20090820/ippTools/src/difftool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/difftool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/difftool.c (revision 25766) @@ -37,4 +37,5 @@ static bool todiffskyfileMode(pxConfig *config); static bool adddiffskyfileMode(pxConfig *config); +static bool advanceMode(pxConfig *config); static bool diffskyfileMode(pxConfig *config); static bool revertdiffskyfileMode(pxConfig *config); @@ -50,5 +51,4 @@ static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked); -static bool diffRunComplete(pxConfig *config); # define MODECASE(caseName, func) \ @@ -76,8 +76,9 @@ MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileMode); MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileMode); + MODECASE(DIFFTOOL_MODE_ADVANCE, advanceMode); MODECASE(DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileMode); MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode); MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN, definepoprunMode); - MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); + MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK, definewarpstackMode); MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP, definewarpwarpMode); MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN, pendingcleanuprunMode); @@ -555,12 +556,4 @@ } - if (!diffRunComplete(config)) { - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - // point of no return if (!psDBCommit(config->dbh)) { @@ -569,4 +562,95 @@ } + + return true; +} + +static bool advanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-diff_id", "diffRun.diff_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + // look for completed diffRuns + psString query = pxDataGet("difftool_completed_runs.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + psString whereString = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&whereString, "\n AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(whereString); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, query, whereString)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + psFree(whereString); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + psFree(whereString); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("difftool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + + psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); + psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); + + // set diffRun.state to 'full' + if (!setdiffRunState(config, diff_id, "full", magicked)) { + psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, + diff_id); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; @@ -581,13 +665,34 @@ PXOPT_COPY_S64(config->args, where, "-diff_id", "diffSkyfile.diff_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "=="); - PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); + PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "=="); PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); - PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); - + PXOPT_COPY_S64(config->args, where, "-magicked", "diffSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE"); + + PXOPT_LOOKUP_BOOL(template, config->args, "-template", false); + if (!template) { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawInput.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawInput.filter", "=="); + } else { + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawTemplate.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawTemplate.filter", "=="); + } + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); psString query = pxDataGet("difftool_skyfile.sql"); @@ -601,6 +706,24 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); + } else if (!all) { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); } psFree(where); + + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(&query, " AND diffSkyfile.magicked = 0"); + } + if (destreaked) { + psStringAppend(&query, " AND diffSkyfile.magicked != 0"); + } + // treat limit == 0 as "no limit" @@ -1810,49 +1933,4 @@ } -static bool diffRunComplete(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // look for completed diffRuns - psString query = pxDataGet("difftool_completed_runs.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("difftool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *row = output->data[i]; - - psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id"); - psS64 magicked = psMetadataLookupS64(NULL, row, "magicked"); - - // set diffRun.state to 'stop' - if (!setdiffRunState(config, diff_id, "full", magicked)) { - psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64, - diff_id); - psFree(output); - return false; - } - } - - return true; -} bool exportrunMode(pxConfig *config) Index: branches/eam_branches/20090820/ippTools/src/difftool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/difftool.h (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/difftool.h (revision 25766) @@ -31,4 +31,5 @@ DIFFTOOL_MODE_TODIFFSKYFILE, DIFFTOOL_MODE_ADDDIFFSKYFILE, + DIFFTOOL_MODE_ADVANCE, DIFFTOOL_MODE_DIFFSKYFILE, DIFFTOOL_MODE_REVERTDIFFSKYFILE, Index: branches/eam_branches/20090820/ippTools/src/difftoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/difftoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/difftoolConfig.c (revision 25766) @@ -115,16 +115,32 @@ psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "define magicked state", false); + // -advance + psMetadata *advanceArgs = psMetadataAlloc(); + psMetadataAddS64(advanceArgs, PS_LIST_TAIL, "-diff_id", 0, "select by diff ID", 0); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "select by label", NULL); + psMetadataAddS32(advanceArgs, PS_LIST_TAIL, "-limit", 0, "limit number of results", 0); + // -diffskyfile psMetadata *diffskyfileArgs = psMetadataAlloc(); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); - psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, "define skycell ID", NULL); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_skyfile_id", 0, "search by diff_skyfile_id ID", 0); - psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "define tessellation ID", NULL); - psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); - psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name", 0, "define exposure name", NULL); - psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id", 0, "define warp_id", NULL); - psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0, "search by diff ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", NULL); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_skyfile_id", 0, "search by diff_skyfile_id ID", 0); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tessellation ID", NULL); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id", 0, "search by warp_id", NULL); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-template", 0, "apply exposure args to template of bothways diff", false); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); + psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); + psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); + psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that are not destreaked", false); + psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL); + psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); + psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -revertdiffskyfile @@ -267,4 +283,5 @@ PXOPT_ADD_MODE("-todiffskyfile", "", DIFFTOOL_MODE_TODIFFSKYFILE, todiffskyfileArgs); PXOPT_ADD_MODE("-adddiffskyfile", "", DIFFTOOL_MODE_ADDDIFFSKYFILE, adddiffskyfileArgs); + PXOPT_ADD_MODE("-advance", "", DIFFTOOL_MODE_ADVANCE, advanceArgs); PXOPT_ADD_MODE("-diffskyfile", "", DIFFTOOL_MODE_DIFFSKYFILE, diffskyfileArgs); PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs); Index: branches/eam_branches/20090820/ippTools/src/disttool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/disttool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/disttool.c (revision 25766) @@ -2,5 +2,5 @@ * disttool.c * - * Copyright (C) 2008 + * Copyright (C) 2008-2009 * * This program is free software; you can redistribute it and/or modify it @@ -37,4 +37,5 @@ static bool pendingcomponentMode(pxConfig *config); static bool addprocessedcomponentMode(pxConfig *config); +static bool revertcomponentMode(pxConfig *config); static bool processedcomponentMode(pxConfig *config); static bool toadvanceMode(pxConfig *config); @@ -51,7 +52,4 @@ static bool listtargetMode(pxConfig *config); -static bool definedsproductMode(pxConfig *config); -static bool updatedsproductMode(pxConfig *config); - static bool definedestinationMode(pxConfig *config); static bool updatedestinationMode(pxConfig *config); @@ -59,4 +57,5 @@ static bool defineinterestMode(pxConfig *config); static bool updateinterestMode(pxConfig *config); +static bool listinterestsMode(pxConfig *config); # define MODECASE(caseName, func) \ @@ -86,4 +85,5 @@ MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode); MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode); + MODECASE(DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentMode); MODECASE(DISTTOOL_MODE_TOADVANCE, toadvanceMode); MODECASE(DISTTOOL_MODE_PENDINGFILESET, pendingfilesetMode); @@ -97,10 +97,9 @@ MODECASE(DISTTOOL_MODE_UPDATETARGET, updatetargetMode); MODECASE(DISTTOOL_MODE_LISTTARGET, listtargetMode); - MODECASE(DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductMode); - MODECASE(DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductMode); MODECASE(DISTTOOL_MODE_DEFINEDESTINATION, definedestinationMode); MODECASE(DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationMode); MODECASE(DISTTOOL_MODE_DEFINEINTEREST, defineinterestMode); MODECASE(DISTTOOL_MODE_UPDATEINTEREST, updateinterestMode); + MODECASE(DISTTOOL_MODE_LISTINTERESTS, listinterestsMode); default: psAbort("invalid option (this should not happen)"); @@ -151,4 +150,5 @@ stage, stage_id, + 0, set_label, outroot, @@ -223,5 +223,6 @@ } } else if (!strcmp(stage, "camera")) { - magicRunType = "chipRun"; // This is used below to set the magicked business + magicRunType = "camRun"; // This is used below to set the magicked business + runJoinStr = "camRun.cam_id"; query = pxDataGet("disttool_definebyquery_camera.sql"); if (!query) { @@ -288,5 +289,5 @@ psStringAppend(&query, " AND (stackRun.label = '%s')", label); } - // stack stage doesn't require magic (perhaps let the script do this? + // stack stage doesn't require magic no_magic = true; } else { @@ -306,24 +307,15 @@ if (!no_magic) { - psStringAppend(&query, " AND (distTarget.clean OR %s.magicked)", magicRunType); + psStringAppend(&query, " AND (%s.magicked)", magicRunType); // is selecting by magic_ds_id really interesting? if (magic_ds_id) { - if (strcmp(stage, "camera")) { - // stage other than camera - if (!runJoinStr) { - psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage); - psFree(query); - return false; - } - psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage" + if (!runJoinStr) { + psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage); + psFree(query); + return false; + } + psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage" " AND magicDSRun.stage_id = %s", runJoinStr); - } else { - // camera masks are magicked when the chipRun is magicked - // XXX: This is confusing. Is it dangerous? - // Maybe I should add a magicked bit to camRun. Note this isn't - psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = 'chip'" - " AND magicDSRun.stage_id = chipRun.chip_id"); - } psStringAppend(&query, " AND (magicDSRun.state = 'full' AND magicDSRun.re_place AND (magic_ds_id = %" PRId64 "))", magic_ds_id); } @@ -377,4 +369,5 @@ psString run_tag = psMetadataLookupStr(NULL, md, "run_tag"); psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id"); + psS64 magic_ds_id = psMetadataLookupS64(NULL, md, "magicked"); psS64 target_id = psMetadataLookupS64(NULL, md, "target_id"); psString target_label = psMetadataLookupStr(NULL, md, "label"); @@ -395,4 +388,5 @@ stage, stage_id, + magic_ds_id, new_label, outroot, @@ -500,8 +494,5 @@ PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - // we need to disambiguate fault so make a copy of the where list before adding fault - psMetadata *whereComponent = psMetadataCopy(NULL, where); PXOPT_COPY_S16(config->args, where, "-fault", "distRun.fault", "=="); - PXOPT_COPY_S16(config->args, whereComponent, "-fault", "distComponent.fault", "=="); // It might be useful to be able to query by the parameters of the underlying runs @@ -513,84 +504,77 @@ } - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psString query = pxDataGet("disttool_revertrun.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + int numUpdated = psDBAffectedRows(config->dbh); + + psLogMsg("disttool", PS_LOG_INFO, "Updated %d dist runs", numUpdated); + + return true; +} + +static bool revertcomponentMode(pxConfig *config) +{ + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dist_id", "distRun.dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; + PXOPT_COPY_STR(config->args, where, "-component", "component", "==");; + PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); + + PXOPT_COPY_S16(config->args, where, "-fault", "distComponent.fault", "=="); + + // It might be useful to be able to query by the parameters of the underlying runs + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - return false; - } - - // Update state to 'new' - int numUpdated; // Number updated - { - psString query = pxDataGet("disttool_revertrun_update.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numUpdated = psDBAffectedRows(config->dbh); - } - - psLogMsg("disttool", PS_LOG_INFO, "Updated %d dist runs", numUpdated); - - // Delete product - int numDeleted; // Number deleted - { - psString query = pxDataGet("disttool_revertrun_delete.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(whereComponent->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(whereComponent, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numDeleted = psDBAffectedRows(config->dbh); - } + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString query = pxDataGet("disttool_revertcomponent.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + + int numDeleted = psDBAffectedRows(config->dbh); psLogMsg("disttool", PS_LOG_INFO, "Deleted %d distComponents", numDeleted); - psFree(where); - psFree(whereComponent); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - return true; } @@ -600,16 +584,18 @@ PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("disttool_pendingcomponent.sql"); + psString queryFile = NULL; + psStringAppend(&queryFile, "disttool_pending_%s.sql", stage); + psString query = pxDataGet(queryFile); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); return false; } @@ -617,5 +603,5 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -629,28 +615,5 @@ } - // the query has where hooks for each stage. - // right now we aren't using them. - // XXX: I think that I want to change the query from a union of selects on the various - // stages to separate queries. As it is pending data at the later stages of the pipline - // will get blocked by pending earlier stages - psString raw_where = ""; - psString raw_clean_where = ""; - psString chip_where = ""; - psString camera_where = ""; - psString fake_where = ""; - psString warp_where = ""; - psString diff_where = ""; - psString stack_where = ""; - - if (!p_psDBRunQueryF(config->dbh, - query, - raw_where, - raw_clean_where, - chip_where, - camera_where, - fake_where, - warp_where, - diff_where, - stack_where)) { + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -868,4 +831,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "distRun.stage", "=="); pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "=="); @@ -939,5 +903,5 @@ // required values PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false); - PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false); + PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -949,5 +913,5 @@ 0, // fs_id dist_id, - prod_id, + dest_id, name, "full", @@ -964,5 +928,5 @@ PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); PXOPT_COPY_S64(config->args, where, "-dist_id", "rcDSFileset.dist_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");; PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "=="); @@ -1092,5 +1056,4 @@ PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "=="); PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); PXOPT_COPY_S64(config->args, where, "-target_id","target_id", "=="); PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "=="); @@ -1433,99 +1396,12 @@ } -static bool definedsproductMode(pxConfig *config) +static bool definedestinationMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); // required - PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); - PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false); - PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false); - - // XXX: should we insure that these names do not contatin any whitespace? - - rcDSProductRow *row = rcDSProductRowAlloc( - 0, // prod_id - name, - dbname, - dbhost - ); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDSProduct object"); - return false; - } - if (!rcDSProductInsertObject(config->dbh, row)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(row); - return false; - } - - // get the assigned target_id - row->prod_id = psDBLastInsertID(config->dbh); - - if (!rcDSProductPrintObject(stdout, row, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(row); - return false; - } - - psFree(row); - - return true; -} -static bool updatedsproductMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); - - PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", false, false); - PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", false, false); - - if (!(dbname || dbhost)) { - psError(PS_ERR_UNKNOWN, true, "one or more of dbname or dbhost is required"); - psFree(where); - return false; - } - psString query = psStringCopy("UPDATE rcDSProduct SET"); - psString sep = ""; - if (dbname) { - psStringAppend(&query, " dbname = '%s'", dbname); - sep = ","; - } - if (dbhost) { - psStringAppend(&query, " %s dbhost = '%s'", sep, dbhost); - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - } else { - psError(PS_ERR_UNKNOWN, true, "search parameters are required"); - psFree(where); - psFree(query); - return false; - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - return true; -} - -static bool definedestinationMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false); PXOPT_LOOKUP_STR(name, config->args, "-name", true, false); + PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false); + PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false); // optional @@ -1539,9 +1415,10 @@ rcDestinationRow *row = rcDestinationRowAlloc( 0, // dest_id - prod_id, name, status_uri, comment, last_fileset, + dbname, + dbhost, state ? state : "enabled" ); @@ -1577,5 +1454,4 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); - PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "=="); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); @@ -1599,5 +1475,5 @@ // last_fileset normally gets set by updatercrunMode // Allowing it to be set here might cause problems - // especially since we are allowing selection by prod_id + // especially since we are allowing selection by dest_id if (last_fileset) { psStringAppend(&query, " %s last_fileset = '%s'", sep, last_fileset); @@ -1631,40 +1507,102 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // required - PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false); - PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", true, false); + // one of these is required + PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", false, false); + PXOPT_LOOKUP_STR(dest_name, config->args, "-dest_name", false, false); + if (!dest_id && !dest_name) { + psError(PS_ERR_UNKNOWN, true, "either dest_id or dest_name is required"); + return false; + } + + // either target_id or stage and label are required + PXOPT_LOOKUP_S64(target_id, config->args, "-target_id", false, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_STR(filter, config->args, "-filter", false, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + if (!target_id) { + bool error = false; + if (!stage) { + psError(PS_ERR_UNKNOWN, true, "stage is required if target_id is not supplied"); + error = true; + } + if (!label) { + psError(PS_ERR_UNKNOWN, !error, "label is required if target_id is not supplied"); + error = true; + } + if (error) { + return false; + } + } // optional + PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - - // XXX: should we insure that these names do not contatin any whitespace? - - rcInterestRow *row = rcInterestRowAlloc( - 0, // int_id - dest_id, - target_id, - state ? state : "enabled" - ); - - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to allocate rcInterest object"); - return false; - } - if (!rcInterestInsertObject(config->dbh, row)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(row); - return false; - } - - // get the assigned target_id - row->int_id = psDBLastInsertID(config->dbh); - - if (!rcInterestPrintObject(stdout, row, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print object"); - psFree(row); - return false; - } - - psFree(row); + if (state) { + if (strcmp(state, "enabled") && strcmp(state, "disabled")) { + psError(PS_ERR_PROGRAMMING, true, "state must be enabled or disabled"); + return false; + } + } else { + // default state + state = "enabled"; + } + + // now that we've done all of our argument checking, copy the values to where + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + + psString query = pxDataGet("disttool_defineinterest.sql"); + + if (!psListLength(where->list)) { + // can't get here + psError(PS_ERR_PROGRAMMING, true, "search parameters are required"); + psFree(where); + psFree(query); + return false; + } + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + psFree(where); + if (clean) { + psStringAppend(&query, " AND (distTarget.clean)"); + } else { + psStringAppend(&query, " AND (!distTarget.clean)"); + } + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + { + // psStringSubstitute fails unless the input is a psString which it determines by + // comparing the memory blocks free function to an expected value. + // pxDataGet uses psSlurp which leaves a different free function on the memory block. + // To work around this make a copy of the query before doing the substitution. + psString queryCopy = psStringCopy(query); + psFree(query); + query = queryCopy; + } + // change the @STATE@ in the sql file to our state + if (!psStringSubstitute(&query, state, "@STATE@")) { + psError(PS_ERR_UNKNOWN, false, "failed to substitute state string"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + int numInserted = psDBAffectedRows(config->dbh); + printf("inserted %d rows into rcInterest\n", numInserted); return true; @@ -1710,3 +1648,83 @@ return true; } - +static bool listinterestsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "=="); + PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "=="); + PXOPT_COPY_STR(config->args, where, "-dest_name", "name", "=="); + PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "=="); + PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-state", "state", "=="); + + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + PXOPT_LOOKUP_BOOL(full, config->args, "-full", false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + if (clean && full) { + psError(PS_ERR_UNKNOWN, false, "can't select both -clean and -full"); + return false; + } + + psString query = pxDataGet("disttool_listinterests.sql"); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + if (clean) { + psStringAppend(&query, " AND (clean)"); + } else if (full) { + psStringAppend(&query, " AND (!clean)"); + } + } else if (clean) { + psStringAppend(&query, " WHERE clean"); + } else if (full) { + psStringAppend(&query, " WHERE !clean"); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("disttool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (!ippdbPrintMetadatas(stdout, output, "rcInterest", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/disttool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/disttool.h (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/disttool.h (revision 25766) @@ -31,4 +31,5 @@ DISTTOOL_MODE_PENDINGCOMPONENT, DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, + DISTTOOL_MODE_REVERTCOMPONENT, DISTTOOL_MODE_PROCESSEDCOMPONENT, DISTTOOL_MODE_TOADVANCE, @@ -42,6 +43,4 @@ DISTTOOL_MODE_DEFINEDESTINATION, DISTTOOL_MODE_UPDATEDESTINATION, - DISTTOOL_MODE_DEFINEDSPRODUCT, - DISTTOOL_MODE_UPDATEDSPRODUCT, DISTTOOL_MODE_DEFINETARGET, DISTTOOL_MODE_UPDATETARGET, @@ -49,4 +48,5 @@ DISTTOOL_MODE_DEFINEINTEREST, DISTTOOL_MODE_UPDATEINTEREST, + DISTTOOL_MODE_LISTINTERESTS, } disttoolMode; Index: branches/eam_branches/20090820/ippTools/src/disttoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/disttoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/disttoolConfig.c (revision 25766) @@ -95,6 +95,6 @@ // -pendingcomponent psMetadata *pendingcomponentArgs = psMetadataAlloc(); + psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage (required)", NULL); psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage", 0, "limit results to runs for stage", NULL); psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "limit results to label", NULL); psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); @@ -111,4 +111,15 @@ psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + // -revertcomponent + psMetadata *revertcomponentArgs = psMetadataAlloc(); + psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-state", 0, "define state", NULL); + psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddS16(revertcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); + psMetadataAddBool(revertcomponentArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); + // -processedcomponent psMetadata *processedcomponentArgs = psMetadataAlloc(); @@ -135,5 +146,5 @@ psMetadata *addfilesetArgs = psMetadataAlloc(); psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); + psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-name", 0, "define file name", NULL); psMetadataAddS32(addfilesetArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); @@ -150,5 +161,5 @@ psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); - psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define dist_id", 0); + psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dist_id", 0); psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0); @@ -162,5 +173,4 @@ psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); - psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-fs_id", 0, "define fs_id", 0); @@ -189,22 +199,9 @@ psMetadataAddBool(revertrcrunArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted runs", NULL); - // -definedsproduct - psMetadata *definedsproductArgs = psMetadataAlloc(); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-name", 0, "define product name", NULL); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); - psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); - - // -updatedsproduct - // does this mode make sense? - psMetadata *updatedsproductArgs = psMetadataAlloc(); - psMetadataAddS64(updatedsproductArgs, PS_LIST_TAIL, "-prod_id", 0, "select by prod_id", 0); - // can't select by name because it isn't necssarily unique - psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL); - psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL); - // -definedestination psMetadata *definedestinationArgs = psMetadataAlloc(); - psMetadataAddS64(definedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id (required)", 0); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name (required)", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name (required)", NULL); + psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host (required)", NULL); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); @@ -215,10 +212,7 @@ psMetadata *updatedestinationArgs = psMetadataAlloc(); psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); - psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-name", 0, "define destination name", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-status_uri", 0, "define status_uri", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-comment", 0, "define comment", NULL); -// last_fileset gets updated by -updatercrun -// psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_last_fileset",0, "define last_fileset", NULL); psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); @@ -254,7 +248,13 @@ // -defineinterest psMetadata *defineinterestArgs = psMetadataAlloc(); - psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id (required)", 0); - psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-dest_name", 0, "define destination name (LIKE comparison)", NULL); + psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-stage", 0, "define stage", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); + psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-filter", 0, "define filter (LIKE comparison)", NULL); + psMetadataAddBool(defineinterestArgs, PS_LIST_TAIL,"-clean", 0, "list clean targets", false); psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL); + psMetadataAddU64(defineinterestArgs, PS_LIST_TAIL, "-limit", 0, "limit number of targets listed to N", 0); // -updateinterest @@ -265,4 +265,19 @@ psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL); + // -listinterests + psMetadata *listinterestsArgs = psMetadataAlloc(); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-target_id", 0, "list interests with target_id", 0); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-dest_id", 0, "list interests with dest_id", 0); + psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-int_id", 0, "list interests with int_id", 0); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dest_name", 0, "list interests for destinationn name)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-label", 0, "list interests for label (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-filter", 0, "list interests by filter (LIKE comparison)", NULL); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-stage", 0, "list interests for stage", NULL); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-clean", 0, "list clean interests", false); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-full", 0, "list full interests", false); + psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-state", 0, "list interests in state", NULL); + psMetadataAddU64(listinterestsArgs, PS_LIST_TAIL, "-limit", 0, "limit number of interests listed to N", 0); + psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadata *argSets = psMetadataAlloc(); psMetadata *modes = psMetadataAlloc(); @@ -272,6 +287,7 @@ PXOPT_ADD_MODE("-updaterun", "", DISTTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-revertrun", "", DISTTOOL_MODE_REVERTRUN, revertrunArgs); - PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); - PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-pendingcomponent", "", DISTTOOL_MODE_PENDINGCOMPONENT, pendingcomponentArgs); + PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs); + PXOPT_ADD_MODE("-revertcomponent", "", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs); PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs); PXOPT_ADD_MODE("-toadvance", "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs); @@ -284,8 +300,4 @@ PXOPT_ADD_MODE("-pendingdest", "", DISTTOOL_MODE_PENDINGDEST, pendingdestArgs); - PXOPT_ADD_MODE("-definedsproduct", "", DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductArgs); - PXOPT_ADD_MODE("-updatedsproduct", "", DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductArgs); -// PXOPT_ADD_MODE("-listdsproduct", "", DISTTOOL_MODE_LISTDSPRODUCT, updatedsproductArgs); - PXOPT_ADD_MODE("-definedestination", "", DISTTOOL_MODE_DEFINEDESTINATION, definedestinationArgs); PXOPT_ADD_MODE("-updatedestination", "", DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationArgs); @@ -298,5 +310,5 @@ PXOPT_ADD_MODE("-defineinterest", "", DISTTOOL_MODE_DEFINEINTEREST, defineinterestArgs); PXOPT_ADD_MODE("-updateinterest", "", DISTTOOL_MODE_UPDATEINTEREST, updateinterestArgs); -// PXOPT_ADD_MODE("-listinterest", "", DISTTOOL_MODE_LISTINTEREST, listinterestArgs); + PXOPT_ADD_MODE("-listinterests", "", DISTTOOL_MODE_LISTINTERESTS, listinterestsArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090820/ippTools/src/faketool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/faketool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/faketool.c (revision 25766) @@ -52,4 +52,5 @@ static bool tofullimfileMode(pxConfig *config); static bool topurgedimfileMode(pxConfig *config); +static bool toscrubbedimfileMode(pxConfig *config); static bool exportrunMode(pxConfig *config); static bool importrunMode(pxConfig *config); @@ -91,4 +92,5 @@ MODECASE(FAKETOOL_MODE_TOFULLIMFILE, tofullimfileMode); MODECASE(FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileMode); + MODECASE(FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileMode); MODECASE(FAKETOOL_MODE_EXPORTRUN, exportrunMode); MODECASE(FAKETOOL_MODE_IMPORTRUN, importrunMode); @@ -1251,4 +1253,8 @@ return change_imfile_data_state(config, "purged", "goto_purged"); } +static bool toscrubbedimfileMode(pxConfig *config) +{ + return change_imfile_data_state(config, "scrubbed", "goto_scrubbed"); +} bool exportrunMode(pxConfig *config) Index: branches/eam_branches/20090820/ippTools/src/faketool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/faketool.h (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/faketool.h (revision 25766) @@ -45,4 +45,5 @@ FAKETOOL_MODE_TOFULLIMFILE, FAKETOOL_MODE_TOPURGEDIMFILE, + FAKETOOL_MODE_TOSCRUBBEDIMFILE, FAKETOOL_MODE_EXPORTRUN, FAKETOOL_MODE_IMPORTRUN Index: branches/eam_branches/20090820/ippTools/src/faketoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/faketoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/faketoolConfig.c (revision 25766) @@ -301,4 +301,9 @@ psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -toscrubbedimfile + psMetadata *toscrubbedimfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-fake_id", 0, "fake ID to update", 0); + psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0, "class ID to update", NULL); + // -exportrun psMetadata *exportrunArgs = psMetadataAlloc(); @@ -337,4 +342,5 @@ PXOPT_ADD_MODE("-tofullimfile", "set imfile state to full", FAKETOOL_MODE_TOFULLIMFILE, tofullimfileArgs); PXOPT_ADD_MODE("-topurgedimfile", "set imfile state to purged", FAKETOOL_MODE_TOPURGEDIMFILE, topurgedimfileArgs); + PXOPT_ADD_MODE("-toscrubbedimfile", "set imfile state to scrubbed", FAKETOOL_MODE_TOSCRUBBEDIMFILE, toscrubbedimfileArgs); PXOPT_ADD_MODE("-exportrun", "export run for import on other database", FAKETOOL_MODE_EXPORTRUN, exportrunArgs); PXOPT_ADD_MODE("-importrun", "import run from metadata file", FAKETOOL_MODE_IMPORTRUN, importrunArgs); Index: branches/eam_branches/20090820/ippTools/src/magicdstool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magicdstool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/magicdstool.c (revision 25766) @@ -37,12 +37,12 @@ static bool todestreakMode(pxConfig *config); static bool adddestreakedfileMode(pxConfig *config); +static bool advancerunMode(pxConfig *config); static bool revertdestreakedfileMode(pxConfig *config); static bool getskycellsMode(pxConfig *config); static bool toremoveMode(pxConfig *config); -static bool torestoreMode(pxConfig *config); static bool torevertMode(pxConfig *config); +static bool completedrevertMode(pxConfig *config); static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state); -static bool magicDSRunComplete(pxConfig *config, bool setmagicked); static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id); @@ -70,9 +70,10 @@ MODECASE(MAGICDSTOOL_MODE_TODESTREAK, todestreakMode); MODECASE(MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileMode); + MODECASE(MAGICDSTOOL_MODE_ADVANCERUN, advancerunMode); MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode); MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS, getskycellsMode); MODECASE(MAGICDSTOOL_MODE_TOREMOVE, toremoveMode); - MODECASE(MAGICDSTOOL_MODE_TORESTORE, torestoreMode); MODECASE(MAGICDSTOOL_MODE_TOREVERT, torevertMode); + MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertMode); default: psAbort("invalid option (this should not happen)"); @@ -105,5 +106,5 @@ PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false); - PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false); + PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false); PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false); PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); @@ -119,5 +120,5 @@ PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "=="); - PXOPT_COPY_S64(config->args, where, "-magic_id","magic_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_id","magicRun.magic_id", "=="); PXOPT_COPY_S32(config->args, where, "-streaks_max","streaks", "<="); @@ -268,5 +269,5 @@ outroot, recoveryroot, - re_place, + noreplace ? 0 :1, // re_place 0); // remove @@ -396,23 +397,28 @@ PS_ASSERT_PTR_NON_NULL(config, false); + PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false); + psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - PXOPT_COPY_STR(config->args, where, "-stage", "stage", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_todestreak.sql"); + psString sql_file = NULL; + psStringAppend(&sql_file, "magicdstool_todestreak_%s.sql", stage); + + psString query = pxDataGet(sql_file); if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file); + psFree(sql_file); + return false; + } + psFree(sql_file); if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } @@ -656,13 +662,4 @@ } - if (!magicDSRunComplete(config, setmagicked)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -738,7 +735,13 @@ } -static bool magicDSRunComplete(pxConfig *config, bool setmagicked) +static bool advancerunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); // look for completed magicDSRuns @@ -749,4 +752,10 @@ } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -766,4 +775,8 @@ return true; } + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } for (long i = 0; i < psArrayLength(output); i++) { psMetadata *row = output->data[i]; @@ -771,8 +784,12 @@ psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); - // if requested, set stageRun.magicked + // if re_place, set stageRun.magicked + bool setmagicked = psMetadataLookupBool(NULL, row, "re_place"); if (setmagicked && !setRunMagicked(config, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64, magic_ds_id); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } @@ -783,8 +800,14 @@ magic_ds_id); psFree(output); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } return false; } } - + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } return true; @@ -795,4 +818,10 @@ { PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", true); + if (!i_am_sure) { + psError(PS_ERR_UNKNOWN, true, "Reverting destreaked files must be done carefully. -i_am_sure is required."); + return false; + } psMetadata *where = psMetadataAlloc(); @@ -802,5 +831,5 @@ pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - psString query = psStringCopy("DELETE FROM magicDSFile USING magicDSFile, magicDSRun WHERE (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id) AND magicDSFile.fault != 0"); + psString query = pxDataGet("magicdstool_revertdestreakedfile.sql"); if (psListLength(where->list)) { @@ -808,4 +837,7 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else { + psError(PS_ERR_UNKNOWN, true, "search arguments are required"); + return false; } psFree(where); @@ -813,40 +845,44 @@ if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "failed to revert"); - return false; - } - return true; -} - -static bool getskycellsMode(pxConfig *config) -{ - // required - PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); + psFree(query); + return false; + } + psFree(query); + return true; +} + +static bool completedrevertMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("magicdstool_getskycells.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - + PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "label", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("magicdstool_completedrevert.sql"); + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + psString whereString = NULL; if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); + psStringAppend(&whereString, "\nAND %s", whereClause); psFree(whereClause); } psFree(where); - if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } + if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) { + psFree(whereString); + psError(PS_ERR_UNKNOWN, false, "failed to revert"); + return false; + } + psFree(whereString); psFree(query); - psArray *output = p_psDBFetchResult(config->dbh); if (!output) { @@ -868,14 +904,24 @@ return true; } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); + for (int i=0; idata[i]; + psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); + psString old_state = psMetadataLookupStr(NULL, row, "state"); + psString new_state; + if (!strcmp(old_state, "goto_censored")) { + new_state = "censored"; + } else if (!strcmp(old_state, "goto_restored")) { + new_state = "restored"; + } else { + psError(PXTOOLS_ERR_PROG, true, "unexpected state found: %s", old_state); psFree(output); return false; } - } - + char *query2 = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query2, new_state, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, "failed to set run magicDSRun.state to %s", new_state); + return false; + } + } psFree(output); @@ -883,43 +929,16 @@ } -static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) -{ - PS_ASSERT_PTR_NON_NULL(state, false); - - // check that state is a valid string value - if (!( - (strncmp(state, "new", 4) == 0) - || (strncmp(state, "full", 5) == 0) - ) - ) { - psError(PS_ERR_UNKNOWN, false, - "invalid magicDSRun state: %s", state); - return false; - } - - char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; - if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { - psError(PS_ERR_UNKNOWN, false, - "failed to change state for magic_id %" PRId64, magic_ds_id); - return false; - } - - return true; -} - -static bool toremoveMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); +static bool getskycellsMode(pxConfig *config) +{ + // required + PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false); psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "=="); - PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_COPY_STR(config->args, where, "-class_id", "warpSkyCellMap.class_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_toremove.sql"); + psString query = pxDataGet("magicdstool_getskycells.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -929,17 +948,10 @@ if (psListLength(where->list)) { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); + psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } psFree(where); - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { + if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -970,5 +982,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -981,5 +993,35 @@ return true; } -static bool torestoreMode(pxConfig *config) + +static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!((strcmp(state, "new") == 0) || + (strcmp(state, "full") == 0) || + (strcmp(state, "restored") == 0) || + (strcmp(state, "censored") == 0) || + (strcmp(state, "purged") == 0) || + (strcmp(state, "goto_restored") == 0) || + (strcmp(state, "goto_censored") == 0) || + (strcmp(state, "goto_purged") == 0)) + ) { + psError(PS_ERR_UNKNOWN, false, + "invalid magicDSRun state: %s", state); + return false; + } + + char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for magic_id %" PRId64, magic_ds_id); + return false; + } + + return true; +} + +static bool toremoveMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -994,5 +1036,5 @@ // look for "inputs" that need to processed - psString query = pxDataGet("magicdstool_torestore.sql"); + psString query = pxDataGet("magicdstool_toremove.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -1043,5 +1085,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -1054,5 +1096,4 @@ return true; } - static bool torevertMode(pxConfig *config) Index: branches/eam_branches/20090820/ippTools/src/magicdstool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/magicdstool.h (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/magicdstool.h (revision 25766) @@ -30,9 +30,10 @@ MAGICDSTOOL_MODE_TODESTREAK, MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, + MAGICDSTOOL_MODE_ADVANCERUN, MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, MAGICDSTOOL_MODE_GETSKYCELLS, MAGICDSTOOL_MODE_TOREMOVE, - MAGICDSTOOL_MODE_TORESTORE, MAGICDSTOOL_MODE_TOREVERT, + MAGICDSTOOL_MODE_COMPLETEDREVERT, } MAGICDStoolMode; Index: branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c (revision 25766) @@ -52,5 +52,5 @@ psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir", 0, "define workdir", NULL); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-replace", 0, "replace input files with the destreaked versions", false); + psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false); psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label", 0, "define label", NULL); @@ -75,6 +75,6 @@ psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-replace", 0, "use the simple output format", false); - psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false); + psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace the input wit with the destreaked versions", false); +// psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false); psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -93,8 +93,8 @@ // -todestreak psMetadata *todestreakArgs = psMetadataAlloc(); + psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage (required)", NULL); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); - psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL); psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -114,9 +114,10 @@ psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL); psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddBool(revertdestreakedfileArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirm that you know what you are doing", false); psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); // -getskycells psMetadata *getskycellsArgs = psMetadataAlloc(); - psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID (required)", 0); + psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID", 0); psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL); psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL); @@ -130,12 +131,4 @@ psMetadataAddU64(toremoveArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(toremoveArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); - - // -torestore - psMetadata *torestoreArgs = psMetadataAlloc(); - psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); - psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0); - psMetadataAddStr(torestoreArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); - psMetadataAddU64(torestoreArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(torestoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -torevert @@ -147,4 +140,16 @@ psMetadataAddU64(torevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(torevertArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + + // -completedrevert + psMetadata *completedrevertArgs = psMetadataAlloc(); + psMetadataAddS64(completedrevertArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddStr(completedrevertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(completedrevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + + // -advancerun + psMetadata *advancerunArgs = psMetadataAlloc(); + psMetadataAddS64(advancerunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0); + psMetadataAddStr(advancerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL); + psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psFree(now); @@ -163,4 +168,6 @@ PXOPT_ADD_MODE("-adddestreakedfile", "add a de-streaked file", MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileArgs); + PXOPT_ADD_MODE("-advancerun", "change state for runs that have finished destrreaking", + MAGICDSTOOL_MODE_ADVANCERUN, advancerunArgs); PXOPT_ADD_MODE("-revertdestreakedfile", " revert a faulted de-streaked file", MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, revertdestreakedfileArgs); @@ -169,8 +176,10 @@ PXOPT_ADD_MODE("-toremove", "backup images pending removal", MAGICDSTOOL_MODE_TOREMOVE, toremoveArgs); - PXOPT_ADD_MODE("-torestore", "images pending restore of exicsed streak pixels", - MAGICDSTOOL_MODE_TORESTORE, torestoreArgs); - PXOPT_ADD_MODE("-torevert", "faulted images to revert", + PXOPT_ADD_MODE("-torevert", "images to restore or revert", MAGICDSTOOL_MODE_TOREVERT, torevertArgs); + + PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting", + MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs); + if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: branches/eam_branches/20090820/ippTools/src/magictool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magictool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/magictool.c (revision 25766) @@ -1393,46 +1393,25 @@ } -static bool censorStage(pxConfig *config, psString stage, psString whereClause) -{ - psString queryFile = NULL; - psStringAppend(&queryFile, "magictool_censor_%s.sql", stage); - psString query = pxDataGet(queryFile); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); - psFree(queryFile); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(queryFile); - - psStringAppend(&query, " WHERE %s", whereClause); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - return true; -} - static bool censorrunMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); - psError(PS_ERR_PROGRAMMING, true, "-censorrun mode not ready yet"); - return false; - psMetadata *where = psMetadataAlloc(); + + PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", false, false); + PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + if (!magic_id) { + if (!exp_id && !label) { + psError(PS_ERR_UNKNOWN, true, "either -magic_id or exp_id and label is required"); + return false; + } + } // at least one of these required PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "=="); PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); if (!psListLength(where->list)) { @@ -1470,22 +1449,22 @@ // Now queue any destreaked files to be re-verted - // note: on failure censorStage issues the rollback - if (!censorStage(config, "raw", whereClause)) { + // note: on failure magicRestoreStage issues the rollback + if (!magicRestoreStage(config, "raw", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "chip", whereClause)) { + if (!magicRestoreStage(config, "chip", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "camera", whereClause)) { + if (!magicRestoreStage(config, "camera", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "warp", whereClause)) { + if (!magicRestoreStage(config, "warp", whereClause, "goto_censored")) { psFree(whereClause); return false; } - if (!censorStage(config, "diff", whereClause)) { + if (!magicRestoreStage(config, "diff", whereClause, "goto_censored")) { psFree(whereClause); return false; Index: branches/eam_branches/20090820/ippTools/src/magictoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/magictoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/magictoolConfig.c (revision 25766) @@ -165,4 +165,5 @@ psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0); psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0); + psMetadataAddStr(censorrunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL); psFree(now); Index: branches/eam_branches/20090820/ippTools/src/pstamptool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pstamptool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pstamptool.c (revision 25766) @@ -44,4 +44,5 @@ static bool pendingjobMode(pxConfig *config); static bool updatejobMode(pxConfig *config); +static bool revertjobMode(pxConfig *config); static bool addprojectMode(pxConfig *config); static bool projectMode(pxConfig *config); @@ -80,4 +81,5 @@ MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingjobMode); MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updatejobMode); + MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode); MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode); MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode); @@ -224,21 +226,21 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - // PXOPT_LOOKUP_STR(outFileset, config->args, "-out_fileset", true, false); - PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); - - char *query ="INSERT INTO pstampRequest" - " (state, uri, ds_id, fault)" - " VALUES( 'new', '%s', %" PRId64 ", 0 )"; - if (!p_psDBRunQueryF(config->dbh, query, uri, ds_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psU64 affected = psDBAffectedRows(config->dbh); - if (affected != 1) { - psError(PS_ERR_UNKNOWN, false, - "should have affected one row but %" PRIu64 " rows were modified", - affected); + PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_STR(name, config->args, "-name", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + PXOPT_LOOKUP_S64(ds_id, config->args, "-ds_id", false, false); + + if (!pstampRequestInsert(config->dbh, + 0, // req_id + ds_id, + "new", //state + name, + NULL, // reqType + label, + NULL, // outProduct + uri, + 0 // fault + )) { + psError(PS_ERR_UNKNOWN, false, "failed to insert request"); return false; } @@ -317,4 +319,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!="); PXOPT_COPY_STR(config->args, where, "-name", "name", "=="); @@ -381,8 +384,5 @@ PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = - "SELECT * FROM pstampRequest WHERE state = 'run' AND " - "(SELECT count(*) FROM pstampJob WHERE pstampJob.req_id = pstampRequest.req_id " - " AND pstampJob.state != 'stop') = 0" ; + psString query = pxDataGet("pstamptool_completedreq.sql"); // treat limit == 0 as "no limit" @@ -491,17 +491,40 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", true, false); - - // printf("Revert request %" PRId64 "\n", req_id); + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "pstampRequest.state", "=="); + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters are required"); + return false; + } + + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psFree(where); + + // delete any jobs that were queued by requests that didn't complete parsing (pstampRequest.state = 'new' + // If state = 'run' was supplied this will be a no-op + psString query = pxDataGet("pstamptool_revertreq_deletejobs.sql"); + psStringAppend(&query, " AND %s", whereClause); + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); + + // clear fault for requests + query = pxDataGet("pstamptool_revertreq.sql"); + psStringAppend(&query, " AND %s", whereClause); - if (!p_psDBRunQueryF(config->dbh, "DELETE FROM pstampJob where req_id = %" PRId64, req_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!p_psDBRunQueryF(config->dbh, - "UPDATE pstampRequest set state ='new', name=NULL, reqType=NULL, fault=0 where req_id = %" PRId64, req_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + psFree(query); return true; @@ -521,4 +544,5 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false); + PXOPT_LOOKUP_S64(options, config->args, "-options", false, false); // unless the job is being inserted with stop state require outputBase @@ -545,5 +569,6 @@ fault, exp_id, - outputBase + outputBase, + options )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -569,34 +594,30 @@ PS_ASSERT_PTR_NON_NULL(config, false); - PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", false, false); - PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false); + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = NULL; - - if (!req_id && !job_id) { - fprintf(stderr, "either req_id or job_id must be specified\n"); + if (!psListLength(where->list)) { + fprintf(stderr, "search arguments are required\n"); exit (1); } - if (req_id) { - psStringAppend(&query, - "SELECT" - " pstampJob.*, pstampRequest.name, pstampRequest.outProduct" - " FROM pstampJob" - " JOIN pstampRequest USING(req_id)" - " WHERE req_id = %" PRId64, req_id - ); - } else { - psStringAppend(&query, - "SELECT" - " pstampJob.*, pstampRequest.name, pstampRequest.outProduct" - " FROM pstampJob" - " JOIN pstampRequest USING(req_id)" - " WHERE job_id = %" PRId64, job_id - ); - } + psString query = pxDataGet("pstamptool_listjob.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // use psDBGenerateWhereSQL because the SQL yields an intermediate table + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); // treat limit == 0 as "no limit" @@ -729,4 +750,38 @@ psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" PRIu64 " rows were modified", affected); + return false; + } + + return true; +} + +static bool revertjobMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + + PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "=="); + PXOPT_COPY_S64(config->args, where, "-fault", "fault", "=="); + PXOPT_COPY_S64(config->args, where, "-req_id_min", "req_id", ">="); + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + + psString query = pxDataGet("pstamptool_revertjob.sql"); + if (!psListLength(where->list) && !all) { + psFree(where); + psError(PXTOOLS_ERR_DATA, false, "search parameters or -all are required"); + return false; + } + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob"); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); return false; } Index: branches/eam_branches/20090820/ippTools/src/pstamptool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pstamptool.h (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pstamptool.h (revision 25766) @@ -39,4 +39,5 @@ PSTAMPTOOL_MODE_JOBRESULT, PSTAMPTOOL_MODE_UPDATEJOB, + PSTAMPTOOL_MODE_REVERTJOB, PSTAMPTOOL_MODE_ADDPROJECT, PSTAMPTOOL_MODE_MODPROJECT, Index: branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c (revision 25766) @@ -63,7 +63,8 @@ // -addreq psMetadata *addreqArgs = psMetadataAlloc(); - psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); - psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); - // psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-out_fileset", 0, "define request output_fileset", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0, "define request file uri (required)", NULL); + psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0, "define request ds_id", 0); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0, "define request name", NULL); + psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-label", 0, "define request label", NULL); // -pendingreq @@ -77,6 +78,7 @@ psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0, "list by req_id", 0); psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0, "list by name", NULL); + psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0, "req_id to not list", 0); psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -completedreq @@ -97,22 +99,26 @@ // -revertreq psMetadata *revertreqArgs = psMetadataAlloc(); - psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id for which to revert", 0); + psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS16(revertreqArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0, "state to revert", NULL); // -addjob psMetadata *addjobArgs = psMetadataAlloc(); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); - psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "new state", "run"); - psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "exposure id", 0); - psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "new result", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0, "define job req_id", 0); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0, "define job rownum", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0, "define job job_type", "stamp"); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0, "define job outputBase", NULL); + psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0, "define state", "run"); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure id", 0); + psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0, "define options", 0); + psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0, "define job result", 0); // -listjob psMetadata *listjobArgs = psMetadataAlloc(); - psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "define request", 0); - psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "define job", 0); - psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); - psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0, "select by request ID", 0); + psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0, "select by job ID", 0); + psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0, "select by fault", 0); + psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -pendingjob @@ -128,4 +134,13 @@ psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0, "new state", NULL); psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0, "new result", NULL); + + // -revertjob + psMetadata *revertjobArgs = psMetadataAlloc(); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id", 0, "req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id_min", 0, "minimum req_id to revert", 0); + psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-job_id", 0, "job_id to revert", 0); + psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault", 0, "fault to revert", 0); + psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0, "revert all faulted jobs", false); + psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -addproject @@ -159,5 +174,5 @@ PXOPT_ADD_MODE("-addreq", "", PSTAMPTOOL_MODE_ADDREQ, addreqArgs); PXOPT_ADD_MODE("-pendingreq", "", PSTAMPTOOL_MODE_PENDINGREQ, pendingreqArgs); - PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); + PXOPT_ADD_MODE("-updatereq", "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs); PXOPT_ADD_MODE("-listreq", "", PSTAMPTOOL_MODE_LISTREQ, listreqArgs); PXOPT_ADD_MODE("-completedreq", "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs); @@ -167,5 +182,6 @@ PXOPT_ADD_MODE("-listjob", "", PSTAMPTOOL_MODE_LISTJOB, listjobArgs); PXOPT_ADD_MODE("-pendingjob", "", PSTAMPTOOL_MODE_PENDINGJOB, pendingjobArgs); - PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + PXOPT_ADD_MODE("-updatejob", "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs); + PXOPT_ADD_MODE("-revertjob", "", PSTAMPTOOL_MODE_REVERTJOB, revertjobArgs); PXOPT_ADD_MODE("-adddatastore", "", PSTAMPTOOL_MODE_ADDDATASTORE, adddatastoreArgs); Index: branches/eam_branches/20090820/ippTools/src/pubtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pubtool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pubtool.c (revision 25766) @@ -259,7 +259,9 @@ // required PXOPT_LOOKUP_S64(pub_id, config->args, "-pub_id", true, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); // optional + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false); @@ -269,5 +271,5 @@ } - if (!publishDoneInsert(config->dbh, pub_id, path_base, fault)) { + if (!publishDoneInsert(config->dbh, pub_id, path_base, hostname, dtime_script, fault)) { psError(PS_ERR_UNKNOWN, false, "Unable to add file"); if (!psDBRollback(config->dbh)) { Index: branches/eam_branches/20090820/ippTools/src/pubtoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pubtoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pubtoolConfig.c (revision 25766) @@ -68,4 +68,6 @@ psMetadataAddS64(addArgs, PS_LIST_TAIL, "-pub_id", 0, "define pub_id (required)", 0); psMetadataAddStr(addArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL); + psMetadataAddStr(addArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL); + psMetadataAddF32(addArgs, PS_LIST_TAIL, "-dtime_script", 0, "define time for script", NAN); psMetadataAddS32(addArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0); Index: branches/eam_branches/20090820/ippTools/src/pxadd.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxadd.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxadd.c (revision 25766) @@ -0,0 +1,262 @@ +/* + * pxadd.c + * + * Copyright (C) 2007 Joshua Hoblitt + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxadd.h" + +bool pxaddSetSearchArgs (psMetadata *md) { + + psMetadataAddS64(md, PS_LIST_TAIL, "-add_id", 0, "search by add_id", 0); + psMetadataAddS64(md, PS_LIST_TAIL, "-cam_id", 0, "search by cam_id", 0); + +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_name", 0, "search by exp_name", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-inst", 0, "search for camera", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-telescope", 0, "search for telescope", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); */ +/* psMetadataAddTime(md, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<)", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_tag", 0, "search by exp_tag", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-exp_type", 0, "search by exp_type", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-comment", 0, "search by comment", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filelevel", 0, "search by filelevel", NULL); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-filter", 0, "search for filter", NULL); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_min", 0, "define min airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-airmass_max", 0, "define max airmass", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_min", 0, "define min RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-ra_max", 0, "define max RA (degrees) ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_min", 0, "define min DEC (degrees)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-decl_max", 0, "define max DEC (degrees)", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_min", 0, "define min exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-exp_time_max", 0, "define max exposure time", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_min", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sat_pixel_frac_max", 0, "define max fraction of saturated pixels", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_min", 0, "define min background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_max", 0, "define max background", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_min", 0, "define min background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_stdev_max", 0, "define max background standard deviation", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_min", 0, "define min background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-bg_mean_stdev_max", 0, "define max background mean standard deviation (across imfiles)", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_min", 0, "define min altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-alt_max", 0, "define max altitude", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_min", 0, "define min azimuth ", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-az_max", 0, "define max azimuth ", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_min", 0, "define min ccd tempature", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-ccd_temp_max", 0, "define max ccd tempature", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_min", 0, "define min rotator position angle", NAN); */ +/* psMetadataAddF64(md, PS_LIST_TAIL, "-posang_max", 0, "define max rotator position angle", NAN); */ +/* psMetadataAddStr(md, PS_LIST_TAIL, "-object", 0, "search by exposure object", NULL); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_min", 0, "define min solar angle", NAN); */ +/* psMetadataAddF32(md, PS_LIST_TAIL, "-sun_angle_max", 0, "define max solar angle", NAN); */ + + return true; +} + +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where) { + + PXOPT_COPY_S64(config->args, where, "-add_id", "addRun.add_id", "=="); +/* PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "=="); */ +/* PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); */ +/* PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); */ +/* PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); */ +/* PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); */ +/* PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); */ +/* PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); */ +/* PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); */ +/* PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); */ + + return true; +} + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid camRun state: %s", state); + return false; + } + + char *query = "UPDATE addRun SET state = '%s', magicked = %" PRId64 " WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, state, magicked, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + + +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + PS_ASSERT_PTR_NON_NULL(state, false); + + // check that state is a valid string value + if (!pxIsValidState(state)) { + psError(PS_ERR_UNKNOWN, false, + "invalid chipRun state: %s", state); + return false; + } + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.state = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + + +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + char *query = "UPDATE addRun SET addRun.label = '%s' WHERE add_id = %" PRId64; + if (!p_psDBRunQueryF(config->dbh, query, label, add_id)) { + psError(PS_ERR_UNKNOWN, false, + "failed to change state for add_id %" PRId64, add_id); + return false; + } + + return true; +} + +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + // note label == NULL should be explicitly allowed + + psString query = psStringCopy("UPDATE addRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET addRun.label = '%s'"); + + if (where) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQueryF(config->dbh, query, label)) { + psFree(query); + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + psFree(query); + + return true; +} + +// Need to think more about this to see what we want it to do. BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // load the SQL to enqueue our exp_ids from disk once + static psString query = NULL; + if (!query) { + query = pxDataGet("addtool_queue_cam_id.sql"); + psMemSetPersistent(query, true); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + } + // fprintf(stderr,"%s",query); + // queue the exp + // XXX chip_id is being cast here work around psS64 have a different type + // different on 32/64 + if (!p_psDBRunQueryF(config->dbh, query, + "new", // state + workdir ? workdir : "NULL", + "dirty", //workdir_state + label ? label : "NULL", + dvodb ? dvodb : "NULL", + (long long)cam_id + )) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + // just to be safe, we should have changed at least one row + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, + "no rows affected - should have changed at least one row"); + return false; + } + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/pxadd.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxadd.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxadd.h (revision 25766) @@ -0,0 +1,44 @@ +/* + * pxadd.h + * + * Copyright (C) 2009 Joshua Hoblitt, Chris Waters + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXADD_H +#define PXADD_H 1 + +#include + +#include "pxtools.h" + +bool pxaddRunSetState(pxConfig *config, psS64 add_id, const char *state, psS64 magicked); +bool pxaddRunSetStateByQuery(pxConfig *config, psMetadata *where, const char *state); +bool pxaddRunSetLabel(pxConfig *config, psS64 add_id, const char *label); +bool pxaddRunSetLabelByQuery(pxConfig *config, psMetadata *where, const char *label); + +bool pxaddSetSearchArgs (psMetadata *md); +bool pxaddGetSearchArgs (pxConfig *config, psMetadata *where); + +// Likely BROKEN +bool pxaddQueueByCamID(pxConfig *config, + psS64 cam_id, + char *workdir, + char *label, + char *recipe, + char *dvodb); + + +#endif // PXADD_H Index: branches/eam_branches/20090820/ippTools/src/pxinject.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxinject.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pxinject.c (revision 25766) @@ -137,7 +137,9 @@ PXOPT_LOOKUP_STR(tmp_class_id, config->args, "-tmp_class_id", true, false); PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); // insert with error flag state set to 0 (no errors) - if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL)) { + if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL, bytes, md5sum)) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; Index: branches/eam_branches/20090820/ippTools/src/pxinjectConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxinjectConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pxinjectConfig.c (revision 25766) @@ -61,4 +61,6 @@ psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-tmp_class_id", 0, "define the class ID (required)", NULL); psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-uri", 0, "define the URI (required)", NULL); + psMetadataAddS32(newImfileArgs, PS_LIST_TAIL, "-bytes", 0, "define the size of the file", 0); + psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-md5sum", 0, "define the size of the file", NULL); // -updatenewExp Index: branches/eam_branches/20090820/ippTools/src/pxmagic.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxmagic.c (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxmagic.c (revision 25766) @@ -0,0 +1,67 @@ +/* + * pxmagic.c + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pxtools.h" +#include "pxmagic.h" + +bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState) +{ + psString queryFile = NULL; + psStringAppend(&queryFile, "magictool_restore_%s.sql", stage); + psString query_temp = pxDataGet(queryFile); + if (!query_temp) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile); + psFree(queryFile); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(queryFile); + + // psStringSubstittute fails on strings created by pxDataGet() + psString query = psStringCopy(query_temp); + // change the @NEW_STATE@ in the sql file to our state + if (!psStringSubstitute(&query, newState, "@NEW_STATE@")) { + psError(PS_ERR_UNKNOWN, false, "failed to substitute state string"); + return false; + } + + psStringAppend(&query, " AND %s", whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + return false; + } + psFree(query); + + return true; +} Index: branches/eam_branches/20090820/ippTools/src/pxmagic.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxmagic.h (revision 25766) +++ branches/eam_branches/20090820/ippTools/src/pxmagic.h (revision 25766) @@ -0,0 +1,29 @@ +/* + * pxmagic.h + * + * Copyright (C) 2009 IfA + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; version 2 of the License. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * program; see the file COPYING. If not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#ifndef PXMAGIC_H +#define PXMAGIC_H 1 + +#include + +#include "pxtools.h" + +extern bool magicRestoreStage(pxConfig *config, psString stage, psString whereClause, psString newState); + +#endif // PXMAGIC_H Index: branches/eam_branches/20090820/ippTools/src/pxtools.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxtools.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pxtools.c (revision 25766) @@ -37,4 +37,6 @@ if (!strcmp(state, "goto_cleaned")) return true; if (!strcmp(state, "error_cleaned")) return true; + if (!strcmp(state, "goto_purged")) return true; + if (!strcmp(state, "error_purged")) return true; if (!strcmp(state, "goto_scrubbed")) return true; if (!strcmp(state, "error_scrubbed")) return true; @@ -42,12 +44,11 @@ if (!strcmp(state, "update")) return true; if (!strcmp(state, "purged")) return true; - if (!strcmp(state, "goto_purged")) return true; - if (!strcmp(state, "error_purged")) return true; - + if (!strcmp(state, "scrubbed")) return true; return false; } -// 'scrubbed' is a virtual state equivalent to cleaned, but allows files to be removed -// even if the config files is missing +// 'scrubbed' is no longer a virtual state equivalent to cleaned, but allows files to be removed +// even if the config files is missing. This change was prompted as files that are cleaned can +// be regenerated, but that is not certain after being scrubbed. Index: branches/eam_branches/20090820/ippTools/src/pxtools.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/pxtools.h (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pxtools.h (revision 25766) @@ -37,4 +37,5 @@ #include "pxtoolsErrorCodes.h" +#include "pxadd.h" #include "pxcam.h" #include "pxchip.h" @@ -45,4 +46,5 @@ #include "pxtag.h" #include "pxtree.h" +#include "pxmagic.h" # define MAX_ROWS 10e9 Index: branches/eam_branches/20090820/ippTools/src/pztool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pztool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pztool.c (revision 25766) @@ -401,4 +401,6 @@ PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); PXOPT_LOOKUP_BOOL(row_lock, config->args, "-row_lock", false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); // default values @@ -457,5 +459,7 @@ fault, NULL, // epoch - hostname + hostname, + bytes, + md5sum )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -595,5 +599,7 @@ " pzDownloadImfile.class_id," // tmp_class_id " pzDownloadImfile.uri," // uri - " NULL" // epoch + " NULL," // epoch + " pzDownloadImfile.bytes," // bytes + " pzDownloadImfile.md5sum" // md5sum " FROM pzDownloadImfile" " WHERE" Index: branches/eam_branches/20090820/ippTools/src/pztoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/pztoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/pztoolConfig.c (revision 25766) @@ -95,4 +95,6 @@ psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-hostname", 0, "define the host that copied the image", NULL); + psMetadataAddS32(copydoneArgs, PS_LIST_TAIL, "-bytes", 0, "define the size in bytes for the copied image", 0); + psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-md5sum", 0, "define the md5 sum for the copied image", NULL); psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0); psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0, "lock pzDownImfile rows while advancing an exposure", false); Index: branches/eam_branches/20090820/ippTools/src/regtool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/regtool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/regtool.c (revision 25766) @@ -224,4 +224,5 @@ PXOPT_LOOKUP_F32(teltemp_extra, config->args, "-teltemp_extra", false, false); PXOPT_LOOKUP_F32(pon_time, config->args, "-pon_time", false, false); + PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false); @@ -238,4 +239,6 @@ PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false); PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false); + PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false); PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); @@ -307,5 +310,8 @@ quality, NULL, - 0 + 0, + bytes, + md5sum, + 0 // burntool_state )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -327,8 +333,15 @@ PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">="); PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "filter", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "magicked", "=="); + + PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(ordered_by_date, config->args, "-ordered_by_date", false); psString query = pxDataGet("regtool_processedimfile.sql"); @@ -352,4 +365,25 @@ // don't list faulted rows psStringAppend(&query, " %s", "AND rawImfile.fault = 0"); + } + + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(&query, " AND rawImfile.magicked = 0"); + } + if (destreaked) { + psStringAppend(&query, " AND rawImfile.magicked != 0"); + } + + + // add the ORDER BY statement if desired + if (ordered_by_date) { + psStringAppend(&query, " ORDER BY dateobs"); } @@ -456,12 +490,13 @@ PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); - - if ((fault == INT16_MAX) && !isfinite(user_1)) { - psError(PS_ERR_UNKNOWN, false, "one of -fault or -user_1 must be selected"); - return false; - } - if ((fault != INT16_MAX) && isfinite(user_1)) { - psError(PS_ERR_UNKNOWN, false, "only one of -fault or -user_1 must be selected"); +/* PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); */ + PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false); + + if ((fault == INT16_MAX) && !isfinite(burntool_state)) { + psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state must be selected"); + return false; + } + if ((fault != INT16_MAX) && isfinite(burntool_state)) { + psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected"); return false; } @@ -480,5 +515,5 @@ } - if (isfinite(user_1)) { + if (isfinite(burntool_state)) { psString query = pxDataGet("regtool_updateprocessedimfile.sql"); if (!query) { @@ -487,5 +522,5 @@ } - if (!p_psDBRunQueryF(config->dbh, query, user_1, exp_id, class_id)) { + if (!p_psDBRunQueryF(config->dbh, query, burntool_state, exp_id, class_id)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); Index: branches/eam_branches/20090820/ippTools/src/regtoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/regtoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/regtoolConfig.c (revision 25766) @@ -125,4 +125,7 @@ ADD_OPT(Time, addprocessedimfileArgs, "-dateobs", "define observation time", NULL); ADD_OPT(Str, addprocessedimfileArgs, "-hostname", "define host name", NULL); + ADD_OPT(Str, addprocessedimfileArgs, "-md5sum", "define md5sum", NULL); + ADD_OPT(S32, addprocessedimfileArgs, "-bytes", "define bytes", 0); + ADD_OPT(S16, addprocessedimfileArgs, "-burntool_state", "set burntool state", 0); ADD_OPT(S16, addprocessedimfileArgs, "-fault", "set fault code", 0); ADD_OPT(S16, addprocessedimfileArgs, "-quality", "set quality flag", 0); @@ -133,6 +136,10 @@ ADD_OPT(Str, processedimfileArgs, "-exp_name", "search by exposure name", NULL); ADD_OPT(Str, processedimfileArgs, "-class_id", "search by class ID", NULL); + ADD_OPT(Str, processedimfileArgs, "-filter", "search by filter", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL); ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL); + ADD_OPT(S64, processedimfileArgs, "-magicked", "search by magicked value", 0); + ADD_OPT(Bool, processedimfileArgs, "-destreaked", "only return imfiles that have been destreaked", false); + ADD_OPT(Bool, processedimfileArgs, "-not_destreaked", "only return imfiles that have not been destreaked", false); ADD_OPT(U64, processedimfileArgs, "-limit", "limit result set to N items", 0); ADD_OPT(Bool, processedimfileArgs, "-faulted", "only return imfiles with a fault status set", false); @@ -153,5 +160,5 @@ ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id", "search by exposure ID", 0); ADD_OPT(Str, updateprocessedimfileArgs, "-class_id", "search by class ID", NULL); - ADD_OPT(F64, updateprocessedimfileArgs, "-user_1", "set user stat (1)", NAN); + ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state", "set burntool state", 0); ADD_OPT(S16, updateprocessedimfileArgs, "-fault", "set fault code", INT16_MAX); Index: branches/eam_branches/20090820/ippTools/src/stacktool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/stacktool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/stacktool.c (revision 25766) @@ -139,4 +139,10 @@ PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_min", "camProcessedExp.fwhm_minor", ">="); PXOPT_COPY_F32(config->args, where, "-select_fwhm_minor_max", "camProcessedExp.fwhm_minor", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_max", "camProcessedExp.iq_m2", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m2_min", "camProcessedExp.iq_m2", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m3_max", "camProcessedExp.iq_m3", "<="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_min", "camProcessedExp.iq_m4", ">="); + PXOPT_COPY_F32(config->args, where, "-select_iq_m4_max", "camProcessedExp.iq_m4", "<="); + PXOPT_COPY_STR(config->args, where, "-select_exp_type", "rawExp.exp_type", "=="); PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "warpSkyfile.good_frac", ">="); @@ -853,10 +859,17 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "=="); PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "=="); + +// The following three selectors are incompatible with the sql so omit them +// PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); +// PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); +// PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -873,5 +886,9 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } + } else if (!all) { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; + } + psFree(where); Index: branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c (revision 25766) @@ -77,4 +77,10 @@ psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_min", 0, "define min fwhm (minor axis)", NAN); psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_max", 0, "define max fwhm (minor axis)", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_min", 0, "define min iq_m2", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_max", 0, "define max iq_m2", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_min", 0, "define min iq_m3", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_max", 0, "define max iq_m3", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_min", 0, "define min iq_m4", NAN); + psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_max", 0, "define max iq_m4", NAN); psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0); psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false); @@ -158,11 +164,18 @@ psMetadata *sumskyfileArgs= psMetadataAlloc(); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0, "search by stack ID", 0); - psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0); +#ifdef notdef + // These don't work so omit (for now) We probably should create a different mode for this type of search. + psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "search by warp ID", 0); psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0, "search by exposure ID", 0); psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure name", NULL); +#endif + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-label", 0, "search by stackRun.label", NULL); + psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-filter", 0, "search by filter (LIKE comparison)", NULL); + psMetadataAddS16(sumskyfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-all", 0, "enable search without arguments", false); // -revertsumskyfile Index: branches/eam_branches/20090820/ippTools/src/warptool.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/warptool.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/warptool.c (revision 25766) @@ -53,4 +53,5 @@ static bool tocleanedskyfileMode(pxConfig *config); static bool topurgedskyfileMode(pxConfig *config); +static bool toscrubbedskyfileMode(pxConfig *config); static bool tofullskyfileMode(pxConfig *config); static bool updateskyfileMode(pxConfig *config); @@ -102,4 +103,5 @@ MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileMode); MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileMode); + MODECASE(WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileMode); MODECASE(WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileMode); MODECASE(WARPTOOL_MODE_UPDATESKYFILE, updateskyfileMode); @@ -1196,4 +1198,15 @@ PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); + + PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false); + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -1211,6 +1224,24 @@ psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); + } else if (!all) { + psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required"); + return false; } psFree(where); + + if (not_destreaked) { + if (destreaked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense"); + return false; + } + if (magicked) { + psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense"); + return false; + } + psStringAppend(&query, " AND warpSkyfile.magicked = 0"); + } + if (destreaked) { + psStringAppend(&query, " AND warpSkyfile.magicked != 0"); + } // treat limit == 0 as "no limit" @@ -1668,4 +1699,8 @@ return change_skyfile_data_state(config, "purged", "goto_purged"); } +static bool toscrubbedskyfileMode(pxConfig *config) +{ + return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); +} static bool updateskyfileMode(pxConfig *config) Index: branches/eam_branches/20090820/ippTools/src/warptool.h =================================================================== --- branches/eam_branches/20090820/ippTools/src/warptool.h (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/warptool.h (revision 25766) @@ -49,4 +49,5 @@ WARPTOOL_MODE_TOCLEANEDSKYFILE, WARPTOOL_MODE_TOPURGEDSKYFILE, + WARPTOOL_MODE_TOSCRUBBEDSKYFILE, WARPTOOL_MODE_TOFULLSKYFILE, WARPTOOL_MODE_UPDATESKYFILE, Index: branches/eam_branches/20090820/ippTools/src/warptoolConfig.c =================================================================== --- branches/eam_branches/20090820/ippTools/src/warptoolConfig.c (revision 25139) +++ branches/eam_branches/20090820/ippTools/src/warptoolConfig.c (revision 25766) @@ -215,4 +215,14 @@ psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0, "search by exposure tag", 0); psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0, "search by phase 3 version of exposure tag", 0); + psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_begin", 0, "search for exposures by time (>=)", NULL); + psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_end", 0, "search for exposures by time (<=)", NULL); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-filter", 0, "search for exposures by filter", NULL); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-destreaked", 0, "search for destreaked images", false); + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-not_destreaked", 0, "search for images that have not been destreaked", false); + psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-magicked", 0, "search by magicked value", 0); + psMetadataAddS16(warpedArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0); + psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by warpRun label", NULL); + + psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-all", 0, "search without arguments", false); psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); @@ -278,4 +288,9 @@ psMetadataAddS64(topurgedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); + + // -toscrubbedskyfile + psMetadata *toscrubbedskyfileArgs = psMetadataAlloc(); + psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0); + psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL); // -tofullskyfile @@ -328,4 +343,5 @@ PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs); PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs); + PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs); PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs); PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs);