Index: /branches/bills_branches/bills_201312/dbconfig/ff.md =================================================================== --- /branches/bills_branches/bills_201312/dbconfig/ff.md (revision 36408) +++ /branches/bills_branches/bills_201312/dbconfig/ff.md (revision 36409) @@ -2,5 +2,4 @@ fullForceRun METADATA ff_id S64 0 - warp_id S64 0 skycal_id S64 0 sources_path_base STR 255 @@ -15,5 +14,21 @@ END +fullForceInput METADATA + ff_id S64 0 + warp_id S64 0 +END + fullForceResult METADATA + ff_id S64 0 + warp_id S64 0 + path_base STR 255 + dtime_script F32 0.0 + quality S16 0 + hostname STR 64 + software_ver STR 16 + fault S16 0 +END + +fullForceSummary METADATA ff_id S64 0 path_base STR 255 Index: /branches/bills_branches/bills_201312/ippTools/share/Makefile.am =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/Makefile.am (revision 36408) +++ /branches/bills_branches/bills_201312/ippTools/share/Makefile.am (revision 36409) @@ -490,4 +490,5 @@ releasetool_pendingrelgroup.sql \ fftool_definebyquery.sql \ + fftool_definebyquery_select_warps.sql \ fftool_result.sql \ fftool_revert.sql \ Index: /branches/bills_branches/bills_201312/ippTools/share/fftool_definebyquery.sql =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/fftool_definebyquery.sql (revision 36408) +++ /branches/bills_branches/bills_201312/ippTools/share/fftool_definebyquery.sql (revision 36409) @@ -1,26 +1,14 @@ SELECT - warpRun.warp_id, skycalRun.skycal_id, skycalResult.path_base, - skycalRun.data_group + skycalRun.data_group, + stackRun.tess_id, + stackRun.skycell_id, + stackRun.filter FROM skycalRun JOIN skycalResult USING(skycal_id) JOIN stackRun USING(stack_id) JOIN skycell USING(tess_id, skycell_id) - JOIN warpRun USING(tess_id) - JOIN warpSkyfile USING(warp_id, skycell_id) - JOIN fakeRun USING(fake_id) - JOIN camRun USING(cam_id) - JOIN chipRun USING(chip_id) - JOIN rawExp ON chipRun.exp_id = rawExp.exp_id and stackRun.filter = rawExp.filter - LEFT JOIN fullForceRun ON fullForceRun.skycal_id = fullForceRun.skycal_id - AND fullForceRun.warp_id = warpRun.warp_id - AND warpSkyfile.tess_id = stackRun.tess_id - AND warpSkyfile.skycell_id = stackRun.skycell_id -- join hook %s + -- join hook %s WHERE skycalRun.state = 'full' AND skycalResult.quality = 0 - AND warpRun.state = 'full' AND warpSkyfile.quality = 0 - --- an optimization might be to do cuts on distance between rawExp.ra,decl and skycell centers within some amount - - Index: /branches/bills_branches/bills_201312/ippTools/share/fftool_definebyquery_select_warps.sql =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/fftool_definebyquery_select_warps.sql (revision 36409) +++ /branches/bills_branches/bills_201312/ippTools/share/fftool_definebyquery_select_warps.sql (revision 36409) @@ -0,0 +1,8 @@ +SELECT warp_id +FROM warpRun join warpSkyfile using(warp_id, tess_id) + JOIN fakeRun using(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) +WHERE warpRun.tess_id = '%s' AND warpSkyfile.skycell_id = '%s' AND rawExp.filter = '%s' + AND warpRun.state = 'full' AND warpSkyfile.quality = 0 and warpSkyfile.fault = 0 Index: /branches/bills_branches/bills_201312/ippTools/share/fftool_result.sql =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/fftool_result.sql (revision 36408) +++ /branches/bills_branches/bills_201312/ippTools/share/fftool_result.sql (revision 36409) @@ -1,5 +1,4 @@ SELECT fullForceResult.*, - fullForceRun.warp_id, fullForceRun.skycal_id, fullForceRun.label, Index: /branches/bills_branches/bills_201312/ippTools/share/fftool_todo.sql =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/fftool_todo.sql (revision 36408) +++ /branches/bills_branches/bills_201312/ippTools/share/fftool_todo.sql (revision 36409) @@ -7,12 +7,13 @@ warpSkyfile.path_base AS warp_path_base FROM fullForceRun - JOIN skycalRun using(skycal_id) - JOIN stackRun using(stack_id) - JOIN warpRun using(warp_id, tess_id) + JOIN fullForceInput USING(ff_id) + JOIN skycalRun USING(skycal_id) + JOIN stackRun USING(stack_id) + JOIN warpRun USING(warp_id, tess_id) JOIN warpSkyfile USING(warp_id, tess_id, skycell_id) - JOIN fakeRun using(fake_id) - JOIN camRun using(cam_id) - JOIN chipRun using(chip_id) - JOIN rawExp using(exp_id) + JOIN fakeRun USING(fake_id) + JOIN camRun USING(cam_id) + JOIN chipRun USING(chip_id) + JOIN rawExp USING(exp_id) LEFT JOIN fullForceResult USING(ff_id) WHERE fullForceRun.state = 'new' Index: /branches/bills_branches/bills_201312/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/bills_branches/bills_201312/ippTools/share/pxadmin_create_tables.sql (revision 36408) +++ /branches/bills_branches/bills_201312/ippTools/share/pxadmin_create_tables.sql (revision 36409) @@ -2309,5 +2309,5 @@ CREATE TABLE fullForceRun ( ff_id BIGINT NOT NULL AUTO_INCREMENT, - warp_id BIGINT, +# warp_id BIGINT, skycal_id BIGINT, sources_path_base VARCHAR(255), @@ -2324,11 +2324,35 @@ KEY(label), KEY(data_group), - KEY(warp_id), +# KEY(warp_id), KEY(skycal_id), - FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id), +# FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id), FOREIGN KEY(skycal_id) REFERENCES skycalRun(skycal_id) ) ENGINE=innodb DEFAULT CHARSET=latin1; +CREATE TABLE fullForceInput ( + ff_id BIGINT, + warp_id BIGINT, + PRIMARY KEY(ff_id, warp_id), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE fullForceResult ( + ff_id BIGINT, + warp_id BIGINT, + path_base VARCHAR(255) NOT NULL, + dtime_script FLOAT, + quality SMALLINT NOT NULL, + hostname VARCHAR(64) NOT NULL, + software_ver VARCHAR(16), + fault SMALLINT NOT NULL, + PRIMARY KEY(ff_id, warp_id), + KEY(fault), + KEY(quality), + FOREIGN KEY(ff_id) REFERENCES fullForceRun(ff_id), + FOREIGN KEY(warp_id) REFERENCES warpRun(warp_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE fullForceSummary ( ff_id BIGINT, path_base VARCHAR(255) NOT NULL, Index: /branches/bills_branches/bills_201312/ippTools/src/fftool.c =================================================================== --- /branches/bills_branches/bills_201312/ippTools/src/fftool.c (revision 36408) +++ /branches/bills_branches/bills_201312/ippTools/src/fftool.c (revision 36409) @@ -40,5 +40,5 @@ static bool updateresultMode(pxConfig *config); -static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state); +// static bool setfullForceRunState(pxConfig *config, psS64 sky_id, const char *state); # define MODECASE(caseName, func) \ @@ -103,23 +103,50 @@ PXOPT_LOOKUP_STR(sources_path_base, config->args, "-set_sources_path_base", false, false); - psMetadata *whereMD = psMetadataAlloc(); - pxAddLabelSearchArgs(config, whereMD, "-select_warp_label", "warpRun.label", "LIKE"); - pxAddLabelSearchArgs(config, whereMD, "-select_warp_data_group","warpRun.data_group", "LIKE"); - PXOPT_COPY_S64(config->args, whereMD, "-select_warp_id", "warpRun.warp_id", "=="); - - pxAddLabelSearchArgs(config, whereMD, "-select_skycal_label", "skycalRun.label", "LIKE"); - pxAddLabelSearchArgs(config, whereMD, "-select_skycal_data_group", "skycalRun.data_group", "LIKE"); - PXOPT_COPY_S64(config->args, whereMD, "-select_skycal_id", "skycalRun.skycal_id", "=="); - - // XXX: REQUIRE at least one of the stack selectors and one of the warp selectors are supplied - - PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); - PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id", "stackRun.tess_id", "=="); - pxAddLabelSearchArgs(config, whereMD, "-select_filter", "stackRun.filter", "LIKE"); - - if (!pxskycellAddWhere(config, whereMD)) { + psMetadata *skycalWhereMD = psMetadataAlloc(); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_label", "skycalRun.label", "LIKE"); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_skycal_data_group", "skycalRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, skycalWhereMD, "-select_skycal_id", "skycalRun.skycal_id", "=="); + + if (!psListLength(skycalWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "skycal search parameters are required"); + psFree(skycalWhereMD); + return false; + } + + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_skycell_id", "stackRun.skycell_id", "LIKE"); + PXOPT_COPY_STR(config->args, skycalWhereMD, "-select_tess_id", "stackRun.tess_id", "=="); + pxAddLabelSearchArgs(config, skycalWhereMD, "-select_filter", "stackRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, skycalWhereMD)) { psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); - return false; - } + psFree(skycalWhereMD); + return false; + } + + psMetadata *warpWhereMD = psMetadataAlloc(); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_label", "warpRun.label", "LIKE"); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_warp_data_group","warpRun.data_group", "LIKE"); + PXOPT_COPY_S64(config->args, warpWhereMD, "-select_warp_id", "warpRun.warp_id", "=="); + if (!psListLength(warpWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "warp search parameters are required"); + psFree(warpWhereMD); + psFree(skycalWhereMD); + return false; + } + if (!psListLength(warpWhereMD->list)) { + psError(PXTOOLS_ERR_CONFIG, false, "skycal search parameters are required"); + psFree(skycalWhereMD); + psFree(warpWhereMD); + return false; + } + + PXOPT_COPY_STR(config->args, warpWhereMD, "-select_tess_id", "warpRun.tess_id", "=="); + pxAddLabelSearchArgs(config, warpWhereMD, "-select_filter", "warpRun.filter", "LIKE"); + if (!pxskycellAddWhere(config, warpWhereMD)) { + psError(PXTOOLS_ERR_CONFIG, false, "failed to add skycell search arguments"); + psFree(skycalWhereMD); + psFree(warpWhereMD); + return false; + } + PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false); @@ -130,23 +157,19 @@ if (!select) { psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - psFree(whereMD); - return false; - } - - if (!psListLength(whereMD->list)) { - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - psFree(whereMD); + psFree(skycalWhereMD); + psFree(warpWhereMD); return false; } psString where = NULL; - psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL); + psString whereClause = psDBGenerateWhereConditionSQL(skycalWhereMD, NULL); psStringAppend(&where, "\nAND %s", whereClause); psStringAppend(&select, where); psFree(whereClause); - psFree(whereMD); + psFree(skycalWhereMD); psString joinHook = NULL; if (!rerun) { + psStringAppend(&joinHook, "\nLEFT JOIN fullForceRun ON fullForceRun.skycal_id = fullForceRun.skycal_id"); psStringAppend(&joinHook, "\n %s\nAND fullForceRun.label = '%s'", where, label); psStringAppend(&select, "\nAND ff_id IS NULL"); @@ -181,7 +204,8 @@ return true; } + if (pretend) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "staticskyInput", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "toFullForce", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -193,4 +217,9 @@ return true; } + + psString warpQueryTemplate = pxDataGet("fftool_definebyquery_select_warps.sql"); + + whereClause = psDBGenerateWhereConditionSQL(warpWhereMD, NULL); + psStringAppend(&warpQueryTemplate, "\nAND %s", whereClause); for (long i = 0; i < output->n; i++) { @@ -204,5 +233,5 @@ } - psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); + // psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); psS64 skycal_id = psMetadataLookupS64(&status, row, "skycal_id"); @@ -216,9 +245,42 @@ psString skycal_data_group = psMetadataLookupStr(&status, row, "data_group"); + psString tess_id = psMetadataLookupStr(&status, row, "tess_id"); + psString skycell_id = psMetadataLookupStr(&status, row, "skycell_id"); + psString filter = psMetadataLookupStr(&status, row, "filter"); + + psString query = NULL; + psStringAppend(&query, warpQueryTemplate, tess_id, skycell_id, filter); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + // Find the warps for this skycell and filter combination + psArray *warpOutput = p_psDBFetchResult(config->dbh); + if (!warpOutput) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + return false; + } + if (!psArrayLength(warpOutput)) { + // no warps for this skycal. Suprise? + psFree(warpOutput); + psFree(query); + continue; + } // create a staticskyRun if (!fullForceRunInsert(config->dbh, 0x0, // ff_id - warp_id, skycal_id, path_base, @@ -241,9 +303,31 @@ } + psS64 ff_id = psDBLastInsertID(config->dbh); + + for (int j = 0; j < warpOutput->n; j++) { + psMetadata *warpRow = warpOutput->data[j]; + psS64 warp_id = psMetadataLookupS64(&status, warpRow, "warp_id"); + + if (!fullForceInputInsert(config->dbh, + ff_id, + warp_id) + ) { + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); + psFree(output); + return false; + } + } + if (!psDBCommit(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(warpOutput); psFree(output); return false; } + psFree(warpOutput); } psFree(output); @@ -365,4 +449,5 @@ // required PXOPT_LOOKUP_S64(ff_id, config->args, "-ff_id", true, false); + PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); // optional @@ -384,4 +469,5 @@ if (!fullForceResultInsert(config->dbh, ff_id, + warp_id, path_base, dtime_script, @@ -398,4 +484,5 @@ } +#ifdef notdef if (!fault) { if (!setfullForceRunState(config, ff_id, "full")) { @@ -407,4 +494,5 @@ } } +#endif // point of no return @@ -557,4 +645,5 @@ psMetadata *where = psMetadataAlloc(); PXOPT_COPY_S64(config->args, where, "-ff_id", "ff_id", "=="); + PXOPT_COPY_S64(config->args, where, "-warp_id", "ff_id", "=="); @@ -567,5 +656,5 @@ return true; } - +#ifdef notyet static bool setfullForceRunState(pxConfig *config, psS64 ff_id, const char *state) { @@ -578,3 +667,4 @@ return true; } - +#endif + Index: /branches/bills_branches/bills_201312/ippTools/src/fftoolConfig.c =================================================================== --- /branches/bills_branches/bills_201312/ippTools/src/fftoolConfig.c (revision 36408) +++ /branches/bills_branches/bills_201312/ippTools/src/fftoolConfig.c (revision 36409) @@ -95,5 +95,6 @@ // -addresult psMetadata *addresultArgs = psMetadataAlloc(); - psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define sky ID (required)", 0); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location(required)", 0); psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_script", 0, "define elapsed time in script (seconds)", NAN); @@ -128,5 +129,6 @@ // -updateresult psMetadata *updateresultArgs = psMetadataAlloc(); - psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define staticksky ID (required)", 0); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-ff_id", 0, "define full force ID (required)", 0); + psMetadataAddS64(updateresultArgs, PS_LIST_TAIL, "-warp_id", 0, "define warp ID (required)", 0); psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code (required)", 0); psMetadataAddS16(updateresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality code", 0);