Index: branches/pap/ippTools/src/bgtool.c =================================================================== --- branches/pap/ippTools/src/bgtool.c (revision 28188) +++ branches/pap/ippTools/src/bgtool.c (revision 28199) @@ -1,6 +1,6 @@ /* - * warptool.c + * bgtool.c * - * Copyright (C) 2006 Joshua Hoblitt + * Copyright (C) 2006-2010 Joshua Hoblitt, Paul Price * * This program is free software; you can redistribute it and/or modify it @@ -29,10 +29,10 @@ #include "pxtools.h" -#include "warptool.h" +#include "bgtool.h" static bool definechipMode(pxConfig *config); static bool updatechipMode(pxConfig *config); static bool tochipMode(pxConfig *config); -static bool chipInputsMode(pxConfig *config); +static bool chipinputsMode(pxConfig *config); static bool addchipMode(pxConfig *config); static bool chipMode(pxConfig *config); @@ -42,5 +42,5 @@ static bool updatewarpMode(pxConfig *config); static bool towarpMode(pxConfig *config); -static bool warpInputsMode(pxConfig *config); +static bool warpinputsMode(pxConfig *config); static bool addwarpMode(pxConfig *config); static bool warpMode(pxConfig *config); @@ -56,4 +56,22 @@ static bool importwarpMode(pxConfig *config); +// Tables to import/export +typedef struct { + const char *name; // Table name + void* (*parse)(); // Parsing function + bool (*insert)(); // Insertion function +} tableData; +static const tableData chipTables[] = { + { "chipBackgroundRun", (void*)&chipBackgroundRunObjectFromMetadata, &chipBackgroundRunInsertObject }, + { "chipBackgroundImfile", (void*)&chipBackgroundImfileObjectFromMetadata, &chipBackgroundImfileInsertObject }, + { NULL, NULL, NULL } +}; +static const tableData warpTables[] = { + { "warpBackgroundRun", (void*)&warpBackgroundRunObjectFromMetadata, &warpBackgroundRunInsertObject }, + { "warpBackgroundImfile", (void*)&warpBackgroundSkyfileObjectFromMetadata, &warpBackgroundSkyfileInsertObject }, + { NULL, NULL, NULL } +}; + + # define MODECASE(caseName, func) \ case caseName: \ @@ -77,5 +95,5 @@ MODECASE(BGTOOL_MODE_UPDATECHIP, updatechipMode); MODECASE(BGTOOL_MODE_TOCHIP, tochipMode); - MODECASE(BGTOOL_MODE_CHIPINPUTS, chipInputsMode); + MODECASE(BGTOOL_MODE_CHIPINPUTS, chipinputsMode); MODECASE(BGTOOL_MODE_ADDCHIP, addchipMode); MODECASE(BGTOOL_MODE_CHIP, chipMode); @@ -85,5 +103,5 @@ MODECASE(BGTOOL_MODE_UPDATEWARP, updatewarpMode); MODECASE(BGTOOL_MODE_TOWARP, towarpMode); - MODECASE(BGTOOL_MODE_WARPINPUTS, warpInputsMode); + MODECASE(BGTOOL_MODE_WARPINPUTS, warpinputsMode); MODECASE(BGTOOL_MODE_ADDWARP, addwarpMode); MODECASE(BGTOOL_MODE_WARP, warpMode); @@ -119,4 +137,130 @@ exit(exit_status); } + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// General functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static bool exportTables(pxConfig *config, // Configuration (with DB handle) + const char *filename, // Filename to which to write + const tableData tables[], // Tables to export (NULL terminated) + const psMetadata *where, // WHERE restrictions + bool clean // Write as cleaned? + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + FILE *file = fopen(filename, "w"); + if (!file) { + psError(PXTOOLS_ERR_SYS, true, "failed to open output file %s", filename); + return false; + } + + if (!pxExportVersion(config, file)) { + psError(psErrorCodeLast(), false, "failed to write dbversion to output file %s", filename); + return false; + } + + for (int i = 0; tables[i].name; i++) { + const char *name = tables[i].name; // Name of table + psString query = NULL; + psStringAppend(&query, "SELECT * FROM %s", name); + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); + psFree(whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psError(PXTOOLS_ERR_CONFIG, true, "no rows found"); + psFree(output); + return false; + } + + if (clean && + (strcmp(name, "chipBackgroundRun") == 0 || + strcmp(name, "warpBackgroundRun") == 0) && + !pxSetStateCleaned(name, "state", output)) { + psFree(output); + psError(psErrorCodeLast(), false, "pxSetStateClean failed for table %s", name); + return false; + } + + if (!ippdbPrintMetadatas(file, output, name, true)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + } + fclose(file); + + return true; +} + +static bool importTables(pxConfig *config, // Configuration + const char *filename, // + const tableData tables[] // Tables to read in + ) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + unsigned int badLines = 0; // Number of bad lines + psMetadata *input = psMetadataConfigRead(NULL, &badLines, filename, false); // Input file contents + if (!input) { + psError(psErrorCodeLast(), false, "Unable to parse input file %s", filename); + return false; + } + if (badLines > 0) { + psWarning("%d bad lines encountered when parsing %s", badLines, filename); + } + + if (!pxCheckImportVersion(config, input)) { + psError(psErrorCodeLast(), false, "pxCheckImportVersion failed"); + return false; + } + + // Import primary table + for (int i = 0; tables[i].name; i++) { + const char *name = tables[i].name; // Name of table + psMetadataItem *item = psMetadataLookup(input, name); // Item from input + psAssert(item, "%s not in input", name); + psAssert(item->type == PS_DATA_METADATA_MULTI, "%s not MULTI type", name); + psAssert(psListLength(item->data.list) == 1, "%s has multiple entries", name); + psMetadataItem *entry = psListGet(item->data.list, PS_LIST_HEAD); // Entry of interest + void *data = tables[i].parse(entry); // Parsed entry + if (!data) { + psError(PXTOOLS_ERR_CONFIG, false, "Unable to parse entry %s", name); + psFree(input); + return false; + } + if (!tables[0].insert(config->dbh, data)) { + psError(psErrorCodeLast(), false, "Unable to insert entry %s", name); + psFree(input); + return false; + } + } + + psFree(input); + + return true; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Functions for chip stage +////////////////////////////////////////////////////////////////////////////////////////////////////////////// static bool definechipMode(pxConfig *config) @@ -293,5 +437,5 @@ psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); PXOPT_COPY_STR(config->args, where, "-state", "chipBackgroundRun.state", "=="); PXOPT_COPY_STR(config->args, where, "-data_group", "chipBackgroundRun.data_group","LIKE"); @@ -312,10 +456,5 @@ } - psString query = psStringCopy("UPDATE chipBackgroundRun " - "JOIN chipBackgroundImfile USING(chip_bg_id) " - "JOIN chipRun USING(chip_id) " - "JOIN rawExp USING(exp_id)"); - - // pxUpdateRun gets parameters from config->args and updates + psString query = psStringCopy("UPDATE chipBackgroundRun JOIN chipBackgroundImfile USING(chip_bg_id)"); bool result = pxUpdateRun(config, where, &query, "chipBackgroundRun", "chip_bg_id", "chipBackgroundImfile", true); @@ -352,8 +491,6 @@ psFree(where); - psString limitString = psStringCopy("\nORDER BY priority DESC, warp_id"); + psString limitString = psStringCopy("\nORDER BY priority DESC, chip_bg_id"); if (limit) { - // We apply the limit to both sides of the UNION to avoid slow queries - // and to the query itself to satisfy the user's requested limit psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit)); psStringAppend(&query, " %s", limitString); @@ -394,40 +531,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -static bool chipMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "=="); +static bool chipinputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -435,12 +543,12 @@ // find all rawImfiles matching the default query - psString query = pxDataGet("warptool_imfile.sql"); + psString query = pxDataGet("bgtool_chipinputs.sql"); if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); return false; } if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun"); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -448,5 +556,4 @@ psFree(where); - // treat limit == 0 as "no limit" if (limit) { psString limitString = psDBGenerateLimitSQL(limit); @@ -456,5 +563,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(psErrorCodeLast(), false, "database error"); psFree(query); return false; @@ -464,29 +571,16 @@ psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - + psError(psErrorCodeLast(), false, "database error"); return false; } if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); + psTrace("bgtool", PS_LOG_INFO, "no rows found"); psFree(output); return true; } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpInputImfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; } @@ -497,30 +591,104 @@ -static bool tooverlapMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - +static bool addchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); + + // optional + + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_ppbackground, config->args, "-ver_ppbackground", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); + PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); + PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); + PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); + PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); + PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false); + PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false); + PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); + + psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules); + ver_code = pxMergeCodeVersions(ver_code, ver_ppbackground); + ver_code = pxMergeCodeVersions(ver_code, ver_ppstats); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script, + hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, + maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool chipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-data_group", "chipBackgroundRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, where, "-dist_group", "chipBackgroundRun.data_group", "LIKE"); + + PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // find all rawImfiles matching the default query - psString query = pxDataGet("warptool_tooverlap.sql"); + psString query = pxDataGet("bgtool_chip.sql"); if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString magicWhere = NULL; + if (!pxmagicAddWhere(config, &magicWhere, "chipBackgroundImfile")) { + psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); + return false; + } + if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) { + psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); return false; } if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun"); - psStringAppend(&query, " AND %s", whereClause); + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } - psFree(where); - - // treat limit == 0 as "no limit" + } else if (!all && !magicWhere) { + psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); + return false; + } + if (magicWhere) { + psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere); + } + psFree(magicWhere); + psFree(where); + if (limit) { psString limitString = psDBGenerateLimitSQL(limit); @@ -530,4 +698,171 @@ if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (psArrayLength(output)) { + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } + psFree(output); + + return true; +} + + +static bool advancechipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); + + psString select = pxDataGet("bgtool_advancechip.sql"); + if (!select) { + psError(psErrorCodeLast(), false, "failed to retrieve SQL statement"); + return false; + } + + psString selectWhere = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&selectWhere, "\n WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(select); + psFree(selectWhere); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(select); + psFree(selectWhere); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = true; // Status of MD lookup + psS64 chip_bg_id = psMetadataLookupS64(&status, row, "chip_bg_id"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for chip_bg_id"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psS64 magicked = psMetadataLookupS64(&status, row, "magicked"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!p_psDBRunQueryF(config->dbh, + "UPDATE chipBackgroundRun " + "SET state = 'full', magicked = %" PRId64 " " + " WHERE chip_bg_id = %" PRId64, + chip_bg_id, magicked)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + psS64 numUpdated = psDBAffectedRows(config->dbh); + if (numUpdated != 1) { + psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool revertchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "chipBackgroundImfile.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + psString query = pxDataGet("bgtool_revertchip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQuery(config->dbh, query)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(query); @@ -536,240 +871,28 @@ psFree(query); - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - - -static bool addoverlapMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_STR(mapfile, config->args, "-mapfile", false, false); - PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - if (fault == 0) { - if (!parseAndInsertSkyCellMap(config, mapfile)) { - psError(PS_ERR_UNKNOWN, false, "failed to inject mapfile: %s into the database", mapfile); - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - } else { - warpSkyCellMapInsert(config->dbh, - warp_id, - "faulted", // skycell_id - "faulted", // tess_id - "faulted", // class_id - fault // fault - ); - } - - // point of no return - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} - -static bool revertoverlapMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyCellMap.tess_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyCellMap.fault", "=="); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { - psFree(where); - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - return false; - } - - int numDeleted; // Number deleted - { - psString query = pxDataGet("warptool_revertoverlap.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return false; - } - psFree(query); - - numDeleted = psDBAffectedRows(config->dbh); - } - - psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkycellMap", numDeleted); - - psFree(where); - - return true; -} - - - -static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile) -{ - unsigned int nFail = 0; - psMetadata *imfiles = psMetadataAlloc(); - psMetadata *skycells = psMetadataConfigRead(NULL, &nFail, mapfile, false); - if (!skycells) { - psError(PS_ERR_UNKNOWN, false, "failed to parse mapfile: %s", mapfile); - return false; - } - if (nFail) { - psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing mapfile: %s", nFail, mapfile); - psFree(skycells); - psFree(imfiles); - return false; - } - - psMetadataItem *item = NULL; - psMetadataIterator *iter = psMetadataIteratorAlloc(skycells, 0, NULL); - while ((item = psMetadataGetAndIncrement(iter))) { - if (item->type != PS_DATA_METADATA) { - psError(PS_ERR_UNKNOWN, false, "mapfile: %s is in the wrong format", mapfile); - psFree(iter); - psFree(skycells); - psFree(imfiles); - return false; - } - - psMetadata *sc = item->data.md; - // this conversion isn't strictly nessicary but it's an easy way of - // validating the format - warpSkyCellMapRow *row = warpSkyCellMapObjectFromMetadata(sc); - if (!row) { - psError(PS_ERR_UNKNOWN, false, "failed to convert mapfile: %s metdata entry into a warpSkyCellMap object", mapfile); - psFree(iter); - psFree(skycells); - psFree(imfiles); - return false; - } - psMetadataAddS64(imfiles, PS_LIST_TAIL, row->skycell_id, PS_META_REPLACE, "", row->warp_id); - - if (!warpSkyCellMapInsertObject(config->dbh, row)) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - psFree(row); - psFree(iter); - psFree(skycells); - psFree(imfiles); - return false; - } - - psFree(row); - } - psFree(iter); - - // create warp_skyfile_ids for the output skyfiles - psString query = "INSERT INTO warpImfile VALUES(%" PRId64 ", '%s', 0)"; - iter = psMetadataIteratorAlloc(imfiles, 0, NULL); - while ((item = psMetadataGetAndIncrement(iter))) { - psString skycell_id = item->name; - psS64 warp_id = item->data.S64; - - if (!p_psDBRunQueryF(config->dbh, query, warp_id, skycell_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - } - psFree(iter); - psFree(skycells); - psFree(imfiles); - - return true; -} - - -static bool scmapMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "=="); - PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "=="); + int numDeleted = psDBAffectedRows(config->dbh); + psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d chipBackgroundImfiles", numDeleted); + + return true; +} + +static bool tocleanchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // find all rawImfiles matching the default query - psString query = pxDataGet("warptool_scmap.sql"); + psMetadata *where = psMetadataAlloc(); + pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "=="); + + psString query = pxDataGet("bgtool_tocleanchip.sql"); if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "warpSkyCellMap"); + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (where && psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); @@ -777,5 +900,4 @@ psFree(where); - // treat limit == 0 as "no limit" if (limit) { psString limitString = psDBGenerateLimitSQL(limit); @@ -785,5 +907,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(psErrorCodeLast(), false, "database error"); psFree(query); return false; @@ -793,51 +915,311 @@ psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - + psError(psErrorCodeLast(), false, "database error"); return false; } if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); + psTrace("bgtool", PS_LOG_INFO, "no rows found"); psFree(output); return true; } - if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + psFree(output); + + return true; +} + +static bool cleanedchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + if (!pxIsValidCleanedState(state)) { + psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); + return false; + } + + psString query = pxDataGet("bgtool_cleanedchip.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + psFree(query); + + return true; +} + +static bool exportchipMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false); + PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); + PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "=="); + + bool status = exportTables(config, outfile, chipTables, where, clean); + + psFree(where); + return status; +} + +static bool importchipMode(pxConfig *config) +{ + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + return importTables(config, infile, chipTables); +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Functions for warp stage +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +static bool definewarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, NULL); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); + PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "=="); + PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "=="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); + PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); + PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "=="); + PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "=="); + PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "=="); + PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "=="); + PXOPT_COPY_F64(config->args, where, "-airmass_min", "rawExp.airmass", ">="); + PXOPT_COPY_F64(config->args, where, "-airmass_max", "rawExp.airmass", "<"); + PXOPT_COPY_RADEC(config->args, where, "-ra_min", "rawExp.ra", ">="); + PXOPT_COPY_RADEC(config->args, where, "-ra_max", "rawExp.ra", "<"); + PXOPT_COPY_RADEC(config->args, where, "-decl_min", "rawExp.decl", ">="); + PXOPT_COPY_RADEC(config->args, where, "-decl_max", "rawExp.decl", "<"); + PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">="); + PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<"); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">="); + PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">="); + PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<"); + PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">="); + PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<"); + PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">="); + PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<"); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">="); + PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<"); + PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">="); + PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<"); + PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "=="); + PXOPT_COPY_STR(config->args, where, "-comment", "rawExp.comment", "LIKE"); + PXOPT_COPY_STR(config->args, where, "-obs_mode", "rawExp.obs_mode", "LIKE"); + PXOPT_COPY_F32(config->args, where, "-sun_angle_min", "rawExp.sun_angle", ">="); + PXOPT_COPY_F32(config->args, where, "-sun_angle_max", "rawExp.sun_angle", "<"); + pxAddLabelSearchArgs(config, where, "-warp_label", "warpRun.label", "=="); + pxAddLabelSearchArgs(config, where, "-bg_label", "chipBackgroundRun.label", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); + PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); + PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false); + PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false); + PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); + PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false); + PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false); + + // Get warp runs to promote to warpBackgroundRun + + psString query = pxDataGet("bgtool_definewarp.sql"); // Query to execute + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + psFree(where); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, "AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); // Matching rows + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return true; + } + + if (pretend) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpSkyCellMap", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - + if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *md = output->data[i]; + + psS64 chip_bg_id = psMetadataLookupS64(NULL, md, "chip_bg_id"); + + warpRunRow *row = warpRunObjectFromMetadata(md); + if (!row) { + psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!warpBackgroundRunInsert(config->dbh, 0, row->warp_id, chip_bg_id, "new", + workdir ? workdir : row->workdir, + label ? label : row->label, + data_group ? data_group : row->data_group, + dist_group ? dist_group : row->dist_group, + reduction ? reduction : row->reduction, + note ? note : row->note, + NULL)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(row); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(row); + } psFree(output); - return true; -} - -static bool towarpedMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + return true; +} + +static bool updatewarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-state", "warpBackgroundRun.state", "=="); + PXOPT_COPY_STR(config->args, where, "-data_group", "warpBackgroundRun.data_group","LIKE"); + PXOPT_COPY_STR(config->args, where, "-dist_group", "warpBackgroundRun.dist_group","LIKE"); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "LIKE"); + PXOPT_COPY_TIME(config->args, where, "-registered_begin", "warpBackgroundRun.registered", ">="); + PXOPT_COPY_TIME(config->args, where, "-registered_end", "warpBackgroundRun.registered", "<"); + + PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false); + if (destreaked) { + psMetadataAddS64(where, PS_LIST_TAIL, "warpBackgroundRun.magicked", PS_META_DUPLICATE_OK, ">", 0); + } + + if (!psListLength(where->list)) { + psFree(where); + psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); + return false; + } + + psString query = psStringCopy("UPDATE warpBackgroundRun JOIN warpBackgroundSkyfile USING(warp_bg_id)"); + bool result = pxUpdateRun(config, where, &query, "warpBackgroundRun", "warp_bg_id", + "warpBackgroundSkyfile", true); + + psFree(query); + psFree(where); + + return result; +} + +static bool towarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + pxAddLabelSearchArgs (config, where, "-label", "warpBackgroundRun.label", "=="); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - // find all rawImfiles matching the default query - psString query = pxDataGet("warptool_towarped.sql"); + psString query = pxDataGet("bgtool_towarp.sql"); if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); return false; } @@ -851,9 +1233,6 @@ psFree(where); - // treat limit == 0 as "no limit" - psString limitString = psStringCopy("\nORDER BY priority DESC, warp_id"); + psString limitString = psStringCopy("\nORDER BY priority DESC, warp_bg_id"); if (limit) { - // We apply the limit to both sides of the UNION to avoid slow queries - // and to the query itself to satisfy the user's requested limit psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit)); psStringAppend(&query, " %s", limitString); @@ -861,5 +1240,5 @@ if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr, limitString)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(psErrorCodeLast(), false, "database error"); psFree(query); return false; @@ -871,18 +1250,9 @@ psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - + psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query); return false; } if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); + psTrace("bgtool", PS_LOG_INFO, "no rows found"); psFree(output); return true; @@ -890,7 +1260,6 @@ if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); psFree(output); return false; @@ -904,34 +1273,85 @@ -static bool addwarpedMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); +static bool warpinputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpProcessedSkyfile.skycell_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // find all rawImfiles matching the default query + psString query = pxDataGet("bgtool_warpinputs.sql"); + if (!query) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundSkyfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + + +static bool addwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); - PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); + PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false); // optional - PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); - PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false); + + PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); + PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); + PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); + PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); + PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); + PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); + PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); + PXOPT_LOOKUP_STR(ver_ppbackground, config->args, "-ver_ppbackground", false, false); + PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false); PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false); - PXOPT_LOOKUP_F32(dtime_warp, config->args, "-dtime_warp", false, false); - PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false); - PXOPT_LOOKUP_S32(xmin, config->args, "-xmin", false, false); - PXOPT_LOOKUP_S32(xmax, config->args, "-xmax", false, false); - PXOPT_LOOKUP_S32(ymin, config->args, "-ymin", false, false); - PXOPT_LOOKUP_S32(ymax, config->args, "-ymax", false, false); - PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false); - PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false); - PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); - - PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false); - PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false); - PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false); - PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false); - PXOPT_LOOKUP_STR(ver_pswarp, config->args, "-ver_pswarp", false, false); - PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false); - PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false); PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false); @@ -940,31 +1360,8 @@ PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false); - psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s pswarp %s streaks %s\n", - ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_pswarp,ver_streaks); - psString ver_code = NULL; - if ((ver_pslib)&&(ver_psmodules)) { - ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules); - } - if (ver_psphot) { - ver_code = pxMergeCodeVersions(ver_code,ver_psphot); - } - if (ver_ppstats) { - ver_code = pxMergeCodeVersions(ver_code,ver_ppstats); - } - if (ver_pswarp) { - ver_code = pxMergeCodeVersions(ver_code,ver_pswarp); - } - if (ver_streaks) { - ver_code = pxMergeCodeVersions(ver_code,ver_streaks); - } - - // default values - PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false); - - - - // we don't want to insert the last skyfile in a run but then not mark the - // run as 'stop' + psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules); + ver_code = pxMergeCodeVersions(ver_code, ver_ppbackground); + ver_code = pxMergeCodeVersions(ver_code, ver_ppstats); + if (!psDBTransaction(config->dbh)) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -972,286 +1369,50 @@ } - // XXX need to validate that this coresponds to an warpInputImfile - if (!warpSkyfileInsert(config->dbh, - warp_id, - skycell_id, - tess_id, - uri, - path_base, - "full", // data_state - bg, - bg_stdev, - dtime_warp, - dtime_script, - hostname, - good_frac, - xmin, - xmax, - ymin, - ymax, - fault, - quality, - magicked, - ver_code, - maskfrac_npix, - maskfrac_static, - maskfrac_dynamic, - maskfrac_magic, - maskfrac_advisory - )) { + if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script, + hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix, + maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { + psError(psErrorCodeLast(), false, "database error"); if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - // point of no return + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} - -static bool advancerunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "label", "=="); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - - psString query = pxDataGet("warptool_finished_run_select.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - psFree(where); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - query = pxDataGet("warptool_finish_run.sql"); - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *row = output->data[i]; - - bool status; - psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); - - psString software_ver = NULL; - psS64 maskfrac_npix = 0; - psF32 maskfrac_static = 0; - psF32 maskfrac_dynamic = 0; - psF32 maskfrac_magic = 0; - psF32 maskfrac_advisory = 0; - - // Calculate run level masking and software state - if (!pxCoalesceRunStatus(config,"warptool_coalesce_run.sql",warp_id, - &software_ver,&maskfrac_npix, - &maskfrac_static,&maskfrac_dynamic, - &maskfrac_magic,&maskfrac_advisory)) { - psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics"); - psFree(output); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return(false); - } - // Set warpRun.software_ver to the appropriate value if ( - if (software_ver) { - if (!pxSetRunSoftware(config, "warpRun", "warp_id", warp_id, software_ver)) { - psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, - warp_id); - psFree(output); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return(false); - } - } - // Set warpRun.maskfrac* to the appropriate values. - if (maskfrac_npix) { - if (!pxSetRunMaskfrac(config, "warpRun", "warp_id", warp_id, maskfrac_npix, maskfrac_static, - maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) { - psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64, - warp_id); - psFree(output); - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - return(false); - } - } - - - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id"); - psFree(output); - psFree(query); - return false; - } - psS64 magicked = psMetadataLookupS64(&status, row, "magicked"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to look up value for magicked"); - psFree(output); - psFree(query); - return false; - } - if (!p_psDBRunQueryF(config->dbh, query, magicked, warp_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(output); - psFree(query); - return false; - } - - psS64 numUpdated = psDBAffectedRows(config->dbh); - - if (numUpdated != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); - psFree(query); - psFree(output); - return false; - } - } - psFree(output); - psFree(query); - - return true; -} - -bool warpCompletedRuns(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psString query = pxDataGet("warptool_finished_run_select.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement"); - return false; - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - query = pxDataGet("warptool_finish_run.sql"); - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *row = output->data[i]; - - bool status; - psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id"); - psFree(output); - psFree(query); - return false; - } - psS32 magicked = psMetadataLookupS64(&status, row, "magicked"); - if (!status) { - psError(PS_ERR_UNKNOWN, false, "failed to look up value for magicked"); - psFree(output); - psFree(query); - return false; - } - if (!p_psDBRunQueryF(config->dbh, query, magicked, warp_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(output); - psFree(query); - return false; - } - - psS64 numUpdated = psDBAffectedRows(config->dbh); - - if (numUpdated != 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected 1 row"); - psFree(query); - psFree(output); - return false; - } - } - psFree(output); - psFree(query); - - return true; -} - -static bool warpedMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); - PXOPT_COPY_S64(config->args, where, "-warp_skyfile_id", "warpImfile.warp_skyfile_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); - PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "=="); - pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); - pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool warpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "LIKE"); + pxAddLabelSearchArgs(config, where, "-data_group", "warpBackgroundRun.data_group", "LIKE"); + pxAddLabelSearchArgs(config, where, "-dist_group", "warpBackgroundRun.data_group", "LIKE"); PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // find all rawImfiles matching the default query - psString query = pxDataGet("warptool_warped.sql"); + psString query = pxDataGet("bgtool_warp.sql"); if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - // generate where strings for arguments that require extra processing - // beyond PXOPT_COPY* - psString where2 = NULL; - if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + psString magicWhere = NULL; + if (!pxmagicAddWhere(config, &magicWhere, "warpBackgroundSkyfile")) { psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); return false; } - if (!pxspaceAddWhere(config, &where2, "rawExp")) { + if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) { psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); return false; @@ -1262,19 +1423,14 @@ psStringAppend(&query, " WHERE %s", whereClause); psFree(whereClause); - } else if (!all && !where2) { + } else if (!all && !magicWhere) { psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); return false; } - - if (where2) { - if (psListLength(where->list)) { - psStringAppend(&query, " %s", where2); - } else { - psStringAppend(&query, " WHERE 1 %s", where2); - } - } - psFree(where); - - // treat limit == 0 as "no limit" + if (magicWhere) { + psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere); + } + psFree(magicWhere); + psFree(where); + if (limit) { psString limitString = psDBGenerateLimitSQL(limit); @@ -1284,5 +1440,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(psErrorCodeLast(), false, "database error"); psFree(query); return false; @@ -1292,31 +1448,19 @@ psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - + psError(psErrorCodeLast(), false, "database error"); return false; } if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); + psTrace("bgtool", PS_LOG_INFO, "no rows found"); psFree(output); return true; } - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpSkyfile", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } + } psFree(output); @@ -1325,31 +1469,131 @@ -static bool revertwarpedMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "=="); - PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "=="); - pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); - PXOPT_COPY_S16(config->args, where, "-fault", "warpSkyfile.fault", "=="); - - if (!psListLength(where->list) - && !psMetadataLookupBool(NULL, config->args, "-all")) { +static bool advancewarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "label", "=="); + + psString select = pxDataGet("bgtool_advancewarp.sql"); + if (!select) { + psError(psErrorCodeLast(), false, "failed to retrieve SQL statement"); + return false; + } + + psString selectWhere = psStringCopy(""); + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereSQL(where, NULL); + psStringAppend(&selectWhere, "\n WHERE %s", whereClause); + psFree(whereClause); + } + psFree(where); + + if (!psDBTransaction(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(select); + psFree(selectWhere); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psFree(select); + psFree(selectWhere); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psError(psErrorCodeLast(), false, "database error"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + if (!psArrayLength(output)) { + psTrace("bgtool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + for (long i = 0; i < psArrayLength(output); i++) { + psMetadata *row = output->data[i]; + bool status = true; // Status of MD lookup + psS64 warp_bg_id = psMetadataLookupS64(&status, row, "warp_bg_id"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for warp_bg_id"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + psS64 magicked = psMetadataLookupS64(&status, row, "magicked"); + if (!status) { + psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + if (!p_psDBRunQueryF(config->dbh, + "UPDATE warpBackgroundRun " + "SET state = 'full', magicked = %" PRId64 " " + " WHERE warp_bg_id = %" PRId64, + warp_bg_id, magicked)) { + psError(psErrorCodeLast(), false, "database error"); + psFree(output); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + return false; + } + + psS64 numUpdated = psDBAffectedRows(config->dbh); + if (numUpdated != 1) { + psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row"); + if (!psDBRollback(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + } + psFree(output); + return false; + } + } + psFree(output); + + if (!psDBCommit(config->dbh)) { + psError(psErrorCodeLast(), false, "database error"); + return false; + } + + return true; +} + +static bool revertwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "=="); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "=="); + PXOPT_COPY_S16(config->args, where, "-fault", "warpBackgroundSkyfile.fault", "=="); + + if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) { psFree(where); - psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required"); - return false; - } - - psString query = pxDataGet("warptool_revertwarped_delete.sql"); + psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required"); + return false; + } + + psString query = pxDataGet("bgtool_revertwarp.sql"); if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - psString query_updated = pxDataGet("warptool_revertwarped_updated.sql"); - if (!query_updated) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); return false; } @@ -1358,5 +1602,4 @@ psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); - psStringAppend(&query_updated, " AND %s", whereClause); psFree(whereClause); } @@ -1371,94 +1614,10 @@ int numDeleted = psDBAffectedRows(config->dbh); - - psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkyfiles", numDeleted); - - // fix any faulted warpSkyfiles in data_state 'update' - - if (!p_psDBRunQuery(config->dbh, query_updated)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query_updated); - return false; - } - psFree(query_updated); - - int numUpdated = psDBAffectedRows(config->dbh); - - psLogMsg("warptool", PS_LOG_INFO, "Updated %d warpSkyfiles", numUpdated); - - return true; -} - - -static bool blockMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); - - if (!warpMaskInsert(config->dbh, label)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} - - -static bool maskedMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = psStringCopy("SELECT * FROM warpMask"); - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("warpool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpMask", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - -static bool unblockMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_STR(label, config->args, "-label", true, false); - - char *query = "DELETE FROM warpMask WHERE label = '%s'"; - - if (!p_psDBRunQueryF(config->dbh, query, label)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} - -static bool pendingcleanuprunMode(pxConfig *config) + psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d warpBackgroundSkyfiles", numDeleted); + + return true; +} + +static bool tocleanwarpMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, NULL); @@ -1468,9 +1627,9 @@ psMetadata *where = psMetadataAlloc(); - pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); - - psString query = pxDataGet("warptool_pendingcleanuprun.sql"); + pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "=="); + + psString query = pxDataGet("bgtool_tocleanwarp.sql"); if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); return false; } @@ -1483,5 +1642,4 @@ psFree(where); - // treat limit == 0 as "no limit" if (limit) { psString limitString = psDBGenerateLimitSQL(limit); @@ -1491,5 +1649,5 @@ if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(psErrorCodeLast(), false, "database error"); psFree(query); return false; @@ -1499,9 +1657,9 @@ psArray *output = p_psDBFetchResult(config->dbh); if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); + psError(psErrorCodeLast(), false, "database error"); return false; } if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); + psTrace("bgtool", PS_LOG_INFO, "no rows found"); psFree(output); return true; @@ -1509,10 +1667,9 @@ // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupRun", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - + if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) { + psError(psErrorCodeLast(), false, "failed to print array"); + psFree(output); + return false; + } psFree(output); @@ -1520,24 +1677,29 @@ } - -static bool pendingcleanupwarpMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "=="); - - psString query = pxDataGet("warptool_pendingcleanupskyfile.sql"); +static bool cleanedwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "=="); + + PXOPT_LOOKUP_STR(state, config->args, "-state", true, false); + if (!pxIsValidCleanedState(state)) { + psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state); + return false; + } + + if (!psListLength(where->list)) { + psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set."); + return false; + } + + psString query = pxDataGet("bgtool_cleanedwarp.sql"); if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { + psError(psErrorCodeLast(), false, "failed to retreive SQL statement"); + return false; + } + + { psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); psStringAppend(&query, " AND %s", whereClause); @@ -1546,639 +1708,34 @@ psFree(where); - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); + if (!p_psDBRunQueryF(config->dbh, query, state)) { + psError(psErrorCodeLast(), false, "database error"); return false; } psFree(query); - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupWarp", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - - -static bool donecleanupMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); - - psString query = pxDataGet("warptool_donecleanup.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " AND %s", whereClause); - psFree(whereClause); - } - psFree(where); - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpDoneCleanup", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - - psFree(output); - - return true; -} - - -// update warpSkyfile.data_state to given value. -// afterwards, if all skfyiles in the run have the new state, update the state for the run as well -// shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile - -static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // warp_id, skycell_id are required - PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); - PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); - - psString query = pxDataGet("warptool_change_skyfile_data_state.sql"); - - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psString set_magicked_skyfile = psStringCopy(""); - psString set_magicked_run = psStringCopy(""); - if (!strcmp(data_state, "full")) { - // magicked is only an argument for for -tofullskyfile - PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false); - if (magicked) { - psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked); - psStringAppend(&set_magicked_run, "\n, warpRun.magicked = %" PRId64, magicked); - } - } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) { - // if magicked is currently nonzero set it to -1 - // Set warpRun.magicked when the first skyfile is cleaned - psStringAppend(&set_magicked_skyfile, "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1), warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)"); - } - - if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); - psFree(set_magicked_skyfile); - - query = pxDataGet("warptool_change_run_state.sql"); - if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(set_magicked_run); - - if (!psDBCommit(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} -static bool tocleanedskyfileMode(pxConfig *config) -{ - return change_skyfile_data_state(config, "cleaned", "goto_cleaned"); -} -static bool tofullskyfileMode(pxConfig *config) -{ - return change_skyfile_data_state(config, "full", "update"); -} -static bool topurgedskyfileMode(pxConfig *config) -{ - return change_skyfile_data_state(config, "purged", "goto_purged"); -} -static bool toscrubbedskyfileMode(pxConfig *config) -{ - return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed"); -} - -static bool updateskyfileMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // warp_id, skycell_id, fault are required - PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); - PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); - PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false); - PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false); - - if (!state) { - psString query = pxDataGet("warptool_updateskyfile.sql"); - - if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - psFree(query); - } - else { - if (strcmp(state,"error_cleaned") == 0) { - change_skyfile_data_state(config,"error_cleaned","goto_cleaned"); - } - else if (strcmp(state, "error_scrubbed") == 0) { - change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed"); - } - else if (strcmp(state, "error_purged") == 0) { - change_skyfile_data_state(config,"error_purged","goto_purged"); - } - else { - psError(PS_ERR_UNKNOWN, false, "unhandled state given"); - return(false); - } - } - - return true; -} - -bool exportrunMode(pxConfig *config) -{ - typedef struct ExportTable { - char tableName[80]; - char sqlFilename[80]; - } ExportTable; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(det_id, config->args, "-warp_id", true, false); + return true; +} + +static bool exportwarpMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false); PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true, false); - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false); - FILE *f = fopen (outfile, "w"); - if (f == NULL) { - psError(PS_ERR_UNKNOWN, false, "failed to open output file"); - return false; - } - - if (!pxExportVersion(config, f)) { - psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file"); - return false; - } - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "=="); - - ExportTable tables [] = { - {"warpRun", "warptool_export_run.sql"}, - {"warpImfile", "warptool_export_imfile.sql"}, - {"warpSkyfile", "warptool_export_skyfile.sql"}, - {"warpSkyCellMap", "warptool_export_skycell_map.sql"}, - }; - - int numTables = sizeof(tables)/sizeof(tables[0]); - - for (int i=0; i < numTables; i++) { - psString query = pxDataGet(tables[i].sqlFilename); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (where && psListLength(where->list)) { - psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); - psFree(whereClause); - } - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - if (!psArrayLength(output)) { - psError(PS_ERR_UNKNOWN, true, "no rows found"); - psFree(output); - return false; - } - - if (clean) { - bool success = true; - if (!strcmp(tables[i].tableName, "warpRun")) { - success = pxSetStateCleaned("warpRun", "state", output); - } else if (!strcmp(tables[i].tableName, "warpSkyfile")) { - success = pxSetStateCleaned("warpSkyfile", "data_state", output); - } - if (!success) { - psFree(output); - psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s", tables[i].tableName); - return false; - } - } - - // we must write the export table in non-simple (true) format - if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - psFree(output); - } - - fclose (f); - - return true; -} - -bool importrunMode(pxConfig *config) -{ - unsigned int nFail; - - int numImportTables = 3; - - char tables[3] [80] = {"warpImfile", "warpSkyfile", "warpSkyCellMap"}; - - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); - - psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false); - -#ifdef notdef - fprintf (stderr, "---- input ----\n"); - psMetadataPrint (stderr, input, 1); -#endif - - if (!pxCheckImportVersion(config, input)) { - psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed"); - return false; - } - - psMetadataItem *item = psMetadataLookup (input, "warpRun"); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - psMetadataItem *entry = psListGet (item->data.list, 0); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - warpRunRow *warpRun = warpRunObjectFromMetadata (entry->data.md); - warpRunInsertObject (config->dbh, warpRun); - - // fprintf (stdout, "---- warp run ----\n"); - // psMetadataPrint (stderr, entry->data.md, 1); - - for (int i = 0; i < numImportTables; i++) { - item = psMetadataLookup (input, tables[i]); - psAssert (item, "entry not in input?"); - psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?"); - - switch (i) { - case 0: - for (int i = 0; i < item->data.list->n; i++) { - entry = psListGet (item->data.list, i); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - warpImfileRow *warpImfile = warpImfileObjectFromMetadata (entry->data.md); - warpImfileInsertObject (config->dbh, warpImfile); - - // fprintf (stdout, "---- row %d ----\n", i); - // psMetadataPrint (stderr, entry->data.md, 1); - } - break; - - case 1: - for (int i = 0; i < item->data.list->n; i++) { - entry = psListGet (item->data.list, i); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - warpSkyfileRow *warpSkyfile = warpSkyfileObjectFromMetadata (entry->data.md); - warpSkyfileInsertObject (config->dbh, warpSkyfile); - - // fprintf (stdout, "---- row %d ----\n", i); - // psMetadataPrint (stderr, entry->data.md, 1); - } - break; - - case 2: - for (int i = 0; i < item->data.list->n; i++) { - entry = psListGet (item->data.list, i); - assert (entry); - assert (entry->type == PS_DATA_METADATA); - warpSkyCellMapRow *warpSkyCellMap = warpSkyCellMapObjectFromMetadata (entry->data.md); - warpSkyCellMapInsertObject (config->dbh, warpSkyCellMap); - - // fprintf (stdout, "---- row %d ----\n", i); - // psMetadataPrint (stderr, entry->data.md, 1); - } - break; - } - } - return true; -} - -static bool runstateMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); - -// PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); - PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - - psString query = pxDataGet("warptool_runstate.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - } else { - psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); - return false; - } - psFree(where); - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpRunState", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - -static bool listrunMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-tess_id", "warpRun.tess_id", "=="); - PXOPT_COPY_STR(config->args, where, "-state", "warpRun.state", "=="); - PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "=="); - PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "=="); - PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "=="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">="); - PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<="); - PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "LIKE"); - PXOPT_COPY_S64(config->args, where, "-magicked", "warpRun.magicked", "=="); - pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "LIKE"); - pxAddLabelSearchArgs (config, where, "-data_group", "warpRun.data_group", "LIKE"); - pxAddLabelSearchArgs (config, where, "-dist_group", "warpRun.dist_group", "LIKE"); - - PXOPT_LOOKUP_BOOL(all, config->args, "-all", false); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); - PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false); - - // find all rawImfiles matching the default query - psString query = pxDataGet("warptool_listrun.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - // generate where strings for arguments that require extra processing - // beyond PXOPT_COPY* - psString where2 = NULL; - if (!pxmagicAddWhere(config, &where2, "warpRun")) { - psError(psErrorCodeLast(), false, "pxMagicAddWhere failed"); - return false; - } - if (!pxspaceAddWhere(config, &where2, "rawExp")) { - psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed"); - return false; - } - - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); - psStringAppend(&query, " WHERE %s", whereClause); - psFree(whereClause); - } else if (!all && !where2) { - psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required"); - return false; - } - - if (where2) { - if (psListLength(where->list)) { - psStringAppend(&query, " %s", where2); - } else { - psStringAppend(&query, " WHERE 1 %s", where2); - } - } - psFree(where); - - if (pstamp_order) { - // put runs in order of exposure id with newest chip Runs first - // The postage stamp parser depends on this behavior - psStringAppend(&query, "\nORDER by exp_id, warp_id DESC"); - } - - - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *output = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); - psFree(output); - return true; - } - - if (psArrayLength(output)) { - // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) { - psError(PS_ERR_UNKNOWN, false, "failed to print array"); - psFree(output); - return false; - } - } - - psFree(output); - - return true; -} - -// a very specfic function to queue a cleaned warpSkyfile to be updated -static bool setskyfiletoupdateMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, NULL); - - PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false); - PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); - - psString query = pxDataGet("warptool_setskyfiletoupdate.sql"); - if (!query) { - psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement"); - return false; - } - - psString setHook = psStringCopy(""); - if (label) { - psStringAppend(&setHook, "\n , warpRun.label = '%s'", label); - } - - if (skycell_id) { - psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id); - } - - if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - psFree(setHook); - psFree(query); - - return true; -} + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "=="); + + bool status = exportTables(config, outfile, warpTables, where, clean); + + psFree(where); + return status; +} + +static bool importwarpMode(pxConfig *config) +{ + PXOPT_LOOKUP_STR(infile, config->args, "-infile", true, false); + + return importTables(config, infile, warpTables); +}