Changeset 27737 for trunk/ippTools/src/warptool.c

Timestamp:

Apr 23, 2010, 11:19:20 AM (16 years ago)

Author:

bills

Message:

Fix very slow query for -towarped. Re-enable update processing;
Add -setskyfiletoupdate mode to queue updates for a single skycell.
Add -listrun.
Various other changes to support the postage stamp server.

File:

• trunk/ippTools/src/warptool.c (modified) (11 diffs)

trunk/ippTools/src/warptool.c

@@ -59 +59 @@
 static bool importrunMode(pxConfig *config);
 static bool runstateMode(pxConfig *config);
+static bool listrunMode(pxConfig *config);
+static bool setskyfiletoupdateMode(pxConfig *config);
 
 static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile);
@@ -110 +112 @@
         MODECASE(WARPTOOL_MODE_IMPORTRUN,          importrunMode);
         MODECASE(WARPTOOL_MODE_RUNSTATE,           runstateMode);
+        MODECASE(WARPTOOL_MODE_LISTRUN,            listrunMode);
+        MODECASE(WARPTOOL_MODE_SETSKYFILETOUPDATE, setskyfiletoupdateMode);
 
         default:
@@ -916 +920 @@
 
     // treat limit == 0 as "no limit"
+    psString limitString = psStringCopy("\n");
     if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
+        // We apply the limit to both sides of the UNION
+        // to avoid slow queries and to the query itself
+        // to satisfy the user's requested limit
+        psStringAppend(&limitString, psDBGenerateLimitSQL(limit));
         psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
+    }
+
+#define TWO_WHERES
 #ifdef TWO_WHERES
-    if (!p_psDBRunQueryF(config->dbh, query, whereStr, whereStr)) {
+    if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr,  limitString)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -935 +943 @@
     }
 #endif
+    psFree(limitString);
     psFree(whereStr);
     psFree(query);
@@ -994 +1003 @@
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
-    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
+    PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
 
     // default values
@@ -1242 +1251 @@
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+        psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
     } else if (!all && !where2) {
@@ -1248 +1257 @@
         return false;
     }
+
+    if (where2) {
+        if (psListLength(where->list)) {
+            psStringAppend(&query, " %s", where2);
+        } else {
+            psStringAppend(&query, " WHERE 1 %s", where2);
+        }
+    }
     psFree(where);
-
-    if (where2) {
-        psStringAppend(&query, " %s", where2);
-    }
 
     // treat limit == 0 as "no limit"
@@ -1665 +1678 @@
     }
 
-    char *set_magicked_skyfile = "";
-    char *set_magicked_run = "";
+    psString set_magicked_skyfile = psStringCopy("");
+    psString set_magicked_run = psStringCopy("");
     if (!strcmp(data_state, "full")) {
-        set_magicked_skyfile = "\n , warpSkyfile.magicked = IF((warpSkyfile.magicked < 0 AND chipRun.magicked = 0), warpSkyfile.magicked, chipRun.magicked)";
-        set_magicked_run = "\n,  warpRun.magicked = IF((warpRun.magicked < 0 AND chipRun.magicked = 0), warpRun.magicked, chipRun.magicked)";
-
+        // magicked is only an argument for for -tofullskyfile
+        PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
+        if (magicked) {
+            psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked);
+            psStringAppend(&set_magicked_run, "\n,  warpRun.magicked = %" PRId64, magicked);
+        }
     } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
-        set_magicked_skyfile = "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1)";
-        set_magicked_run = "\n, warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)";
+        // if magicked is currently nonzero set it to -1
+        // Set warpRun.magicked when the first skyfile is cleaned
+        psStringAppend(&set_magicked_skyfile, "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1), warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)");
     }
 
@@ -1686 +1703 @@
     }
     psFree(query);
+    psFree(set_magicked_skyfile);
 
     query = pxDataGet("warptool_change_run_state.sql");
@@ -1696 +1714 @@
         return false;
     }
+    psFree(set_magicked_run);
 
     if (!psDBCommit(config->dbh)) {
@@ -2026 +2045 @@
     return true;
 }
+
+static bool listrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpRun.warp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-tess_id",    "warpRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-state",      "warpRun.state", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
+    PXOPT_COPY_S64(config->args, where, "-fake_id",    "fakeRun.fake_id", "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs",  ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawExp.dateobs",  "<=");
+    PXOPT_COPY_STR(config->args, where, "-filter",    "rawExp.filter", "LIKE");
+    PXOPT_COPY_S64(config->args, where, "-magicked", "warpRun.magicked", "==");
+    pxAddLabelSearchArgs (config, where, "-label",   "warpRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group",   "warpRun.data_group", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-dist_group",   "warpRun.dist_group", "LIKE");
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false);
+
+    // find all rawImfiles matching the default query
+    psString query = pxDataGet("warptool_listrun.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // generate where strings for arguments that require extra processing
+    // beyond PXOPT_COPY*
+    psString where2 = NULL;
+    if (!pxmagicAddWhere(config, &where2, "warpRun")) {
+        psError(psErrorCodeLast(), false, "pxMagicAddWhere failed");
+        return false;
+    }
+    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
+        psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else if (!all && !where2) {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+        return false;
+    }
+
+    if (where2) {
+        if (psListLength(where->list)) {
+            psStringAppend(&query, " %s", where2);
+        } else {
+            psStringAppend(&query, " WHERE 1 %s", where2);
+        }
+    }
+    psFree(where);
+
+    if (pstamp_order) {
+        // put runs in order of exposure id with newest chip Runs first
+        // The postage stamp parser depends on this behavior
+        psStringAppend(&query, "\nORDER by exp_id, warp_id DESC");
+    }
+
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("warptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+// a very specfic function to queue a cleaned warpSkyfile to be updated
+static bool setskyfiletoupdateMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
+    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+
+    psString query = pxDataGet("warptool_setskyfiletoupdate.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString setHook = psStringCopy("");
+    if (label) {
+        psStringAppend(&setHook, "\n , warpRun.label = '%s'", label);
+    }
+
+    if (skycell_id) {
+        psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psFree(setHook);
+    psFree(query);
+
+    return true;
+}