Changeset 29717 for trunk/ippTools

Timestamp:

Nov 9, 2010, 12:31:36 PM (16 years ago)

Author:

bills

Message:

rework warptool -towarped to first query for labels that have outstanding
warpRuns and loop over them in priority order. Much much faster when there are lots of warpSkyfiles to
process

Location:

trunk/ippTools

Files:

• share/Makefile.am (modified) (1 diff)
• share/warptool_towarped_labels.sql (added)
• src/warptool.c (modified) (2 diffs)

trunk/ippTools/share/Makefile.am

@@ -385 +385 @@
         warptool_tooverlap.sql \
         warptool_towarped.sql \
+        warptool_towarped_labels.sql \
         warptool_updateskyfile.sql \
         warptool_warped.sql \

trunk/ippTools/src/warptool.c

@@ -924 +924 @@
 
     psMetadata *where = psMetadataAlloc();
+
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    // find all rawImfiles matching the default query
-    psString query = pxDataGet("warptool_towarped.sql");
-    if (!query) {
+    // First find Label's with warpRuns in state new in priority order
+    psMetadata *labelWhere = psMetadataAlloc();
+    pxAddLabelSearchArgs (config, labelWhere, "-label", "warpRun.label", "==");
+
+    psString labelQuery = pxDataGet("warptool_towarped_labels.sql");
+    if (!labelQuery) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
-
-    psString whereStr = psStringCopy("");
+    psString labelWhereStr = psStringCopy("");
+    if (psListLength(labelWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(labelWhere, NULL);
+        psStringAppend(&labelWhereStr, "\n AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(labelWhere);
+
+    if (!p_psDBRunQueryF(config->dbh, labelQuery, labelWhereStr)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(labelQuery);
+        return false;
+    }
+    psFree(labelQuery);
+    psFree(labelWhereStr);
+
+    psArray *labelOutput = p_psDBFetchResult(config->dbh);
+    if (psArrayLength(labelOutput) == 0) {
+        psTrace("warptool", PS_LOG_INFO, "no labels with pending warps found.");
+        return true;
+    }
+
+    psString originalQuery = pxDataGet("warptool_towarped.sql");
+    if (!originalQuery) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString whereStr = NULL;
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
@@ -945 +975 @@
     psFree(where);
 
-    // treat limit == 0 as "no limit"
-    psString limitString = psStringCopy("\nORDER BY priority DESC, warp_id");
-    if (limit) {
-        // We apply the limit to both sides of the UNION to avoid slow queries
-        // and to the query itself to satisfy the user's requested limit
-        psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit));
-        psStringAppend(&query, " %s", limitString);
-    }
-
-    if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr,  limitString)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+    // Now loop over the labels and query each individually
+    for (long i = 0; i < psArrayLength(labelOutput); i++) {
+        psMetadata *labelRow = labelOutput->data[i];
+
+        bool status;
+        psString label = psMetadataLookupStr(&status, labelRow, "label");
+
+        psString query = psStringCopy(originalQuery);
+
+        // treat limit == 0 as "no limit"
+        psString limitString = psStringCopy("");
+        if (limit) {
+            // We apply the limit to both sides of the UNION to avoid slow queries
+            // and to the query itself to satisfy the user's requested limit
+            psStringAppend(&limitString, "\n%s", psDBGenerateLimitSQL(limit));
+            psStringAppend(&query, "%s", limitString);
+        }
+        psString thisWhere = NULL;
+        if (whereStr) {
+            psStringAppend(&thisWhere, whereStr);
+        }
+        psStringAppend(&thisWhere, "\nAND warpRun.label = '%s'", label); 
+
+        if (!p_psDBRunQueryF(config->dbh, query, thisWhere, limitString, thisWhere,  limitString)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(query);
+            return false;
+        }
+        psFree(limitString);
+        psFree(thisWhere);
         psFree(query);
-        return false;
-    }
-    psFree(limitString);
+
+        psArray *output = p_psDBFetchResult(config->dbh);
+        if (!output) {
+            psErrorCode err = psErrorCodeLast();
+            switch (err) {
+                case PS_ERR_DB_CLIENT:
+                    psError(PXTOOLS_ERR_SYS, false, "database error");
+                    case PS_ERR_DB_SERVER:
+                    psError(PXTOOLS_ERR_PROG, false, "database error");
+                default:
+                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
+            }
+
+            return false;
+        }
+        long outputLength = psArrayLength(output);
+        if (outputLength) {
+            // negative simple so the default is true
+            if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) {
+                psError(PS_ERR_UNKNOWN, false, "failed to print array");
+                psFree(output);
+                return false;
+            }
+        } else {
+            psTrace("warptool", PS_LOG_INFO, "no rows found for %s", label);
+            psFree(output);
+            continue;
+        }
+        psFree(output);
+
+        if (limit) {
+            limit -= outputLength;
+            if (limit <= 0) {
+                // All done
+                break;
+            }
+        }
+    }
+    psFree(labelOutput);
     psFree(whereStr);
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    if (psArrayLength(output)) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
-    }
-
-    psFree(output);
+    psFree(originalQuery);
 
     return true;