Changeset 24949

Timestamp:

Jul 30, 2009, 4:06:23 PM (17 years ago)

Author:

bills

Message:

to reduce query time get list of nodes to process for each pending magicRun individually

Location:

trunk/ippTools

Files:

• share/magictool_toprocess_runs.sql (added)
• src/magictool.c (modified) (6 diffs)

trunk/ippTools/src/magictool.c

@@ -804 +804 @@
     // treat limit == 0 as "no limit"
     if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
+        // cut limit in half
+        // hack to prevent pending leaf nodes from blocking branch nodes
+        psString limitString = psDBGenerateLimitSQL((limit + 1) / 2);
         psStringAppend(&query, " %s", limitString);
         psFree(limitString);
@@ -850 +852 @@
 
     // look for tree nodes that need to be processed
-    // XXX: This gets all nodes from all magicRuns that are in 'new'state
-    // That doens't seem particularly efficient
-    query = pxDataGet("magictool_toprocess_tree.sql");
+    query = pxDataGet("magictool_toprocess_runs.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -858 +858 @@
     }
 
+    {
+        psString limitString = psDBGenerateLimitSQL( 100 );
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
 
     if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString :  "")) {
@@ -868 +873 @@
     psFree(query);
 
-    psArray *magicTree = p_psDBFetchResult(config->dbh);
-    if (!magicTree) {
+    psArray *magicRuns = p_psDBFetchResult(config->dbh);
+    if (!magicRuns) {
         psErrorCode err = psErrorCodeLast();
         switch (err) {
@@ -882 +887 @@
         return false;
     }
-    if (!psArrayLength(magicTree)) {
+    if (!psArrayLength(magicRuns)) {
         psTrace("magictool", PS_LOG_INFO, "no rows found");
-        psFree(magicTree);
+        psFree(magicRuns);
         return true;
     }
 
-    // entries are ordered by magic_id
-    long index = 0;
-    while (index <  psArrayLength(magicTree)) {
-        bool status;
+    query = pxDataGet("magictool_toprocess_tree.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    for (psS64 index = 0; index < psArrayLength(magicRuns); index++) {
         if (limit && (psArrayLength(output) >= limit)) {
             break;
         }
-        psS64 current_magic_id = psMetadataLookupS64(&status, magicTree->data[index], "magic_id");
+        bool status;
+        psS64 magic_id = psMetadataLookupS64(&status, magicRuns->data[index], "magic_id");
         if (!status) {
             psAbort("failed to lookup value for magic_id column");
         }
-
-        // find the end of this block
-        long first = index;
-        long last = index;
-        for (long i = index + 1; i < psArrayLength(magicTree); i++) {
-            psS64 magic_id = psMetadataLookupS64(&status, magicTree->data[i], "magic_id");
-            if (!status) {
-                psAbort("failed to lookup value for magic_id column");
+        
+        whereString = NULL;
+        psStringAppend(&whereString, "\nAND (magic_id = %" PRId64 ")", magic_id);
+        if (!p_psDBRunQueryF(config->dbh, query, whereString )) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(whereString);
+            psFree(query);
+            return false;
+        }
+        psFree(whereString);
+        psArray *magicTree = p_psDBFetchResult(config->dbh);
+        if (!magicTree) {
+            psErrorCode err = psErrorCodeLast();
+            switch (err) {
+                case PS_ERR_DB_CLIENT:
+                    psError(PXTOOLS_ERR_SYS, false, "database error");
+                case PS_ERR_DB_SERVER:
+                    psError(PXTOOLS_ERR_PROG, false, "database error");
+                default:
+                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
             }
-            if (magic_id != current_magic_id) {
-                break;
-            }
-            last = i;
-        }
-
-        index = last + 1;
-
-        psHash *forest = psHashAlloc(last - first + 1);
+
+            return false;
+        }
+        psS64 length = psArrayLength(magicTree);
+        if (!length) {
+            psTrace("magictool", PS_LOG_INFO, "no rows found for magic_id %" PRId64, magic_id);
+            psFree(magicTree);
+            continue;
+        }
+
+        psHash *forest = psHashAlloc(length);
 
         // convert the array of metadata into a pxTree structure
-        for (long i = first; i <= last; i++) {
+        for (long i = 0; i < length; i++) {
             bool status;
             psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
@@ -944 +967 @@
         pxTreeCrawl(root, findReadyNodes, output);
         psFree(root);
-
-    }
-    psFree(magicTree);
+        psFree(magicTree);
+    }
 
     if (psArrayLength(output)) {