Changeset 25429

Timestamp:

Sep 17, 2009, 10:04:08 AM (17 years ago)

Author:

bills

Message:

Speed up disttool -pendingcomponent by having separate queries for each stage.

Location:

trunk

Files:

• ippTasks/dist.pro (modified) (3 diffs)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/disttool_pending_camera.sql (added)
• ippTools/share/disttool_pending_chip.sql (added)
• ippTools/share/disttool_pending_diff.sql (added)
• ippTools/share/disttool_pending_fake.sql (added)
• ippTools/share/disttool_pending_raw.sql (added)
• ippTools/share/disttool_pending_stack.sql (added)
• ippTools/share/disttool_pending_warp.sql (added)
• ippTools/share/disttool_pendingcomponent.sql (deleted)
• ippTools/src/disttool.c (modified) (4 diffs)
• ippTools/src/disttoolConfig.c (modified) (1 diff)

trunk/ippTasks/dist.pro

@@ -16 +16 @@
 $distQueue_DB = 0
 
+### list of stages
+#list of stages
+$STAGE:n = 0
+list STAGE -add "raw"
+list STAGE -add "chip"
+list STAGE -add "camera"
+list STAGE -add "fake"
+list STAGE -add "warp"
+list STAGE -add "diff"
+list STAGE -add "stack"
+
+$currentStage = 0
+
 ### Check status of tasks
 macro dist.status
@@ -78 +91 @@
 
   task.exec
-    $run = disttool -pendingcomponent
+    $run = disttool -pendingcomponent -stage $STAGE:$currentStage
+    $currentStage ++
+    if ($currentStage >= $STAGE:n) set currentStage = 0
     if ($DB:n == 0)
       option DEFAULT
@@ -85 +100 @@
       option $DB:$distToProcess_DB
       $run = $run -dbname $DB:$distToProcess_DB
-      $distToProcess_DB ++
-      if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0
+
+      # only increment the database number after we have gone through all of
+      # the stages
+      if ($currentStage == 0)
+          $distToProcess_DB ++
+          if ($distToProcess_DB >= $DB:n) set distToProcess_DB = 0
+      end
     end
     add_poll_args run

trunk/ippTools/share/Makefile.am

@@ -110 +110 @@
      disttool_definebyquery_stack.sql \
      disttool_definebyquery_warp.sql \
-     disttool_pendingcomponent.sql \
+     disttool_pending_camera.sql \
+     disttool_pending_chip.sql \
+     disttool_pending_diff.sql \
+     disttool_pending_fake.sql \
+     disttool_pending_raw.sql \
+     disttool_pending_stack.sql \
+     disttool_pending_warp.sql \
      disttool_pendingfileset.sql \
      disttool_pendingdest.sql \

trunk/ippTools/src/disttool.c

@@ -2 +2 @@
  * disttool.c
  *
- * Copyright (C) 2008
+ * Copyright (C) 2008-2009
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -600 +600 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
+
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    // look for "inputs" that need to processed
-    psString query = pxDataGet("disttool_pendingcomponent.sql");
+    psString queryFile = NULL;
+    psStringAppend(&queryFile, "disttool_pending_%s.sql", stage);
+    psString query = pxDataGet(queryFile);
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile);
         return false;
     }
@@ -617 +619 @@
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " WHERE %s", whereClause);
+        psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
@@ -629 +631 @@
     }
 
-    // the query has where hooks for each stage.
-    // right now we aren't using them.
-    // XXX: I think that I want to change the query from a union of selects on the various
-    // stages to separate queries. As it is pending data at the later stages of the pipline
-    // will get blocked by pending earlier stages
-    psString    raw_where = "";
-    psString    raw_clean_where = "";
-    psString    chip_where = "";
-    psString    camera_where = "";
-    psString    fake_where = "";
-    psString    warp_where = "";
-    psString    diff_where = "";
-    psString    stack_where = "";
-
-    if (!p_psDBRunQueryF(config->dbh,
-            query,
-            raw_where,
-            raw_clean_where,
-            chip_where,
-            camera_where,
-            fake_where,
-            warp_where,
-            diff_where,
-            stack_where)) {
+    if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);

trunk/ippTools/src/disttoolConfig.c

@@ -95 +95 @@
     // -pendingcomponent
     psMetadata *pendingcomponentArgs = psMetadataAlloc();
+    psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage",    0, "limit results to runs for stage (required)", NULL);
     psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
-    psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage",    0, "limit results to runs for stage", NULL);
     psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label",    PS_META_DUPLICATE_OK, "limit results to label", NULL);
     psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);