Changeset 20734

Timestamp:

Nov 13, 2008, 2:56:00 PM (17 years ago)

Author:

Paul Price

Message:

Cleaning up magictool -definebyquery so that part1 and part2 queries aren't required (psDBRunQuery takes %s etc). Adding '-available' flag which says 'don't worry if all the diffs didn't make it, just do it with whatever's immediately available.'

Location:

trunk/ippTools

Files:

• share/Makefile.am (modified) (1 diff)
• share/magictool_definebyquery_select.sql (added)
• share/magictool_definebyquery_select_part1.sql (deleted)
• share/magictool_definebyquery_select_part2.sql (deleted)
• share/magictool_definebyquery_temp_insert.sql (modified) (1 diff)
• share/magictool_definebyquery_temp_insert_groupby.sql (deleted)
• src/magictool.c (modified) (5 diffs)
• src/magictoolConfig.c (modified) (1 diff)

trunk/ippTools/share/Makefile.am

@@ -101 +101 @@
      magictool_definebyquery.sql \
      magictool_definebyquery_insert.sql \
-     magictool_definebyquery_select_part1.sql \
-     magictool_definebyquery_select_part2.sql \
+     magictool_definebyquery_select.sql \
      magictool_definebyquery_select_test.sql \
      magictool_definebyquery_temp_create.sql \
      magictool_definebyquery_temp_insert.sql \
-     magictool_definebyquery_temp_insert_groupby.sql \
      magictool_inputs.sql \
      magictool_inputskyfile.sql \

trunk/ippTools/share/magictool_definebyquery_temp_insert.sql

@@ -21 +21 @@
 WHERE
     diffSkyfile.fault = 0
+---- WHERE hook %s
+GROUP BY
+    exp_id,
+    warpSkyfile.skycell_id

trunk/ippTools/src/magictool.c

@@ -155 +155 @@
         PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
 
+        psString whereClause = NULL;    // WHERE conditions
         if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&query, " AND %s", whereClause);
+            whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringPrepend(&whereClause, "\n AND ");
+        }
+        psFree(where);
+
+        if (!p_psDBRunQuery(config->dbh, query, whereClause)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(whereClause);
-        }
-        psFree(where);
-
-        psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql");
-        if (!groupby) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
             psFree(query);
             return false;
         }
-
-        psStringAppend(&query, " %s", groupby);
-        psFree(groupby);
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(query);
-            return false;
-        }
+        psFree(whereClause);
         psFree(query);
     }
@@ -182 +174 @@
     // Get list of exposures ready to magic
     {
-        psString query = pxDataGet("magictool_definebyquery_select_part1.sql");
+        psString query = pxDataGet("magictool_definebyquery_select.sql");
         if (!query) {
             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -188 +180 @@
         }
 
+        psString whereClause = NULL;    // WHERE conditions
         {
             psMetadata *where = psMetadataAlloc();
@@ -194 +187 @@
 
             if (psListLength(where->list)) {
-                psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-                psStringAppend(&query, " AND %s", whereClause);
-                psFree(whereClause);
+                whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+                psStringPrepend(&whereClause, "\n AND ");
             }
             psFree(where);
         }
 
-        psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql");
-        if (!part2) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
-        }
-
-        psStringAppend(&query, " %s", part2);
-        psFree(part2);
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
+        // Only concern ourselves with exposures that have all diffs completed, unless we're told to only take
+        // what's available.
+        PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+        if (!available) {
+            psStringAppend(&query, " AND num_done = num_todo");
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query, whereClause ? whereClause : "")) {
             psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(whereClause);
             psFree(query);
             return false;
         }
-        psFree(query);
-    }
+        psFree(whereClause);
+        psFree(query);
+    }
+
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
@@ -236 +229 @@
         return true;
     }
+
+    // Parse the list of exposures ready to magic
 
     if (!psDBTransaction(config->dbh)) {

trunk/ippTools/src/magictoolConfig.c

@@ -55 +55 @@
     psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
     psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);