Changeset 18663 for trunk/ippTools/src/magictool.c

Timestamp:

Jul 22, 2008, 4:36:48 PM (18 years ago)

Author:

Paul Price

Message:

Reworking magictool -definebyquery to support automation: want to be able to run the query over and over, adding new runs as they become available. Be warned, the new queries are pretty complicated (at least, for me). Some files are included in CVS to help with testing and debugging which are not actually installed and used by magictool. Adding exp_id to magicRun.

File:

• trunk/ippTools/src/magictool.c (modified) (6 diffs)

trunk/ippTools/src/magictool.c

@@ -110 +110 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+    // Required
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
+
+    // Optional
     PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
     PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
     PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
-
-    // create warped skycells temp table
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // Create temporary table of the best diffs
     {
-        psString query = pxDataGet("magictool_create_tmp_warpcomplete.sql");
+        psString query = pxDataGet("magictool_definebyquery_temp_create.sql");
         if (!query) {
             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -127 +131 @@
             return false;
         }
-    }
-
-    // find warped skycells
+        psFree(query);
+    }
+
+    // Insert list of best diffs into temporary table
     {
-        psString query = pxDataGet("magictool_find_complete_warpruns.sql");
+        psString query = pxDataGet("magictool_definebyquery_temp_insert.sql");
         if (!query) {
             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
             return false;
         }
+
+        psMetadata *where = psMetadataAlloc();
+        PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+        PXOPT_COPY_S64(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
+        PXOPT_COPY_S64(config->args, where, "-good_frac", "diffSkyfile.good_frac", ">=");
+
+        if (psListLength(where->list)) {
+            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+            psStringAppend(&query, " AND %s", whereClause);
+            psFree(whereClause);
+        }
+        psFree(where);
+
+        psString groupby = pxDataGet("magictool_definebyquery_temp_insert_groupby.sql");
+        if (!groupby) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            psFree(query);
+            return false;
+        }
+
+        psStringAppend(&query, " %s", groupby);
+        psFree(groupby);
 
         if (!p_psDBRunQuery(config->dbh, query)) {
@@ -145 +172 @@
     }
 
-    // find the diff_id's of the warped skycells
+    // Get list of exposures ready to magic
     {
-        psString query = pxDataGet("magictool_find_complete_diffed_exposures.sql");
+        psString query = pxDataGet("magictool_definebyquery_select_part1.sql");
         if (!query) {
             psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
             return false;
         }
+
+        {
+            psMetadata *where = psMetadataAlloc();
+            PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+            PXOPT_COPY_S64(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
+
+            if (psListLength(where->list)) {
+                psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+                psStringAppend(&query, " AND %s", whereClause);
+                psFree(whereClause);
+            }
+            psFree(where);
+        }
+
+        psString part2 = pxDataGet("magictool_definebyquery_select_part2.sql");
+        if (!part2) {
+            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+            return false;
+        }
+
+        psStringAppend(&query, " %s", part2);
+        psFree(part2);
 
         if (!p_psDBRunQuery(config->dbh, query)) {
@@ -180 +229 @@
     }
 
-    // pre-allocate enough hash buckets for 1 unique warp_id per row
-    psHash *groups = psHashAlloc(psArrayLength(output));
-    // iterate over result set, and group by warp_id
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query
+
+    // iterate over array of grouped warp_ids
     for (long i = 0; i < psArrayLength(output); i++) {
-        // lookup the key we're grouping by
-        bool status = false;
-        psS64 warp_id = psMetadataLookupS64(&status, output->data[i], "warp_id");
-        if (!status) {
-            psAbort("failed to lookup value for warp_id");
-        }
-
-        psString key = psDBIntToString(warp_id);
-        // look to see if there is a bin for this key
-        psArray *groupBin = psHashLookup(groups, key);
-        if (!groupBin) {
-            // if not, create one
-            groupBin = psArrayAlloc(0);
-            psHashAdd(groups, key, groupBin);
-        }
-        psFree(key);
-
-        // add this row to the bin
-        psArrayAdd(groupBin, 0, output->data[i]);
-        psFree(groupBin);
-    }
-    psFree(output);
-
-    // create a magic run
-    psArray *grouped =  psHashToArray(groups);
-    psFree(groups);
-
-    // iterate over array of grouped warp_ids
-    for (long i = 0; i < psArrayLength(grouped); i++) {
-        psArray *group = grouped->data[i];
+        psMetadata *row = output->data[i]; // Row of interest
+        psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
 
         // create a new magicRun for this group
-        magicRunRow *run = magicRunRowAlloc(
-            0,                          // ID
-            "reg",                      // state
-            workdir,                    // workdir
-            "dirty",                    // workdir_state
-            label,                      // label
-            dvodb,                      // dvodb
-            registered,                 // registered
-            0                           // fault
-            );
+        magicRunRow *run = magicRunRowAlloc(0, exp_id, "reg", workdir, "dirty", label, dvodb, registered, 0);
         if (!run) {
             psAbort("failed to alloc magicRun object");
         }
+
         if (!magicRunInsertObject(config->dbh, run)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(run);
-            psFree(grouped);
-            return false;
-        }
-
-        // get the assigned warp_id
-        psS64 magic_id = psDBLastInsertID(config->dbh);
-
-        // insert all rows in this bin into the new magicRun
-        for (long j = 0; j < psArrayLength(group); j++) {
-            psMetadata *row = group->data[j];
-
-            bool status = false;
-            psS64 diff_id = psMetadataLookupS64(&status, row, "diff_id");
-            if (!status) {
-                psAbort("failed to lookup value for diff_id");
+            psFree(insert);
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
             }
-            psString node = psMetadataLookupStr(&status, row, "skycell_id");
-            if (!status) {
-                psAbort("failed to lookup value for diff_id");
+            return false;
+        }
+        if (!magicRunPrintObject(stdout, run, !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print object");
+            psFree(run);
+            psFree(insert);
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
             }
-
-            if (!magicInputSkyfileInsert(config->dbh,
-                                         magic_id,
-                                         diff_id,
-                                         node)) {
+            return false;
+        }
+        psFree(run);
+
+        psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier
+
+        // Create a suitable insertion query for this run
+        psString thisInsert = psStringCopy(insert);
+        psString idString = NULL;
+        psStringAppend(&idString, "%" PRId64, magic_id);
+        psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@");
+        psFree(idString);
+
+        if (!p_psDBRunQuery(config->dbh, thisInsert, magic_id)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(thisInsert);
+            psFree(insert);
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
                 psError(PS_ERR_UNKNOWN, false, "database error");
-                psFree(grouped);
-                return false;
             }
-        }
-
-        // set magicRun.state to 'run'
-        return setmagicRunState(config, magic_id, "run");
-    }
-    psFree(grouped);
+            return false;
+        }
+        psFree(thisInsert);
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(output);
 
     return true;
@@ -277 +306 @@
     // required
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
+    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", true, false);
 
     // optional
@@ -286 +316 @@
     magicRunRow *run = magicRunRowAlloc(
             0,          // ID
+            exp_id,
             "reg",      // state
             workdir,