Changeset 15499

Timestamp:

Nov 7, 2007, 6:06:07 PM (19 years ago)

Author:

jhoblitt

Message:

add magictool -queue mode

Location:

trunk/ippTools

Files:

• share/Makefile.am (modified) (1 diff)
• share/magictool_find_complete_diffed_exposures.sql (added)
• share/magictool_find_complete_warpruns.sql (modified) (1 diff)
• share/magictool_find_unmagiced.sql (modified) (3 diffs)
• src/magictool.c (modified) (3 diffs)
• src/magictool.h (modified) (1 diff)
• src/magictoolConfig.c (modified) (2 diffs)

trunk/ippTools/share/Makefile.am

@@ -42 +42 @@
         magictool_tomask.sql \
         magictool_toskyfilemask.sql \
+        magictool_find_complete_diffed_exposures.sql \
+        magictool_find_complete_warpruns.sql \
         pztool_find_completed_exp.sql \
         pztool_pendingimfile.sql \

trunk/ippTools/share/magictool_find_complete_warpruns.sql

@@ -1 +1 @@
 -- find stopped warp runsthat have had all of their skycell's warped without
 -- any faults
-SELECT 
-    warp_id
+INSERT INTO warpComplete
+SELECT
+    warp_id,
+    skycell_id,
+    tess_id
 FROM (
     SELECT 

trunk/ippTools/share/magictool_find_unmagiced.sql

@@ -8 +8 @@
         warpSkyfile.fault as f1,
         diffSkyfile.fault as f2,
-        warpSkyCellMap.skycell_id as s1,
+        warpSkyCellMap.skycell_id as wscm_s,
+        warpSkyfile.skycell_id as wsf_s,
         diffInputSkyfile.skycell_id as s2
     FROM warpRun
@@ -26 +27 @@
     LEFT JOIN magicInputSkyfile
         USING(diff_id)
-    LEFT JOIN magicRun
-        USING (magic_id)
     WHERE
         warpRun.state = 'stop'
@@ -42 +41 @@
     --    warpRun.warp_id, warpSkyCellMap.skycell_id, warpSkyCellMap.tess_id
     HAVING
-        COUNT(warpSkyCellMap.skycell_id) = COUNT(diffInputSkyfile.skycell_id)
+        COUNT(warpSkyCellMap.skycell_id) = COUNT(warpSkyfile.skycell_id)
+        AND COUNT(warpSkyCellMap.skycell_id) = COUNT(diffInputSkyfile.skycell_id)
         AND SUM(warpSkyfile.fault) = 0
         AND SUM(diffSkyfile.fault) = 0 

trunk/ippTools/src/magictool.c

@@ -31 +31 @@
 #include "magictool.h"
 
+static bool queueMode(pxConfig *config);
 static psS64 definerunMode(pxConfig *config);
 static bool updaterunMode(pxConfig *config);
@@ -63 +64 @@
 
     switch (config->mode) {
+        MODECASE(MAGICTOOL_MODE_QUEUE,          queueMode);
         MODECASE(MAGICTOOL_MODE_DEFINERUN,      definerunMode);
         MODECASE(MAGICTOOL_MODE_UPDATERUN,      updaterunMode);
@@ -92 +94 @@
 
     exit(exit_status);
+}
+
+static bool queueMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // create warped skycells temp table
+{
+    char *query =
+        "CREATE TEMPORARY TABLE warpComplete\n"
+        " (warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64),\n"
+        " PRIMARY KEY(warp_id, skycell_id, tess_id)) ENGINE=MEMORY\n";
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+}
+
+    // find warped skycells
+{
+    psString query = pxDataGet("magictool_find_complete_warpruns.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+}
+
+    // find the diff_id's of the warped skycells
+{
+    psString query = pxDataGet("magictool_find_complete_diffed_exposures.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+}
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // pre-allocate enough hash buckets for 1 unique warp_id per row
+    psHash *groups = psHashAlloc(psArrayLength(output));
+    // iterate over result set, and group by warp_id
+    for (long i = 0; i < psArrayLength(output); i++) {
+        // lookup the key we're grouping by
+        bool status = false;
+        psS64 warp_id = psMetadataLookupS64(&status, output->data[i], "warp_id");
+        if (!status) {
+            psAbort("failed to lookup value for warp_id");
+        }
+
+        psString key = psDBIntToString(warp_id);
+        // look to see if there is a bin for this key
+        psArray *groupBin = psHashLookup(groups, key);
+        if (!groupBin) {
+            // if not, create one
+            groupBin = psArrayAlloc(0);
+            psHashAdd(groups, key, groupBin);
+        }
+        psFree(key);
+
+        // add this row to the bin
+        psArrayAdd(groupBin, 0, output->data[i]);
+        psFree(groupBin);
+    }
+    psFree(output);
+
+    // create a magic run
+    psArray *grouped =  psHashToArray(groups);
+    psFree(groups);
+
+    // iterate over array of grouped warp_ids
+    for (long i = 0; i < psArrayLength(grouped); i++) {
+        psArray *group = grouped->data[i];
+
+        // create a new magicRun for this group
+        // XXX This need to have options passed through to it
+
+        magicRunRow *run = magicRunRowAlloc(
+                0,          // ID
+                "reg",      // state
+                NULL,       // workdir
+                "dirty",    // workdir_state
+                NULL,       // label
+                NULL,       // dvodb
+                NULL        // registered
+        );
+        if (!run) {
+            psAbort("failed to alloc magicRun object");
+        }
+        if (!magicRunInsertObject(config->dbh, run)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(run);
+            psFree(grouped);
+            return false;
+        }
+
+        // get the assigned warp_id
+        psS64 magic_id = psDBLastInsertID(config->dbh);
+
+        // insert all rows in this bin into the new magicRun
+        for (long j = 0; j < psArrayLength(group); j++) {
+            psMetadata *row = group->data[j];
+
+            bool status = false;
+            psS64 diff_id = psMetadataLookupS64(&status, row, "diff_id");
+            if (!status) {
+                psAbort("failed to lookup value for diff_id");
+            }
+            psString node = psMetadataLookupStr(&status, row, "skycell_id");
+            if (!status) {
+                psAbort("failed to lookup value for diff_id");
+            }
+
+            if (!magicInputSkyfileInsert(config->dbh,
+                magic_id,
+                diff_id,
+                node)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(grouped);
+                return false;
+            }
+        }
+
+        // set magicRun.state to 'run'
+        return setmagicRunState(config, magic_id, "run");
+    }
+
+    return true;
 }
 

trunk/ippTools/src/magictool.h

@@ -25 +25 @@
 typedef enum {
     MAGICTOOL_MODE_NONE           = 0x0,
+    MAGICTOOL_MODE_QUEUE,
     MAGICTOOL_MODE_DEFINERUN,
     MAGICTOOL_MODE_UPDATERUN,

trunk/ippTools/src/magictoolConfig.c

@@ -48 +48 @@
     psString now = psTimeToISO(time);
     psFree(time);
+
+    // -queue
+    psMetadata *queueArgs = psMetadataAlloc();
 
     // -definerun
@@ -146 +149 @@
     psMetadata *modes   = psMetadataAlloc();
 
+    PXTOOL_ADD_MODE("-queue",           "", MAGICTOOL_MODE_QUEUE,           queueArgs);
     PXTOOL_ADD_MODE("-definerun",       "", MAGICTOOL_MODE_DEFINERUN,       definerunArgs);
     PXTOOL_ADD_MODE("-updaterun",       "", MAGICTOOL_MODE_UPDATERUN,       updaterunArgs);