Changeset 18530

Timestamp:

Jul 14, 2008, 8:54:45 PM (18 years ago)

Author:

eugene

Message:

updates from HEAD

Location:

branches/eam_branch_20080706

Files:

• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/magictool_addmask.sql (added)
• ippTools/share/magictool_inputs.sql (added)
• ippTools/share/magictool_mask.sql (added)
• ippTools/share/magictool_toprocess_inputs.sql (modified) (1 diff)
• ippTools/share/magictool_toprocess_tree.sql (modified) (1 diff)
• ippTools/share/magictool_totree.sql (added)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff)
• ippTools/share/stacktool_tosum.sql (modified) (1 diff)
• ippTools/src/magictool.c (modified) (21 diffs)
• ippTools/src/magictool.h (modified) (1 diff)
• ippTools/src/magictoolConfig.c (modified) (5 diffs)
• ippTools/src/pxwarp.c (modified) (2 diffs)
• psModules/src/imcombine/pmPSFEnvelope.c (modified) (3 diffs)
• psModules/src/imcombine/pmStackReject.c (modified) (1 diff)

branches/eam_branch_20080706/ippTools/share/Makefile.am

@@ -80 +80 @@
         magictool_find_unmagiced.sql \
         magictool_inputskyfile.sql \
+        magictool_totree.sql \
         magictool_tomask.sql \
         magictool_toprocess_inputs.sql \
         magictool_toprocess_tree.sql \
         magictool_toskyfilemask.sql \
+        magictool_inputs.sql \
+        magictool_addmask.sql \
+        magictool_mask.sql \
         pstamptool_addjob_otherjob.sql \
         pstamptool_addjob_stampjob.sql \

branches/eam_branch_20080706/ippTools/share/magictool_toprocess_inputs.sql

@@ -3 +3 @@
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    diffSkyfile.uri
+    magicNodeResult.magic_id IS TRUE as done
 FROM magicTree
 JOIN magicRun

branches/eam_branch_20080706/ippTools/share/magictool_toprocess_tree.sql

@@ -3 +3 @@
     -- convert magic_id into a boolean value (1 or 0)
     -- note that the type stays a 64 bit int
-    magicNodeResult.magic_id IS TRUE as done,
-    magicNodeResult.uri
+    magicNodeResult.magic_id IS TRUE as done
 FROM magicTree
 JOIN magicRun

branches/eam_branch_20080706/ippTools/share/pxadmin_create_tables.sql

@@ -719 +719 @@
 CREATE TABLE diffInputSkyfile (diff_id BIGINT, template TINYINT, stack_id BIGINT, warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), kind VARCHAR(64), PRIMARY KEY(diff_id, template), KEY(stack_id), KEY(warp_id), KEY(skycell_id), KEY(tess_id), KEY(kind), FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id), FOREIGN KEY (stack_id)  REFERENCES  stackSumSkyfile(stack_id), FOREIGN KEY (warp_id, skycell_id, tess_id)  REFERENCES  warpSkyfile(warp_id, skycell_id, tess_id)) ENGINE=innodb DEFAULT CHARSET=latin1;
 CREATE TABLE diffSkyfile (diff_id BIGINT, uri VARCHAR(255), path_base VARCHAR(255), bg DOUBLE, bg_stdev DOUBLE, stamps_num INT, stamps_rms FLOAT, sources INT, dtime_diff FLOAT, hostname VARCHAR(64), good_frac FLOAT, fault SMALLINT, PRIMARY KEY(diff_id), KEY(good_frac), KEY(fault), FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id)) ENGINE=innodb DEFAULT CHARSET=latin1;
-CREATE TABLE magicRun (magic_id BIGINT AUTO_INCREMENT, state VARCHAR(64), workdir VARCHAR(255), workdir_state VARCHAR(255), label VARCHAR(64), dvodb VARCHAR(255), registered DATETIME, PRIMARY KEY(magic_id), KEY(magic_id), KEY(state), KEY(workdir_state), KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1;
-CREATE TABLE magicInputSkyfile (magic_id BIGINT, diff_id BIGINT, node VARCHAR(64), PRIMARY KEY(magic_id, diff_id), FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id), FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id)) ENGINE=innodb DEFAULT CHARSET=latin1;
-CREATE TABLE magicTree (magic_id BIGINT, node VARCHAR(64), dep VARCHAR(64), KEY(magic_id), KEY(node), KEY(dep), INDEX(magic_id, node), FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id)) ENGINE=innodb DEFAULT CHARSET=latin1;
-CREATE TABLE magicNodeResult (magic_id BIGINT, node VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(magic_id, node), FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id), FOREIGN KEY (magic_id, node)  REFERENCES  magicTree(magic_id, node)) ENGINE=innodb DEFAULT CHARSET=latin1;
-CREATE TABLE magicMask (magic_id BIGINT, uri VARCHAR(255), PRIMARY KEY(magic_id), FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id)) ENGINE=innodb DEFAULT CHARSET=latin1;
-CREATE TABLE magicSkyfileMask (magic_id BIGINT, diff_id BIGINT, uri VARCHAR(255), PRIMARY KEY(magic_id, diff_id), KEY(uri), FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id), FOREIGN KEY (magic_id, diff_id)  REFERENCES  magicInputSkyfile(magic_id, diff_id), FOREIGN KEY (magic_id)  REFERENCES  magicMask(magic_id)) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE magicRun (
+        magic_id BIGINT AUTO_INCREMENT,
+        state VARCHAR(64),
+        workdir VARCHAR(255),
+        workdir_state VARCHAR(255),
+        label VARCHAR(64),
+        dvodb VARCHAR(255),
+        registered DATETIME,
+        fault SMALLINT,
+        PRIMARY KEY(magic_id),
+        KEY(magic_id),
+        KEY(state),
+        KEY(workdir_state),
+        KEY(label),
+        KEY(fault)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE magicInputSkyfile (
+        magic_id BIGINT,
+        diff_id BIGINT,
+        node VARCHAR(64),
+        PRIMARY KEY(magic_id, diff_id),
+        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id),
+        FOREIGN KEY (diff_id)  REFERENCES  diffRun(diff_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE magicTree (
+        magic_id BIGINT,
+        node VARCHAR(64),
+        dep VARCHAR(64),
+        KEY(magic_id),
+        KEY(node),
+        KEY(dep),
+        INDEX(magic_id, node),
+        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE magicNodeResult (
+        magic_id BIGINT,
+        node VARCHAR(64),
+        uri VARCHAR(255),
+        fault SMALLINT,
+        PRIMARY KEY(magic_id, node),
+        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id),
+        FOREIGN KEY (magic_id, node)  REFERENCES  magicTree(magic_id, node),
+        KEY(fault)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+CREATE TABLE magicMask (
+        magic_id BIGINT,
+        uri VARCHAR(255),
+        streaks INT,
+        fault SMALLINT,
+        PRIMARY KEY(magic_id),
+        FOREIGN KEY (magic_id)  REFERENCES  magicRun(magic_id),
+        KEY(fault)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
 CREATE TABLE calDB (cal_id BIGINT AUTO_INCREMENT, dvodb VARCHAR(64), state VARCHAR(64), PRIMARY KEY(cal_id), KEY(cal_id)) ENGINE=innodb DEFAULT CHARSET=latin1;
 CREATE TABLE calRun (cal_id BIGINT AUTO_INCREMENT, region VARCHAR(64), last_step VARCHAR(64), state VARCHAR(64), PRIMARY KEY(cal_id), KEY(cal_id), KEY(last_step), FOREIGN KEY (cal_id)  REFERENCES  calDB(cal_id)) ENGINE=innodb DEFAULT CHARSET=latin1;

branches/eam_branch_20080706/ippTools/share/stacktool_tosum.sql

@@ -1 @@
-SELECT
+SELECT DISTINCT
     stackRun.stack_id,
     stackRun.tess_id,
     stackRun.skycell_id,
-    stackRun.workdir
+    stackRun.workdir,
+    rawExp.camera
 FROM stackRun
+JOIN stackInputSkyfile
+    USING(stack_id)
+JOIN warpSkyfile
+    ON  stackInputSkyfile.warp_id = warpSkyfile.warp_id
+    AND stackRun.skycell_id       = warpSkyfile.skycell_id
+    AND stackRun.tess_id          = warpSkyfile.tess_id
+JOIN warpRun
+    ON warpRun.warp_id = warpSkyfile.warp_id
+JOIN fakeRun
+    USING(fake_id)
+JOIN camRun
+    USING(cam_id)
+JOIN chipRun
+    USING(chip_id)
+JOIN rawExp
+    USING(exp_id)
 LEFT JOIN stackSumSkyfile
     USING(stack_id)

branches/eam_branch_20080706/ippTools/src/magictool.c

@@ -36 +36 @@
 static bool addinputskyfileMode(pxConfig *config);
 static bool inputskyfileMode(pxConfig *config);
+static bool totreeMode(pxConfig *config);
 static bool inputtreeMode(pxConfig *config);
 static bool toprocessMode(pxConfig *config);
 static bool addresultMode(pxConfig *config);
+static bool inputsMode(pxConfig *config);
 static bool tomaskMode(pxConfig *config);
 static bool addmaskMode(pxConfig *config);
-static bool toskyfilemaskMode(pxConfig *config);
-static bool addskyfilemaskMode(pxConfig *config);
+static bool maskMode(pxConfig *config);
 
 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
@@ -65 +66 @@
 
     switch (config->mode) {
-        MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY,          definebyqueryMode);
+        MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY,  definebyqueryMode);
         MODECASE(MAGICTOOL_MODE_DEFINERUN,      definerunMode);
         MODECASE(MAGICTOOL_MODE_UPDATERUN,      updaterunMode);
         MODECASE(MAGICTOOL_MODE_ADDINPUTSKYFILE,addinputskyfileMode);
         MODECASE(MAGICTOOL_MODE_INPUTSKYFILE,   inputskyfileMode);
+        MODECASE(MAGICTOOL_MODE_TOTREE,         totreeMode);
         MODECASE(MAGICTOOL_MODE_INPUTTREE,      inputtreeMode);
         MODECASE(MAGICTOOL_MODE_TOPROCESS,      toprocessMode);
         MODECASE(MAGICTOOL_MODE_ADDRESULT,      addresultMode);
+        MODECASE(MAGICTOOL_MODE_INPUTS,         inputsMode);
         MODECASE(MAGICTOOL_MODE_TOMASK,         tomaskMode);
         MODECASE(MAGICTOOL_MODE_ADDMASK,        addmaskMode);
-        MODECASE(MAGICTOOL_MODE_TOSKYFILEMASK,  toskyfilemaskMode);
-        MODECASE(MAGICTOOL_MODE_ADDSKYFILEMASK, addskyfilemaskMode);
+        MODECASE(MAGICTOOL_MODE_MASK,           maskMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -212 +214 @@
             NULL,       // label
             NULL,       // dvodb
-            NULL        // registered
+            NULL,       // registered
+            0           // fault
             );
         if (!run) {
@@ -263 +266 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    // required 
+    // required
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
 
@@ -279 +282 @@
             label,
             dvodb,
-            registered
+            registered,
+            0
     );
-    psFree(registered);
+
     if (!run) {
         psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object");
-        return true;
+        return false;
     }
     if (!magicRunInsertObject(config->dbh, run)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(run);
-        return true;
+        return false;
     }
 
@@ -400 +404 @@
     }
     if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
@@ -420 +424 @@
 
 
-static bool inputtreeMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    // required
-    PXOPT_LOOKUP_STR(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_STR(dep_file, config->args, "-dep_file", true, false);
-
-    if (!parseAndInsertNodeDeps(config, (psS64)atoll(magic_id), dep_file)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to parse file");
-        return false;
-    }
-
-    return true;
-}
-
-
-bool findBaseNodes(void *arg, pxNode *node)
-{
-    bool status = false;
-    psS64 done = psMetadataLookupS64(&status, node->data, "done");
-    if (!status) {
-        psAbort("failed to lookup value for done column");
-    }
-
-    if ((!pxNodeHasChildren(node)) && (!done)) {
-        // if this node has no child and it's not 'done', then push it's data
-        // onto the void *array
-        psArrayAdd((psArray *)arg, 0, node->data);
-        return false;
-    }
-    
-    return true;
-}
-
-
-bool findReadyNodes(void *arg, pxNode *node)
-{
-
-    if (pxNodeHasChildren(node)) {
-        psListIterator *iter = psListIteratorAlloc(node->children, 0, false);
-        psMetadata *work = psMetadataCopy(NULL, node->data);
-        psMetadataRemoveKey(work, "dep");
-        psMetadataRemoveKey(work, "done");
-        psMetadataRemoveKey(work, "uri");
-        pxNode *child = NULL;
-        while ((child = psListGetAndIncrement(iter))) {
-            psMetadata *data = child->data;
-
-            bool status = false;
-            psS32 done = psMetadataLookupS32(&status, data, "done");
-            if (!status) {
-                psAbort("failed to lookup value for done column");
-            }
-
-            if (!done) {
-                // if a child isn't "done", give up on this node and continue
-                // to crawl the tree
-                psFree(iter);
-                psFree(work);
-                return true;
-            }
-
-            char *uri = psMetadataLookupStr(&status, data, "uri");
-            if (!status) {
-                psAbort("failed to lookup value for uri column");
-            }
-
-            psMetadataAddStr(work, PS_LIST_TAIL, "uri", PS_META_DUPLICATE_OK, NULL, uri);
-
-        }
-        psFree(iter);
-        // if all this nodes children are done, then push it's data onto the
-        // void *array
-        psArrayAdd((psArray *)arg, 0, work);
-        psFree(work);
-        return false;
-    }
-    
-    return true;
-}
-
-
-static bool toprocessMode(pxConfig *config)
+static bool totreeMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -510 +431 @@
     PXOPT_COPY_STR(config->args, where, "-magic_id", "magic_id", "==");
 
-    psString whereClause = NULL;
-    if (psListLength(where->list)) {
-        whereClause = psDBGenerateWhereSQL(where, NULL);
-    }
-    psFree(where);
-
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
     // look for "inputs" that need to processed
-    psString query = pxDataGet("magictool_toprocess_inputs.sql");
+    psString query = pxDataGet("magictool_totree.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -525 +441 @@
     }
 
-    if (whereClause) {
-        psStringAppend(&query, " %s", whereClause);
+XXX : add where test
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
     }
 
@@ -552 +473 @@
     if (!psArrayLength(output)) {
         psTrace("magictool", PS_LOG_INFO, "no rows found");
-        // psFree(output);
-        // return true;
-    }
-
-    // look for tree nodes that need to be processed
-    query = pxDataGet("magictool_toprocess_tree.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (whereClause) {
-        psStringAppend(&query, " %s", whereClause);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *magicTree = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
-        return false;
-    }
-    if (!psArrayLength(magicTree)) {
-        psTrace("magictool", PS_LOG_INFO, "no rows found");
-        psFree(magicTree);
+        psFree(output);
         return true;
     }
-     
-    psHash *forest = psHashAlloc(psArrayLength(magicTree));
-
-    // convert the array of metadata into a pxTree structure
-    for (long i = 0; i < psArrayLength(magicTree); i++) {
-        bool status;
-        psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
-        if (!status) {
-            psAbort("failed to lookup value for node column");
-        }
-
-        psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
-        if (!status) {
-            psAbort("failed to lookup value for dep column");
-        }
-
-        pxTreeBuilder(forest, node, dep, magicTree->data[i]);
-
-    }
-    psFree(magicTree);
-
-    // find the root of the tree
-    pxNode *root = psHashLookup(forest, "root");
-    psFree(forest);
-    // pxTreePrint(stdout, root);
-
-    // crawl through the tree and looking for nodes with children that are all
-    // "done"
-    pxTreeCrawl(root, findReadyNodes, output);
-    psFree(root);
 
     if (psArrayLength(output)) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "totree", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -634 +487 @@
 
     psFree(output);
-    psFree(whereClause);
-
-    return true;
-}
-
-
-static bool addresultMode(pxConfig *config)
+
+    return true;
+}
+
+
+static bool inputtreeMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -646 +498 @@
     // required
     PXOPT_LOOKUP_STR(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_STR(node, config->args, "-node", true, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false);
-
-    if (!magicNodeResultInsert(config->dbh,
-                (psS64)atoll(magic_id),
-                node,
-                uri
-        )) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-
-
-static bool tomaskMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_STR(dep_file, config->args, "-dep_file", true, false);
+
+    if (!parseAndInsertNodeDeps(config, (psS64)atoll(magic_id), dep_file)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to parse file");
+        return false;
+    }
+
+    return true;
+}
+
+
+bool findBaseNodes(void *arg, pxNode *node)
+{
+    bool status = false;
+    psS64 done = psMetadataLookupS64(&status, node->data, "done");
+    if (!status) {
+        psAbort("failed to lookup value for done column");
+    }
+
+    if ((!pxNodeHasChildren(node)) && (!done)) {
+        // if this node has no child and it's not 'done', then push it's data
+        // onto the void *array
+        psArrayAdd((psArray *)arg, 0, node->data);
+        return false;
+    }
+
+    return true;
+}
+
+
+bool findReadyNodes(void *arg, pxNode *node)
+{
+    if (psMetadataLookupBool(NULL, node->data, "done")) {
+        // It's already done
+        return true;
+    }
+
+    if (pxNodeHasChildren(node)) {
+        psListIterator *iter = psListIteratorAlloc(node->children, 0, false);
+        psMetadata *work = psMetadataCopy(NULL, node->data);
+        psMetadataRemoveKey(work, "dep");
+        psMetadataRemoveKey(work, "done");
+#if 0
+        psMetadataRemoveKey(work, "uri");
+#endif
+        pxNode *child = NULL;
+        while ((child = psListGetAndIncrement(iter))) {
+            psMetadata *data = child->data;
+
+            bool status = false;
+            psS32 done = psMetadataLookupS32(&status, data, "done");
+            if (!status) {
+                psAbort("failed to lookup value for done column");
+            }
+
+            if (!done) {
+                // if a child isn't "done", give up on this node and continue
+                // to crawl the tree
+                psFree(iter);
+                psFree(work);
+                return true;
+            }
+#if 0
+            char *uri = psMetadataLookupStr(&status, data, "uri");
+            if (!status) {
+                psAbort("failed to lookup value for uri column");
+            }
+
+            psMetadataAddStr(work, PS_LIST_TAIL, "uri", PS_META_DUPLICATE_OK, NULL, uri);
+#endif
+        }
+        psFree(iter);
+        // if all this nodes children are done, then push it's data onto the
+        // void *array
+        psArrayAdd((psArray *)arg, 0, work);
+        psFree(work);
+        return false;
+    }
+
+    return true;
+}
+
+
+static bool toprocessMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_STR(config->args, where, "-magic_id", "magic_id", "==");
+
+    psString whereClause = NULL;
+    if (psListLength(where->list)) {
+        whereClause = psDBGenerateWhereSQL(where, NULL);
+    }
+    psFree(where);
+
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
     // look for "inputs" that need to processed
-    psString query = pxDataGet("magictool_tomask.sql");
+    psString query = pxDataGet("magictool_toprocess_inputs.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -676 +603 @@
     }
 
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
+    if (whereClause) {
+        psStringAppend(&query, " %s", whereClause);
     }
 
@@ -706 +630 @@
     if (!psArrayLength(output)) {
         psTrace("magictool", PS_LOG_INFO, "no rows found");
-        psFree(output);
+        // psFree(output);
+        // return true;
+    }
+
+    // look for tree nodes that need to be processed
+    query = pxDataGet("magictool_toprocess_tree.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (whereClause) {
+        psStringAppend(&query, " %s", whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *magicTree = p_psDBFetchResult(config->dbh);
+    if (!magicTree) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(magicTree)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(magicTree);
         return true;
     }
+
+    psHash *forest = psHashAlloc(psArrayLength(magicTree));
+
+    // convert the array of metadata into a pxTree structure
+    for (long i = 0; i < psArrayLength(magicTree); i++) {
+        bool status;
+        psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
+        if (!status) {
+            psAbort("failed to lookup value for node column");
+        }
+
+        psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
+        if (!status) {
+            psAbort("failed to lookup value for dep column");
+        }
+
+        pxTreeBuilder(forest, node, dep, magicTree->data[i]);
+
+    }
+    psFree(magicTree);
+
+    // find the root of the tree
+    pxNode *root = psHashLookup(forest, "root");
+    psFree(forest);
+    // pxTreePrint(stdout, root);
+
+    // crawl through the tree and looking for nodes with children that are all
+    // "done"
+    pxTreeCrawl(root, findReadyNodes, output);
+    psFree(root);
 
     if (psArrayLength(output)) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "toprocess", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -720 +712 @@
 
     psFree(output);
-
-    return true;
-}
-
-
-static bool addmaskMode(pxConfig *config)
+    psFree(whereClause);
+
+    return true;
+}
+
+
+static bool addresultMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -731 +724 @@
     // required
     PXOPT_LOOKUP_STR(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false);
-
-    if (!magicMaskInsert(config->dbh,
-                (psS64)atoll(magic_id),
-                uri
+    PXOPT_LOOKUP_STR(node, config->args, "-node", true, false);
+
+    // optional
+    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+
+    // default values
+    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+
+    if (!magicNodeResultInsert(config->dbh,
+                               (psS64)atoll(magic_id),
+                               node,
+                               uri,
+                               code
         )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -744 +745 @@
 }
 
-
-static bool toskyfilemaskMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-magic_id", "magic_id", "==");
+static bool inputsMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = pxDataGet("magictool_toskyfilemask.sql");
+    psString query = pxDataGet("magictool_inputs.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -761 +758 @@
     }
 
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, "magicRun");
+    if (config->where) {
+        psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL);
         psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
-    psFree(where);
 
     // treat limit == 0 as "no limit"
@@ -804 +800 @@
     if (psArrayLength(output)) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "toskyfilemask", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "magicNode", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -817 +813 @@
 
 
-static bool addskyfilemaskMode(pxConfig *config)
+static bool tomaskMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // look for "inputs" that need to processed
+    psString query = pxDataGet("magictool_tomask.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "toprocess", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool addmaskMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -823 +882 @@
     // required
     PXOPT_LOOKUP_STR(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_STR(diff_id, config->args, "diff_id", true, false);
-    PXOPT_LOOKUP_STR(uri, config->args, "uri", true, false);
-
-    if (!magicSkyfileMaskInsert(config->dbh,
-                (psS64)atoll(magic_id),
-                (psS64)atoll(diff_id),
-                uri
+
+    // optional
+    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+    PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false);
+
+    // default values
+    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!magicMaskInsert(config->dbh,
+                         (psS64)atoll(magic_id),
+                         uri,
+                         streaks,
+                         code
         )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
     }
+
+    // Set the magicRun state
+    psString query = pxDataGet("magictool_addmask.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // Add "magic_id = value"
+    psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+
+static bool maskMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_STR(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("magictool_mask.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "magicNode", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
 
     return true;

branches/eam_branch_20080706/ippTools/src/magictool.h

@@ -30 +30 @@
     MAGICTOOL_MODE_ADDINPUTSKYFILE,
     MAGICTOOL_MODE_INPUTSKYFILE,
+    MAGICTOOL_MODE_TOTREE,
     MAGICTOOL_MODE_INPUTTREE,
     MAGICTOOL_MODE_TOPROCESS,
+    MAGICTOOL_MODE_INPUTS,
     MAGICTOOL_MODE_ADDRESULT,
     MAGICTOOL_MODE_TOMASK,
     MAGICTOOL_MODE_ADDMASK,
-    MAGICTOOL_MODE_TOSKYFILEMASK,
-    MAGICTOOL_MODE_ADDSKYFILEMASK,
+    MAGICTOOL_MODE_MASK,
 } MAGICtoolMode;
 

branches/eam_branch_20080706/ippTools/src/magictoolConfig.c

@@ -77 +77 @@
     psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
 
+    // -totree
+    psMetadata *totreeArgs = psMetadataAlloc();
+    psMetadataAddStr(totreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", NULL);
+    psMetadataAddU64(totreeArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(totreeArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
     // -inputtree
     psMetadata *inputtreeArgs = psMetadataAlloc();
     psMetadataAddStr(inputtreeArgs, PS_LIST_TAIL, "-magic_id", 0,            "define magictool ID (required)", NULL);
     psMetadataAddStr(inputtreeArgs, PS_LIST_TAIL, "-dep_file", 0,            "order of operations dep. file (required)", NULL);
+    psMetadataAddS16(inputtreeArgs, PS_LIST_TAIL, "-code",  0, "set fault code", 0);
+
+    // -inputs
+    psMetadata *inputsArgs = psMetadataAlloc();
+    psMetadataAddStr(inputsArgs, PS_LIST_TAIL, "-magic_id", 0,            "search by magictool ID", NULL);
+    psMetadataAddStr(inputsArgs, PS_LIST_TAIL, "-node",  0,            "search by symbolic node name", NULL);
+    psMetadataAddU64(inputsArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddBool(inputsArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
 
     // -tooprocess
@@ -93 +107 @@
     psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node",  0,            "define symbolic node name (required)", NULL);
     psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri",  0,            "define URI (required)", NULL);
+    psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-code",  0, "set fault code", 0);
 
     // -tomask
@@ -101 +116 @@
     // -addmask
     psMetadata *addmaskArgs = psMetadataAlloc();
-    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0,            "define magictool ID (required)", NULL);
-    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri",  0,            "define URI (required)", NULL);
+    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0,          "define magictool ID (required)", NULL);
+    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri",  0, "define URI", NULL);
+    psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks",  0, "define number of streaks", 0);
+    psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-code",  0, "set fault code", 0);
 
+# if (0)
     // -toskyfilemask
     psMetadata *toskyfilemaskArgs = psMetadataAlloc();
@@ -115 +133 @@
     psMetadataAddStr(addskyfilemaskArgs, PS_LIST_TAIL, "-diff_id",  0,            "define difftool ID (required)", NULL);
     psMetadataAddStr(addskyfilemaskArgs, PS_LIST_TAIL, "-uri",  0,            "define URI (required)", NULL);
+# endif
+
+    // -mask
+    psMetadata *maskArgs = psMetadataAlloc();
+    psMetadataAddStr(maskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", NULL);
+    psMetadataAddU64(maskArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(maskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
     psFree(now);
@@ -121 +146 @@
     psMetadata *modes   = psMetadataAlloc();
 
-    PXOPT_ADD_MODE("-definebyquery",           "", MAGICTOOL_MODE_DEFINEBYQUERY,           queueArgs);
+    PXOPT_ADD_MODE("-definebyquery",   "", MAGICTOOL_MODE_DEFINEBYQUERY,   queueArgs);
     PXOPT_ADD_MODE("-definerun",       "", MAGICTOOL_MODE_DEFINERUN,       definerunArgs);
     PXOPT_ADD_MODE("-updaterun",       "", MAGICTOOL_MODE_UPDATERUN,       updaterunArgs);
     PXOPT_ADD_MODE("-addinputskyfile", "", MAGICTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs);
     PXOPT_ADD_MODE("-inputskyfile",    "", MAGICTOOL_MODE_INPUTSKYFILE,    inputskyfileArgs);
+    PXOPT_ADD_MODE("-totree",          "", MAGICTOOL_MODE_TOTREE,          totreeArgs);
     PXOPT_ADD_MODE("-inputtree",       "", MAGICTOOL_MODE_INPUTTREE,       inputtreeArgs);
     PXOPT_ADD_MODE("-toprocess",       "", MAGICTOOL_MODE_TOPROCESS,       toprocessArgs);
+    PXOPT_ADD_MODE("-inputs",          "", MAGICTOOL_MODE_INPUTS,          inputsArgs);
     PXOPT_ADD_MODE("-addresult",       "", MAGICTOOL_MODE_ADDRESULT,       addresultArgs);
     PXOPT_ADD_MODE("-tomask",          "", MAGICTOOL_MODE_TOMASK,          tomaskArgs);
     PXOPT_ADD_MODE("-addmask",         "", MAGICTOOL_MODE_ADDMASK,         addmaskArgs);
-    PXOPT_ADD_MODE("-toskyfilemask",   "", MAGICTOOL_MODE_TOSKYFILEMASK,   toskyfilemaskArgs);
-    PXOPT_ADD_MODE("-addskyfilemask",  "", MAGICTOOL_MODE_ADDSKYFILEMASK,  addskyfilemaskArgs);
+    PXOPT_ADD_MODE("-mask",            "", MAGICTOOL_MODE_MASK,            maskArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/eam_branch_20080706/ippTools/src/pxwarp.c

@@ -74 +74 @@
     }
 
-    psString query = psStringCopy("UPDATE warpRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET state = '%s'");
+    psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.state = '%s'");
 
     if (where) {
         psString whereClause = psDBGenerateWhereSQL(where, NULL);
-        psStringAppend(&query, " %s", whereClause);
+        if (whereClause && strlen(whereClause) > 0) {
+            psStringAppend(&query, " %s", whereClause);
+        }
         psFree(whereClause);
     }
@@ -115 +117 @@
     // note label == NULL should be explicitly allowed
 
-    psString query = psStringCopy("UPDATE warpRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.label = '%s'");
+    psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.label = '%s'");
 
     if (where) {

branches/eam_branch_20080706/psModules/src/imcombine/pmPSFEnvelope.c

@@ -34 +34 @@
 #define PEAK_FLUX 1.0e4                 // Peak flux for each source
 #define SKY_VALUE 0.0e0                 // Sky value for fake image
-#define WEIGHT_VAL 1.0                  // Weighting for image
+#define WEIGHT_VAL 10.0                 // Weighting for image
+#define WEIGHT_FACTOR 10.0              // Factor to multiply image by to get weighting
 #define PSF_STATS PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV // Statistics options for measuring PSF
 
@@ -199 +200 @@
     psFree(envelope);
 
-    // XXX This seems the best way to set the weight image so that pixels aren't rejected as "insignificant"
-    readout->weight = (psImage*)psBinaryOp(NULL, readout->image, "*", readout->image);
+    // XXX Setting the weight seems to be an art
+    // Can't set it too high so that pixels are rejected as insignificant
+    // Can't set it too low so that it's hard to get to the minimum
+    // Have also tried:
+    // *** readout->weight = (psImage*)psBinaryOp(NULL, readout->image, "*", readout->image);
+    // *** readout->weight = (psImage*)psBinaryOp(NULL, readout->image, "*", psScalarAlloc(WEIGHT_FACTOR, PS_TYPE_F32));
+    readout->weight = (psImage*)psBinaryOp(NULL, readout->image, "+", psScalarAlloc(PS_SQR(WEIGHT_VAL), PS_TYPE_F32));
     readout->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK);
     psImageInit(readout->mask, 0);
@@ -258 +264 @@
     options->psfFieldYo = 0;
 
-    pmSourceFitModelInit (15, 0.01, WEIGHT_VAL, options->poissonErrorsPhotLMM);
+    pmSourceFitModelInit(50, 0.01, WEIGHT_VAL, false);
 
     pmPSFtry *try = pmPSFtryModel(fakes, modelName, options, 0, 0xff);

branches/eam_branch_20080706/psModules/src/imcombine/pmStackReject.c

@@ -23 +23 @@
     PS_ASSERT_ARRAY_NON_NULL(kernels, NULL);
     PS_ASSERT_ARRAYS_SIZE_EQUAL(subRegions, kernels, NULL);
+
+    // Trivial case
+    if (in->n == 0) {
+        return psPixelsAllocEmpty(0);
+    }
 
     // Get the original image size