Index: /branches/eam_branch_20080706/ippTools/share/Makefile.am =================================================================== --- /branches/eam_branch_20080706/ippTools/share/Makefile.am (revision 18529) +++ /branches/eam_branch_20080706/ippTools/share/Makefile.am (revision 18530) @@ -80,8 +80,12 @@ magictool_find_unmagiced.sql \ magictool_inputskyfile.sql \ + magictool_totree.sql \ magictool_tomask.sql \ magictool_toprocess_inputs.sql \ magictool_toprocess_tree.sql \ magictool_toskyfilemask.sql \ + magictool_inputs.sql \ + magictool_addmask.sql \ + magictool_mask.sql \ pstamptool_addjob_otherjob.sql \ pstamptool_addjob_stampjob.sql \ Index: /branches/eam_branch_20080706/ippTools/share/magictool_addmask.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/magictool_addmask.sql (revision 18530) +++ /branches/eam_branch_20080706/ippTools/share/magictool_addmask.sql (revision 18530) @@ -0,0 +1,6 @@ +UPDATE + magicRun +SET + state = 'stop' +WHERE + state != 'stop' Index: /branches/eam_branch_20080706/ippTools/share/magictool_inputs.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/magictool_inputs.sql (revision 18530) +++ /branches/eam_branch_20080706/ippTools/share/magictool_inputs.sql (revision 18530) @@ -0,0 +1,29 @@ +SELECT * +FROM ( +-- Single skycells +SELECT + magicInputSkyfile.node, + magicRun.state, + diffSkyfile.uri, + diffSkyfile.fault +FROM magicInputSkyfile +JOIN diffSkyfile + using(diff_id) +JOIN magicRun + USING(magic_id) +UNION +-- Merged skycells +SELECT + magicTree.node, + magicRun.state, + magicNodeResult.uri, + magicNodeResult.fault +FROM magicTree +JOIN magicRun + USING(magic_id) +JOIN magicNodeResult + ON magicTree.magic_id = magicNodeResult.magic_id + AND magicTree.dep = magicNodeResult.node +) as Foo +WHERE + fault = 0 Index: /branches/eam_branch_20080706/ippTools/share/magictool_mask.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/magictool_mask.sql (revision 18530) +++ /branches/eam_branch_20080706/ippTools/share/magictool_mask.sql (revision 18530) @@ -0,0 +1,8 @@ +SELECT + magicMask.* +FROM magicMask +JOIN magicRun + USING(magic_id) +WHERE + magicRun.state = 'stop' + AND magicMask.fault = 0 Index: /branches/eam_branch_20080706/ippTools/share/magictool_toprocess_inputs.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/magictool_toprocess_inputs.sql (revision 18529) +++ /branches/eam_branch_20080706/ippTools/share/magictool_toprocess_inputs.sql (revision 18530) @@ -3,6 +3,5 @@ -- convert magic_id into a boolean value (1 or 0) -- note that the type stays a 64 bit int - magicNodeResult.magic_id IS TRUE as done, - diffSkyfile.uri + magicNodeResult.magic_id IS TRUE as done FROM magicTree JOIN magicRun Index: /branches/eam_branch_20080706/ippTools/share/magictool_toprocess_tree.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/magictool_toprocess_tree.sql (revision 18529) +++ /branches/eam_branch_20080706/ippTools/share/magictool_toprocess_tree.sql (revision 18530) @@ -3,6 +3,5 @@ -- convert magic_id into a boolean value (1 or 0) -- note that the type stays a 64 bit int - magicNodeResult.magic_id IS TRUE as done, - magicNodeResult.uri + magicNodeResult.magic_id IS TRUE as done FROM magicTree JOIN magicRun Index: /branches/eam_branch_20080706/ippTools/share/magictool_totree.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/magictool_totree.sql (revision 18530) +++ /branches/eam_branch_20080706/ippTools/share/magictool_totree.sql (revision 18530) @@ -0,0 +1,9 @@ +SELECT + magic_id +FROM magicRun +LEFT JOIN magicTree + USING(magic_id) +WHERE + magicRun.state = 'run' + AND magicTree.node IS NULL + AND magicRun.fault = 0 Index: /branches/eam_branch_20080706/ippTools/share/pxadmin_create_tables.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/pxadmin_create_tables.sql (revision 18529) +++ /branches/eam_branch_20080706/ippTools/share/pxadmin_create_tables.sql (revision 18530) @@ -719,10 +719,63 @@ CREATE TABLE diffInputSkyfile (diff_id BIGINT, template TINYINT, stack_id BIGINT, warp_id BIGINT, skycell_id VARCHAR(64), tess_id VARCHAR(64), kind VARCHAR(64), PRIMARY KEY(diff_id, template), KEY(stack_id), KEY(warp_id), KEY(skycell_id), KEY(tess_id), KEY(kind), FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id), FOREIGN KEY (stack_id) REFERENCES stackSumSkyfile(stack_id), FOREIGN KEY (warp_id, skycell_id, tess_id) REFERENCES warpSkyfile(warp_id, skycell_id, tess_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE diffSkyfile (diff_id BIGINT, uri VARCHAR(255), path_base VARCHAR(255), bg DOUBLE, bg_stdev DOUBLE, stamps_num INT, stamps_rms FLOAT, sources INT, dtime_diff FLOAT, hostname VARCHAR(64), good_frac FLOAT, fault SMALLINT, PRIMARY KEY(diff_id), KEY(good_frac), KEY(fault), FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE magicRun (magic_id BIGINT AUTO_INCREMENT, state VARCHAR(64), workdir VARCHAR(255), workdir_state VARCHAR(255), label VARCHAR(64), dvodb VARCHAR(255), registered DATETIME, PRIMARY KEY(magic_id), KEY(magic_id), KEY(state), KEY(workdir_state), KEY(label)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE magicInputSkyfile (magic_id BIGINT, diff_id BIGINT, node VARCHAR(64), PRIMARY KEY(magic_id, diff_id), FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE magicTree (magic_id BIGINT, node VARCHAR(64), dep VARCHAR(64), KEY(magic_id), KEY(node), KEY(dep), INDEX(magic_id, node), FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE magicNodeResult (magic_id BIGINT, node VARCHAR(64), uri VARCHAR(255), PRIMARY KEY(magic_id, node), FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), FOREIGN KEY (magic_id, node) REFERENCES magicTree(magic_id, node)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE magicMask (magic_id BIGINT, uri VARCHAR(255), PRIMARY KEY(magic_id), FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id)) ENGINE=innodb DEFAULT CHARSET=latin1; -CREATE TABLE magicSkyfileMask (magic_id BIGINT, diff_id BIGINT, uri VARCHAR(255), PRIMARY KEY(magic_id, diff_id), KEY(uri), FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), FOREIGN KEY (magic_id, diff_id) REFERENCES magicInputSkyfile(magic_id, diff_id), FOREIGN KEY (magic_id) REFERENCES magicMask(magic_id)) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicRun ( + magic_id BIGINT AUTO_INCREMENT, + state VARCHAR(64), + workdir VARCHAR(255), + workdir_state VARCHAR(255), + label VARCHAR(64), + dvodb VARCHAR(255), + registered DATETIME, + fault SMALLINT, + PRIMARY KEY(magic_id), + KEY(magic_id), + KEY(state), + KEY(workdir_state), + KEY(label), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicInputSkyfile ( + magic_id BIGINT, + diff_id BIGINT, + node VARCHAR(64), + PRIMARY KEY(magic_id, diff_id), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY (diff_id) REFERENCES diffRun(diff_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicTree ( + magic_id BIGINT, + node VARCHAR(64), + dep VARCHAR(64), + KEY(magic_id), + KEY(node), + KEY(dep), + INDEX(magic_id, node), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicNodeResult ( + magic_id BIGINT, + node VARCHAR(64), + uri VARCHAR(255), + fault SMALLINT, + PRIMARY KEY(magic_id, node), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + FOREIGN KEY (magic_id, node) REFERENCES magicTree(magic_id, node), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + +CREATE TABLE magicMask ( + magic_id BIGINT, + uri VARCHAR(255), + streaks INT, + fault SMALLINT, + PRIMARY KEY(magic_id), + FOREIGN KEY (magic_id) REFERENCES magicRun(magic_id), + KEY(fault) +) ENGINE=innodb DEFAULT CHARSET=latin1; + CREATE TABLE calDB (cal_id BIGINT AUTO_INCREMENT, dvodb VARCHAR(64), state VARCHAR(64), PRIMARY KEY(cal_id), KEY(cal_id)) ENGINE=innodb DEFAULT CHARSET=latin1; CREATE TABLE calRun (cal_id BIGINT AUTO_INCREMENT, region VARCHAR(64), last_step VARCHAR(64), state VARCHAR(64), PRIMARY KEY(cal_id), KEY(cal_id), KEY(last_step), FOREIGN KEY (cal_id) REFERENCES calDB(cal_id)) ENGINE=innodb DEFAULT CHARSET=latin1; Index: /branches/eam_branch_20080706/ippTools/share/stacktool_tosum.sql =================================================================== --- /branches/eam_branch_20080706/ippTools/share/stacktool_tosum.sql (revision 18529) +++ /branches/eam_branch_20080706/ippTools/share/stacktool_tosum.sql (revision 18530) @@ -1,8 +1,25 @@ -SELECT +SELECT DISTINCT stackRun.stack_id, stackRun.tess_id, stackRun.skycell_id, - stackRun.workdir + stackRun.workdir, + rawExp.camera FROM stackRun +JOIN stackInputSkyfile + USING(stack_id) +JOIN warpSkyfile + ON stackInputSkyfile.warp_id = warpSkyfile.warp_id + AND stackRun.skycell_id = warpSkyfile.skycell_id + AND stackRun.tess_id = warpSkyfile.tess_id +JOIN warpRun + ON warpRun.warp_id = warpSkyfile.warp_id +JOIN fakeRun + USING(fake_id) +JOIN camRun + USING(cam_id) +JOIN chipRun + USING(chip_id) +JOIN rawExp + USING(exp_id) LEFT JOIN stackSumSkyfile USING(stack_id) Index: /branches/eam_branch_20080706/ippTools/src/magictool.c =================================================================== --- /branches/eam_branch_20080706/ippTools/src/magictool.c (revision 18529) +++ /branches/eam_branch_20080706/ippTools/src/magictool.c (revision 18530) @@ -36,11 +36,12 @@ static bool addinputskyfileMode(pxConfig *config); static bool inputskyfileMode(pxConfig *config); +static bool totreeMode(pxConfig *config); static bool inputtreeMode(pxConfig *config); static bool toprocessMode(pxConfig *config); static bool addresultMode(pxConfig *config); +static bool inputsMode(pxConfig *config); static bool tomaskMode(pxConfig *config); static bool addmaskMode(pxConfig *config); -static bool toskyfilemaskMode(pxConfig *config); -static bool addskyfilemaskMode(pxConfig *config); +static bool maskMode(pxConfig *config); static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state); @@ -65,16 +66,17 @@ switch (config->mode) { - MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); + MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY, definebyqueryMode); MODECASE(MAGICTOOL_MODE_DEFINERUN, definerunMode); MODECASE(MAGICTOOL_MODE_UPDATERUN, updaterunMode); MODECASE(MAGICTOOL_MODE_ADDINPUTSKYFILE,addinputskyfileMode); MODECASE(MAGICTOOL_MODE_INPUTSKYFILE, inputskyfileMode); + MODECASE(MAGICTOOL_MODE_TOTREE, totreeMode); MODECASE(MAGICTOOL_MODE_INPUTTREE, inputtreeMode); MODECASE(MAGICTOOL_MODE_TOPROCESS, toprocessMode); MODECASE(MAGICTOOL_MODE_ADDRESULT, addresultMode); + MODECASE(MAGICTOOL_MODE_INPUTS, inputsMode); MODECASE(MAGICTOOL_MODE_TOMASK, tomaskMode); MODECASE(MAGICTOOL_MODE_ADDMASK, addmaskMode); - MODECASE(MAGICTOOL_MODE_TOSKYFILEMASK, toskyfilemaskMode); - MODECASE(MAGICTOOL_MODE_ADDSKYFILEMASK, addskyfilemaskMode); + MODECASE(MAGICTOOL_MODE_MASK, maskMode); default: psAbort("invalid option (this should not happen)"); @@ -212,5 +214,6 @@ NULL, // label NULL, // dvodb - NULL // registered + NULL, // registered + 0 // fault ); if (!run) { @@ -263,5 +266,5 @@ PS_ASSERT_PTR_NON_NULL(config, false); - // required + // required PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); @@ -279,15 +282,16 @@ label, dvodb, - registered + registered, + 0 ); - psFree(registered); + if (!run) { psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object"); - return true; + return false; } if (!magicRunInsertObject(config->dbh, run)) { psError(PS_ERR_UNKNOWN, false, "database error"); psFree(run); - return true; + return false; } @@ -400,5 +404,5 @@ } if (!psArrayLength(output)) { - psTrace("warptool", PS_LOG_INFO, "no rows found"); + psTrace("magictool", PS_LOG_INFO, "no rows found"); psFree(output); return true; @@ -420,88 +424,5 @@ -static bool inputtreeMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // required - PXOPT_LOOKUP_STR(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_STR(dep_file, config->args, "-dep_file", true, false); - - if (!parseAndInsertNodeDeps(config, (psS64)atoll(magic_id), dep_file)) { - psError(PS_ERR_UNKNOWN, false, "failed to parse file"); - return false; - } - - return true; -} - - -bool findBaseNodes(void *arg, pxNode *node) -{ - bool status = false; - psS64 done = psMetadataLookupS64(&status, node->data, "done"); - if (!status) { - psAbort("failed to lookup value for done column"); - } - - if ((!pxNodeHasChildren(node)) && (!done)) { - // if this node has no child and it's not 'done', then push it's data - // onto the void *array - psArrayAdd((psArray *)arg, 0, node->data); - return false; - } - - return true; -} - - -bool findReadyNodes(void *arg, pxNode *node) -{ - - if (pxNodeHasChildren(node)) { - psListIterator *iter = psListIteratorAlloc(node->children, 0, false); - psMetadata *work = psMetadataCopy(NULL, node->data); - psMetadataRemoveKey(work, "dep"); - psMetadataRemoveKey(work, "done"); - psMetadataRemoveKey(work, "uri"); - pxNode *child = NULL; - while ((child = psListGetAndIncrement(iter))) { - psMetadata *data = child->data; - - bool status = false; - psS32 done = psMetadataLookupS32(&status, data, "done"); - if (!status) { - psAbort("failed to lookup value for done column"); - } - - if (!done) { - // if a child isn't "done", give up on this node and continue - // to crawl the tree - psFree(iter); - psFree(work); - return true; - } - - char *uri = psMetadataLookupStr(&status, data, "uri"); - if (!status) { - psAbort("failed to lookup value for uri column"); - } - - psMetadataAddStr(work, PS_LIST_TAIL, "uri", PS_META_DUPLICATE_OK, NULL, uri); - - } - psFree(iter); - // if all this nodes children are done, then push it's data onto the - // void *array - psArrayAdd((psArray *)arg, 0, work); - psFree(work); - return false; - } - - return true; -} - - -static bool toprocessMode(pxConfig *config) +static bool totreeMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -510,14 +431,9 @@ PXOPT_COPY_STR(config->args, where, "-magic_id", "magic_id", "=="); - psString whereClause = NULL; - if (psListLength(where->list)) { - whereClause = psDBGenerateWhereSQL(where, NULL); - } - psFree(where); - + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // look for "inputs" that need to processed - psString query = pxDataGet("magictool_toprocess_inputs.sql"); + psString query = pxDataGet("magictool_totree.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -525,6 +441,11 @@ } - if (whereClause) { - psStringAppend(&query, " %s", whereClause); +XXX : add where test + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); } @@ -552,79 +473,11 @@ if (!psArrayLength(output)) { psTrace("magictool", PS_LOG_INFO, "no rows found"); - // psFree(output); - // return true; - } - - // look for tree nodes that need to be processed - query = pxDataGet("magictool_toprocess_tree.sql"); - if (!query) { - psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); - return false; - } - - if (whereClause) { - psStringAppend(&query, " %s", whereClause); - } - - if (!p_psDBRunQuery(config->dbh, query)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(query); - return false; - } - psFree(query); - - psArray *magicTree = p_psDBFetchResult(config->dbh); - if (!output) { - psErrorCode err = psErrorCodeLast(); - switch (err) { - case PS_ERR_DB_CLIENT: - psError(PXTOOLS_ERR_SYS, false, "database error"); - case PS_ERR_DB_SERVER: - psError(PXTOOLS_ERR_PROG, false, "database error"); - default: - psError(PXTOOLS_ERR_PROG, false, "unknown error"); - } - - return false; - } - if (!psArrayLength(magicTree)) { - psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(magicTree); + psFree(output); return true; } - - psHash *forest = psHashAlloc(psArrayLength(magicTree)); - - // convert the array of metadata into a pxTree structure - for (long i = 0; i < psArrayLength(magicTree); i++) { - bool status; - psString node = psMetadataLookupStr(&status, magicTree->data[i], "node"); - if (!status) { - psAbort("failed to lookup value for node column"); - } - - psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep"); - if (!status) { - psAbort("failed to lookup value for dep column"); - } - - pxTreeBuilder(forest, node, dep, magicTree->data[i]); - - } - psFree(magicTree); - - // find the root of the tree - pxNode *root = psHashLookup(forest, "root"); - psFree(forest); - // pxTreePrint(stdout, root); - - // crawl through the tree and looking for nodes with children that are all - // "done" - pxTreeCrawl(root, findReadyNodes, output); - psFree(root); if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "totree", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -634,11 +487,10 @@ psFree(output); - psFree(whereClause); - - return true; -} - - -static bool addresultMode(pxConfig *config) + + return true; +} + + +static bool inputtreeMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -646,29 +498,104 @@ // required PXOPT_LOOKUP_STR(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_STR(node, config->args, "-node", true, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - - if (!magicNodeResultInsert(config->dbh, - (psS64)atoll(magic_id), - node, - uri - )) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - return true; -} - - -static bool tomaskMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_STR(dep_file, config->args, "-dep_file", true, false); + + if (!parseAndInsertNodeDeps(config, (psS64)atoll(magic_id), dep_file)) { + psError(PS_ERR_UNKNOWN, false, "failed to parse file"); + return false; + } + + return true; +} + + +bool findBaseNodes(void *arg, pxNode *node) +{ + bool status = false; + psS64 done = psMetadataLookupS64(&status, node->data, "done"); + if (!status) { + psAbort("failed to lookup value for done column"); + } + + if ((!pxNodeHasChildren(node)) && (!done)) { + // if this node has no child and it's not 'done', then push it's data + // onto the void *array + psArrayAdd((psArray *)arg, 0, node->data); + return false; + } + + return true; +} + + +bool findReadyNodes(void *arg, pxNode *node) +{ + if (psMetadataLookupBool(NULL, node->data, "done")) { + // It's already done + return true; + } + + if (pxNodeHasChildren(node)) { + psListIterator *iter = psListIteratorAlloc(node->children, 0, false); + psMetadata *work = psMetadataCopy(NULL, node->data); + psMetadataRemoveKey(work, "dep"); + psMetadataRemoveKey(work, "done"); +#if 0 + psMetadataRemoveKey(work, "uri"); +#endif + pxNode *child = NULL; + while ((child = psListGetAndIncrement(iter))) { + psMetadata *data = child->data; + + bool status = false; + psS32 done = psMetadataLookupS32(&status, data, "done"); + if (!status) { + psAbort("failed to lookup value for done column"); + } + + if (!done) { + // if a child isn't "done", give up on this node and continue + // to crawl the tree + psFree(iter); + psFree(work); + return true; + } +#if 0 + char *uri = psMetadataLookupStr(&status, data, "uri"); + if (!status) { + psAbort("failed to lookup value for uri column"); + } + + psMetadataAddStr(work, PS_LIST_TAIL, "uri", PS_META_DUPLICATE_OK, NULL, uri); +#endif + } + psFree(iter); + // if all this nodes children are done, then push it's data onto the + // void *array + psArrayAdd((psArray *)arg, 0, work); + psFree(work); + return false; + } + + return true; +} + + +static bool toprocessMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-magic_id", "magic_id", "=="); + + psString whereClause = NULL; + if (psListLength(where->list)) { + whereClause = psDBGenerateWhereSQL(where, NULL); + } + psFree(where); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); // look for "inputs" that need to processed - psString query = pxDataGet("magictool_tomask.sql"); + psString query = pxDataGet("magictool_toprocess_inputs.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -676,9 +603,6 @@ } - // treat limit == 0 as "no limit" - if (limit) { - psString limitString = psDBGenerateLimitSQL(limit); - psStringAppend(&query, " %s", limitString); - psFree(limitString); + if (whereClause) { + psStringAppend(&query, " %s", whereClause); } @@ -706,11 +630,79 @@ if (!psArrayLength(output)) { psTrace("magictool", PS_LOG_INFO, "no rows found"); - psFree(output); + // psFree(output); + // return true; + } + + // look for tree nodes that need to be processed + query = pxDataGet("magictool_toprocess_tree.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (whereClause) { + psStringAppend(&query, " %s", whereClause); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *magicTree = p_psDBFetchResult(config->dbh); + if (!magicTree) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(magicTree)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(magicTree); return true; } + + psHash *forest = psHashAlloc(psArrayLength(magicTree)); + + // convert the array of metadata into a pxTree structure + for (long i = 0; i < psArrayLength(magicTree); i++) { + bool status; + psString node = psMetadataLookupStr(&status, magicTree->data[i], "node"); + if (!status) { + psAbort("failed to lookup value for node column"); + } + + psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep"); + if (!status) { + psAbort("failed to lookup value for dep column"); + } + + pxTreeBuilder(forest, node, dep, magicTree->data[i]); + + } + psFree(magicTree); + + // find the root of the tree + pxNode *root = psHashLookup(forest, "root"); + psFree(forest); + // pxTreePrint(stdout, root); + + // crawl through the tree and looking for nodes with children that are all + // "done" + pxTreeCrawl(root, findReadyNodes, output); + psFree(root); if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "toprocess", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -720,10 +712,11 @@ psFree(output); - - return true; -} - - -static bool addmaskMode(pxConfig *config) + psFree(whereClause); + + return true; +} + + +static bool addresultMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -731,9 +724,17 @@ // required PXOPT_LOOKUP_STR(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false); - - if (!magicMaskInsert(config->dbh, - (psS64)atoll(magic_id), - uri + PXOPT_LOOKUP_STR(node, config->args, "-node", true, false); + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!magicNodeResultInsert(config->dbh, + (psS64)atoll(magic_id), + node, + uri, + code )) { psError(PS_ERR_UNKNOWN, false, "database error"); @@ -744,16 +745,12 @@ } - -static bool toskyfilemaskMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - psMetadata *where = psMetadataAlloc(); - PXOPT_COPY_STR(config->args, where, "-magic_id", "magic_id", "=="); +static bool inputsMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); - psString query = pxDataGet("magictool_toskyfilemask.sql"); + psString query = pxDataGet("magictool_inputs.sql"); if (!query) { psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); @@ -761,10 +758,9 @@ } - if (psListLength(where->list)) { - psString whereClause = psDBGenerateWhereConditionSQL(where, "magicRun"); + if (config->where) { + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); psStringAppend(&query, " AND %s", whereClause); psFree(whereClause); } - psFree(where); // treat limit == 0 as "no limit" @@ -804,5 +800,5 @@ if (psArrayLength(output)) { // negative simple so the default is true - if (!ippdbPrintMetadatas(stdout, output, "toskyfilemask", !simple)) { + if (!ippdbPrintMetadatas(stdout, output, "magicNode", !simple)) { psError(PS_ERR_UNKNOWN, false, "failed to print array"); psFree(output); @@ -817,5 +813,68 @@ -static bool addskyfilemaskMode(pxConfig *config) +static bool tomaskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + // look for "inputs" that need to processed + psString query = pxDataGet("magictool_tomask.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "toprocess", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); + + return true; +} + + +static bool addmaskMode(pxConfig *config) { PS_ASSERT_PTR_NON_NULL(config, false); @@ -823,15 +882,122 @@ // required PXOPT_LOOKUP_STR(magic_id, config->args, "-magic_id", true, false); - PXOPT_LOOKUP_STR(diff_id, config->args, "diff_id", true, false); - PXOPT_LOOKUP_STR(uri, config->args, "uri", true, false); - - if (!magicSkyfileMaskInsert(config->dbh, - (psS64)atoll(magic_id), - (psS64)atoll(diff_id), - uri + + // optional + PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false); + PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false); + + // default values + PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); + + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!magicMaskInsert(config->dbh, + (psS64)atoll(magic_id), + uri, + streaks, + code )) { psError(PS_ERR_UNKNOWN, false, "database error"); return false; } + + // Set the magicRun state + psString query = pxDataGet("magictool_addmask.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + // Add "magic_id = value" + psString whereClause = psDBGenerateWhereConditionSQL(config->where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + if (!psDBCommit(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + return true; +} + + +static bool maskMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + psMetadata *where = psMetadataAlloc(); + PXOPT_COPY_STR(config->args, where, "-magic_id", "magicRun.magic_id", "=="); + + PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); + + psString query = pxDataGet("magictool_mask.sql"); + if (!query) { + psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement"); + return false; + } + + if (psListLength(where->list)) { + psString whereClause = psDBGenerateWhereConditionSQL(where, NULL); + psStringAppend(&query, " AND %s", whereClause); + psFree(whereClause); + } + psFree(where); + + // treat limit == 0 as "no limit" + if (limit) { + psString limitString = psDBGenerateLimitSQL(limit); + psStringAppend(&query, " %s", limitString); + psFree(limitString); + } + + if (!p_psDBRunQuery(config->dbh, query)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + psFree(query); + return false; + } + psFree(query); + + psArray *output = p_psDBFetchResult(config->dbh); + if (!output) { + psErrorCode err = psErrorCodeLast(); + switch (err) { + case PS_ERR_DB_CLIENT: + psError(PXTOOLS_ERR_SYS, false, "database error"); + case PS_ERR_DB_SERVER: + psError(PXTOOLS_ERR_PROG, false, "database error"); + default: + psError(PXTOOLS_ERR_PROG, false, "unknown error"); + } + + return false; + } + if (!psArrayLength(output)) { + psTrace("magictool", PS_LOG_INFO, "no rows found"); + psFree(output); + return true; + } + + if (psArrayLength(output)) { + // negative simple so the default is true + if (!ippdbPrintMetadatas(stdout, output, "magicNode", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + } + + psFree(output); return true; Index: /branches/eam_branch_20080706/ippTools/src/magictool.h =================================================================== --- /branches/eam_branch_20080706/ippTools/src/magictool.h (revision 18529) +++ /branches/eam_branch_20080706/ippTools/src/magictool.h (revision 18530) @@ -30,11 +30,12 @@ MAGICTOOL_MODE_ADDINPUTSKYFILE, MAGICTOOL_MODE_INPUTSKYFILE, + MAGICTOOL_MODE_TOTREE, MAGICTOOL_MODE_INPUTTREE, MAGICTOOL_MODE_TOPROCESS, + MAGICTOOL_MODE_INPUTS, MAGICTOOL_MODE_ADDRESULT, MAGICTOOL_MODE_TOMASK, MAGICTOOL_MODE_ADDMASK, - MAGICTOOL_MODE_TOSKYFILEMASK, - MAGICTOOL_MODE_ADDSKYFILEMASK, + MAGICTOOL_MODE_MASK, } MAGICtoolMode; Index: /branches/eam_branch_20080706/ippTools/src/magictoolConfig.c =================================================================== --- /branches/eam_branch_20080706/ippTools/src/magictoolConfig.c (revision 18529) +++ /branches/eam_branch_20080706/ippTools/src/magictoolConfig.c (revision 18530) @@ -77,8 +77,22 @@ psMetadataAddBool(inputskyfileArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -totree + psMetadata *totreeArgs = psMetadataAlloc(); + psMetadataAddStr(totreeArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", NULL); + psMetadataAddU64(totreeArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(totreeArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + // -inputtree psMetadata *inputtreeArgs = psMetadataAlloc(); psMetadataAddStr(inputtreeArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", NULL); psMetadataAddStr(inputtreeArgs, PS_LIST_TAIL, "-dep_file", 0, "order of operations dep. file (required)", NULL); + psMetadataAddS16(inputtreeArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); + + // -inputs + psMetadata *inputsArgs = psMetadataAlloc(); + psMetadataAddStr(inputsArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magictool ID", NULL); + psMetadataAddStr(inputsArgs, PS_LIST_TAIL, "-node", 0, "search by symbolic node name", NULL); + psMetadataAddU64(inputsArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(inputsArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); // -tooprocess @@ -93,4 +107,5 @@ psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-node", 0, "define symbolic node name (required)", NULL); psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-uri", 0, "define URI (required)", NULL); + psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); // -tomask @@ -101,7 +116,10 @@ // -addmask psMetadata *addmaskArgs = psMetadataAlloc(); - psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", NULL); - psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI (required)", NULL); + psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", NULL); + psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL); + psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks", 0, "define number of streaks", 0); + psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); +# if (0) // -toskyfilemask psMetadata *toskyfilemaskArgs = psMetadataAlloc(); @@ -115,4 +133,11 @@ psMetadataAddStr(addskyfilemaskArgs, PS_LIST_TAIL, "-diff_id", 0, "define difftool ID (required)", NULL); psMetadataAddStr(addskyfilemaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI (required)", NULL); +# endif + + // -mask + psMetadata *maskArgs = psMetadataAlloc(); + psMetadataAddStr(maskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", NULL); + psMetadataAddU64(maskArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddBool(maskArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); psFree(now); @@ -121,16 +146,17 @@ psMetadata *modes = psMetadataAlloc(); - PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, queueArgs); + PXOPT_ADD_MODE("-definebyquery", "", MAGICTOOL_MODE_DEFINEBYQUERY, queueArgs); PXOPT_ADD_MODE("-definerun", "", MAGICTOOL_MODE_DEFINERUN, definerunArgs); PXOPT_ADD_MODE("-updaterun", "", MAGICTOOL_MODE_UPDATERUN, updaterunArgs); PXOPT_ADD_MODE("-addinputskyfile", "", MAGICTOOL_MODE_ADDINPUTSKYFILE, addinputskyfileArgs); PXOPT_ADD_MODE("-inputskyfile", "", MAGICTOOL_MODE_INPUTSKYFILE, inputskyfileArgs); + PXOPT_ADD_MODE("-totree", "", MAGICTOOL_MODE_TOTREE, totreeArgs); PXOPT_ADD_MODE("-inputtree", "", MAGICTOOL_MODE_INPUTTREE, inputtreeArgs); PXOPT_ADD_MODE("-toprocess", "", MAGICTOOL_MODE_TOPROCESS, toprocessArgs); + PXOPT_ADD_MODE("-inputs", "", MAGICTOOL_MODE_INPUTS, inputsArgs); PXOPT_ADD_MODE("-addresult", "", MAGICTOOL_MODE_ADDRESULT, addresultArgs); PXOPT_ADD_MODE("-tomask", "", MAGICTOOL_MODE_TOMASK, tomaskArgs); PXOPT_ADD_MODE("-addmask", "", MAGICTOOL_MODE_ADDMASK, addmaskArgs); - PXOPT_ADD_MODE("-toskyfilemask", "", MAGICTOOL_MODE_TOSKYFILEMASK, toskyfilemaskArgs); - PXOPT_ADD_MODE("-addskyfilemask", "", MAGICTOOL_MODE_ADDSKYFILEMASK, addskyfilemaskArgs); + PXOPT_ADD_MODE("-mask", "", MAGICTOOL_MODE_MASK, maskArgs); if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) { Index: /branches/eam_branch_20080706/ippTools/src/pxwarp.c =================================================================== --- /branches/eam_branch_20080706/ippTools/src/pxwarp.c (revision 18529) +++ /branches/eam_branch_20080706/ippTools/src/pxwarp.c (revision 18530) @@ -74,9 +74,11 @@ } - psString query = psStringCopy("UPDATE warpRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET state = '%s'"); + psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.state = '%s'"); if (where) { psString whereClause = psDBGenerateWhereSQL(where, NULL); - psStringAppend(&query, " %s", whereClause); + if (whereClause && strlen(whereClause) > 0) { + psStringAppend(&query, " %s", whereClause); + } psFree(whereClause); } @@ -115,5 +117,5 @@ // note label == NULL should be explicitly allowed - psString query = psStringCopy("UPDATE warpRun JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.label = '%s'"); + psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) SET warpRun.label = '%s'"); if (where) { Index: /branches/eam_branch_20080706/psModules/src/imcombine/pmPSFEnvelope.c =================================================================== --- /branches/eam_branch_20080706/psModules/src/imcombine/pmPSFEnvelope.c (revision 18529) +++ /branches/eam_branch_20080706/psModules/src/imcombine/pmPSFEnvelope.c (revision 18530) @@ -34,5 +34,6 @@ #define PEAK_FLUX 1.0e4 // Peak flux for each source #define SKY_VALUE 0.0e0 // Sky value for fake image -#define WEIGHT_VAL 1.0 // Weighting for image +#define WEIGHT_VAL 10.0 // Weighting for image +#define WEIGHT_FACTOR 10.0 // Factor to multiply image by to get weighting #define PSF_STATS PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV // Statistics options for measuring PSF @@ -199,6 +200,11 @@ psFree(envelope); - // XXX This seems the best way to set the weight image so that pixels aren't rejected as "insignificant" - readout->weight = (psImage*)psBinaryOp(NULL, readout->image, "*", readout->image); + // XXX Setting the weight seems to be an art + // Can't set it too high so that pixels are rejected as insignificant + // Can't set it too low so that it's hard to get to the minimum + // Have also tried: + // *** readout->weight = (psImage*)psBinaryOp(NULL, readout->image, "*", readout->image); + // *** readout->weight = (psImage*)psBinaryOp(NULL, readout->image, "*", psScalarAlloc(WEIGHT_FACTOR, PS_TYPE_F32)); + readout->weight = (psImage*)psBinaryOp(NULL, readout->image, "+", psScalarAlloc(PS_SQR(WEIGHT_VAL), PS_TYPE_F32)); readout->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); psImageInit(readout->mask, 0); @@ -258,5 +264,5 @@ options->psfFieldYo = 0; - pmSourceFitModelInit (15, 0.01, WEIGHT_VAL, options->poissonErrorsPhotLMM); + pmSourceFitModelInit(50, 0.01, WEIGHT_VAL, false); pmPSFtry *try = pmPSFtryModel(fakes, modelName, options, 0, 0xff); Index: /branches/eam_branch_20080706/psModules/src/imcombine/pmStackReject.c =================================================================== --- /branches/eam_branch_20080706/psModules/src/imcombine/pmStackReject.c (revision 18529) +++ /branches/eam_branch_20080706/psModules/src/imcombine/pmStackReject.c (revision 18530) @@ -23,4 +23,9 @@ PS_ASSERT_ARRAY_NON_NULL(kernels, NULL); PS_ASSERT_ARRAYS_SIZE_EQUAL(subRegions, kernels, NULL); + + // Trivial case + if (in->n == 0) { + return psPixelsAllocEmpty(0); + } // Get the original image size