Changeset 12174

Timestamp:

Mar 1, 2007, 4:57:45 PM (19 years ago)

Author:

jhoblitt

Message:

first pass working chiptool -queuerawexp
normalize chipPending[Exp|Imfile] & chipProcessed[Exp|Imfile]

Location:

trunk/ippTools

Files:

• scripts/chiptest.sh (modified) (1 diff)
• share/Makefile.am (modified) (1 diff)
• share/chiptool_completely_processed_exp.sql (modified) (1 diff)
• share/chiptool_find_unprocessed_imfile.pl (modified) (1 diff)
• share/chiptool_pendingimfile.sql (modified) (1 diff)
• share/chiptool_queuerawexp.sql (added)
• share/chiptool_queuerawimfile.sql (added)
• src/chiptool.c (modified) (7 diffs)
• src/chiptoolConfig.c (modified) (2 diffs)
• src/regtool.c (modified) (1 diff)

trunk/ippTools/scripts/chiptest.sh

@@ -8 +8 @@
 
 for ID in `seq 0 3`; do
-    chiptool -addprocessedimfile -chip_id 1 -class_id $ID -uri file://chipp-t10.$ID -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1
+    chiptool -addprocessedimfile -chip_id 1 -class_id $ID -uri file://chipp-t10.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1
 done;
 
 for ID in `seq 0 3`; do
-    chiptool -addprocessedimfile -chip_id 2 -class_id $ID -uri file://chipp-t11.$ID -recip myrecipe -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1
+    chiptool -addprocessedimfile -chip_id 2 -class_id $ID -uri file://chipp-t11.$ID -bg 1 -bg_stdev 2 -bg_mean_stdev 3 -path_base file:///foo || exit 1
 done;
 

trunk/ippTools/share/Makefile.am

@@ -9 +9 @@
         regtool_find_unprocessed_imfile.sql \
         regtool_processedimfile.sql \
-        regtool_pendingexp.sql
+        regtool_pendingexp.sql \
+        chiptool_queuerawexp.sql

trunk/ippTools/share/chiptool_completely_processed_exp.sql

@@ -12 +12 @@
     ON chipPendingExp.exp_tag = rawExp.exp_tag
 LEFT JOIN chipProcessedExp
-    ON chipPendingExp.exp_tag = chipProcessedExp.exp_tag
+    ON chipPendingExp.chip_id = chipProcessedExp.chip_id
 LEFT JOIN chipPendingImfile
-    ON chipPendingExp.exp_tag = chipPendingImfile.exp_tag
+    ON chipPendingExp.chip_id = chipPendingImfile.chip_id
 LEFT JOIN chipProcessedImfile
-    ON chipPendingExp.exp_tag = chipProcessedImfile.exp_tag
+    ON chipPendingExp.chip_id = chipProcessedImfile.chip_id
 WHERE
-    chipProcessedExp.exp_tag IS NULL
-    AND chipPendingImfile.exp_tag IS NULL
-    AND chipProcessedImfile.exp_tag IS NOT NULL
+    chipProcessedExp.chip_id IS NULL
+    AND chipPendingImfile.chip_id IS NULL
+    AND chipProcessedImfile.chip_id IS NOT NULL
 GROUP BY
-    chipPendingExp.exp_tag
+    chipPendingExp.chip_id
 HAVING rawExp.imfiles = COUNT(chipProcessedImfile.class_id)

trunk/ippTools/share/chiptool_find_unprocessed_imfile.pl

@@ -3 +3 @@
 FROM chipPendingImfile
 LEFT JOIN chipProcessedImfile
-    USING(exp_tag, class_id)
+    USING(chip_id, class_id)
 WHERE
-    chipProcessedImfile.exp_tag IS NULL
+    chipProcessedImfile.chip_id IS NULL
     AND chipProcessedImfile.class_id IS NULL

trunk/ippTools/share/chiptool_pendingimfile.sql

@@ -5 +5 @@
 FROM chipPendingImfile
 JOIN chipPendingExp
-    USING(exp_tag)
+    USING(chip_id)
 JOIN rawExp
-    USING(exp_tag)
+    ON chipPendingExp.exp_tag = rawExp.exp_tag
 LEFT JOIN chipMask
     ON chipPendingExp.label = chipMask.label

trunk/ippTools/src/chiptool.c

@@ -111 +111 @@
 { \
     bool status = false; \
-    ps##type var = psMetadataLookup##type(&status, from, "-" #name); \
+    ps##type var = psMetadataLookup##type(&status, from, "-" name); \
     if (!status) { \
         psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -" name); \
@@ -118 +118 @@
     if (!isnan(var)) { \
         if (!psMetadataAdd##type(to, PS_LIST_TAIL, #name, 0, "==", var)) { \
-            psError(PS_ERR_UNKNOWN, false, "failed to add item " #name); \
+            psError(PS_ERR_UNKNOWN, false, "failed to add item " name); \
             psFree(to); \
             return false; \
@@ -153 +153 @@
     ADDPARAMSTR(config->args, where, "object");
 
-    if (config->where->list->n < 1) {
-        psFree(config->where);
-        config->where = NULL;
+    if (where->list->n < 1) {
+        psFree(where);
+        where = NULL;
     }
 
@@ -165 +165 @@
     }
 
-    // XXX this does not work!!!
+    psString recipe = psMetadataLookupStr(&status, config->args, "-set_recipe");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_recipe");
+        return false;
+    }
+
+    psString expgroup = psMetadataLookupStr(&status, config->args, "-set_expgroup");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_expgroup");
+        return false;
+    }
+
+    psString dvodb = psMetadataLookupStr(&status, config->args, "-set_dvodb");
+    if (!status) {
+        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -set_dvodb");
+        return false;
+    }
+
+    // find the exp_tag of all the exposures that we want to queue up.
     psString query = psStringCopy(
-            "INSERT INTO chipPendingExp\n"
-            "   SElECT\n"
-            "       0\n"            // chip_id
-            "       exp_tag,\n"
-            "       'my recipe',\n"  // recipe
-            "       255\n"         // guide_version
-            "       255\n"         // chip_version
-            "       %s\n"
-            "   FROM rawExp"
-            "   LEFT JOIN chipProcessedExp"
-            "       USING(exp_tag)"
-            "   WHERE"
-            "       rawExp.fault = 0"
-        );
-
-    if (config->where) {
-        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp");
+            "SELECT exp_tag FROM rawExp WHERE rawExp.fault = 0");
+    if (where) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, "rawExp");
+        psFree(where);
         psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
 
-    if (!p_psDBRunQuery(config->dbh, query, label ? label : "NULL")) {
+    if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -201 +206 @@
     }
     if (!psArrayLength(output)) {
-        // XXX check psError here
-        psError(PS_ERR_UNKNOWN, false, "no chipPendingImfile rows found");
-        psFree(output);
+        psTrace("chiptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // load the SQL to enqueue our exp_tags from disk once
+    psString queuerawexp_query = pxDataGet("chiptool_queuerawexp.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psFree(output);
+        return false;
+    }
+
+    psString queuerawimfile_query = pxDataGet("chiptool_queuerawimfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psFree(queuerawexp_query);
+        psFree(output);
+        return false;
+    }
+
+    // start a transaction so we don't end up with an exp without any associted
+    // imfiles
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(queuerawimfile_query);
+        psFree(queuerawexp_query);
+        psFree(output);
+        return false;
+    }
+
+    // loop over our list of exp_tags
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        psString exp_tag = psMetadataLookupStr(&status, md, "exp_tag");
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for exp_tag");
+            psFree(queuerawimfile_query);
+            psFree(queuerawexp_query);
+            psFree(output);
+            return false;
+        }
+
+        // queue the exp
+        if (!p_psDBRunQuery(config->dbh, queuerawexp_query,
+                    label    ? label    : "NULL",
+                    recipe   ? recipe   : "NULL",
+                    expgroup ? expgroup : "NULL",
+                    dvodb    ? dvodb    : "NULL",
+                    exp_tag
+        )) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(queuerawimfile_query);
+            psFree(queuerawexp_query);
+            psFree(output);
+            return false;
+        }
+
+        // just to be safe, we should have changed at least one row
+        if (psDBAffectedRows(config->dbh) < 1) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false,
+                    "no rows affected - should have changed at least one row");
+            psFree(queuerawimfile_query);
+            psFree(queuerawexp_query);
+            psFree(output);
+            return false;
+        }
+
+        // queue the imfiles for the exp we just queued
+        if (!p_psDBRunQuery(config->dbh, queuerawimfile_query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(queuerawimfile_query);
+            psFree(queuerawexp_query);
+            psFree(output);
+            return false;
+        }
+
+        // just to be safe, we should have changed at least one row
+        if (psDBAffectedRows(config->dbh) < 1) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false,
+                    "no rows affected - should have changed at least one row");
+            psFree(queuerawimfile_query);
+            psFree(queuerawexp_query);
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(queuerawimfile_query);
+    psFree(queuerawexp_query);
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
     }
@@ -776 +878 @@
     }
 
-    psString recipe = psMetadataLookupStr(&status, config->args, "-recip");
-    if (!status) {
-        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -recip");
-        return false;
-    }
-    if (!recipe) {
-        psError(PS_ERR_UNKNOWN, true, "-recip is required");
-        return false;
-    }
-
     psF64 bg = psMetadataLookupF64(&status, config->args, "-bg");
     if (!status) {
@@ -820 +912 @@
     return chipProcessedImfileRowAlloc(
         imfile->chip_id,
-        imfile->exp_tag,
-        imfile->guide_id,
         imfile->class_id,
-        recipe,
         uri,
         bg,

trunk/ippTools/src/chiptoolConfig.c

@@ -86 +86 @@
             "search by exposure object", NULL);
     psMetadataAddStr(queuerawexpArgs, PS_LIST_TAIL, "-set_label",  0,
-            "define label for phase 2 processing (non-detrend data only)", NULL);
+            "define label", NULL);
+    psMetadataAddStr(queuerawexpArgs, PS_LIST_TAIL, "-set_recipe",  0,
+            "define recipe", NULL);
+    psMetadataAddStr(queuerawexpArgs, PS_LIST_TAIL, "-set_expgroup",  0,
+            "define exposure group", NULL);
+    psMetadataAddStr(queuerawexpArgs, PS_LIST_TAIL, "-set_dvodb",  0,
+            "define DBO db", NULL);
 
 
@@ -114 +120 @@
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-chip_id",  0,
             "define chip ID (required)", NULL);
-#if 0
-    psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-exp_tag",  0,
-            "define exposure tag (required) (XXX redudant with chip_id)", NULL);
-#endif
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-class_id",  0,
             "define class ID", NULL);
     psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-uri",  0,
             "define URL", NULL);
-    psMetadataAddStr(addprocessedimfileArgs, PS_LIST_TAIL, "-recip",  0,
-            "define recipe (required)", NULL);
     psMetadataAddF64(addprocessedimfileArgs, PS_LIST_TAIL, "-bg",  0,
             "define exposue background", NAN);

trunk/ippTools/src/regtool.c

@@ -910 +910 @@
     return chipPendingImfileRowAlloc(
             chip_id,
-            rawImfile->exp_tag,
-            0xdeadbeef,     // guide version
             rawImfile->class_id,
-            "my recipe",    // recipe
             rawImfile->uri
     );