Changeset 27840 for branches/simtest_nebulous_branches/ippTools/src/chiptool.c

Timestamp:

May 3, 2010, 8:50:52 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simtest_nebulous_branches

Files:

• . (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/chiptool.c (modified) (39 diffs)

branches/simtest_nebulous_branches/ippTools/src

@@ +1 @@
+*.la
+*.lo
 .deps
 .gdb_history
@@ -4 +6 @@
 Makefile
 Makefile.in
+addtool
+caltool
+camtool
+chiptool
 config.h
 config.h.in
-stamp-h1
-*.la
-*.lo
+detselect
+dettool
+difftool
+disttool
+dqstatstool
+faketool
+flatcorr
+guidetool
+magicdstool
+magictool
+pstamptool
+pubtool
+pxadmin
+pxdata.c
+pxinject
 pxtoolsErrorCodes.c
 pxtoolsErrorCodes.h
-pxadmin
-pxinject
-pztool
 pzgetexp
 pzgetimfiles
+pztool
+receivetool
 regtool
-guidetool
-chiptool
-camtool
+stacktool
+stamp-h1
 warptool
-difftool
-stacktool
-faketool
-dettool
-detselect
-pxdata.c
-magictool
-magicdstool
-caltool
-flatcorr
-pstamptool
-disttool
-receivetool
-
-pubtool

@@ +1 @@
+*.la
+*.lo
 .deps
 .gdb_history
@@ -4 +6 @@
 Makefile
 Makefile.in
+addtool
+caltool
+camtool
+chiptool
 config.h
 config.h.in
-stamp-h1
-*.la
-*.lo
+detselect
+dettool
+difftool
+disttool
+dqstatstool
+faketool
+flatcorr
+guidetool
+magicdstool
+magictool
+pstamptool
+pubtool
+pxadmin
+pxdata.c
+pxinject
 pxtoolsErrorCodes.c
 pxtoolsErrorCodes.h
-pxadmin
-pxinject
-pztool
 pzgetexp
 pzgetimfiles
+pztool
+receivetool
 regtool
-guidetool
-chiptool
-camtool
+stacktool
+stamp-h1
 warptool
-difftool
-stacktool
-faketool
-dettool
-detselect
-pxdata.c
-magictool
-magicdstool
-caltool
-flatcorr
-pstamptool
-disttool
-receivetool
-
-pubtool

branches/simtest_nebulous_branches/ippTools/src/chiptool.c

@@ -36 +36 @@
 
 static bool definebyqueryMode(pxConfig *config);
+static bool definecopyMode(pxConfig *config);
 static bool updaterunMode(pxConfig *config);
 static bool pendingimfileMode(pxConfig *config);
@@ -54 +55 @@
 static bool tofullimfileMode(pxConfig *config);
 static bool topurgedimfileMode(pxConfig *config);
+static bool toscrubbedimfileMode(pxConfig *config);
 static bool exportrunMode(pxConfig *config);
 static bool importrunMode(pxConfig *config);
+static bool runstateMode(pxConfig *config);
+static bool setimfiletoupdateMode(pxConfig *config);
+static bool listrunMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -75 +80 @@
     switch (config->mode) {
         MODECASE(CHIPTOOL_MODE_DEFINEBYQUERY,           definebyqueryMode);
+        MODECASE(CHIPTOOL_MODE_DEFINECOPY,              definecopyMode);
         MODECASE(CHIPTOOL_MODE_UPDATERUN,               updaterunMode);
         MODECASE(CHIPTOOL_MODE_PENDINGIMFILE,           pendingimfileMode);
@@ -93 +99 @@
         MODECASE(CHIPTOOL_MODE_TOFULLIMFILE,            tofullimfileMode);
         MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE,          topurgedimfileMode);
+        MODECASE(CHIPTOOL_MODE_TOSCRUBBEDIMFILE,        toscrubbedimfileMode);
         MODECASE(CHIPTOOL_MODE_EXPORTRUN,               exportrunMode);
         MODECASE(CHIPTOOL_MODE_IMPORTRUN,               importrunMode);
+        MODECASE(CHIPTOOL_MODE_RUNSTATE,                runstateMode);
+        MODECASE(CHIPTOOL_MODE_SETIMFILETOUPDATE,       setimfiletoupdateMode);
+        MODECASE(CHIPTOOL_MODE_LISTRUN,                 listrunMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -116 +126 @@
 }
 
+// Queue exposures for chip processing
+static bool queue_exposures(pxConfig *config,  // Configuration
+                            const psArray *exps, // Exposures information
+                            const char *workdir, // Working directory, or NULL to inherit
+                            const char *label,   // Label, or NULL to inherit
+                            const char *data_group, // Data group, or NULL to inherit
+                            const char *dist_group, // Distribution group, or NULL to inherit
+                            const char *reduction, // Reduction class, or NULL to inherit
+                            const char *expgroup,  // Exposure group
+                            const char *dvodb,     // DVO database, or NULL to inherit
+                            const char *tess_id,   // Tessellation identifier, or NULL to inherit
+                            const char *end_stage,  // End stage, or NULL to inherit
+                            const char *note        // Note
+    )
+{
+    // start a transaction so we don't end up with an exp without any associted
+    // imfiles
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    // if end_stage is warp (or NULL), check for valid tess_id
+    for (long i = 0; i < psArrayLength(exps); i++) {
+        psMetadata *md = exps->data[i];
+
+        bool status;
+        char *end_stage = psMetadataLookupStr(&status, md, "end_stage");
+        if (end_stage && strcasecmp(end_stage, "warp")) continue;
+
+        char *raw_tess_id   = psMetadataLookupStr(&status, md, "tess_id");
+        if (raw_tess_id || tess_id) continue;
+
+        char *label  = psMetadataLookupStr(&status, md, "label");
+        psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id");
+
+        if (!status) {
+            psError(PS_ERR_UNKNOWN, false,
+                    "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64,
+                    label, exp_id);
+            return false;
+        }
+    }
+
+
+# define GET_VALUE(PTYPE,CTYPE,VALUE,NAME)                              \
+    PTYPE VALUE;                                                        \
+    {                                                                   \
+        bool status;                                                    \
+        VALUE = psMetadataLookup##CTYPE(&status, md, NAME);             \
+        if (!status) {                                                  \
+            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", NAME); \
+            return false;                                               \
+        }                                                               \
+    }
+
+    // loop over our list of exp_ids
+    for (long i = 0; i < psArrayLength(exps); i++) {
+        psMetadata *md = exps->data[i];
+
+        rawExpRow *row = rawExpObjectFromMetadata(md);
+        if (!row) {
+            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun");
+            return false;
+        }
+
+        GET_VALUE (psS64,    S64, exp_id,        "exp_id");
+        GET_VALUE (psString, Str, raw_workdir,   "workdir");
+        GET_VALUE (psString, Str, raw_label,     "label");
+        GET_VALUE (psString, Str, raw_reduction, "reduction");
+        // GET_VALUE (psString, Str, raw_expgroup,  "expgroup");
+        GET_VALUE (psString, Str, raw_dvodb,     "dvodb");
+        GET_VALUE (psString, Str, raw_tess_id,   "tess_id");
+        GET_VALUE (psString, Str, raw_end_stage, "end_stage");
+
+        if (!row->exp_id) {
+            psError(PS_ERR_UNKNOWN, false, "failed to find value for exp_id");
+            return false;
+        }
+
+        // queue the exp
+        if (!pxchipQueueByExpTag(config,
+                                 exp_id,
+                                 workdir     ? workdir   : raw_workdir,
+                                 label       ? label     : raw_label,
+                                 data_group  ? data_group : (label ? label : raw_label),
+                                 dist_group,
+                                 reduction   ? reduction : raw_reduction,
+                                 // expgroup    ? expgroup  : raw_expgroup,
+                                 // XXX how does expgroup get defined?
+                                 expgroup,
+                                 dvodb       ? dvodb     : raw_dvodb,
+                                 tess_id     ? tess_id   : raw_tess_id,
+                                 end_stage   ? end_stage : raw_end_stage,
+                                 note
+                                 )) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            psError(PS_ERR_UNKNOWN, false,
+                    "failed to trying to queue exp_id: %" PRId64, exp_id);
+            return false;
+        }
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
 
 static bool definebyqueryMode(pxConfig *config)
@@ -125 +247 @@
     psMetadata *where = psMetadataAlloc();
     pxchipGetSearchArgs (config, where); // rawExp only
-    pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-label", "newExp.label", "LIKE"); // define using newExp label
 
     // psListLength(where->list) is at least 1 because exp_type defaults to "object"
@@ -131 +253 @@
     if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) {
         psFree(where);
-        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
         return false;
     }
@@ -142 +264 @@
     PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false);
     PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false);
+    PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
 
     // default
@@ -150 +275 @@
     psString query = pxDataGet("chiptool_find_rawexp.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         psFree(where);
         return false;
@@ -190 +315 @@
     }
 
-    // start a transaction so we don't end up with an exp without any associted
-    // imfiles
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(output);
-        return false;
-    }
-
-    // if end_stage is warp (or NULL), check for valid tess_id
-    for (long i = 0; i < psArrayLength(output); i++) {
-        psMetadata *md = output->data[i];
-
-        bool status;
-        char *end_stage = psMetadataLookupStr(&status, md, "end_stage");
-        if (end_stage && strcasecmp(end_stage, "warp")) continue;
-
-        char *raw_tess_id   = psMetadataLookupStr(&status, md, "tess_id");
-        if (raw_tess_id || tess_id) continue;
-
-        char *label  = psMetadataLookupStr(&status, md, "label");
-        psS64 exp_id = psMetadataLookupS64(&status, md, "exp_id");
-
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "cannot queue analysis to WARP without a defined tess id: label: %s, exp_id %" PRId64, label, exp_id);
+    if (!queue_exposures(config, output, workdir, label, data_group, dist_group, reduction, expgroup,
+                         dvodb, tess_id, end_stage, note)) {
+        psError(psErrorCodeLast(), false, "Unable to queue exposures for chip.");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool definecopyMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
+
+    psMetadata *where = psMetadataAlloc();
+    pxchipGetSearchArgs (config, where); // rawExp only
+    pxAddLabelSearchArgs (config, where, "-label", "chipOld.label", "LIKE");
+
+    // psListLength(where->list) is at least 1 because exp_type defaults to "object"
+    // so we require a list longer than 1 entry
+    if ((psListLength(where->list) <= 1) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", true, false);
+    PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false);
+    PXOPT_LOOKUP_STR(expgroup, config->args, "-set_expgroup", false, false);
+    PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false);
+    PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false);
+    PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false);
+    PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+
+    // default
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // find the exp_id of all the exposures that we want to queue up.
+    psString query = pxDataGet("chiptool_definecopy.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+
+    psFree(whereClause);
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, label)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("chiptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
             return false;
         }
-    }
-
-
-# define GET_VALUE(PTYPE,CTYPE,VALUE,NAME)                              \
-    PTYPE VALUE;                                                        \
-    { bool status;                                                      \
-        VALUE = psMetadataLookup##CTYPE(&status, md, NAME);             \
-        if (!status) {                                                  \
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for %s", NAME); \
-            psFree(output);                                             \
-            return false;                                               \
-        } }
-
-    // loop over our list of exp_ids
-    for (long i = 0; i < psArrayLength(output); i++) {
-        psMetadata *md = output->data[i];
-
-        rawExpRow *row = rawExpObjectFromMetadata(md);
-        if (!row) {
-            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into chipRun");
-            psFree(output);
-            return false;
-        }
-
-        GET_VALUE (psS64,    S64, exp_id,        "exp_id");
-        GET_VALUE (psString, Str, raw_workdir,   "workdir");
-        GET_VALUE (psString, Str, raw_label,     "label");
-        GET_VALUE (psString, Str, raw_reduction, "reduction");
-        // GET_VALUE (psString, Str, raw_expgroup,  "expgroup");
-        GET_VALUE (psString, Str, raw_dvodb,     "dvodb");
-        GET_VALUE (psString, Str, raw_tess_id,   "tess_id");
-        GET_VALUE (psString, Str, raw_end_stage, "end_stage");
-
-        if (!row->exp_id) {
-            psError(PS_ERR_UNKNOWN, false, "failed to find value for exp_id");
-            psFree(output);
-            return false;
-        }
-
-        // queue the exp
-        if (!pxchipQueueByExpTag(config, 
-                                 exp_id, 
-                                 workdir     ? workdir   : raw_workdir,
-                                 label       ? label     : raw_label,
-                                 reduction   ? reduction : raw_reduction,
-                                 // expgroup    ? expgroup  : raw_expgroup,
-                                 // XXX how does expgroup get defined?
-                                 expgroup,
-                                 dvodb       ? dvodb     : raw_dvodb,
-                                 tess_id     ? tess_id   : raw_tess_id,
-                                 end_stage   ? end_stage : raw_end_stage
-                                 )) {
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false,
-                    "failed to trying to queue exp_id: %" PRId64, exp_id);
-            psFree(output);
-            return false;
-        }
+        psFree(output);
+        return true;
+    }
+
+    if (!queue_exposures(config, output, workdir, label, data_group, dist_group, reduction, expgroup,
+                         dvodb, tess_id, end_stage, note)) {
+        psError(psErrorCodeLast(), false, "Unable to queue exposures for chip.");
+        psFree(output);
+        return false;
     }
     psFree(output);
-
-    if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
 
     return true;
@@ -299 +424 @@
     PXOPT_COPY_STR(config->args,  where, "-label",   "chipRun.label",   "==");
     PXOPT_COPY_STR(config->args,  where, "-state",   "chipRun.state",   "==");
-
-    if (!psListLength(where->list) &&
-        !psMetadataLookupBool(NULL, config->args, "-all")) {
+    PXOPT_COPY_STR(config->args,  where, "-data_group", "chipRun.data_group",   "==");
+    PXOPT_COPY_STR(config->args,  where, "-dist_group", "chipRun.dist_group",   "==");
+
+    if (!psListLength(where->list)) {
         psFree(where);
-        where = NULL;
-        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
-        return false;
-    }
-
-    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
-    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
-
-    if ((!state) && (!label)) {
-        psError(PXTOOLS_ERR_DATA, false, "parameters are required");
-        psFree(where);
-        return false;
-    }
-
-    if (state) {
-        // set chipRun.state to state
-        if (!pxchipRunSetStateByQuery(config, where, state)) {
-            psFree(where);
-            return false;
-        }
-    }
-
-    if (label) {
-        // set chipRun.label to label
-        if (!pxchipRunSetLabelByQuery(config, where, label)) {
-            psFree(where);
-            return false;
-        }
-    }
-
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+    psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) -- join hook %s\n");
+
+    // pxUpdateRun gets parameters from config->args and updates
+    bool result = pxUpdateRun(config, where, &query, "chipRun", "chip_id", "chipProcessedImfile", true);
+    if (!result) {
+        psError(psErrorCodeLast(), false, "pxUpdateRun failed");
+    }
+
+    psFree(query);
     psFree(where);
 
-    return true;
+    return result;
 }
 
@@ -353 +461 @@
     psString query = pxDataGet("chiptool_pendingimfile.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -593 +701 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
+    PXOPT_LOOKUP_BOOL(allfiles, config->args, "-allfiles", false);
+    if (allfiles) {
+        faulted = false;
+    }
 
     psMetadata *where = psMetadataAlloc();
@@ -601 +713 @@
     PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
     pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE");
-    PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "==");
-
-    if (!psListLength(where->list) &&
-        !psMetadataLookupBool(NULL, config->args, "-all")) {
-        psFree(where);
-        psError(PXTOOLS_ERR_DATA, false, "search parameters (or -all) are required");
+    pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE");
+    PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "==");
+
+    psString where2 = NULL;
+    pxmagicAddWhere(config, &where2, "chipProcessedImfile");
+    // add cuts on ra and decl if supplied
+    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
+        psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed");
         return false;
     }
@@ -612 +726 @@
     psString query = pxDataGet("chiptool_processedimfile.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (where && psListLength(where->list)) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " WHERE %s", whereClause);
+        psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : "");
         psFree(whereClause);
+    } else if (psMetadataLookupBool(NULL, config->args, "-all") || (faulted || where2)) {
+        psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : "");
+    } else {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required");
+        return false;
     }
     psFree(where);
@@ -626 +746 @@
         // list only faulted rows
         psStringAppend(&query, " %s", "AND chipProcessedImfile.fault != 0");
-    } else {
+    } else if (!allfiles) {
         // don't list faulted rows
         psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0");
@@ -683 +803 @@
         && !psMetadataLookupBool(NULL, config->args, "-all")) {
         psFree(where);
-        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
         return false;
     }
@@ -689 +809 @@
     psString query = pxDataGet("chiptool_revertprocessedimfile.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+    psString query_update = pxDataGet("chiptool_revertupdatedimfile.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         psFree(where);
         return false;
@@ -697 +823 @@
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
         psStringAppend(&query, " AND %s", whereClause);
+        psStringAppend(&query_update, " AND %s", whereClause);
         psFree(whereClause);
     }
@@ -708 +835 @@
     }
     psFree(query);
+    if (!p_psDBRunQuery(config->dbh, query_update)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query_update);
+        return false;
+    }
+    psFree(query_update);
 
     return true;
@@ -719 +852 @@
     PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
     PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
-    PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
-
-    if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) {
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, NULL);
+    if (psListLength(where->list) == 0) {
+        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
+        return false;
+    }
+
+    if (!state) {
+      PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
+
+      if (!pxSetFaultCode(config->dbh, "chipProcessedImfile", where, fault)) {
         psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
         return false;
-    }
-    psFree(where);
+      }
+      psFree(where);
+    }
+    else {
+      if (!pxchipProcessedImfileSetStateByQuery(config,where,state)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to set chipProcessedImfile state");
+        return(false);
+      }
+    }
+
 
     return true;
@@ -828 +976 @@
     psString query = pxDataGet("chiptool_unmasked.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -906 +1054 @@
     psString query = pxDataGet("chiptool_pendingcleanuprun.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -970 +1118 @@
     psString query = pxDataGet("chiptool_pendingcleanupimfile.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1031 +1179 @@
     psString query = pxDataGet("chiptool_donecleanup.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1089 +1237 @@
     // make sure that the state string is valid
     if (!pxIsValidState(state)) {
-        psError(PXTOOLS_ERR_DATA, false, "%s is not a valid state", state);
+        psError(PXTOOLS_ERR_CONFIG, false, "%s is not a valid state", state);
         return false;
     }
 
     psMetadata *where = psMetadataAlloc();
+    pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
     PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
@@ -1099 +1250 @@
     psString query = pxDataGet("chiptool_run.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1161 +1312 @@
     psString query = pxDataGet("chiptool_completely_processed_exp.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         return false;
     }
@@ -1233 +1384 @@
                     chipRun->workdir,
                     chipRun->label,
+                    chipRun->data_group,
+                    chipRun->dist_group,
                     chipRun->reduction,
                     chipRun->expgroup,
                     chipRun->dvodb,
                     chipRun->tess_id,
-                    chipRun->end_stage
+                    chipRun->end_stage,
+                    chipRun->magicked,
+                    NULL    // note does not propagate
         )) {
            if (!psDBRollback(config->dbh)) {
@@ -1278 +1433 @@
     }
 
-    psString set_magic = "";
+    psString set_magicked_imfile = psStringCopy("");
+    psString set_magicked_run = psStringCopy("");
     if (!strcmp(data_state, "full")) {
-        // copy the magicked state from the input to the output when transitioning to full state
-        set_magic = "\n , chipProcessedImfile.magicked = rawImfile.magicked";
-    }
-
-    // note only updates if chipRun.state = run_state
-    // XXX note that we have removed this constraint for now
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magic, chip_id, class_id)) {
+        // if (chipProcessedImfile.magicked < 0 and rawImfile.magicked = 0) leave magicked unchanged. This will
+        // block warp processing until destreaking has been done
+        // otherwise copy magicked from the rawImfile
+        // Same thing for chipRun/rawExp
+        psStringAppend(&set_magicked_imfile, "\n , chipProcessedImfile.magicked = IF((chipProcessedImfile.magicked < 0"
+                                      " AND rawImfile.magicked = 0), chipProcessedImfile.magicked, rawImfile.magicked)");
+        psStringAppend(&set_magicked_run, "\n , chipRun.magicked = IF((chipRun.magicked < 0 AND rawExp.magicked = 0), "
+                                      " chipRun.magicked, rawExp.magicked)");
+
+    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
+        // if magicked is non-zero set it to -1
+        // Once one imfile has been cleaned, the chipRun is no longer 'magicked'
+        psStringAppend(&set_magicked_imfile, "\n, chipProcessedImfile.magicked = IF(chipProcessedImfile.magicked = 0, 0, -1),"
+                                             " chipRun.magicked = IF(chipRun.magicked = 0, 0, -1)");
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_imfile, chip_id, class_id)) {
         psFree(query);
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1297 +1463 @@
     }
     psFree(query);
+    psFree(set_magicked_imfile);
+    psFree(set_magicked_run);
     if (psDBAffectedRows(config->dbh) < 1) {
         psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
@@ -1303 +1471 @@
 
     query = pxDataGet("chiptool_change_exp_state.sql");
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, chip_id, data_state)) {
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, chip_id, data_state)) {
         psFree(query);
         // rollback
@@ -1333 +1501 @@
     return change_imfile_data_state(config, "purged", "goto_purged");
 }
-
+static bool toscrubbedimfileMode(pxConfig *config)
+{
+  return change_imfile_data_state(config, "scrubbed", "goto_scrubbed");
+}
 bool exportrunMode(pxConfig *config)
 {
@@ -1357 +1528 @@
   }
 
+  if (!pxExportVersion(config, f)) {
+    psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file");
+    return false;
+  }
+
   psMetadata *where = psMetadataAlloc();
   PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
@@ -1370 +1546 @@
     psString query = pxDataGet(tables[i].sqlFilename);
     if (!query) {
-      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+      psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
       return false;
     }
@@ -1406 +1582 @@
 
     if (clean) {
-        bool success = true; 
+        bool success = true;
         if (!strcmp(tables[i].tableName, "chipRun")) {
             success = pxSetStateCleaned("chipRun", "state", output);
@@ -1448 +1624 @@
     psVector *identifiers = psVectorAllocEmpty(16, PS_TYPE_U64); // Identifiers inserted
 
+    psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL);       // Iterator
+
+    if (!pxCheckImportVersion(config, input)) {
+        psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed");
+        return false;
+    }
+    // first item is the dbversion, skip it
+    psMetadataItem *dbversion =  psMetadataGetAndIncrement(iter);
+    (void) dbversion;
+
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1453 +1639 @@
     }
 
-    psMetadataIterator *iter = psMetadataIteratorAlloc(input, PS_LIST_HEAD, NULL);       // Iterator
     psMetadataItem *item;               // Item from iteration
     while ((item = psMetadataGetAndIncrement(iter))) {
@@ -1482 +1667 @@
     return true;
 }
+
+static bool runstateMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_id",    "chipRun.chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
+    pxAddLabelSearchArgs (config, where, "-label",     "chipRun.label", "LIKE");
+
+//    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+    PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("chiptool_runstate.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+        return false;
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("chiptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "chipRunState", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+// a very specfic function to queue a cleaned chipProcessedImfile to be updated
+static bool setimfiletoupdateMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false);
+    PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+
+    psString query = pxDataGet("chiptool_setimfiletoupdate.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString setHook = psStringCopy("");
+    if (label) {
+        psStringAppend(&setHook, "\n , chipRun.label = '%s'", label);
+    }
+
+    if (class_id) {
+        psStringAppend(&query, " AND (chipProcessedImfile.class_id = '%s')", class_id);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psFree(setHook);
+    psFree(query);
+
+    return true;
+}
+
+static bool listrunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false);
+
+    psMetadata *where = psMetadataAlloc();
+    pxchipGetSearchArgs (config, where); // chipRun, chipProcessedImfile, rawExp
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "chipRun.state", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-dist_group", "chipRun.dist_group", "LIKE");
+    PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "==");
+
+    psString where2 = NULL;
+    pxmagicAddWhere(config, &where2, "chipRun");
+    // add cuts on ra and decl if supplied
+    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
+        psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed");
+        return false;
+    }
+
+    psString query = pxDataGet("chiptool_listrun.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s %s", whereClause, where2 ? where2 : "");
+        psFree(whereClause);
+    } else if (psMetadataLookupBool(NULL, config->args, "-all") || where2) {
+        psStringAppend(&query, " WHERE chipRun.chip_id IS NOT NULL %s", where2 ? where2 : "");
+    } else {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters (or -all) are required");
+        return false;
+    }
+    psFree(where);
+
+    if (pstamp_order) {
+        // put runs in order of exposure id with newest chip Runs first
+        // The postage stamp parser depends on this behavior
+        psStringAppend(&query, "\nORDER by exp_id, chip_id DESC");
+    }
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("chiptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "chipRun", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}

@@ +1 @@
+*.la
+*.lo
 .deps
 .gdb_history
@@ -4 +6 @@
 Makefile
 Makefile.in
+addtool
+caltool
+camtool
+chiptool
 config.h
 config.h.in
-stamp-h1
-*.la
-*.lo
+detselect
+dettool
+difftool
+disttool
+dqstatstool
+faketool
+flatcorr
+guidetool
+magicdstool
+magictool
+pstamptool
+pubtool
+pxadmin
+pxdata.c
+pxinject
 pxtoolsErrorCodes.c
 pxtoolsErrorCodes.h
-pxadmin
-pxinject
-pztool
 pzgetexp
 pzgetimfiles
+pztool
+receivetool
 regtool
-guidetool
-chiptool
-camtool
+stacktool
+stamp-h1
 warptool
-difftool
-stacktool
-faketool
-dettool
-detselect
-pxdata.c
-magictool
-magicdstool
-caltool
-flatcorr
-pstamptool
-disttool
-receivetool
-
-pubtool