Changeset 27391

Timestamp:

Mar 22, 2010, 1:58:02 PM (16 years ago)

Author:

bills

Message:

changes to support updating only a subset of the components of a run

Location:

trunk/ippTools

Files:

• share/Makefile.am (modified) (1 diff)
• share/chiptool_pendingimfile.sql (modified) (1 diff)
• share/chiptool_setimfiletoupdate.sql (added)
• share/difftool_todiffskyfile.sql (modified) (2 diffs)
• share/warptool_towarped.sql (modified) (1 diff)
• src/addtool.c (modified) (2 diffs)
• src/camtool.c (modified) (1 diff)
• src/chiptool.c (modified) (4 diffs)
• src/chiptool.h (modified) (1 diff)
• src/chiptoolConfig.c (modified) (2 diffs)
• src/difftool.c (modified) (2 diffs)
• src/faketool.c (modified) (1 diff)
• src/pxtools.c (modified) (4 diffs)
• src/pxtools.h (modified) (1 diff)
• src/stacktool.c (modified) (1 diff)
• src/warptool.c (modified) (3 diffs)

trunk/ippTools/share/Makefile.am

@@ -38 +38 @@
      chiptool_run.sql \
      chiptool_runstate.sql \
+     chiptool_setimfiletoupdate.sql \
      chiptool_export_imfile.sql \
      chiptool_export_processed_imfile.sql \

trunk/ippTools/share/chiptool_pendingimfile.sql

@@ -33 +33 @@
     OR
     (chipRun.state = 'update'
-    AND chipProcessedImfile.data_state = 'cleaned'))
+    AND chipProcessedImfile.data_state = 'update'))
 -- // Restriction to prevent the processing of bad chips. Not needed now.
 --     AND

trunk/ippTools/share/difftool_todiffskyfile.sql

@@ -67 +67 @@
     OR (diffRun.state = 'update'
     AND diffSkyfile.fault = 0
-    AND diffSkyfile.data_state = 'cleaned')
+    AND diffSkyfile.data_state = 'update')
     )
 -- Ensure input warps are available
     AND (diffInputSkyfile.warp1 IS NULL
-    OR (warpRun.state = 'full'
+    OR (warpSkyfile.data_state = 'full'
     AND warpSkyfile.fault = 0
+    AND warpSkyfile.magicked >= 0
     AND warpSkyfile.quality = 0))
 -- Ensure input stacks are available
@@ -81 +82 @@
 -- Ensure template warps are available
     AND (diffInputSkyfile.warp2 IS NULL
-    OR (warpTemplateRun.state = 'full'
+    OR (warpTemplateSkyfile.data_state = 'full'
     AND warpTemplateSkyfile.fault = 0
+    AND warpSkyfile.magicked >= 0
     AND warpTemplateSkyfile.quality = 0))
 -- Ensure template stacks are available

trunk/ippTools/share/warptool_towarped.sql

@@ -1 @@
-SELECT DISTINCT
-    warpSkyCellMap.warp_id,
-    warpImfile.warp_skyfile_id,
-    warpSkyCellMap.skycell_id,
-    warpSkyCellMap.tess_id,
-    warpRun.fake_id,
-    warpRun.state,
-    warpRun.reduction,
-    camRun.cam_id,
-    rawExp.camera,
-    rawExp.exp_tag,
-    warpRun.workdir,
-    chipRun.magicked
-FROM warpRun
-JOIN warpSkyCellMap
-    USING(warp_id)
-JOIN warpImfile
-    ON warpSkyCellMap.warp_id = warpImfile.warp_id
-    AND warpSkyCellMap.skycell_id = warpImfile.skycell_id
-JOIN fakeRun
-    USING(fake_id)
-JOIN camRun
-    USING(cam_id)
-JOIN chipRun
-    USING(chip_id)
-JOIN chipProcessedImfile
-    USING(chip_id)
-JOIN rawExp
-    ON chipRun.exp_id = rawExp.exp_id
-LEFT JOIN warpSkyfile
-    ON warpRun.warp_id = warpSkyfile.warp_id
-    AND warpSkyCellMap.skycell_id = warpSkyfile.skycell_id
-    AND warpSkyCellMap.tess_id = warpSkyfile.tess_id
-LEFT JOIN warpMask
-    ON warpRun.label = warpMask.label
-WHERE
-    ((warpRun.state = 'new'
+SELECT 
+    warp_id,
+    warp_skyfile_id,
+    skycell_id,
+    tess_id,
+    fake_id,
+    state,
+    reduction,
+    cam_id,
+    camera,
+    exp_tag,
+    workdir,
+    magicked
+FROM (
+    SELECT
+        warpSkyCellMap.warp_id,
+        warpImfile.warp_skyfile_id,
+        warpSkyCellMap.skycell_id,
+        warpSkyCellMap.tess_id,
+        warpRun.fake_id,
+        warpRun.state,
+        warpRun.reduction,
+        camRun.cam_id,
+        rawExp.camera,
+        rawExp.exp_tag,
+        warpRun.workdir,
+        chipRun.magicked
+    FROM warpRun
+    JOIN warpSkyCellMap
+        USING(warp_id)
+    JOIN warpImfile
+        ON warpSkyCellMap.warp_id = warpImfile.warp_id
+        AND warpSkyCellMap.skycell_id = warpImfile.skycell_id
+    JOIN fakeRun
+        USING(fake_id)
+    JOIN camRun
+        USING(cam_id)
+    JOIN chipRun
+        USING(chip_id)
+    JOIN chipProcessedImfile
+        USING(chip_id)
+    JOIN rawExp
+        ON chipRun.exp_id = rawExp.exp_id
+    LEFT JOIN warpSkyfile
+        ON warpRun.warp_id = warpSkyfile.warp_id
+        AND warpSkyCellMap.skycell_id = warpSkyfile.skycell_id
+        AND warpSkyCellMap.tess_id = warpSkyfile.tess_id
+    LEFT JOIN warpMask
+        ON warpRun.label = warpMask.label
+    WHERE
+        warpRun.state = 'new'
         AND warpSkyfile.warp_id IS NULL
         AND warpSkyfile.skycell_id IS NULL
-        AND warpSkyfile.tess_id IS NULL)
-    OR (warpRun.state = 'update'
-        AND chipRun.magicked >= 0
-        AND warpSkyfile.fault = 0
-        AND warpSkyfile.data_state = 'cleaned')
-    )
-    AND fakeRun.state = 'full'
-    AND camRun.state = 'full'
-    AND chipRun.state = 'full'
-    AND warpMask.label IS NULL
-    AND warpSkyCellMap.fault = 0
+        AND warpSkyfile.tess_id IS NULL
+        AND fakeRun.state = 'full'
+        AND camRun.state = 'full'
+        AND chipRun.state = 'full'
+        AND warpMask.label IS NULL
+        AND warpSkyCellMap.fault = 0
+        -- where hook 1 %s
+    GROUP BY warp_id, skycell_id
+UNION
+    SELECT
+        warpSkyCellMap.warp_id,
+        warpImfile.warp_skyfile_id,
+        warpSkyCellMap.skycell_id,
+        warpSkyCellMap.tess_id,
+        warpRun.fake_id,
+        warpRun.state,
+        warpRun.reduction,
+        camRun.cam_id,
+        rawExp.camera,
+        rawExp.exp_tag,
+        warpRun.workdir,
+        MIN(chipProcessedImfile.magicked) AS magicked
+    FROM warpRun
+    JOIN warpSkyCellMap USING(warp_id)
+    JOIN warpImfile USING(warp_id, skycell_id)
+    JOIN warpSkyfile USING(warp_id, skycell_id)
+    JOIN fakeRun USING(fake_id)
+    JOIN camRun USING(cam_id)
+    JOIN chipRun USING(chip_id)
+    JOIN rawExp USING(exp_id)
+    LEFT JOIN chipProcessedImfile USING(chip_id, class_id)
+    WHERE warpRun.state = 'update'
+        AND warpSkyfile.data_state = 'update'
+        AND camRun.state = 'full'
+        AND chipProcessedImfile.data_state = 'full'
+        -- if warpSkyfile was magicked previously require inputs to be magicked
+        -- this blocks processing until all the chip inputs have been destreaked
+        AND (warpSkyfile.magicked = 0 OR chipProcessedImfile.magicked >= 0)
+        -- where hook 2 %s
+    GROUP BY warp_id, skycell_id
+    HAVING COUNT(warpSkyCellMap.class_id) = COUNT(chipProcessedImfile.class_id)
+) as towarped
+

trunk/ippTools/src/addtool.c

@@ -290 +290 @@
 
     // pxUpdateRun gets parameters from config->args and runs the update query
-    bool result = pxUpdateRun(config, where, &query, "addRun", false);
+    bool result = pxUpdateRun(config, where, &query, "addRun", "add_id", 
+        "addProcessedExp", false);
 
     psFree(query);
@@ -296 +297 @@
 
     return result;
-
-
-#ifdef notdef
-    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
-    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
-
-    if ((!state) && (!label)) {
-        psError(PXTOOLS_ERR_CONFIG, false, "parameters are required");
-        psFree(where);
-        return false;
-    }
-
-    if (state) {
-        // set addRun.state to state
-        if (!pxaddRunSetStateByQuery(config, where, state)) {
-            psFree(where);
-            return false;
-        }
-    }
-
-    if (label) {
-        // set addRun.label to label
-        if (!pxaddRunSetLabelByQuery(config, where, label)) {
-            psFree(where);
-            return false;
-        }
-    }
-
-    psFree(where);
-
-    return true;
-#endif
 }
 

trunk/ippTools/src/camtool.c

@@ -267 +267 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "camRun", true);
+    bool result = pxUpdateRun(config, where, &query, "camRun", "cam_id", "camProcessedExp", true);
     if (!result) {
         psError(psErrorCodeLast(), false, "pxUpdateRun failed");

trunk/ippTools/src/chiptool.c

@@ -59 +59 @@
 static bool importrunMode(pxConfig *config);
 static bool runstateMode(pxConfig *config);
+static bool setimfiletoupdateMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -101 +102 @@
         MODECASE(CHIPTOOL_MODE_IMPORTRUN,               importrunMode);
         MODECASE(CHIPTOOL_MODE_RUNSTATE,                runstateMode);
+        MODECASE(CHIPTOOL_MODE_SETIMFILETOUPDATE,       setimfiletoupdateMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -428 +430 @@
         return false;
     }
-    psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id)");
+    psString query = psStringCopy("UPDATE chipRun JOIN rawExp USING(exp_id) -- join hook %s\n");
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "chipRun", true);
+    bool result = pxUpdateRun(config, where, &query, "chipRun", "chip_id", "chipProcessedImfile", true);
     if (!result) {
         psError(psErrorCodeLast(), false, "pxUpdateRun failed");
@@ -1720 +1722 @@
     return true;
 }
+
+// a very specfic function to queue a cleaned chipProcessedImfile to be updated
+static bool setimfiletoupdateMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_S64(chip_id, config->args, "-chip_id", true, false);
+    PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+
+    psString query = pxDataGet("chiptool_setimfiletoupdate.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString setHook = psStringCopy("");
+    if (label) {
+        psStringAppend(&setHook, "\n , chipRun.label = '%s'", label);
+    }
+
+    if (class_id) {
+        psStringAppend(&query, " AND (chipProcessedImfile.class_id = '%s')", class_id);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, setHook, chip_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psFree(setHook);
+    psFree(query);
+
+    return true;
+}

trunk/ippTools/src/chiptool.h

@@ -33 +33 @@
     CHIPTOOL_MODE_REVERTPROCESSEDIMFILE,
     CHIPTOOL_MODE_UPDATEPROCESSEDIMFILE,
+    CHIPTOOL_MODE_SETIMFILETOUPDATE,
     CHIPTOOL_MODE_ADVANCEEXP,
     CHIPTOOL_MODE_BLOCK,

trunk/ippTools/src/chiptoolConfig.c

@@ -97 +97 @@
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_dist_group", 0,   "set dist group", NULL);
     psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note", 0,         "set note", NULL);
+
+    // -setimfiletoupdate
+    psMetadata *setimfiletoupdateArgs = psMetadataAlloc();
+    psMetadataAddS64(setimfiletoupdateArgs, PS_LIST_TAIL, "-chip_id", 0,   "search by chip ID (required)", 0);
+    psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-class_id", 0,  "search by class_id (required)", NULL);
+    psMetadataAddStr(setimfiletoupdateArgs, PS_LIST_TAIL, "-set_label",  0,"new value for label", NULL);
 
     // -pendingimfile
@@ -315 +321 @@
     PXOPT_ADD_MODE("-definecopy",           "create copy runs",                     CHIPTOOL_MODE_DEFINECOPY,        definecopyArgs);
     PXOPT_ADD_MODE("-updaterun",            "change chip run properties",           CHIPTOOL_MODE_UPDATERUN,            updaterunArgs);
+    PXOPT_ADD_MODE("-setimfiletoupdate",    "set imfile state to update",           CHIPTOOL_MODE_SETIMFILETOUPDATE,    setimfiletoupdateArgs);
     PXOPT_ADD_MODE("-pendingimfile",        "show pending imfiles",                 CHIPTOOL_MODE_PENDINGIMFILE,        pendingimfileArgs);
     PXOPT_ADD_MODE("-addprocessedimfile",   "add a processed imfile",               CHIPTOOL_MODE_ADDPROCESSEDIMFILE,   addprocessedimfileArgs);

trunk/ippTools/src/difftool.c

@@ -204 +204 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "diffRun", true);
+    bool result = pxUpdateRun(config, where, &query, "diffRun", "diff_id", "diffSkyfile", true);
 
     psFree(query);
@@ -210 +210 @@
 
     return result;
-
-#ifdef notdef
-    // required options
-    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false);
-    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
-    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
-
-    // Copy of my hacky work around from stacktool.c
-    if ((state)&&(diff_id)) {
-        // set detRun.state to state
-        return setdiffRunState(config, diff_id, state, false);
-    }
-
-    if ((state)&&(label)) {
-      return setdiffRunStateByLabel(config, label, state);
-    }
-
-    psError(PS_ERR_UNKNOWN, false, "Required options not found.");
-
-    return false;
-#endif
 }
 

trunk/ippTools/src/faketool.c

@@ -348 +348 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "fakeRun", true);
+    bool result = pxUpdateRun(config, where, &query, "fakeRun", "fake_id", "fakeProcessedImfile", true);
     if (!result) {
         psError(psErrorCodeLast(), false, "pxUpdateRun failed");

trunk/ippTools/src/pxtools.c

@@ -110 +110 @@
 
 // shared code for updating the various strings for a Run
-bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString table, bool has_dist_group)
+bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -116 +116 @@
     PS_ASSERT_PTR_NON_NULL(pQuery, false);
     PS_ASSERT_PTR_NON_NULL(*pQuery, false);
+    PS_ASSERT_PTR_NON_NULL(runTable, false);
+    PS_ASSERT_PTR_NON_NULL(idColumn, false);
+    PS_ASSERT_PTR_NON_NULL(fileTable, false);
 
     // make sure that -state is not the only selection parameter
@@ -158 +161 @@
         } while (0)
 
-    addColumn(table, state);
-    addColumn(table, data_group);
+    addColumn(runTable, state);
+    addColumn(runTable, data_group);
     if (has_dist_group) {
-        addColumn(table, dist_group);
-    }
-    addColumn(table, note);
-    addColumn(table, label);
+        addColumn(runTable, dist_group);
+    }
+    addColumn(runTable, note);
+    addColumn(runTable, label);
+
+    psString joinHook = psStringCopy("");
+    psString fileWhere = NULL;
+    if (state && !strcmp(state, "update")) {
+        psStringAppend(&joinHook, "\n JOIN %s USING(%s)", fileTable, idColumn);
+        psStringAppend(pQuery, ", %s.data_state = 'update'", fileTable);
+        psStringAppend(&fileWhere, "AND %s.data_state = 'cleaned'", fileTable);
+    }
 
     psString whereClause =  psDBGenerateWhereSQL(where, NULL);
     psStringAppend(pQuery, " %s", whereClause);
     psFree(whereClause);
+    if (fileWhere) {
+        psStringAppend(pQuery, fileWhere);
+    }
 
     bool mdok;                          // Status of MD lookup
@@ -176 +190 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, *pQuery)) {
+    if (!p_psDBRunQueryF(config->dbh, *pQuery, joinHook)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;

trunk/ippTools/src/pxtools.h

@@ -65 +65 @@
 bool pxGetOptions(FILE *stream, int argc, char **argv, pxConfig *config, psMetadata *modes, psMetadata *argSets);
 
-bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString table, bool has_dist_group);
+bool pxUpdateRun(pxConfig *config, psMetadata *where, psString *pQuery, psString runTable, psString idColumn, psString fileTable, bool has_dist_group);
 
 #define PXOPT_ADD_MODE(option, comment, modeval, argset) \

trunk/ippTools/src/stacktool.c

@@ -612 +612 @@
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "stackRun", true);
+    bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true);
 
     psFree(query);

trunk/ippTools/src/warptool.c

@@ -402 +402 @@
     }
 
-    psString query = psStringCopy("UPDATE warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)");
+    psString query = psStringCopy("UPDATE warpRun JOIN warpSkyfile USING(warp_id) JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id)");
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "warpRun", true);
+    bool result = pxUpdateRun(config, where, &query, "warpRun", "warp_id", "warpSkyfile", true);
 
     psFree(query);
@@ -907 +907 @@
     }
 
+    psString whereStr = psStringCopy("");
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+        psStringAppend(&whereStr, "\n AND %s", whereClause);
         psFree(whereClause);
     }
@@ -921 +922 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    if (!p_psDBRunQueryF(config->dbh, query, whereStr, whereStr)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
         return false;
     }
+    psFree(whereStr);
     psFree(query);