Changeset 24683

Timestamp:

Jul 6, 2009, 12:52:22 PM (17 years ago)

Author:

bills

Message:

Add hooks for carefully reverting faulted destreaked files. Support "destreak" operation at
camera stage. (For censoring detections)

Location:

trunk/ippTools

Files:

• share/Makefile.am (modified) (1 diff)
• share/magicdstool_completed_runs.sql (modified) (3 diffs)
• share/magicdstool_todestreak.sql (modified) (1 diff)
• share/magicdstool_torevert_chip.sql (added)
• share/magicdstool_torevert_diff.sql (added)
• share/magicdstool_torevert_raw.sql (added)
• share/magicdstool_torevert_warp.sql (added)
• src/magicdstool.c (modified) (7 diffs)
• src/magicdstool.h (modified) (1 diff)
• src/magicdstoolConfig.c (modified) (2 diffs)

trunk/ippTools/share/Makefile.am

@@ -159 +159 @@
      magicdstool_toremove.sql \
      magicdstool_torestore.sql \
+     magicdstool_torevert_raw.sql \
+     magicdstool_torevert_chip.sql \
+     magicdstool_torevert_warp.sql \
+     magicdstool_torevert_diff.sql \
      pstamptool_addjob_otherjob.sql \
      pstamptool_addjob_stampjob.sql \

trunk/ippTools/share/magicdstool_completed_runs.sql

@@ -16 +16 @@
     GROUP BY
         magicDSRun.magic_ds_id,
-        rawImfile.exp_id
+        stage_id
     HAVING
         COUNT(rawImfile.class_id) = COUNT(magicDSFile.component)
@@ -35 +35 @@
     GROUP BY
         magic_ds_id,
-        chip_id
+        stage_id
     HAVING
         COUNT(chipProcessedImfile.class_id) = COUNT(magicDSFile.component)
         AND SUM(magicDSFile.fault) = 0
+UNION
+-- camera stage
+SELECT
+    magicDSRun.magic_ds_id
+    FROM magicDSRun
+    JOIN camProcessedExp ON stage_id = camProcessedExp.cam_id
+    LEFT JOIN magicDSFile
+        ON magicDSFile.magic_ds_id = magicDSRun.magic_ds_id
+        AND magicDSFile.component IS NOT NULL
+    WHERE
+        magicDSRun.state = 'new'
+        AND magicDSRun.stage = 'camera'
+        AND camProcessedExp.quality = 0
+        AND magicDSFile.fault = 0
+    GROUP BY
+        magic_ds_id,
+        stage_id
 UNION
 -- warp stage
@@ -55 +72 @@
     GROUP BY
         magicDSRun.magic_ds_id,
-        warp_id
+        stage_id
     HAVING
         COUNT(warpSkyfile.skycell_id) = COUNT(magicDSFile.component)

trunk/ippTools/share/magicdstool_todestreak.sql

@@ -67 +67 @@
     AND chipProcessedImfile.fault = 0
     AND chipProcessedImfile.quality = 0
+    AND magicDSFile.component IS NULL
+UNION
+  -- camProcessedExp
+SELECT DISTINCT
+    magicDSRun.magic_ds_id,
+    magicDSRun.magic_id,
+    chipRun.exp_id,
+    magicDSRun.label,
+    camera,
+    magicMask.uri as streaks_uri,
+    stage,
+    stage_id,
+    'exposure' as component,
+    NULL AS uri,
+    camProcessedExp.path_base,
+    magicRun.inverse,
+    camProcessedExp.path_base as cam_path_base,
+    outroot,
+    recoveryroot,
+    re_place,
+    remove
+FROM magicDSRun
+JOIN magicMask USING (magic_id)
+JOIN magicRun USING(magic_id)
+JOIN camRun ON magicDSRun.stage_id = camRun.cam_id
+JOIN camProcessedExp ON camRun.cam_id = camProcessedExp.cam_id
+JOIN chipRun USING(chip_id)
+JOIN rawExp ON chipRun.exp_id = rawExp.exp_id
+LEFT JOIN magicDSFile
+    ON magicDSRun.magic_ds_id = magicDSFile.magic_ds_id
+WHERE
+    magicDSRun.state = 'new'
+    AND magicDSRun.stage = 'camera'
+    AND camRun.state = 'full'
+    AND chipRun.state = 'full'
+    AND chipRun.magicked
+    AND camProcessedExp.fault = 0
+    AND camProcessedExp.quality = 0
     AND magicDSFile.component IS NULL
 UNION

trunk/ippTools/src/magicdstool.c

@@ -40 +40 @@
 static bool toremoveMode(pxConfig *config);
 static bool torestoreMode(pxConfig *config);
+static bool torevertMode(pxConfig *config);
 
 static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state);
@@ -72 +73 @@
         MODECASE(MAGICDSTOOL_MODE_TOREMOVE,            toremoveMode);
         MODECASE(MAGICDSTOOL_MODE_TORESTORE,           torestoreMode);
+        MODECASE(MAGICDSTOOL_MODE_TOREVERT,            torevertMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -502 +504 @@
     psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
 
+    if (!strcmp(stage, "camera")) {
+        // no there is no magicked column in camProcessedExp
+        psFree(output);
+        return true;
+    }
 
     // chose the appropriate query based on the stage
@@ -570 +577 @@
     } else if (!strcmp(stage, "chip")) {
         query = "UPDATE chipRun SET magicked = %" PRId64 " where chip_id = %" PRId64;
+    } else if (!strcmp(stage, "camera")) {
+        query = "UPDATE camRun SET magicked = %" PRId64 " where cam_id = %" PRId64;
     } else if (!strcmp(stage, "warp")) {
         query = "UPDATE warpRun SET magicked = %" PRId64 " where warp_id = %" PRId64;
@@ -674 +683 @@
     } else if (!strcmp(stage, "chip")) {
         stageNum = 1;
+    } else if (!strcmp(stage, "camera")) {
+        stageNum = 2;
     } else if (!strcmp(stage, "warp")) {
-        stageNum = 2;
+        stageNum = 3;
     } else {
         psError(PXTOOLS_ERR_DATA, true, "%s is not a valid value for stage", stage);
@@ -718 +729 @@
         *stage_id = psMetadataLookupS64(NULL, row, "chip_id");
     } else if (stageNum == 2) {
+        *stage_id = *cam_id;
+    } else if (stageNum == 3) {
         *stage_id = psMetadataLookupS64(NULL, row, "warp_id");
     }
@@ -1039 +1052 @@
     return true;
 }
+
+
+static bool torevertMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
+
+    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString sql_file = NULL;
+    psStringAppend(&sql_file, "magicdstool_torevert_%s.sql", stage);
+
+    psString query = pxDataGet(sql_file);
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file);
+        psFree(sql_file);
+        return false;
+    }
+    psFree(sql_file);
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magicdstool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "torevert", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+

trunk/ippTools/src/magicdstool.h

@@ -34 +34 @@
     MAGICDSTOOL_MODE_TOREMOVE,
     MAGICDSTOOL_MODE_TORESTORE,
+    MAGICDSTOOL_MODE_TOREVERT,
 } MAGICDStoolMode;
 

trunk/ippTools/src/magicdstoolConfig.c

@@ -130 +130 @@
     psMetadataAddBool(torestoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
+    // -torevert
+    psMetadata *torevertArgs = psMetadataAlloc();
+    psMetadataAddS64(torevertArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
+    psMetadataAddS64(torevertArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
+    psMetadataAddStr(torevertArgs, PS_LIST_TAIL, "-stage", 0, "define output directory (required)", NULL);
+    psMetadataAddStr(torevertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
+    psMetadataAddU64(torevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(torevertArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
     psFree(now);
 
@@ -153 +162 @@
     PXOPT_ADD_MODE("-torestore", "images pending restore of exicsed streak pixels",
                     MAGICDSTOOL_MODE_TORESTORE, torestoreArgs);
+    PXOPT_ADD_MODE("-torevert", "faulted images to revert",
+                    MAGICDSTOOL_MODE_TOREVERT, torevertArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {