Changeset 29561 for trunk/ippTools

Timestamp:

Oct 26, 2010, 8:39:32 AM (16 years ago)

Author:

bills

Message:

Change the destreak cleanup to preserve the rows in the magicDSFile table

Location:

trunk/ippTools

Files:

• share/Makefile.am (modified) (2 diffs)
• share/chiptool_setimfiletoupdate.sql (modified) (1 diff)
• share/magicdstool_change_file_data_state.sql (added)
• share/magicdstool_change_run_state.sql (added)
• share/magicdstool_revertupdated.sql (added)
• share/magicdstool_setfiletoupdate.sql (added)
• share/magicdstool_todestreak_camera.sql (modified) (1 diff)
• share/magicdstool_todestreak_chip.sql (modified) (2 diffs)
• share/magicdstool_todestreak_diff.sql (modified) (2 diffs)
• share/magicdstool_todestreak_raw.sql (modified) (1 diff)
• share/magicdstool_todestreak_warp.sql (modified) (1 diff)
• share/magicdstool_torevert_chip.sql (modified) (1 diff)
• src/magicdstool.c (modified) (9 diffs)
• src/magicdstool.h (modified) (1 diff)
• src/magicdstoolConfig.c (modified) (3 diffs)

trunk/ippTools/share/Makefile.am

@@ -238 +238 @@
         magictool_exposure.sql \
         magicdstool_clearstatefaults.sql \
+        magicdstool_change_file_data_state.sql \
+        magicdstool_change_run_state.sql \
         magicdstool_completed_runs.sql \
         magicdstool_completedrevert.sql \
@@ -252 +254 @@
         magicdstool_getskycells.sql \
         magicdstool_revertdestreakedfile.sql \
+        magicdstool_revertupdated.sql \
+        magicdstool_setfiletoupdate.sql \
         magicdstool_tocleanup.sql \
         magicdstool_todestreak_camera.sql \

trunk/ippTools/share/chiptool_setimfiletoupdate.sql

@@ -11 +11 @@
     AND (chipRun.state = 'cleaned' OR chipRun.state = 'update')
     AND (chipProcessedImfile.data_state = 'cleaned')
-    -- don't queue update if the associated magicDSFile exists
+    -- don't queue update if the associated magicDSFile exists and isn't cleaned
     AND (chipRun.magicked = 0 
-      OR (magicDSRun.state = 'cleaned' OR magicDSRun.state = 'new')
-            AND magicDSFile.component IS NULL)
+      OR ((magicDSRun.state = 'cleaned' OR magicDSRun.state = 'update')
+            AND (magicDSFile.data_state = 'cleaned`' OR magicDSFile.data_state = 'update'))
+    )

trunk/ippTools/share/magicdstool_todestreak_camera.sql

@@ -2 +2 @@
     magicDSRun.magic_ds_id,
     magicDSRun.magic_id,
+    magicDSRun.state,
     chipRun.exp_id,
     camRun.magicked,

trunk/ippTools/share/magicdstool_todestreak_chip.sql

@@ -2 +2 @@
     magicDSRun.magic_ds_id,
     magicDSRun.magic_id,
+    magicDSRun.state,
     chipRun.exp_id,
     chipRun.magicked,
@@ -37 +38 @@
     ON magicDSRun.label = Label.label
 WHERE
-    magicDSRun.state = 'new'
+    ((magicDSRun.state = 'new' AND magicDSFile.component IS NULL)
+     OR (magicDSRun.state = 'update' AND magicDSFile.data_state = 'update'
+         AND magicDSFile.fault = 0))
     AND magicDSRun.stage = 'chip'
     AND (chipRun.state = 'full' OR (chipRun.state = 'update' and chipProcessedImfile.data_state = 'full'))
     AND chipProcessedImfile.fault = 0
     AND chipProcessedImfile.quality = 0
-    AND magicDSFile.component IS NULL
     AND (Label.active OR Label.active IS NULL)

trunk/ippTools/share/magicdstool_todestreak_diff.sql

@@ -3 +3 @@
     magicDSRun.magic_ds_id,
     magicRun.magic_id,
+    magicDSRun.state,
     magicRun.exp_id,
     magicDSRun.label,
@@ -50 +51 @@
     magicDSRun.magic_ds_id,
     magicRun.magic_id,
+    magicDSRun.state,
     magicRun.exp_id,
     magicDSRun.label,

trunk/ippTools/share/magicdstool_todestreak_raw.sql

@@ -2 +2 @@
     magicDSRun.magic_ds_id,
     magicRun.magic_id,
+    magicDSRun.state,
     magicRun.exp_id,
     magicDSRun.label,

trunk/ippTools/share/magicdstool_todestreak_warp.sql

@@ -2 +2 @@
     magicDSRun.magic_ds_id,
     magicRun.magic_id,
+    magicDSRun.state,
     magicRun.exp_id,
     magicDSRun.label,

trunk/ippTools/share/magicdstool_torevert_chip.sql

@@ -22 +22 @@
     JOIN rawExp ON chipRun.exp_id = rawExp.exp_id
 WHERE magicDSRun.stage = 'chip'
-    AND ((magicDSRun.state = 'new' AND magicDSFile.fault > 0)
+    AND (((magicDSRun.state = 'new' OR (magicDSRun.state = 'update'))
+            AND magicDSFile.fault > 0)
+            -- why don't we require a fault for these states?
          OR ((magicDSRun.state = 'goto_censored' OR magicDSRun.state = 'goto_restored')
               AND ((backup_path_base IS NOT NULL) OR (recovery_path_base IS NOT NULL))

trunk/ippTools/src/magicdstool.c

@@ -39 +39 @@
 static bool advancerunMode(pxConfig *config);
 static bool revertdestreakedfileMode(pxConfig *config);
+static bool updatedestreakedfileMode(pxConfig *config);
 static bool clearstatefaultsMode(pxConfig *config);
 static bool getskycellsMode(pxConfig *config);
@@ -45 +46 @@
 static bool completedrevertMode(pxConfig *config);
 static bool tocleanupMode(pxConfig *config);
+static bool tofullfileMode(pxConfig *config);
+static bool tocleanedfileMode(pxConfig *config);
+static bool setfiletoupdateMode(pxConfig *config);
 
 static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, psString extraSetString, psMetadata *where, const char *state);
@@ -73 +77 @@
         MODECASE(MAGICDSTOOL_MODE_ADVANCERUN,          advancerunMode);
         MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode);
+        MODECASE(MAGICDSTOOL_MODE_UPDATEDESTREAKEDFILE,updatedestreakedfileMode);
         MODECASE(MAGICDSTOOL_MODE_CLEARSTATEFAULTS,    clearstatefaultsMode);
         MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS,         getskycellsMode);
@@ -79 +84 @@
         MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT,     completedrevertMode);
         MODECASE(MAGICDSTOOL_MODE_TOCLEANUP,           tocleanupMode);
+        MODECASE(MAGICDSTOOL_MODE_TOFULLFILE,          tofullfileMode);
+        MODECASE(MAGICDSTOOL_MODE_TOCLEANEDFILE,       tocleanedfileMode);
+        MODECASE(MAGICDSTOOL_MODE_SETFILETOUPDATE,     setfiletoupdateMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -558 +566 @@
     PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
 
+    if (!strcmp(state, "update")) {
+        fprintf(stderr, "'-updaterun -set_state update' is not supported. Use -setfiletoupdate");
+        return false;
+    }
+
     // optional
     PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
@@ -907 +920 @@
         return false;
     }
+
+    return true;
+}
+static bool updatedestreakedfileMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required values
+    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
+    PXOPT_LOOKUP_STR(component, config->args, "-component", true, false);
+
+    // default values
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_STR(backup_path_base, config->args, "-backup_path_base", false, false);
+    PXOPT_LOOKUP_STR(recovery_path_base, config->args, "-recovery_path_base", false, false);
+    PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", false, false);
+    PXOPT_LOOKUP_F32(streak_frac, config->args, "-streak_frac", false, false);
+    PXOPT_LOOKUP_F32(nondiff_frac, config->args, "-nondiff_frac", false, false);
+    PXOPT_LOOKUP_F32(run_time, config->args, "-run_time", false, false);
+
+    psString query = psStringCopy("UPDATE magicDSFile");
+
+
+    char *initial_separator = " SET";
+    char *sep = initial_separator;
+    if (fault) {
+        psStringAppend(&query, "%s fault = %d", sep, fault);
+        sep = ", ";
+    }
+    if (backup_path_base) {
+        psStringAppend(&query, "%s backup_path_base = '%s'", sep, backup_path_base);
+        sep = ", ";
+    }
+    if (recovery_path_base) {
+        psStringAppend(&query, "%s recovery_path_base = '%s'", sep, recovery_path_base);
+        sep = ", ";
+    }
+    if (data_state) {
+        psStringAppend(&query, "%s data_state = '%s'", sep, data_state);
+        sep = ", ";
+    }
+    if (streak_frac) {
+        psStringAppend(&query, "%s streak_frac = '%f'", sep, streak_frac);
+        sep = ", ";
+    }
+    if (nondiff_frac) {
+        psStringAppend(&query, "%s nondiff_frac = '%f'", sep, nondiff_frac);
+        sep = ", ";
+    }
+    if (run_time) {
+        psStringAppend(&query, "%s run_time = '%f'", sep, run_time);
+        sep = ", ";
+    }
+    if (sep == initial_separator) {
+        psFree(query);
+        psError(PS_ERR_UNKNOWN, true, "must set at least one value");
+        return false;
+    }
+
+    psStringAppend(&query, "\nWHERE magic_ds_id = %" PRId64 " AND component = '%s'\n", magic_ds_id, component);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
 
     return true;
@@ -1019 +1099 @@
     PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "==");
     PXOPT_COPY_STR(config->args, where, "-component", "component", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "fault", "==");
     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
-    psString query = pxDataGet("magicdstool_revertdestreakedfile.sql");
+    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
+
+    psString queryFile = NULL;
+    if (state) {
+        if (! strcmp(state, "new") ) {
+            queryFile = "magicdstool_revertdestreakedfile.sql";
+        } else if (!strcmp(state, "update")) {
+            queryFile = "magicdstool_revertupdated.sql";
+        } else {
+            psError(PXTOOLS_ERR_SYS, true, "%s is not a valid value for state", state);
+            return false;
+        }
+    } else {
+        queryFile = "magicdstool_revertdestreakedfile.sql";
+    }
+    psString query = pxDataGet(queryFile);
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
 
     if (psListLength(where->list)) {
@@ -1241 +1341 @@
           (strcmp(state, "censored") == 0) ||
           (strcmp(state, "cleaned") == 0) ||
+          (strcmp(state, "update") == 0) ||
           (strcmp(state, "goto_restored") == 0) ||
           (strcmp(state, "goto_censored") == 0) ||
@@ -1520 +1621 @@
     return true;
 }
+
+// update magicDSFile.data_state to given value.
+// afterwards, if all files in the magicDSRun have the new state, 
+// update the state for it as well
+// shared code for the modes -tocleanedfile -tofullfile
+
+static bool change_file_data_state(pxConfig *config, psString data_state)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // magic_id, component
+    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
+    PXOPT_LOOKUP_STR(component, config->args, "-component", true, false);
+
+    psString query = pxDataGet("magicdstool_change_file_data_state.sql");
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!strcmp(data_state, "full")) {
+        // if -tofullfile optoinally set the magicked value for the component
+        PXOPT_LOOKUP_BOOL(setmagicked, config->args, "-setmagicked", false);
+        // set the image file's magicked flag
+        if (!setMagicked(config, magic_ds_id, component)) {
+            psError(PS_ERR_UNKNOWN, false, "setMagicked failed");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, magic_ds_id, component)) {
+        psFree(query);
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(query);
+    if (psDBAffectedRows(config->dbh) < 1) {
+        psError(PS_ERR_UNKNOWN, false, "should have affected atleast 1 row");
+        return false;
+    }
+
+    query = pxDataGet("magicdstool_change_run_state.sql");
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, magic_ds_id, data_state)) {
+        psFree(query);
+        // rollback
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(query);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    return true;
+}
+static bool tocleanedfileMode(pxConfig *config)
+{
+    return change_file_data_state(config, "cleaned");
+}
+static bool tofullfileMode(pxConfig *config)
+{
+    return change_file_data_state(config, "full");
+}
+/*
+static bool topurgedimfileMode(pxConfig *config)
+{
+    return change_imfile_data_state(config, "purged", "goto_purged");
+}
+static bool toscrubbedfileMode(pxConfig *config)
+{
+  return change_file_data_state(config, "scrubbed", "goto_scrubbed");
+}
+*/
+
+// a very specfic function to queue a cleaned magicDSFile to be updated
+static bool setfiletoupdateMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
+    PXOPT_LOOKUP_STR(component, config->args, "-component", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+
+    psString query = pxDataGet("magicdstool_setfiletoupdate.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString setHook = psStringCopy("");
+    if (label) {
+        psStringAppend(&setHook, "\n , magicDSRun.label = '%s'", label);
+    }
+
+    if (component) {
+        psStringAppend(&query, " AND (magicDSFile.component = '%s')", component);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, setHook, magic_ds_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    psFree(setHook);
+    psFree(query);
+
+    return true;
+}
+
+

trunk/ippTools/src/magicdstool.h

@@ -38 +38 @@
     MAGICDSTOOL_MODE_COMPLETEDREVERT,
     MAGICDSTOOL_MODE_TOCLEANUP,
+    MAGICDSTOOL_MODE_TOCLEANEDFILE,
+    MAGICDSTOOL_MODE_TOFULLFILE,
+    MAGICDSTOOL_MODE_SETFILETOUPDATE,
+    MAGICDSTOOL_MODE_UPDATEDESTREAKEDFILE,
 } MAGICDStoolMode;
 

trunk/ippTools/src/magicdstoolConfig.c

@@ -137 +137 @@
     psMetadataAddS16(adddestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
 
+    // -updatedestreakedfile
+    psMetadata *updatedestreakedfileArgs = psMetadataAlloc();
+    psMetadataAddS64(updatedestreakedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magictool ID (required)", 0);
+    psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "define component name (required)", NULL);
+    psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-state", 0, "current value for magicDSRun.state", NULL);
+    psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-backup_path_base", 0, "define backup URI", NULL);
+    psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-recovery_path_base", 0, "define recovery pixels URI", NULL);
+    psMetadataAddStr(updatedestreakedfileArgs, PS_LIST_TAIL, "-data_state", 0, "change data_state", NULL);
+    psMetadataAddBool(updatedestreakedfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false);
+    psMetadataAddF32(updatedestreakedfileArgs, PS_LIST_TAIL, "-streak_frac", 0, "set fraction of pixels masked by streaks", 0);
+    psMetadataAddF32(updatedestreakedfileArgs, PS_LIST_TAIL, "-nondiff_frac", 0, "set fraction of pixels masked because nondiffed", 0);
+    psMetadataAddF32(updatedestreakedfileArgs, PS_LIST_TAIL, "-run_time", 0, "set the streaksremove run time for component ", 0);
+    psMetadataAddS16(updatedestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
+
     // -revertdestreakedfile
     psMetadata *revertdestreakedfileArgs = psMetadataAlloc();
     psMetadataAddS64(revertdestreakedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magictool de-streak ID", 0);
+    psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-state", 0, "current value for magicDSRun.state", NULL);
     psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL);
     psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
@@ -197 +212 @@
     psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -tocleanedfile
+    psMetadata *tocleanedfileArgs = psMetadataAlloc();
+    psMetadataAddS64(tocleanedfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic_ds_id", 0);
+    psMetadataAddStr(tocleanedfileArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL);
+
+    // -tofullfile
+    psMetadata *tofullfileArgs = psMetadataAlloc();
+    psMetadataAddS64(tofullfileArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic_ds_id", 0);
+    psMetadataAddStr(tofullfileArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL);
+    psMetadataAddBool(tofullfileArgs, PS_LIST_TAIL, "-setmagicked", 0, "update the magicked state of the file", false);
+
+    // -setfiletoupdate
+    psMetadata *setfiletoupdateArgs = psMetadataAlloc();
+    psMetadataAddS64(setfiletoupdateArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic_ds_id", 0);
+    psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL);
+    psMetadataAddStr(setfiletoupdateArgs, PS_LIST_TAIL, "-set_label", 0, "set new label", NULL);
+
     psFree(now);
 
@@ -230 +263 @@
                     MAGICDSTOOL_MODE_TOCLEANUP, tocleanupArgs);
 
+    PXOPT_ADD_MODE("-tofullfile", "set component's data_state to full",
+                    MAGICDSTOOL_MODE_TOFULLFILE, tofullfileArgs);
+    PXOPT_ADD_MODE("-tocleanedfile", "set component's data_state to cleaned",
+                    MAGICDSTOOL_MODE_TOCLEANEDFILE, tocleanedfileArgs);
+    PXOPT_ADD_MODE("-setfiletoupdate", "set component's data_state to update",
+                    MAGICDSTOOL_MODE_SETFILETOUPDATE, setfiletoupdateArgs);
+    PXOPT_ADD_MODE("-updatedestreakedfile", "update parameters of destreaked file",
+                    MAGICDSTOOL_MODE_UPDATEDESTREAKEDFILE, updatedestreakedfileArgs);
+
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {