Changeset 30417

Timestamp:

Jan 28, 2011, 2:31:03 PM (15 years ago)

Author:

bills

Message:

merge in magic cleanup code.

Location:

tags/ipp-20101215/ippTools

Files:

• share (modified) (1 prop)
• share/Makefile.am (modified) (2 diffs)
• share/magictool_setgotocleaned.sql (copied) (copied from trunk/ippTools/share/magictool_setgotocleaned.sql )
• share/magictool_tocleanup.sql (copied) (copied from trunk/ippTools/share/magictool_tocleanup.sql )
• share/pubtool_pending.sql (modified) (1 prop)
• share/regtool_checkstatus.sql (copied) (copied from trunk/ippTools/share/regtool_checkstatus.sql )
• src/magictool.c (modified) (7 diffs, 1 prop)
• src/magictool.h (modified) (1 diff, 1 prop)
• src/magictoolConfig.c (modified) (2 diffs, 1 prop)

tags/ipp-20101215/ippTools/share/Makefile.am

@@ -246 +246 @@
         magictool_exposure.sql \
         magictool_setfull.sql \
+        magictool_setgotocleaned.sql \
+        magictool_tocleanup.sql \
         magicdstool_clearstatefaults.sql \
         magicdstool_change_file_data_state.sql \
@@ -333 +335 @@
         regtool_updatebyqueryimfile.sql \
         regtool_pendingburntoolimfile.sql \
+        regtool_checkstatus.sql \
         stacktool_associationdefine_select.sql \
         stacktool_definebyquery_insert.sql \

tags/ipp-20101215/ippTools/src/magictool.c

@@ -49 +49 @@
 static bool censorrunMode(pxConfig *config);
 static bool exposureMode(pxConfig *config);
+static bool setgotocleanedMode(pxConfig *config);
+static bool tocleanupMode(pxConfig *config);
+static bool setworkdirstateMode(pxConfig *config);
 
 static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString);
@@ -89 +92 @@
         MODECASE(MAGICTOOL_MODE_CENSORRUN,           censorrunMode);
         MODECASE(MAGICTOOL_MODE_EXPOSURE,            exposureMode);
+        MODECASE(MAGICTOOL_MODE_SETGOTOCLEANED,      setgotocleanedMode);
+        MODECASE(MAGICTOOL_MODE_TOCLEANUP,           tocleanupMode);
+        MODECASE(MAGICTOOL_MODE_SETWORKDIRSTATE,     setworkdirstateMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -837 +843 @@
     psFree(where);
 
-
-    // First look for tree nodes that need to be processed
+    // First look for branch nodes that need to be processed.
     // These get priority over skycells because they are from runs
-    // that are already in progress
+    // that are already in progress and there are fewer of them.
+    // When we looked for skycells first we got starved.
 
     // first find incomplete magicRuns
@@ -849 +855 @@
     }
 
-    // we limit the query even though it is cheap (only magic_id is selected)
-
-    // XXX: The following comment no longer applies
-    // XXX: if the first 1000 unfinished magicRuns have no ready nodes
-    // that haven't faulted, later runs won't get returned even though
-    // they have work to do. When we used a limit of 100 we actually ran
-    // into this problem. Since we're using labels a limit of 1000 will
-    // probably be ok.
+    // Find outstanding magicRuns in new state.
+    // We limit the query, but this is problematic. In practice do
+    // we need to?
+    // XXX: If the first 1000 magicRuns have no branch nodes ready
+    // but higher runs do we they won't be noticed.
+    // Perhaps have this limit be an argument.
     {
         psString limitString = psDBGenerateLimitSQL( 1000 );
@@ -886 +890 @@
         return false;
     }
+
     if (!psArrayLength(magicRuns)) {
-        // no magicRuns pending. Nothing to do
+        // no pending magicRuns nothing to do
         psTrace("magictool", PS_LOG_INFO, "no rows found");
-        psFree(magicRuns);
         return true;
     }
-
     psArray *output = psArrayAllocEmpty(100);
-    // the following is indented because I removed a conditional and I decided not to shift over
+
+    {
         query = pxDataGet("magictool_toprocess_tree.sql");
         if (!query) {
@@ -994 +998 @@
             }
         }
-    // the above code is indented because I removed a conditional block and I decided not to shift it over
-
-    // look for "inputs" that need to processed
+    }
+
+    // look for "inputs" (skycells) that need to processed
     query = pxDataGet("magictool_toprocess_inputs.sql");
     if (!query) {
@@ -1633 +1637 @@
     return true;
 }
+static bool setgotocleanedMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-exp_id", "magicRun.exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "magicRun.label", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "magicRun.data_group", "LIKE");
+
+    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
+
+    psString query = psStringCopy("UPDATE magicRun SET workdir_state = 'goto_cleaned'\n");
+    if (set_label) {
+        psStringAppend(&query, ", label = '%s'", set_label);
+    }
+    // This mode doubles as a revert function for cleanup errors
+    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false);
+    if (!clearfault) {
+        psStringAppend(&query, "WHERE workdir_state = 'dirty'");
+    } else {
+        psStringAppend(&query, "WHERE workdir_state = 'error_cleaned'");
+    }
+    psStringAppend(&query, "\nAND (magicRun.state = 'full' OR magicRun.state = 'drop')");
+
+    // Require search parameters unless we're just clearing faults
+    if (psListLength(where->list)) {
+        psString clause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\nAND %s", clause);
+        psFree(clause);
+        psFree(where);
+    } else if ( !clearfault) {
+        psError(PS_ERR_UNKNOWN, false, "search parameters are required");
+        psFree(where);
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
+static bool setworkdirstateMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required
+    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
+    PXOPT_LOOKUP_STR(workdir_state, config->args, "-set_workdir_state", true, false);
+
+    if (strcmp(workdir_state, "cleaned") && strcmp(workdir_state, "error_cleaned")) {
+        psError(PS_ERR_UNKNOWN, true, "%s is not a valid value for workdir_state", workdir_state);
+        return false;
+    }
+    
+    psString query = NULL;
+    psStringAppend(&query, "UPDATE magicRun SET workdir_state = '%s' WHERE magic_id = %" PRId64, workdir_state, magic_id);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+static bool tocleanupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
+    pxAddLabelSearchArgs (config, where, "-data_group", "magicRun.data_group", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("magictool_tocleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    psStringAppend(&query, "\nORDER BY priority DESC, magic_id");
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("magictool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (psArrayLength(output)) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "tocleanup", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+
+    psFree(output);
+
+    return true;
+}

tags/ipp-20101215/ippTools/src/magictool.h

@@ -43 +43 @@
     MAGICTOOL_MODE_CENSORRUN,
     MAGICTOOL_MODE_EXPOSURE,
+    MAGICTOOL_MODE_SETGOTOCLEANED,
+    MAGICTOOL_MODE_TOCLEANUP,
+    MAGICTOOL_MODE_SETWORKDIRSTATE,
 } MAGICtoolMode;
 

tags/ipp-20101215/ippTools/src/magictoolConfig.c

@@ -178 +178 @@
     psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-inverse", 0, "select the inverse subtraction?", false);
     psMetadataAddBool(exposureArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -setgotocleaned
+    psMetadata *setgotocleanedArgs = psMetadataAlloc();
+    psMetadataAddS64(setgotocleanedArgs, PS_LIST_TAIL, "-magic_id", 0, "select by magictool ID", 0);
+    psMetadataAddS64(setgotocleanedArgs, PS_LIST_TAIL, "-exp_id", 0, "select by exposure ID)", 0);
+    psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-label",  0, "select by label", NULL);
+    psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-data_group",  0, "select by data_group", NULL);
+    psMetadataAddStr(setgotocleanedArgs, PS_LIST_TAIL, "-set_label",  0, "set new label", NULL);
+    psMetadataAddBool(setgotocleanedArgs, PS_LIST_TAIL, "-clearfault",  0, "clear cleanup errors", NULL);
+
+    // -tocleanup
+    psMetadata *tocleanupArgs = psMetadataAlloc();
+    psMetadataAddS64(tocleanupArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
+    psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-label",    PS_META_DUPLICATE_OK, "search by label", NULL);
+    psMetadataAddStr(tocleanupArgs, PS_LIST_TAIL, "-data_group",    PS_META_DUPLICATE_OK, "search by label", NULL);
+    psMetadataAddU64(tocleanupArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(tocleanupArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -setworkdirstate
+    psMetadata *setworkdirstateArgs = psMetadataAlloc();
+    psMetadataAddS64(setworkdirstateArgs, PS_LIST_TAIL, "-magic_id", 0, "select by magictool ID (required)", 0);
+    psMetadataAddStr(setworkdirstateArgs, PS_LIST_TAIL, "-set_workdir_state", 0, "new workdir_state (required)", 0);
 
     psFree(now);
@@ -202 +224 @@
     PXOPT_ADD_MODE("-censorrun",           "", MAGICTOOL_MODE_CENSORRUN,           censorrunArgs);
     PXOPT_ADD_MODE("-exposure",            "", MAGICTOOL_MODE_EXPOSURE,            exposureArgs);
+    PXOPT_ADD_MODE("-setgotocleaned",      "", MAGICTOOL_MODE_SETGOTOCLEANED,      setgotocleanedArgs);
+    PXOPT_ADD_MODE("-tocleanup",           "", MAGICTOOL_MODE_TOCLEANUP,           tocleanupArgs);
+    PXOPT_ADD_MODE("-setworkdirstate",     "", MAGICTOOL_MODE_SETWORKDIRSTATE,     setworkdirstateArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {