Changeset 29498

Timestamp:

Oct 20, 2010, 1:00:59 PM (16 years ago)

Author:

bills

Message:

From trunk. save magic outputs in nebulous. -revertcleanup modes

Location:

tags/ipp-20100823

Files:

• dbconfig (modified) (1 prop)
• dbconfig/background.md (modified) (2 diffs)
• dbconfig/changes.txt (modified) (1 diff)
• dbconfig/config.md (modified) (1 diff)
• dbconfig/magic.md (modified) (1 diff)
• dbconfig/pstamp.md (modified) (1 prop)
• dbconfig/stack.md (modified) (1 prop)
• ippTasks (modified) (1 prop)
• ippTasks/destreak.pro (modified) (4 diffs, 1 prop)
• ippTasks/diskbalance.pro (copied) (copied from trunk/ippTasks/diskbalance.pro )
• ippTasks/dist.pro (modified) (1 prop)
• ippTasks/magic.pro (modified) (3 diffs)
• ippTasks/pstamp.pro (modified) (1 prop)
• ippTasks/replicate.pro (modified) (2 diffs)
• ippTasks/science.cleanup.pro (modified) (6 diffs)
• ippTools (modified) (1 prop)
• ippTools/configure.ac (modified) (2 diffs)
• ippTools/share (modified) (1 prop)
• ippTools/share/Makefile.am (modified) (3 diffs)
• ippTools/share/addtool_find_cam_id.sql (modified) (3 diffs)
• ippTools/share/addtool_find_cam_id_dvo.sql (modified) (1 diff)
• ippTools/share/chiptool_revertcleanup.sql (copied) (copied from trunk/ippTools/share/chiptool_revertcleanup.sql )
• ippTools/share/difftool_revertcleanup.sql (copied) (copied from trunk/ippTools/share/difftool_revertcleanup.sql )
• ippTools/share/difftool_todiffskyfile.sql (modified) (1 prop)
• ippTools/share/magicdstool_todestreak_camera.sql (modified) (1 diff)
• ippTools/share/magicdstool_todestreak_chip.sql (modified) (1 diff)
• ippTools/share/magicdstool_todestreak_diff.sql (modified) (2 diffs)
• ippTools/share/magicdstool_todestreak_raw.sql (modified) (1 diff)
• ippTools/share/magicdstool_todestreak_warp.sql (modified) (1 diff)
• ippTools/share/magictool_definebyquery_select.sql (modified) (1 prop)
• ippTools/share/magictool_revertnode.sql (modified) (1 prop)
• ippTools/share/magictool_toprocess_inputs.sql (modified) (1 diff)
• ippTools/share/magictool_toprocess_tree.sql (modified) (1 diff)
• ippTools/share/pubtool_pending.sql (modified) (1 prop)
• ippTools/share/pxadmin_create_tables.sql (modified) (2 diffs)
• ippTools/share/stacktool_tosum.sql (modified) (1 diff)
• ippTools/share/warptool_revertcleanup.sql (copied) (copied from trunk/ippTools/share/warptool_revertcleanup.sql )
• ippTools/share/warptool_scmap.sql (modified) (1 prop)
• ippTools/src/addtool.c (modified) (1 diff)
• ippTools/src/bgtool.c (modified) (2 diffs)
• ippTools/src/chiptool.c (modified) (4 diffs)
• ippTools/src/chiptool.h (modified) (1 diff)
• ippTools/src/chiptoolConfig.c (modified) (2 diffs)
• ippTools/src/difftool.c (modified) (3 diffs, 1 prop)
• ippTools/src/difftool.h (modified) (1 diff)
• ippTools/src/difftoolConfig.c (modified) (4 diffs)
• ippTools/src/magicdstool.c (modified) (1 prop)
• ippTools/src/magictool.c (modified) (7 diffs)
• ippTools/src/magictoolConfig.c (modified) (2 diffs)
• ippTools/src/pubtool.c (modified) (1 prop)
• ippTools/src/stacktool.c (modified) (1 diff)
• ippTools/src/warptool.c (modified) (3 diffs)
• ippTools/src/warptool.h (modified) (1 diff)
• ippTools/src/warptoolConfig.c (modified) (2 diffs)
• magic/remove/src (modified) (1 prop)
• magic/remove/src/streaksio.c (modified) (1 prop)
• pstamp (modified) (1 prop)

tags/ipp-20100823/dbconfig/background.md

@@ -21 +21 @@
     class_id            STR     64
     path_base           STR     255
+    data_state          STR     64
     magicked            S64     0
     dtime_script        F32     0.0
@@ -57 +58 @@
     skycell_id          STR     64
     path_base           STR     255
+    data_state          STR     64
     magicked            S64     0
     dtime_script        F32     0.0

tags/ipp-20100823/dbconfig/changes.txt

@@ -1945 +1945 @@
 ALTER TABLE diffSummary DROP PRIMARY KEY, ADD PRIMARY KEY (diff_id, projection_cell);
 ALTER TABLE stackSummary DROP PRIMARY KEY, ADD PRIMARY KEY (sass_id, projection_cell);
+
+ALTER TABLE stackSumSkyfile ADD COLUMN mjd_obs double AFTER good_frac;
+
+-- changes to support cleanup of the background preserved images
+ALTER TABLE chipBackgroundImfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
+ALTER TABLE chipBackgroundImfile ADD KEY(data_state);
+ALTER TABLE warpBackgroundSkyfile ADD COLUMN data_state VARCHAR(64) AFTER path_base;
+ALTER TABLE warpBackgroundSkyfile  ADD KEY(data_state);
+
+ALTER TABLE magicMask ADD COLUMN path_base VARCHAR(255) AFTER uri;
+
+-- diff_id was removed from magicInputSkyfile ages ago, but
+-- pxadmin_create_tables.sql still had it this command didn't work due to
+-- a mysql error.
+-- ALTER TABLE magicInputSkyfile DROP COLUMN diff_id;
+UPDATE dbversion set schema_version = '1.1.64',  updated= CURRENT_TIMESTAMP();
+
+-- Version 1.1.65

tags/ipp-20100823/dbconfig/config.md

@@ -2 +2 @@
     pkg_name        STR     ippdb
     pkg_namespace   STR     ippdb
-    pkg_version     STR     1.1.63
+    pkg_version     STR     1.1.64
 END

tags/ipp-20100823/dbconfig/magic.md

@@ -42 +42 @@
     magic_id    S64         0       # Primary Key fkey(magic_id) ref magicRun(magic_id)
     uri         STR         255
+    path_base   STR         255
     streaks     S32         0
     fault       S16         0       # Key

tags/ipp-20100823/ippTasks/destreak.pro

@@ -29 +29 @@
 $magicDSRevertStage = 0
 
+macro destreak.show.stages
+    echo $DS_STAGE:n stages enabled for destreak processing
+    for i 0 $DS_STAGE:n
+        echo $i $DS_STAGE:$i
+    end
+    echo current magicDSStage = $magicDSStage
+    echo current magicDSRevertStage = $magicDSRevertStage
+end
+
 ### Check status of tasks
 macro destreak.status
@@ -178 +187 @@
     book getword magicToDS $pageName camera -var CAMERA
     book getword magicToDS $pageName streaks_uri -var STREAKS
+    book getword magicToDS $pageName streaks_path_base -var STREAKS_PATH_BASE
     book getword magicToDS $pageName inv_streaks_uri -var INV_STREAKS
+    book getword magicToDS $pageName inv_streaks_path_base -var INV_STREAKS_PATH_BASE
     book getword magicToDS $pageName stage -var STAGE
     book getword magicToDS $pageName stage_id -var STAGE_ID
@@ -204 +215 @@
     # TODO: do not add recoveryroot or replace if they are null or zero
 
-    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
+    $run = magic_destreak.pl --magic_ds_id $MAGIC_DS_ID --camera $CAMERA --exp_id $EXP_ID --streaks_path_base $STREAKS_PATH_BASE --inv_streaks_path_base $INV_STREAKS_PATH_BASE--streaks $STREAKS --inv_streaks $INV_STREAKS --stage $STAGE --stage_id $STAGE_ID --component $COMPONENT --uri $URI --path_base $PATH_BASE --cam_path_base $CAM_PATH_BASE --cam_reduction $CAM_REDUCTION --outroot $OUTROOT --logfile $logfile --recoveryroot $RECROOT --replace $REPLACE --magicked $MAGICKED
 
     add_standard_args run
@@ -249 +260 @@
 
   periods      -poll $LOADPOLL
-  #periods      -exec $LOADEXEC
-  periods      -exec 60
+  periods      -exec 30
   periods      -timeout 300
   npending     1

tags/ipp-20100823/ippTasks/magic.pro

@@ -288 +288 @@
     book getword magicToProcess $pageName camera -var CAMERA
     book getword magicToProcess $pageName workdir -var WORKDIR_TEMPLATE
+    book getword magicToProcess $pageName raw_workdir -var RAW_TEMPLATE
+    book getword magicToProcess $pageName exp_name -var EXP_NAME
     book getword magicToProcess $pageName dbname -var DBNAME
 
@@ -308 +310 @@
     sprintf baseroot "%s/%s/%s.mgc.%s" $WORKDIR $EXP_ID $EXP_ID $MAGIC_ID
 
+    if ("$NODE" == "root")
+        # This is the root node, put the streaks file into the rawExps workdir
+        # which is presumably in nebulous and thus will be replicatable
+        set.workdir.by.camera $CAMERA FPA $RAW_TEMPLATE $default_host RAW_WORKDIR
+        sprintf final_outroot "%s/%s.%s/%s.%s.mgc.%s" $RAW_WORKDIR $EXP_NAME $EXP_ID $EXP_NAME $EXP_ID $MAGIC_ID
+
+        $EXTRA_ARGS = --final-outroot $final_outroot
+    else
+        $EXTRA_ARGS = ""
+    end
+
     $logfile = $baseroot.$NODE.log
     if ("$logfile" == "") 
@@ -319 +332 @@
     mkdir $outpath
 
-    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile
+    $run = magic_process.pl --magic_id $MAGIC_ID --camera $CAMERA --node $NODE --baseroot $baseroot --logfile $logfile $EXTRA_ARGS
     add_standard_args run
 

tags/ipp-20100823/ippTasks/replicate.pro

@@ -173 +173 @@
     book getword replicatePending $pageName volume_name -var VOLUME_NAME
     book getword replicatePending $pageName volume_host -var VOLUME_HOST
+    book getword replicatePending $pageName command     -var COMMAND
 
     set.host.for.replicate $VOLUME_HOST
@@ -180 +181 @@
 
     # these operations do not require a database to be specified
-    $run = neb-replicate --copies $NEED_COPIES $KEY
+    $run = $COMMAND $KEY
 
     # save the pageName for future reference below

tags/ipp-20100823/ippTasks/science.cleanup.pro

@@ -113 +113 @@
   task.exec
     book npages chipPendingCleanup -var N
-    if ($N == 0) break
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
     if ($NETWORK == 0) break
     
@@ -143 +146 @@
       echo command $run
     end
+    periods -exec 0.05
     command $run
   end
@@ -569 +573 @@
   task.exec
     book npages warpPendingCleanup -var N
-    if ($N == 0) break
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
     if ($NETWORK == 0) break
     
@@ -599 +606 @@
       echo command $run
     end
+    periods -exec 0.05
     command $run
   end
@@ -721 +729 @@
   task.exec
     book npages diffCleanup -var N
-    if ($N == 0) break
+    if ($N == 0) 
+        periods -exec $RUNEXEC
+        break
+    end
     if ($NETWORK == 0) break
     
@@ -751 +762 @@
       echo command $run
     end
+    periods -exec 0.05
     command $run
   end

tags/ipp-20100823/ippTools/configure.ac

@@ -1 +1 @@
 AC_PREREQ(2.61)
 
-AC_INIT([ipptools], [1.1.63], [ipp-support@ifa.hawaii.edu])
+AC_INIT([ipptools], [1.1.64], [ipp-support@ifa.hawaii.edu])
 AC_CONFIG_SRCDIR([autogen.sh])
 
@@ -18 +18 @@
 PKG_CHECK_MODULES([PSLIB], [pslib >= 1.1.0])
 PKG_CHECK_MODULES([PSMODULES], [psmodules >= 1.1.0])
-PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.63]) 
+PKG_CHECK_MODULES([IPPDB], [ippdb >= 1.1.64]) 
 PKG_CHECK_MODULES([PPSTAMP], [ppstamp >= 0.1.1]) 
 

tags/ipp-20100823/ippTools/share/Makefile.am

@@ -61 +61 @@
         chiptool_pendingimfile.sql \
         chiptool_processedimfile.sql \
+        chiptool_revertcleanup.sql \
         chiptool_revertprocessedimfile.sql \
         chiptool_revertupdatedimfile.sql \
@@ -143 +144 @@
         difftool_revertdiffskyfile_delete.sql \
         difftool_revertdiffskyfile_updated.sql \
+        difftool_revertcleanup.sql \
         difftool_setskyfiletoupdate.sql \
         difftool_skyfile.sql \
@@ -358 +360 @@
         warptool_revertwarped_delete.sql \
         warptool_revertwarped_updated.sql \
+        warptool_revertcleanup.sql \
         warptool_runstate.sql \
         warptool_scmap.sql \

tags/ipp-20100823/ippTools/share/addtool_find_cam_id.sql

@@ -2 +2 @@
     camRun.*
 FROM camRun
+JOIN camProcessedExp
+    USING(cam_id)
 JOIN chipRun
     USING(chip_id)
@@ -10 +12 @@
            FROM addRun
            JOIN camRun USING(cam_id)
-           JOIN chipRun USING(chip_id)
+           JOIN chipRun USING(chip_id)
           ) as foo
      ON exp_id = added_exp_id
@@ -17 +19 @@
 WHERE
     camRun.state = 'full'
+    AND camProcessedExp.quality = 0
     AND added_exp_id IS NULL
     -- addtool adds checks on exposure being added to the dvodb previously

tags/ipp-20100823/ippTools/share/addtool_find_cam_id_dvo.sql

@@ -1 +1 @@
 SELECT camRun.* FROM camRun
+JOIN camProcessedExp USING(cam_id)
 JOIN chipRun USING(chip_id)
 JOIN rawExp USING(exp_id)
-WHERE camRun.state = 'full'
+WHERE camRun.state = 'full' and camProcessedExp.quality = 0
     AND exp_id NOT IN (SELECT exp_id
        FROM addRun

tags/ipp-20100823/ippTools/share/magicdstool_todestreak_camera.sql

@@ -7 +7 @@
     camera,
     magicMask.uri AS streaks_uri,
+    magicMask.path_base AS streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     stage_id,

tags/ipp-20100823/ippTools/share/magicdstool_todestreak_chip.sql

@@ -7 +7 @@
     camera,
     magicMask.uri AS streaks_uri,
+    magicMask.path_base AS streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     stage_id,

tags/ipp-20100823/ippTools/share/magicdstool_todestreak_diff.sql

@@ -7 +7 @@
     rawExp.camera,
     magicMask.uri AS streaks_uri,
+    magicMask.path_base AS streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     magicRun.diff_id AS stage_id,
@@ -52 +54 @@
     rawExp.camera,
     magicMask.uri AS streaks_uri,
+    magicMask.path_base AS streaks_path_base,
     (SELECT uri from magicMask where magic_id = inv_magic_id) AS inv_streaks_uri,
+    (SELECT path_base from magicMask where magic_id = inv_magic_id) AS inv_streaks_path_base,
     stage,
     magicRun.diff_id AS stage_id,

tags/ipp-20100823/ippTools/share/magicdstool_todestreak_raw.sql

@@ -6 +6 @@
     rawExp.camera,
     magicMask.uri as streaks_uri,
+    magicMask.path_base as streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     stage_id,

tags/ipp-20100823/ippTools/share/magicdstool_todestreak_warp.sql

@@ -6 +6 @@
     camera,
     magicMask.uri as streaks_uri,
+    magicMask.path_base as streaks_path_base,
     CAST(NULL AS CHAR(255)) AS inv_streaks_uri,
+    CAST(NULL AS CHAR(255)) AS inv_streaks_path_base,
     stage,
     stage_id,

tags/ipp-20100823/ippTools/share/magictool_toprocess_inputs.sql

@@ -3 +3 @@
     magicRun.workdir,
     rawExp.exp_id,
+    rawExp.exp_name,
+    rawExp.workdir AS raw_workdir,
     rawExp.camera,
     -- convert magic_id into a boolean value (1 or 0)

tags/ipp-20100823/ippTools/share/magictool_toprocess_tree.sql

@@ -3 +3 @@
     magicRun.workdir,
     rawExp.exp_id,
+    rawExp.exp_name,
+    rawExp.workdir AS raw_workdir,
     rawExp.camera,
     -- convert magic_id into a boolean value (1 or 0)

tags/ipp-20100823/ippTools/share/pxadmin_create_tables.sql

@@ -1221 +1221 @@
 CREATE TABLE magicInputSkyfile (
         magic_id BIGINT,
-        diff_id BIGINT,
         node VARCHAR(64),
-        PRIMARY KEY(magic_id, diff_id, node),
-        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id),
-        FOREIGN KEY(diff_id) REFERENCES diffRun(diff_id)
+        PRIMARY KEY(magic_id, node),
+        FOREIGN KEY(magic_id) REFERENCES magicRun(magic_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
@@ -1253 +1251 @@
         magic_id BIGINT,
         uri VARCHAR(255),
+        path_base VARCHAR(255),
         streaks INT,
         fault SMALLINT,

tags/ipp-20100823/ippTools/share/stacktool_tosum.sql

@@ -7 +7 @@
     stackRun.label,
     stackRun.state,
-    stackSumSkyfile.path_base
+    stackSumSkyfile.path_base,
+    IFNULL(Label.priority, 10000) AS priority
 FROM stackRun
 JOIN stackInputSkyfile USING(stack_id)
 JOIN warpRun USING(warp_id)
 LEFT JOIN stackSumSkyfile USING(stack_id)
+LEFT JOIN Label ON Label.label = stackRun.label
 WHERE
     ((stackRun.state = 'new' AND stackSumSkyfile.stack_id IS NULL)
     OR (stackRun.state = 'update' AND stackSumSkyfile.fault = 0 AND stackSumSkyfile.quality = 0))
+    AND (Label.active OR Label.active IS NULL)
     -- WHERE hook %s
 GROUP BY stack_id

tags/ipp-20100823/ippTools/src/addtool.c

@@ -493 +493 @@
 
     // since there is only one exp per 'new' set addRun.state = 'full'
-    if (!pxaddRunSetState(config, row->add_id, "full")) {
+    // but check to make sure there are no faults
+
+    if (!fault) {
+      if (!pxaddRunSetState(config, row->add_id, "full")) {
         psError(PS_ERR_UNKNOWN, false, "failed to change addRun.state for add_id: %" PRId64, row->add_id);
         psFree(row);
         psFree(pendingRow);
         return false;
+      }
     }
     psFree(row);

tags/ipp-20100823/ippTools/src/bgtool.c

@@ -658 +658 @@
     }
 
-    if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script,
+    if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, "full", magicked, dtime_script,
                                     hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
                                     maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
@@ -1497 +1497 @@
     }
 
-    if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script,
+    if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, "full", magicked, dtime_script,
                                     hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
                                     maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {

tags/ipp-20100823/ippTools/src/chiptool.c

@@ -50 +50 @@
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupimfileMode(pxConfig *config);
+static bool revertcleanupMode(pxConfig *config);
 static bool donecleanupMode(pxConfig *config);
 static bool runMode(pxConfig *config);
@@ -94 +95 @@
         MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPRUN,       pendingcleanuprunMode);
         MODECASE(CHIPTOOL_MODE_PENDINGCLEANUPIMFILE,    pendingcleanupimfileMode);
+        MODECASE(CHIPTOOL_MODE_REVERTCLEANUP,           revertcleanupMode);
         MODECASE(CHIPTOOL_MODE_DONECLEANUP,             donecleanupMode);
         MODECASE(CHIPTOOL_MODE_RUN,                     runMode);
@@ -859 +861 @@
     }
     psString query_update = pxDataGet("chiptool_revertupdatedimfile.sql");
-    if (!query) {
+    if (!query_update) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         psFree(where);
         return false;
     }
+
 
     if (where && psListLength(where->list)) {
@@ -886 +889 @@
     }
     psFree(query_update);
+
+    return true;
+}
+static bool revertcleanupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group", "chipRun.data_group", "LIKE");
+    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
+
+    char* newState = NULL;
+    if (!state) {
+        state = "error_cleaned";
+    }
+    if (!strcmp(state, "error_cleaned")) {
+        newState = "goto_cleaned";
+    } else if (!strcmp(state, "error_purged")) {
+        newState = "goto_purged";
+    } else if (!strcmp(state, "error_scrubbed")) {
+        newState = "goto_scrubbed";
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed");
+        return false;
+    }
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = pxDataGet("chiptool_revertcleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (where && psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
 
     return true;

tags/ipp-20100823/ippTools/src/chiptool.h

@@ -42 +42 @@
     CHIPTOOL_MODE_PENDINGCLEANUPRUN,
     CHIPTOOL_MODE_PENDINGCLEANUPIMFILE,
+    CHIPTOOL_MODE_REVERTCLEANUP,
     CHIPTOOL_MODE_DONECLEANUP,
     CHIPTOOL_MODE_RUN,

tags/ipp-20100823/ippTools/src/chiptoolConfig.c

@@ -290 +290 @@
     psMetadataAddU64(donecleanupArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 
+    // -revertcleanup
+    psMetadata *revertcleanupArgs = psMetadataAlloc();
+    psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-chip_id", 0,            "search by chip ID", 0);
+    psMetadataAddStr(revertcleanupArgs,  PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL);
+    psMetadataAddStr(revertcleanupArgs,  PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by chipRun data_group (LIKE comparison)", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state", 0,             "search by current state", NULL);
+    psMetadataAddS16(revertcleanupArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
+
     // -run
     psMetadata *runArgs = psMetadataAlloc();
@@ -371 +379 @@
     PXOPT_ADD_MODE("-pendingcleanupimfile", "show runs that need to be cleaned up", CHIPTOOL_MODE_PENDINGCLEANUPIMFILE, pendingcleanupimfileArgs);
     PXOPT_ADD_MODE("-donecleanup",          "show runs that have been cleaned",     CHIPTOOL_MODE_DONECLEANUP,          donecleanupArgs);
+    PXOPT_ADD_MODE("-revertcleanup",        "show runs that have been cleaned",     CHIPTOOL_MODE_REVERTCLEANUP,        revertcleanupArgs);
     PXOPT_ADD_MODE("-run",                  "show runs",                            CHIPTOOL_MODE_RUN,                  runArgs);
     PXOPT_ADD_MODE("-tocleanedimfile",      "set imfile state to cleaned",          CHIPTOOL_MODE_TOCLEANEDIMFILE,      tocleanedimfileArgs);

tags/ipp-20100823/ippTools/src/difftool.c

@@ -48 +48 @@
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupskyfileMode(pxConfig *config);
+static bool revertcleanupMode(pxConfig *config);
 static bool donecleanupMode(pxConfig *config);
 static bool updatediffskyfileMode(pxConfig *config);
@@ -100 +101 @@
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
+        MODECASE(DIFFTOOL_MODE_REVERTCLEANUP,         revertcleanupMode);
         MODECASE(DIFFTOOL_MODE_DONECLEANUP,           donecleanupMode);
         MODECASE(DIFFTOOL_MODE_UPDATEDIFFSKYFILE,     updatediffskyfileMode);
@@ -2531 +2533 @@
 
 
+static bool revertcleanupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-diff_id",    "diffSkyfile.diff_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label",     "diffRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group",   "diffRun.data_group", "LIKE");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+    if (!state) {
+        state = "error_cleaned";
+    }
+    char *newState = NULL;
+    if (!strcmp(state, "error_cleaned")) {
+        newState = "goto_cleaned";
+    } else if (!strcmp(state, "error_purged")) {
+        newState = "goto_purged";
+    } else if (!strcmp(state, "error_scrubbed")) {
+        newState = "goto_scrubbed";
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed");
+        return false;
+    }
+
+    psString query = pxDataGet("difftool_revertcleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    int numDeleted = psDBAffectedRows(config->dbh);
+
+    psLogMsg("difftool", PS_LOG_INFO, "Reverted %d diffRuns and diffSkyfiles", numDeleted);
+
+    return true;
+}
 static bool donecleanupMode(pxConfig *config)
 {

tags/ipp-20100823/ippTools/src/difftool.h

@@ -42 +42 @@
     DIFFTOOL_MODE_PENDINGCLEANUPRUN,
     DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE,
+    DIFFTOOL_MODE_REVERTCLEANUP,
     DIFFTOOL_MODE_DONECLEANUP,
     DIFFTOOL_MODE_UPDATEDIFFSKYFILE,

tags/ipp-20100823/ippTools/src/difftoolConfig.c

@@ -173 +173 @@
     psMetadata *listrunArgs = psMetadataAlloc();
     psMetadataAddS64(listrunArgs, PS_LIST_TAIL,  "-diff_id", 0,           "search by diff ID", 0);
-    psMetadataAddStr(listrunArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID", NULL);
+    psMetadataAddStr(listrunArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID (LIKE comparison)", NULL);
     psMetadataAddStr(listrunArgs , PS_LIST_TAIL, "-warp_id",  0,         "search by warp_id", NULL);
     psMetadataAddBool(listrunArgs, PS_LIST_TAIL, "-template",  0,        "apply exposure args to template of bothways diff", false);
@@ -199 +199 @@
     psMetadata *listssrunArgs = psMetadataAlloc();
     psMetadataAddS64(listssrunArgs, PS_LIST_TAIL,  "-diff_id", 0,          "search by diff ID", 0);
-    psMetadataAddStr(listssrunArgs, PS_LIST_TAIL,  "-tess_id",  0,         "search by tessellation ID", NULL);
+    psMetadataAddStr(listssrunArgs, PS_LIST_TAIL,  "-tess_id",  0,         "search by tessellation ID (LIKE comparison)", NULL);
     psMetadataAddS64(listssrunArgs , PS_LIST_TAIL, "-stack_id",  0,        "search by input stack_id", 0);
     psMetadataAddS64(listssrunArgs , PS_LIST_TAIL, "-template_stack_id",0, "search by template stack_id", 0);
@@ -390 +390 @@
     psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 
+    // -revertcleanup
+    psMetadata *revertcleanupArgs = psMetadataAlloc();
+    psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-diff_id", 0,            "search by difftool ID", 0);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by diffRun label", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by diffRun data_group", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state",  0,            "search by state", NULL);
+
+
     // -donecleanup
     psMetadata *donecleanupArgs = psMetadataAlloc();
@@ -464 +472 @@
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
     PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);
+    PXOPT_ADD_MODE("-revertcleanup",           "revert cleanup runs with errors",     DIFFTOOL_MODE_REVERTCLEANUP,          revertcleanupArgs);
     PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     DIFFTOOL_MODE_DONECLEANUP,          donecleanupArgs);
     PXOPT_ADD_MODE("-updatediffskyfile",     "update fault code for a diffskyfile",  DIFFTOOL_MODE_UPDATEDIFFSKYFILE,          updatediffskyfileArgs);

tags/ipp-20100823/ippTools/src/magictool.c

@@ -50 +50 @@
 static bool exposureMode(pxConfig *config);
 
-static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
+static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString);
 static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename);
 
@@ -131 +131 @@
 
     psMetadata *queryWhere = psMetadataAlloc(); // WHERE conditions for everything else
-    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_S64(config->args, queryWhere, "-exp_id", "rawExp.exp_id", "==");
     PXOPT_COPY_STR(config->args, queryWhere, "-select_filter", "rawExp.filter", "==");
 
@@ -394 +394 @@
     // required
     PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
-    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
+    PXOPT_LOOKUP_STR(state, config->args, "-set_state", true, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-set_fault", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_BOOL(clearfault, config->args, "-clearfault", false);
+
+    psString setString = NULL;
+    if (fault || clearfault) {
+        psStringAppend(&setString, ", fault = %d", fault);
+    }
+    if (note) {
+        psStringAppend(&setString, ", note = '%s'", note);
+    }
 
     if (state) {
         // set detRun.state to state
-        return setmagicRunState(config, magic_id, state);
+        return setmagicRunState(config, magic_id, state, setString);
     }
 
@@ -1171 +1182 @@
 
     // optional
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+//    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
     PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false);
 
@@ -1184 +1196 @@
     if (!magicMaskInsert(config->dbh,
                          magic_id,
-                         uri,
+                         NULL,
+                         path_base,
                          streaks,
                          fault
@@ -1364 +1377 @@
 }
 
-static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state)
+static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state, psString setString)
 {
     PS_ASSERT_PTR_NON_NULL(state, false);
@@ -1380 +1393 @@
         return false;
     }
-
-    char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
-    if (!p_psDBRunQueryF(config->dbh, query, state, magic_id)) {
+    psString query = NULL;
+    psStringAppend(&query, "UPDATE magicRun SET state = '%s'", state);
+    if (setString) {
+        psStringAppend(&query, setString);
+    }
+    psStringAppend(&query, " WHERE magic_id = %" PRId64, magic_id);;
+
+//    char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
+    if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false,
                 "failed to change state for magic_id %" PRId64, magic_id);

tags/ipp-20100823/ippTools/src/magictoolConfig.c

@@ -79 +79 @@
     psMetadata *updaterunArgs = psMetadataAlloc();
     psMetadataAddS64(updaterunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
-    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-state", 0, "set state (required)", NULL);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_state", 0, "set state (required)", NULL);
+    psMetadataAddS16(updaterunArgs, PS_LIST_TAIL, "-set_fault", 0, "set fault code", 0);
+    psMetadataAddStr(updaterunArgs, PS_LIST_TAIL, "-set_note",  0, "define note", NULL);
+    psMetadataAddBool(updaterunArgs, PS_LIST_TAIL, "-clearfault",  0, "set fault to zero", NULL);
 
     // -addinputskyfile
@@ -149 +152 @@
     psMetadata *addmaskArgs = psMetadataAlloc();
     psMetadataAddS64(addmaskArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID (required)", 0);
-    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-uri", 0, "define URI", NULL);
+    psMetadataAddStr(addmaskArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL);
     psMetadataAddS32(addmaskArgs, PS_LIST_TAIL, "-streaks", 0, "define number of streaks", 0);
     psMetadataAddS16(addmaskArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);

tags/ipp-20100823/ippTools/src/stacktool.c

@@ -900 +900 @@
     }
     psFree(where);
+
+    psStringAppend(&query, "\nORDER by priority DESC, stack_id");
 
     // treat limit == 0 as "no limit"

tags/ipp-20100823/ippTools/src/warptool.c

@@ -52 +52 @@
 static bool pendingcleanuprunMode(pxConfig *config);
 static bool pendingcleanupwarpMode(pxConfig *config);
+static bool revertcleanupMode(pxConfig *config);
 static bool donecleanupMode(pxConfig *config);
 static bool tocleanedskyfileMode(pxConfig *config);
@@ -107 +108 @@
         MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN,  pendingcleanuprunMode);
         MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode);
+        MODECASE(WARPTOOL_MODE_REVERTCLEANUP,      revertcleanupMode);
         MODECASE(WARPTOOL_MODE_DONECLEANUP,        donecleanupMode);
         MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE,   tocleanedskyfileMode);
@@ -1805 +1807 @@
 
 
+static bool revertcleanupMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label",     "warpRun.label", "LIKE");
+    pxAddLabelSearchArgs (config, where, "-data_group",   "warpRun.data_group", "LIKE");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+    if (!state) {
+        state = "error_cleaned";
+    }
+    char *newState = NULL;
+    if (!strcmp(state, "error_cleaned")) {
+        newState = "goto_cleaned";
+    } else if (!strcmp(state, "error_purged")) {
+        newState = "goto_purged";
+    } else if (!strcmp(state, "error_scrubbed")) {
+        newState = "goto_scrubbed";
+    } else {
+        psError(PXTOOLS_ERR_CONFIG, true, "-state must be either error_cleaned, error_purged, or error_scrubbed");
+        return false;
+    }
+
+    psString query = pxDataGet("warptool_revertcleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, newState, state, state)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    int numDeleted = psDBAffectedRows(config->dbh);
+
+    psLogMsg("warptool", PS_LOG_INFO, "Reverted %d warpRuns and warpSkyfiles", numDeleted);
+
+    return true;
+}
 static bool donecleanupMode(pxConfig *config)
 {

tags/ipp-20100823/ippTools/src/warptool.h

@@ -48 +48 @@
     WARPTOOL_MODE_PENDINGCLEANUPRUN,
     WARPTOOL_MODE_PENDINGCLEANUPSKYFILE,
+    WARPTOOL_MODE_REVERTCLEANUP,
     WARPTOOL_MODE_DONECLEANUP,
     WARPTOOL_MODE_TOCLEANEDSKYFILE,

tags/ipp-20100823/ippTools/src/warptoolConfig.c

@@ -362 +362 @@
     psMetadataAddU64(pendingcleanupskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
 
+    // -revertcleanup
+    psMetadata *revertcleanupArgs = psMetadataAlloc();
+    psMetadataAddS64(revertcleanupArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warptool ID", 0);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by warpRun label", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-data_group",  PS_META_DUPLICATE_OK, "search by warpRun data_group", NULL);
+    psMetadataAddStr(revertcleanupArgs, PS_LIST_TAIL, "-state",  0,            "search by state", NULL);
+
     // -donecleanup
     psMetadata *donecleanupArgs = psMetadataAlloc();
@@ -451 +458 @@
     PXOPT_ADD_MODE("-pendingcleanuprun",     "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPRUN,    pendingcleanuprunArgs);
     PXOPT_ADD_MODE("-pendingcleanupskyfile", "show runs that need to be cleaned up", WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileArgs);
+    PXOPT_ADD_MODE("-revertcleanup",         "revert cleanup runs with errors",     WARPTOOL_MODE_REVERTCLEANUP,          revertcleanupArgs);
     PXOPT_ADD_MODE("-donecleanup",           "show runs that have been cleaned",     WARPTOOL_MODE_DONECLEANUP,          donecleanupArgs);
     PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs);