Changeset 25766 for branches/eam_branches/20090820/ippTools/src

Timestamp:

Oct 2, 2009, 5:10:19 PM (17 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/20090820

Files:

• . (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/Makefile.am (modified) (7 diffs)
• ippTools/src/addtool.c (copied) (copied from trunk/ippTools/src/addtool.c )
• ippTools/src/addtool.h (copied) (copied from trunk/ippTools/src/addtool.h )
• ippTools/src/addtoolConfig.c (copied) (copied from trunk/ippTools/src/addtoolConfig.c )
• ippTools/src/camtool.c (modified) (1 diff)
• ippTools/src/chiptool.c (modified) (5 diffs)
• ippTools/src/chiptool.h (modified) (1 diff)
• ippTools/src/chiptoolConfig.c (modified) (3 diffs)
• ippTools/src/dettool.c (modified) (6 diffs)
• ippTools/src/dettool.h (modified) (2 diffs)
• ippTools/src/dettoolConfig.c (modified) (14 diffs)
• ippTools/src/dettool_detrunsummary.c (modified) (3 diffs)
• ippTools/src/dettool_processedimfile.c (modified) (5 diffs)
• ippTools/src/difftool.c (modified) (8 diffs)
• ippTools/src/difftool.h (modified) (1 diff)
• ippTools/src/difftoolConfig.c (modified) (2 diffs)
• ippTools/src/disttool.c (modified) (28 diffs)
• ippTools/src/disttool.h (modified) (3 diffs)
• ippTools/src/disttoolConfig.c (modified) (12 diffs)
• ippTools/src/faketool.c (modified) (3 diffs)
• ippTools/src/faketool.h (modified) (1 diff)
• ippTools/src/faketoolConfig.c (modified) (2 diffs)
• ippTools/src/magicdstool.c (modified) (24 diffs)
• ippTools/src/magicdstool.h (modified) (1 diff)
• ippTools/src/magicdstoolConfig.c (modified) (8 diffs)
• ippTools/src/magictool.c (modified) (2 diffs)
• ippTools/src/magictoolConfig.c (modified) (1 diff)
• ippTools/src/pstamptool.c (modified) (10 diffs)
• ippTools/src/pstamptool.h (modified) (1 diff)
• ippTools/src/pstamptoolConfig.c (modified) (6 diffs)
• ippTools/src/pubtool.c (modified) (2 diffs)
• ippTools/src/pubtoolConfig.c (modified) (1 diff)
• ippTools/src/pxadd.c (copied) (copied from trunk/ippTools/src/pxadd.c )
• ippTools/src/pxadd.h (copied) (copied from trunk/ippTools/src/pxadd.h )
• ippTools/src/pxinject.c (modified) (1 diff)
• ippTools/src/pxinjectConfig.c (modified) (1 diff)
• ippTools/src/pxmagic.c (copied) (copied from trunk/ippTools/src/pxmagic.c )
• ippTools/src/pxmagic.h (copied) (copied from trunk/ippTools/src/pxmagic.h )
• ippTools/src/pxtools.c (modified) (2 diffs)
• ippTools/src/pxtools.h (modified) (2 diffs)
• ippTools/src/pztool.c (modified) (3 diffs)
• ippTools/src/pztoolConfig.c (modified) (1 diff)
• ippTools/src/regtool.c (modified) (8 diffs)
• ippTools/src/regtoolConfig.c (modified) (3 diffs)
• ippTools/src/stacktool.c (modified) (3 diffs)
• ippTools/src/stacktoolConfig.c (modified) (2 diffs)
• ippTools/src/warptool.c (modified) (5 diffs)
• ippTools/src/warptool.h (modified) (1 diff)
• ippTools/src/warptoolConfig.c (modified) (3 diffs)

branches/eam_branches/20090820/ippTools/src

@@ -34 +34 @@
 disttool
 receivetool
-
+addtool
 pubtool

@@ -34 +34 @@
 disttool
 receivetool
-
+addtool
 pubtool

branches/eam_branches/20090820/ippTools/src/Makefile.am

@@ -1 +1 @@
 bin_PROGRAMS = \
+        addtool \
         caltool \
         camtool \
@@ -23 +24 @@
         pubtool
 
-
-bin_SCRIPTS = \
-        fakemagic
-
 pkginclude_HEADERS = \
+        pxadd.h \
         pxadmin.h \
         pxcam.h \
@@ -34 +32 @@
         pxdata.h \
         pxfake.h \
+        pxmagic.h \
         pxregister.h \
         pxtag.h \
@@ -42 +41 @@
 
 noinst_HEADERS = \
+        addtool.h \
         caltool.h \
         camtool.h \
@@ -68 +68 @@
 libpxtools_la_LDFLAGS   = -release $(PACKAGE_VERSION)
 libpxtools_la_SOURCES   = \
+        pxadd.c \
         pxcam.c \
         pxchip.c \
@@ -75 +76 @@
         pxfake.c \
         pxfault.c \
+        pxmagic.c \
         pxregister.c \
         pxtag.c \
@@ -96 +98 @@
     pstamptool.c \
     pstamptoolConfig.c
+
+addtool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+addtool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+addtool_SOURCES = \
+    addtool.c \
+    addtoolConfig.c
 
 caltool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)

branches/eam_branches/20090820/ippTools/src/camtool.c

@@ -656 +656 @@
         return false;
     }
+
+/*     if (!pxaddQueueByCamID(config, */
+/*                         pendingRow->cam_id, */
+/*                         pendingRow->workdir, */
+/*                         pendingRow->label, */
+/*                         pendingRow->reduction, */
+/*                         pendingRow->dvodb */
+/*     )) { */
+/*         // rollback */
+/*         if (!psDBRollback(config->dbh)) { */
+/*             psError(PS_ERR_UNKNOWN, false, "database error"); */
+/*         } */
+/*         psError(PS_ERR_UNKNOWN, false, "failed to queue new addRun"); */
+/*         psFree(pendingRow); */
+/*         return false; */
+/*     } */
+
     psFree(pendingRow);
 

branches/eam_branches/20090820/ippTools/src/chiptool.c

@@ -54 +54 @@
 static bool tofullimfileMode(pxConfig *config);
 static bool topurgedimfileMode(pxConfig *config);
+static bool toscrubbedimfileMode(pxConfig *config);
 static bool exportrunMode(pxConfig *config);
 static bool importrunMode(pxConfig *config);
@@ -93 +94 @@
         MODECASE(CHIPTOOL_MODE_TOFULLIMFILE,            tofullimfileMode);
         MODECASE(CHIPTOOL_MODE_TOPURGEDIMFILE,          topurgedimfileMode);
+        MODECASE(CHIPTOOL_MODE_TOSCRUBBEDIMFILE,        toscrubbedimfileMode);
         MODECASE(CHIPTOOL_MODE_EXPORTRUN,               exportrunMode);
         MODECASE(CHIPTOOL_MODE_IMPORTRUN,               importrunMode);
@@ -601 +603 @@
     PXOPT_COPY_STR(config->args, where, "-reduction", "chipRun.reduction", "==");
     pxAddLabelSearchArgs (config, where, "-label", "chipRun.label", "LIKE");
-    PXOPT_COPY_S64(config->args, where, "-magicked", "chipRun.magicked", "==");
+    PXOPT_COPY_S64(config->args, where, "-magicked", "chipProcessedImfile.magicked", "==");
+
+    PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false);
+    PXOPT_LOOKUP_BOOL(destreaked, config->args,     "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false);
 
     if (!psListLength(where->list) &&
@@ -629 +635 @@
         // don't list faulted rows
         psStringAppend(&query, " %s", "AND chipProcessedImfile.fault = 0");
+    }
+    if (not_destreaked) {
+        if (destreaked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense");
+            return false;
+        }
+        if (magicked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense");
+            return false;
+        }
+        psStringAppend(&query, " AND chipProcessedImfile.magicked = 0");
+    }
+    if (destreaked) {
+        psStringAppend(&query, " AND chipProcessedImfile.magicked != 0");
     }
 
@@ -1333 +1353 @@
     return change_imfile_data_state(config, "purged", "goto_purged");
 }
-
+static bool toscrubbedimfileMode(pxConfig *config)
+{
+  return change_imfile_data_state(config, "scrubbed", "goto_scrubbed");
+}
 bool exportrunMode(pxConfig *config)
 {

branches/eam_branches/20090820/ippTools/src/chiptool.h

@@ -45 +45 @@
     CHIPTOOL_MODE_TOFULLIMFILE,
     CHIPTOOL_MODE_TOPURGEDIMFILE,
+    CHIPTOOL_MODE_TOSCRUBBEDIMFILE,
     CHIPTOOL_MODE_EXPORTRUN,
     CHIPTOOL_MODE_IMPORTRUN

branches/eam_branches/20090820/ippTools/src/chiptoolConfig.c

@@ -158 +158 @@
     psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-reduction",          0, "search by reduction class", NULL);
     psMetadataAddStr(processedimfileArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by chipRun label (LIKE comparison)", NULL);
-    psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "search by magicked status", false);
+
+    psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-destreaked",  0,      "search for destreaked images", false);
+    psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-not_destreaked",  0,  "search for images that have not been destreaked", false);
+    psMetadataAddS64(processedimfileArgs, PS_LIST_TAIL, "-magicked",  0,        "search by magicked value", 0);
     psMetadataAddU64(processedimfileArgs,  PS_LIST_TAIL, "-limit",  0,           "limit result set to N items", 0);
     psMetadataAddBool(processedimfileArgs, PS_LIST_TAIL, "-all",  0,            "list everything without search terms", false);
@@ -247 +250 @@
     psMetadataAddS64(topurgedimfileArgs, PS_LIST_TAIL, "-chip_id", 0,          "chip ID to update", 0);
     psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id",  0,        "class ID to update", NULL);
+
+    // -toscrubbedimfile
+    psMetadata *toscrubbedimfileArgs = psMetadataAlloc();
+    psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-chip_id", 0,        "chip ID to update", 0);
+    psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0,       "class ID to update", NULL);
 
     // -exportrun
@@ -282 +290 @@
     PXOPT_ADD_MODE("-tofullimfile",         "set imfile state to full",              CHIPTOOL_MODE_TOFULLIMFILE,         tofullimfileArgs);
     PXOPT_ADD_MODE("-topurgedimfile",       "set imfile state to purged",            CHIPTOOL_MODE_TOPURGEDIMFILE,       topurgedimfileArgs);
+    PXOPT_ADD_MODE("-toscrubbedimfile",     "set imfile state to scrubbed",          CHIPTOOL_MODE_TOSCRUBBEDIMFILE,     toscrubbedimfileArgs);
 
     PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", CHIPTOOL_MODE_EXPORTRUN, exportrunArgs);

branches/eam_branches/20090820/ippTools/src/dettool.c

@@ -150 +150 @@
         MODECASE(DETTOOL_MODE_UPDATEDETRUN,     updatedetrunMode);
         MODECASE(DETTOOL_MODE_RERUN,            rerunMode);
+        MODECASE(DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryMode);
         // register
         MODECASE(DETTOOL_MODE_REGISTER_DETREND, register_detrendMode);
@@ -1827 +1828 @@
     if (!strcmp(data_state, "drop")) return true;
     if (!strcmp(data_state, "register")) return true;
+    // These are valid data states, and are necessary for the cleanup to work correctly.
+    if (!strcmp(data_state, "full")) return true;
+    if (!strcmp(data_state, "goto_cleaned")) return true;
+    if (!strcmp(data_state, "goto_scrubbed")) return true;
+    if (!strcmp(data_state, "goto_purged")) return true;
+    if (!strcmp(data_state, "cleaned")) return true;
+    if (!strcmp(data_state, "scrubbed")) return true;
+    if (!strcmp(data_state, "purged")) return true;
+    if (!strcmp(data_state, "error_cleaned")) return true;
+    if (!strcmp(data_state, "error_scrubbed")) return true;
+    if (!strcmp(data_state, "error_purged")) return true;
 
     psError(PS_ERR_UNKNOWN, true, "invalid data state: %s", data_state);
@@ -1853 +1865 @@
 
     if (!isValidDataState (data_state)) return false;
-
+    
     char *query = "UPDATE detProcessedImfile SET data_state = '%s'"
         " WHERE det_id = %" PRId64
@@ -1900 +1912 @@
         " WHERE det_id = %" PRId64
         " AND iteration = %" PRId32
-        " AND class_id = %s";
+        " AND class_id = '%s'";
     if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) {
         psError(PS_ERR_UNKNOWN, false,
@@ -1922 +1934 @@
         " WHERE det_id = %" PRId64
         " AND iteration = %" PRId32
-        " AND class_id = %s";
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration)) {
+        " AND class_id = '%s'";
+/*     fprintf(stderr,"DETTOOL SAYS: %s\n",query); */
+    if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration,class_id)) {
         psError(PS_ERR_UNKNOWN, false,
                 "failed to change state for det_id %" PRId64 ", iteration %" PRId32, 
@@ -1944 +1957 @@
         " WHERE det_id = %" PRId64
         " AND iteration = %" PRId32
-        " AND class_id = %s";
+        " AND class_id = '%s'";
     if (!p_psDBRunQueryF(config->dbh, query, data_state, det_id, iteration, class_id)) {
         psError(PS_ERR_UNKNOWN, false,

branches/eam_branches/20090820/ippTools/src/dettool.h

@@ -111 +111 @@
     DETTOOL_MODE_REVERTDETRUNSUMMARY,
     DETTOOL_MODE_UPDATEDETRUNSUMMARY,
+    DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY,
     DETTOOL_MODE_UPDATEDETRUN,
     DETTOOL_MODE_RERUN,
@@ -209 +210 @@
 bool revertdetrunsummaryMode(pxConfig *config);
 bool updatedetrunsummaryMode(pxConfig *config);
+bool pendingcleanup_detrunsummaryMode(pxConfig *config);
 
 // other utilities

branches/eam_branches/20090820/ippTools/src/dettoolConfig.c

@@ -41 +41 @@
     // XXX EAM : is this used?  does it make sense?
     psMetadata *pendingArgs = psMetadataAlloc();
-    psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id",  0,            "search by exposure ID", 0);
-    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type",  0,            "search by exposure type", NULL);
-    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst",  0,            "search by camera", NULL);
-    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope",  0,            "search by telescope", NULL);
-    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter",  0,            "search by filter", NULL);
+    psMetadataAddS64(pendingArgs, PS_LIST_TAIL, "-exp_id",  0,         "search by exposure ID", 0);
+    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-exp_type",  0,       "search by exposure type", NULL);
+    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-inst",  0,           "search by camera", NULL);
+    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-telescope",  0,      "search by telescope", NULL);
+    psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-filter",  0,         "search by filter", NULL);
     psMetadataAddStr(pendingArgs, PS_LIST_TAIL, "-uri",  0,            "search by URL", NULL);
-    psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
+    psMetadataAddBool(pendingArgs, PS_LIST_TAIL, "-simple",  0,        "use the simple output format", false);
 
     // -definebytag
@@ -125 +125 @@
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_min",  0,            "define min rotator position angle", NAN);
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_posang_max",  0,            "define max rotator position angle", NAN);
+
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_min",  0,            "define min solar angle", NAN);
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_sun_angle_max",  0,            "define max solar angle", NAN);
@@ -137 +138 @@
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_moon_phase_max",  0,          "define max moon phase", NAN);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-comment",                0,          "search by comment field (LIKE comparison)", NULL);
+
 
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend",  0,            "print the exposures that would be included in the detrend run and exit", false);
@@ -207 +209 @@
     // -addcorrectimfile
     psMetadata *addcorrectimfileArgs = psMetadataAlloc();
-    psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id",  0,            "define detrend ID (required)", 0);
-    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id",  0,            "search for class ID (required)", NULL);
-    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri",  0,            "define resid file URI", NULL);
+    psMetadataAddS64(addcorrectimfileArgs, PS_LIST_TAIL, "-det_id",  0,        "define detrend ID (required)", 0);
+    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-class_id",  0,      "search for class ID (required)", NULL);
+    psMetadataAddStr(addcorrectimfileArgs, PS_LIST_TAIL, "-uri",  0,           "define resid file URI", NULL);
     psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg",  0,            "define exposure background", NAN);
     psMetadataAddF64(addcorrectimfileArgs, PS_LIST_TAIL, "-bg_stdev",  0,            "define exposure background stdev", NAN);
@@ -306 +308 @@
     psMetadataAddS64(updateprocessedimfileArgs, PS_LIST_TAIL, "-exp_id",               0,            "search by exp_id", 0);
     psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
-    psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updateprocessedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_processedimfile
@@ -366 +368 @@
     psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-det_id",               0,            "search by chip ID", 0);
     psMetadataAddS64(updateprocessedexpArgs, PS_LIST_TAIL, "-exp_id",               0,            "search by exp_id", 0);
-    psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updateprocessedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_processedexp
@@ -426 +428 @@
     psMetadataAddS32(updatestackedArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
     psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-class_id",  0,            "search by class ID", NULL);
-    psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updatestackedArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_stacked
@@ -478 +480 @@
     psMetadataAddS32(updatenormalizedstatArgs, PS_LIST_TAIL, "-iteration",  0,            "search by iteration number", 0);
     psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
-    psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updatenormalizedstatArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_normalizedstat
@@ -540 +542 @@
     psMetadataAddS32(updatenormalizedimfileArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
     psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
-    psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updatenormalizedimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_normalizedimfile
@@ -598 +600 @@
     psMetadataAddS64(updatenormalizedexpArgs, PS_LIST_TAIL, "-det_id",               0,            "search by chip ID", 0);
     psMetadataAddS32(updatenormalizedexpArgs, PS_LIST_TAIL, "-iteration", 0,            "search by iteration number", 0);
-    psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updatenormalizedexpArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_normalizedexp
@@ -678 +680 @@
     psMetadataAddS64(updateresidimfileArgs, PS_LIST_TAIL, "-exp_id",               0,            "search by exp_id", 0);
     psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-class_id",             0,            "search by exp_name", NULL);
-    psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updateresidimfileArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_residimfile
@@ -759 +761 @@
     psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-path_base",  0,            "define base output location", NULL);
     psMetadataAddBool(updateresidexpArgs, PS_LIST_TAIL, "-reject",  0,            "exposure is not to be stacked in the next iteration", false);
-    psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state",           0,            "search for telescope", NULL);
+    psMetadataAddStr(updateresidexpArgs, PS_LIST_TAIL, "-data_state",           0,            "set state", NULL);
 
     // -pendingcleanup_residexp
@@ -813 +815 @@
     psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-accept",  0,            "declare that this detrun iteration is accepted as a master", false);
     psMetadataAddBool(updatedetrunsummaryArgs, PS_LIST_TAIL, "-reject",  0,            "reject this detrun iteration as a master", false);
+    psMetadataAddStr(updatedetrunsummaryArgs, PS_LIST_TAIL, "-data_state", 0,          "set the data state", NULL);
+    psMetadataAddS32(updatedetrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0,           "search by iteration number", 0);
+
+    // -pendingcleanup_detrunsummary
+    psMetadata *pendingcleanup_detrunsummaryArgs = psMetadataAlloc();
+    psMetadataAddS64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-det_id", 0,     "search by detrend ID (required)", 0);
+    psMetadataAddS32(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-iteration", 0,  "search by iteration number", 0);
+    psMetadataAddU64(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-limit",  0,     "limit result set to N items", 0);
+    psMetadataAddBool(pendingcleanup_detrunsummaryArgs, PS_LIST_TAIL, "-simple", 0,    "use the simple output format", false);
 
     // -updatedetrun
@@ -968 +979 @@
     PXOPT_ADD_MODE("-updatedetrunsummary", "", DETTOOL_MODE_UPDATEDETRUNSUMMARY, updatedetrunsummaryArgs);
 
+    PXOPT_ADD_MODE("-pendingcleanup_detrunsummary", "", DETTOOL_MODE_PENDINGCLEANUP_DETRUNSUMMARY, pendingcleanup_detrunsummaryArgs);
+
     PXOPT_ADD_MODE("-updatedetrun", "", DETTOOL_MODE_UPDATEDETRUN, updatedetrunArgs);
     PXOPT_ADD_MODE("-rerun",           "", DETTOOL_MODE_RERUN,         rerunArgs);

branches/eam_branches/20090820/ippTools/src/dettool_detrunsummary.c

@@ -336 +336 @@
 }
 
-// XXX need to add -data_state here
+bool pendingcleanup_detrunsummaryMode(pxConfig *config) {
+  PS_ASSERT_PTR_NON_NULL(config, false);
+
+  PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "==");
+  PXOPT_COPY_S32(config->args, where, "-iteration", "iteration", "==");
+
+  psString query = pxDataGet("dettool_pendingcleanup_detrunsummary.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+    return(false);
+  }
+  
+  if (psListLength(where->list)) {
+    psString whereClause = psDBGenerateWhereConditionSQL(where, "detRunSummary");
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+  }
+  psFree(where);
+
+  // treat limit == 0 as "no limit"
+  if (limit) {
+    psString limitString = psDBGenerateLimitSQL(limit);
+    psStringAppend(&query, " %s", limitString);
+    psFree(limitString);
+  }
+  //  fprintf(stderr,">>>%s<<<\n",query);
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    psFree(query);
+    return(false);
+  }
+  psFree(query);
+
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psError(PS_ERR_UNKNOWN, false, "database error");
+    return(false);
+  }
+  if (!psArrayLength(output)) {
+    psTrace("dettool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return(true);
+  }
+
+  // negative simple so the default is true
+  if (!ippdbPrintMetadatas(stdout, output, "pendingCleanupDetRun", !simple)) {
+    psError(PS_ERR_UNKNOWN, false, "failed to print array");
+    psFree(output);
+    return(false);
+  }
+  psFree(output);
+
+  return(true);
+}
+
+
+// preliminary code now.
 bool updatedetrunsummaryMode(pxConfig *config)
 {
@@ -344 +404 @@
     PXOPT_LOOKUP_BOOL(accept, config->args, "-accept", false);
     PXOPT_LOOKUP_BOOL(reject, config->args, "-reject", false);
+    PXOPT_LOOKUP_STR(data_state, config->args, "-data_state", ((accept == 0)&&(reject == 0)), false);
+    PXOPT_LOOKUP_S32(iteration, config->args, "-iteration", ((accept == 0)&&(reject == 0)), false);
+
 
     if (accept && reject) {
@@ -350 +413 @@
     }
 
-    if (!(accept || reject)) {
-        psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required");
-        return false;
-    }
-
-    char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64;
-    if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
+    if (!(accept || reject || (data_state != NULL))) {
+        psError(PS_ERR_UNKNOWN, true, "either -accept or -reject is required if -data_state is not supplied");
+        return false;
+    }
+
+    if (accept || reject) {
+      char *query = "UPDATE detRunSummary SET accept = %d WHERE det_id = %"PRId64;
+      if (!p_psDBRunQueryF(config->dbh, query, accept, det_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+      }
+    }
+    else {
+      PS_ASSERT_PTR_NON_NULL(data_state,false);
+
+/*       if (!isValidDataState(data_state)) return false; */
+      
+      
+      
+      char *query_detRunSummary = "UPDATE detRunSummary SET data_state = '%s'"
+        " WHERE det_id = %" PRId64
+        " AND iteration = %" PRId32;
+      if (!p_psDBRunQueryF(config->dbh, query_detRunSummary, data_state, det_id,iteration)) {
+        psError(PS_ERR_UNKNOWN, false,
+                "failed to change state for det_id %" PRId64 ", iteration %" PRId32, det_id,iteration);
+        return false;
+      }
+
+      /* This check allows the one update to flag everything for cleanup.  The check for full is only temporary while I test for bugs. */
+      if ((!strncmp(data_state,"goto_",5)
+           //      || (!strcmp(data_state,"full"))
+           //      || (!strcmp(data_state,"cleaned"))
+           ))   {
+        char *query_detProcessedImfile = "UPDATE detProcessedImfile SET data_state = '%s'"
+          " WHERE det_id = %" PRId64;
+        if (!p_psDBRunQueryF(config->dbh, query_detProcessedImfile,data_state,det_id)) {
+          psError(PS_ERR_UNKNOWN, false,
+                  "failed to change state for detProcessedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration);
+          return(false);
+        }
+        
+        char *query_detProcessedExp = "UPDATE detProcessedExp SET data_state = '%s'"
+          " WHERE det_id = %" PRId64;
+        if (!p_psDBRunQueryF(config->dbh, query_detProcessedExp,data_state,det_id)) {
+          psError(PS_ERR_UNKNOWN, false,
+                  "failed to change state for detProcessedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration);
+          return(false);
+        }
+        
+        char *query_detNormalizedImfile = "UPDATE detNormalizedImfile SET data_state = '%s'"
+          " WHERE det_id = %" PRId64
+          " AND iteration = %" PRId32;
+        if (!p_psDBRunQueryF(config->dbh, query_detNormalizedImfile,data_state,det_id,iteration)) {
+          psError(PS_ERR_UNKNOWN, false,
+                  "failed to change state for detNormalizedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration);
+          return(false);
+        }
+        
+        char *query_detNormalizedStatImfile = "UPDATE detNormalizedStatImfile SET data_state = '%s'"
+          " WHERE det_id = %" PRId64
+          " AND iteration = %" PRId32;
+        if (!p_psDBRunQueryF(config->dbh, query_detNormalizedStatImfile,data_state,det_id,iteration)) {
+          psError(PS_ERR_UNKNOWN, false,
+                  "failed to change state for detNormalizedStatImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration);
+          return(false);
+        }
+
+        char *query_detNormalizedExp = "UPDATE detNormalizedExp SET data_state = '%s'"
+          " WHERE det_id = %" PRId64
+          " AND iteration = %" PRId32;
+        if (!p_psDBRunQueryF(config->dbh, query_detNormalizedExp,data_state,det_id,iteration)) {
+          psError(PS_ERR_UNKNOWN, false,
+                  "failed to change state for detNormalizedExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration);
+          return(false);
+        }
+        
+        char *query_detResidImfile = "UPDATE detResidImfile SET data_state = '%s'"
+          " WHERE det_id = %" PRId64
+          " AND iteration = %" PRId32;
+        if (!p_psDBRunQueryF(config->dbh, query_detResidImfile,data_state,det_id,iteration)) {
+          psError(PS_ERR_UNKNOWN, false,
+                  "failed to change state for detResidImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration);
+          return(false);
+        }
+        
+        char *query_detResidExp = "UPDATE detResidExp SET data_state = '%s'"
+          " WHERE det_id = %" PRId64
+          " AND iteration = %" PRId32;
+        if (!p_psDBRunQueryF(config->dbh, query_detResidExp,data_state,det_id,iteration)) {
+          psError(PS_ERR_UNKNOWN, false,
+                  "failed to change state for detResidExp det_id %" PRId64 ", iteration %" PRId32, det_id,iteration);
+          return(false);
+        }
+        
+        char *query_detStackedImfile = "UPDATE detStackedImfile SET data_state = '%s'"
+          " WHERE det_id = %" PRId64
+          " AND iteration = %" PRId32;
+        if (!p_psDBRunQueryF(config->dbh, query_detStackedImfile,data_state,det_id,iteration)) {
+          psError(PS_ERR_UNKNOWN, false,
+                  "failed to change state for detStackedImfile det_id %" PRId64 ", iteration %" PRId32, det_id,iteration);
+          return(false);
+        }
+      }
+      /* End if */
+      
     }
 

branches/eam_branches/20090820/ippTools/src/dettool_processedimfile.c

@@ -314 +314 @@
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
+/*     int i; */
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-det_id", "det_id", "==");
@@ -345 +345 @@
         return false;
     }
+/*     fprintf(stderr,"DETTOOL:procimfile: %s\n",query); */
     psFree(query);
 
@@ -350 +351 @@
     if (!output) {
         psError(PS_ERR_UNKNOWN, false, "database error");
+/*      fprintf(stderr,"WTF !output?\n"); */
         return false;
     }
     if (!psArrayLength(output)) {
         psTrace("dettool", PS_LOG_INFO, "no rows found");
+/*      fprintf(stderr,"WTF no rows??\n"); */
         psFree(output);
         return true;
@@ -359 +362 @@
 
     // negative simple so the default is true
+/*     i = (int) ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple); */
+/*     fprintf(stderr,">>%d<<\n",i); */
     if (!ippdbPrintMetadatas(stdout, output, "detCleanupProcessedImfile", !simple)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print array");
@@ -364 +369 @@
         return false;
     }
+/*     fprintf(stderr,"DETTOOL:procimfile: %s\n",output); */
+/*     psFree(output); */
 
     psFree(output);

branches/eam_branches/20090820/ippTools/src/difftool.c

@@ -37 +37 @@
 static bool todiffskyfileMode(pxConfig *config);
 static bool adddiffskyfileMode(pxConfig *config);
+static bool advanceMode(pxConfig *config);
 static bool diffskyfileMode(pxConfig *config);
 static bool revertdiffskyfileMode(pxConfig *config);
@@ -50 +51 @@
 
 static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state, psS64 magicked);
-static bool diffRunComplete(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -76 +76 @@
         MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE,         todiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE,        adddiffskyfileMode);
+        MODECASE(DIFFTOOL_MODE_ADVANCE,               advanceMode);
         MODECASE(DIFFTOOL_MODE_DIFFSKYFILE,           diffskyfileMode);
         MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
-        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,         definewarpstackMode);
+        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,       definewarpstackMode);
         MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
@@ -555 +556 @@
     }
 
-    if (!diffRunComplete(config)) {
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
     // point of no return
     if (!psDBCommit(config->dbh)) {
@@ -569 +562 @@
     }
 
+
+    return true;
+}
+
+static bool advanceMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffRun.diff_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+
+    // look for completed diffRuns
+    psString query = pxDataGet("difftool_completed_runs.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString whereString = psStringCopy("");
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&whereString, "\n AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(whereString);
+        return false;
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, whereString)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        psFree(whereString);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+    psFree(whereString);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("difftool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+
+        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");
+        psS64 magicked = psMetadataLookupS64(NULL, row, "magicked");
+
+        // set diffRun.state to 'full'
+        if (!setdiffRunState(config, diff_id, "full", magicked)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
+                diff_id);
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+    }
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
 
     return true;
@@ -581 +665 @@
     PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffSkyfile.diff_id", "==");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
-    PXOPT_COPY_S64(config->args, where,  "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
     PXOPT_COPY_STR(config->args, where, "-tess_id", "diffRun.tess_id", "==");
     PXOPT_COPY_S16(config->args, where, "-fault", "diffSkyfile.fault", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "==");
-    PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "==");
-
+    PXOPT_COPY_S64(config->args, where,  "-magicked", "diffSkyfile.magicked", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "diffRun.label", "LIKE");
+
+    PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
+    if (!template) {
+        PXOPT_COPY_S64(config->args, where, "-exp_id", "rawInput.exp_id", "==");
+        PXOPT_COPY_STR(config->args, where, "-exp_name", "rawInput.exp_name", "==");
+        PXOPT_COPY_STR(config->args, where, "-warp_id", "warpInput.warp_id", "==");
+        PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawInput.dateobs",  ">=");
+        PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawInput.dateobs",  "<=");
+        PXOPT_COPY_STR(config->args, where, "-filter",     "rawInput.filter", "==");
+    } else {
+        PXOPT_COPY_S64(config->args, where, "-exp_id", "rawTemplate.exp_id", "==");
+        PXOPT_COPY_STR(config->args, where, "-exp_name", "rawTemplate.exp_name", "==");
+        PXOPT_COPY_STR(config->args, where, "-warp_id", "warpTemplate.warp_id", "==");
+        PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawTemplate.dateobs",  ">=");
+        PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawTemplate.dateobs",  "<=");
+        PXOPT_COPY_STR(config->args, where, "-filter",     "rawTemplate.filter", "==");
+    }
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    PXOPT_LOOKUP_U64(magicked, config->args,        "-magicked", false, false);
+    PXOPT_LOOKUP_BOOL(destreaked, config->args,     "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false);
 
     psString query = pxDataGet("difftool_skyfile.sql");
@@ -601 +706 @@
         psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
+    } else if (!all) {
+        psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required");
     }
     psFree(where);
+
+    if (not_destreaked) {
+        if (destreaked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense");
+            return false;
+        }
+        if (magicked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense");
+            return false;
+        }
+        psStringAppend(&query, " AND diffSkyfile.magicked = 0");
+    }
+    if (destreaked) {
+        psStringAppend(&query, " AND diffSkyfile.magicked != 0");
+    }
+
 
     // treat limit == 0 as "no limit"
@@ -1810 +1933 @@
 }
 
-static bool diffRunComplete(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    // look for completed diffRuns
-    psString query = pxDataGet("difftool_completed_runs.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("difftool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-    for (long i = 0; i < psArrayLength(output); i++) {
-        psMetadata *row = output->data[i];
-
-        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");
-        psS64 magicked = psMetadataLookupS64(NULL, row, "magicked");
-
-        // set diffRun.state to 'stop'
-        if (!setdiffRunState(config, diff_id, "full", magicked)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
-                diff_id);
-            psFree(output);
-            return false;
-        }
-    }
-
-    return true;
-}
 
 bool exportrunMode(pxConfig *config)

branches/eam_branches/20090820/ippTools/src/difftool.h

@@ -31 +31 @@
     DIFFTOOL_MODE_TODIFFSKYFILE,
     DIFFTOOL_MODE_ADDDIFFSKYFILE,
+    DIFFTOOL_MODE_ADVANCE,
     DIFFTOOL_MODE_DIFFSKYFILE,
     DIFFTOOL_MODE_REVERTDIFFSKYFILE,

branches/eam_branches/20090820/ippTools/src/difftoolConfig.c

@@ -115 +115 @@
     psMetadataAddBool(adddiffskyfileArgs, PS_LIST_TAIL, "-magicked",  0, "define magicked state", false);
 
+    // -advance
+    psMetadata *advanceArgs = psMetadataAlloc();
+    psMetadataAddS64(advanceArgs, PS_LIST_TAIL, "-diff_id", 0, "select by diff ID", 0);
+    psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "select by label", NULL);
+    psMetadataAddS32(advanceArgs, PS_LIST_TAIL, "-limit", 0, "limit number of results", 0);
+
     // -diffskyfile
     psMetadata *diffskyfileArgs = psMetadataAlloc();
-    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_id", 0,            "search by diff ID", 0);
-    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id",  0,       "define skycell ID", NULL);
-    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-diff_skyfile_id", 0,    "search by diff_skyfile_id ID", 0);
-    psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL, "-tess_id",  0,            "define tessellation ID", NULL);
-    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL, "-exp_id",  0,            "define exposure ID", 0);
-    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name",  0,         "define exposure name", NULL);
-    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id",  0,         "define warp_id", NULL);
-    psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
-    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
-    psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "define fault code", 0);
+    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL,  "-diff_id", 0,           "search by diff ID", 0);
+    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-skycell_id",  0,      "search by skycell ID", NULL);
+    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL,  "-diff_skyfile_id", 0,   "search by diff_skyfile_id ID", 0);
+    psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL,  "-tess_id",  0,          "search by tessellation ID", NULL);
+    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-warp_id",  0,         "search by warp_id", NULL);
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-template",  0,        "apply exposure args to template of bothways diff", false);
+    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL,  "-exp_id",  0,           "search by exposure ID", 0);
+    psMetadataAddStr(diffskyfileArgs , PS_LIST_TAIL, "-exp_name",  0,        "search by exposure name", NULL);
+    psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_begin", 0,    "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(diffskyfileArgs, PS_LIST_TAIL, "-dateobs_end", 0,      "search for exposures by time (<=)", NULL);
+    psMetadataAddStr(diffskyfileArgs, PS_LIST_TAIL,  "-filter", 0,           "search for filter", NULL);
+    psMetadataAddS64(diffskyfileArgs, PS_LIST_TAIL,  "-magicked", 0,         "search by magicked value", 0);
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-destreaked",  0,      "search for destreaked images", false);
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-not_destreaked",  0,  "search for images that are not destreaked", false);
+    psMetadataAddStr(diffskyfileArgs,  PS_LIST_TAIL, "-label",  PS_META_DUPLICATE_OK, "search by diffRun label (LIKE comparison)", NULL);
+    psMetadataAddS16(diffskyfileArgs, PS_LIST_TAIL,  "-fault",  0,           "search by fault code", 0);
+
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
+    psMetadataAddU64(diffskyfileArgs, PS_LIST_TAIL,  "-limit",  0,            "limit result set to N items", 0);
+    psMetadataAddBool(diffskyfileArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
 
     // -revertdiffskyfile
@@ -267 +283 @@
     PXOPT_ADD_MODE("-todiffskyfile",    "", DIFFTOOL_MODE_TODIFFSKYFILE,     todiffskyfileArgs);
     PXOPT_ADD_MODE("-adddiffskyfile",   "", DIFFTOOL_MODE_ADDDIFFSKYFILE,    adddiffskyfileArgs);
+    PXOPT_ADD_MODE("-advance",          "", DIFFTOOL_MODE_ADVANCE,           advanceArgs);
     PXOPT_ADD_MODE("-diffskyfile",      "", DIFFTOOL_MODE_DIFFSKYFILE,       diffskyfileArgs);
     PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs);

branches/eam_branches/20090820/ippTools/src/disttool.c

@@ -2 +2 @@
  * disttool.c
  *
- * Copyright (C) 2008
+ * Copyright (C) 2008-2009
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -37 +37 @@
 static bool pendingcomponentMode(pxConfig *config);
 static bool addprocessedcomponentMode(pxConfig *config);
+static bool revertcomponentMode(pxConfig *config);
 static bool processedcomponentMode(pxConfig *config);
 static bool toadvanceMode(pxConfig *config);
@@ -51 +52 @@
 static bool listtargetMode(pxConfig *config);
 
-static bool definedsproductMode(pxConfig *config);
-static bool updatedsproductMode(pxConfig *config);
-
 static bool definedestinationMode(pxConfig *config);
 static bool updatedestinationMode(pxConfig *config);
@@ -59 +57 @@
 static bool defineinterestMode(pxConfig *config);
 static bool updateinterestMode(pxConfig *config);
+static bool listinterestsMode(pxConfig *config);
 
 # define MODECASE(caseName, func) \
@@ -86 +85 @@
         MODECASE(DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentMode);
         MODECASE(DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentMode);
+        MODECASE(DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentMode);
         MODECASE(DISTTOOL_MODE_TOADVANCE, toadvanceMode);
         MODECASE(DISTTOOL_MODE_PENDINGFILESET, pendingfilesetMode);
@@ -97 +97 @@
         MODECASE(DISTTOOL_MODE_UPDATETARGET, updatetargetMode);
         MODECASE(DISTTOOL_MODE_LISTTARGET, listtargetMode);
-        MODECASE(DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductMode);
-        MODECASE(DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductMode);
         MODECASE(DISTTOOL_MODE_DEFINEDESTINATION, definedestinationMode);
         MODECASE(DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationMode);
         MODECASE(DISTTOOL_MODE_DEFINEINTEREST, defineinterestMode);
         MODECASE(DISTTOOL_MODE_UPDATEINTEREST, updateinterestMode);
+        MODECASE(DISTTOOL_MODE_LISTINTERESTS, listinterestsMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -151 +150 @@
             stage,
             stage_id,
+            0,
             set_label,
             outroot,
@@ -223 +223 @@
         }
     } else if (!strcmp(stage, "camera")) {
-        magicRunType = "chipRun";    // This is used below to set the magicked business
+        magicRunType = "camRun";    // This is used below to set the magicked business
+        runJoinStr = "camRun.cam_id";
         query = pxDataGet("disttool_definebyquery_camera.sql");
         if (!query) {
@@ -288 +289 @@
             psStringAppend(&query, " AND (stackRun.label = '%s')", label);
         }
-        // stack stage doesn't require magic (perhaps let the script do this?
+        // stack stage doesn't require magic
         no_magic = true;
     } else {
@@ -306 +307 @@
 
     if (!no_magic) {
-        psStringAppend(&query, " AND (distTarget.clean OR %s.magicked)", magicRunType);
+        psStringAppend(&query, " AND (%s.magicked)", magicRunType);
 
         // is selecting by magic_ds_id really interesting?
         if (magic_ds_id) {
-            if (strcmp(stage, "camera")) {
-                // stage other than camera
-                if (!runJoinStr) {
-                    psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage);
-                    psFree(query);
-                    return false;
-                }
-                psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage"
+            if (!runJoinStr) {
+                psError(PS_ERR_PROGRAMMING, true, "cannot select by magic_ds_id for stage: %s", stage);
+                psFree(query);
+                return false;
+            }
+            psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = distTarget.stage"
                                               " AND magicDSRun.stage_id = %s", runJoinStr);
-            } else {
-                // camera masks are magicked when the chipRun is magicked
-                // XXX: This is confusing. Is it dangerous?
-                // Maybe I should add a magicked bit to camRun. Note this isn't
-                psStringAppend(&joinHook, "\nJOIN magicDSRun ON magicDSRun.stage = 'chip'"
-                                              " AND magicDSRun.stage_id = chipRun.chip_id");
-            }
             psStringAppend(&query, " AND (magicDSRun.state = 'full' AND magicDSRun.re_place AND (magic_ds_id = %" PRId64 "))", magic_ds_id);
         }
@@ -377 +369 @@
         psString run_tag = psMetadataLookupStr(NULL, md, "run_tag");
         psS64 stage_id = psMetadataLookupS64(NULL, md, "stage_id");
+        psS64 magic_ds_id = psMetadataLookupS64(NULL, md, "magicked");
         psS64 target_id = psMetadataLookupS64(NULL, md, "target_id");
         psString target_label = psMetadataLookupStr(NULL, md, "label");
@@ -395 +388 @@
                 stage,
                 stage_id,
+                magic_ds_id,
                 new_label,
                 outroot,
@@ -500 +494 @@
     PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
 
-    // we need to disambiguate fault so make a copy of the where list before adding fault
-    psMetadata *whereComponent = psMetadataCopy(NULL, where);
     PXOPT_COPY_S16(config->args, where,  "-fault", "distRun.fault", "==");
-    PXOPT_COPY_S16(config->args, whereComponent, "-fault", "distComponent.fault", "==");
 
     // It might be useful to be able to query by the parameters of the underlying runs
@@ -513 +504 @@
     }
 
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+    psString query = pxDataGet("disttool_revertrun.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+
+    int numUpdated = psDBAffectedRows(config->dbh);
+
+    psLogMsg("disttool", PS_LOG_INFO, "Updated %d dist runs", numUpdated);
+
+    return true;
+}
+
+static bool revertcomponentMode(pxConfig *config)
+{
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-dist_id", "distRun.dist_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");;
+    PXOPT_COPY_STR(config->args, where, "-component", "component", "==");;
+    PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
+
+    PXOPT_COPY_S16(config->args, where,  "-fault", "distComponent.fault", "==");
+
+    // It might be useful to be able to query by the parameters of the underlying runs
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
         psFree(where);
-        return false;
-    }
-
-    // Update state to 'new'
-    int numUpdated;                     // Number updated
-    {
-        psString query = pxDataGet("disttool_revertrun_update.sql");
-        if (!query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&query, " AND %s", whereClause);
-            psFree(whereClause);
-        }
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psFree(query);
-
-        numUpdated = psDBAffectedRows(config->dbh);
-    }
-
-    psLogMsg("disttool", PS_LOG_INFO, "Updated %d dist runs", numUpdated);
-
-    // Delete product
-    int numDeleted;                     // Number deleted
-    {
-        psString query = pxDataGet("disttool_revertrun_delete.sql");
-        if (!query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        if (psListLength(whereComponent->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(whereComponent, NULL);
-            psStringAppend(&query, " AND %s", whereClause);
-            psFree(whereClause);
-        }
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psFree(query);
-
-        numDeleted = psDBAffectedRows(config->dbh);
-    }
+        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = pxDataGet("disttool_revertcomponent.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+
+    int numDeleted = psDBAffectedRows(config->dbh);
 
     psLogMsg("disttool", PS_LOG_INFO, "Deleted %d distComponents", numDeleted);
 
-    psFree(where);
-    psFree(whereComponent);
-
-    if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
     return true;
 }
@@ -600 +584 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
+
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    // look for "inputs" that need to processed
-    psString query = pxDataGet("disttool_pendingcomponent.sql");
+    psString queryFile = NULL;
+    psStringAppend(&queryFile, "disttool_pending_%s.sql", stage);
+    psString query = pxDataGet(queryFile);
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile);
         return false;
     }
@@ -617 +603 @@
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " WHERE %s", whereClause);
+        psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
@@ -629 +615 @@
     }
 
-    // the query has where hooks for each stage.
-    // right now we aren't using them.
-    // XXX: I think that I want to change the query from a union of selects on the various
-    // stages to separate queries. As it is pending data at the later stages of the pipline
-    // will get blocked by pending earlier stages
-    psString    raw_where = "";
-    psString    raw_clean_where = "";
-    psString    chip_where = "";
-    psString    camera_where = "";
-    psString    fake_where = "";
-    psString    warp_where = "";
-    psString    diff_where = "";
-    psString    stack_where = "";
-
-    if (!p_psDBRunQueryF(config->dbh,
-            query,
-            raw_where,
-            raw_clean_where,
-            chip_where,
-            camera_where,
-            fake_where,
-            warp_where,
-            diff_where,
-            stack_where)) {
+    if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -868 +831 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-dist_id", "dist_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-stage", "distRun.stage", "==");
     pxAddLabelSearchArgs (config, where, "-label", "distRun.label", "==");
 
@@ -939 +903 @@
     // required values
     PXOPT_LOOKUP_S64(dist_id, config->args, "-dist_id", true, false);
-    PXOPT_LOOKUP_S64(prod_id, config->args, "-prod_id", true, false);
+    PXOPT_LOOKUP_S64(dest_id, config->args, "-dest_id", true, false);
 
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
@@ -949 +913 @@
             0,          // fs_id
             dist_id,
-            prod_id,
+            dest_id,
             name,
             "full",
@@ -964 +928 @@
     PXOPT_COPY_S64(config->args, where, "-fs_id", "fs_id", "==");
     PXOPT_COPY_S64(config->args, where, "-dist_id", "rcDSFileset.dist_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "==");
     PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");;
     PXOPT_COPY_S64(config->args, where, "-stage_id", "stage_id", "==");
@@ -1092 +1056 @@
     PXOPT_COPY_S64(config->args, where, "-dist_id",  "dist_id", "==");
     PXOPT_COPY_S64(config->args, where, "-dest_id",  "dest_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-prod_id",  "prod_id", "==");
     PXOPT_COPY_S64(config->args, where, "-target_id","target_id", "==");
     PXOPT_COPY_S64(config->args, where, "-fs_id",    "fs_id", "==");
@@ -1433 +1396 @@
 }
 
-static bool definedsproductMode(pxConfig *config)
+static bool definedestinationMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
     // required
-    PXOPT_LOOKUP_STR(name, config->args,   "-name", true, false);
-    PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", true, false);
-    PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", true, false);
-
-    // XXX: should we insure that these names do not contatin any whitespace?
-
-    rcDSProductRow *row = rcDSProductRowAlloc(
-            0,          // prod_id
-            name,
-            dbname,
-            dbhost
-            );
-            
-    if (!row) {
-        psError(PS_ERR_UNKNOWN, false, "failed to allocate rcDSProduct object");
-        return false;
-    }
-   if (!rcDSProductInsertObject(config->dbh, row)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(row);
-        return false;
-    }
-
-    // get the assigned target_id
-    row->prod_id = psDBLastInsertID(config->dbh);
-
-    if (!rcDSProductPrintObject(stdout, row, true)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print object");
-        psFree(row);
-        return false;
-    }
-
-    psFree(row);
-
-    return true;
-}
-static bool updatedsproductMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "==");
-
-    PXOPT_LOOKUP_STR(dbname, config->args, "-ds_dbname", false, false);
-    PXOPT_LOOKUP_STR(dbhost, config->args, "-ds_dbhost", false, false);
-
-    if (!(dbname || dbhost)) {
-        psError(PS_ERR_UNKNOWN, true, "one or more of dbname or dbhost is required");
-        psFree(where);
-        return false;
-    }
-    psString query = psStringCopy("UPDATE rcDSProduct SET");
-    psString sep = "";
-    if (dbname) {
-        psStringAppend(&query, " dbname = '%s'", dbname);
-        sep = ",";
-    }
-    if (dbhost) {
-        psStringAppend(&query, " %s dbhost = '%s'", sep, dbhost);
-    }
-
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " WHERE %s", whereClause);
-        psFree(whereClause);
-    } else {
-        psError(PS_ERR_UNKNOWN, true, "search parameters are required");
-        psFree(where);
-        psFree(query);
-        return false;
-    }
-    psFree(where);
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    return true;
-}
-
-static bool definedestinationMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    // required
-    PXOPT_LOOKUP_S64(prod_id, config->args,      "-prod_id", true, false);
     PXOPT_LOOKUP_STR(name, config->args,         "-name", true, false);
+    PXOPT_LOOKUP_STR(dbname, config->args,       "-ds_dbname", true, false);
+    PXOPT_LOOKUP_STR(dbhost, config->args,       "-ds_dbhost", true, false);
 
     // optional
@@ -1539 +1415 @@
     rcDestinationRow *row = rcDestinationRowAlloc(
             0,          // dest_id
-            prod_id,
             name,
             status_uri,
             comment,
             last_fileset,
+            dbname,
+            dbhost,
             state ? state : "enabled"
             );
@@ -1577 +1454 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-prod_id", "prod_id", "==");
 
     PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
@@ -1599 +1475 @@
     // last_fileset normally gets set by updatercrunMode
     // Allowing it to be set here might cause problems
-    // especially since we are allowing selection by prod_id
+    // especially since we are allowing selection by dest_id
     if (last_fileset) {
         psStringAppend(&query, " %s last_fileset = '%s'", sep, last_fileset);
@@ -1631 +1507 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    // required
-    PXOPT_LOOKUP_S64(dest_id, config->args,      "-dest_id", true, false);
-    PXOPT_LOOKUP_S64(target_id, config->args,    "-target_id", true, false);
+    // one of these is required
+    PXOPT_LOOKUP_S64(dest_id, config->args,      "-dest_id", false, false);
+    PXOPT_LOOKUP_STR(dest_name, config->args,    "-dest_name", false, false);
+    if (!dest_id && !dest_name) {
+        psError(PS_ERR_UNKNOWN, true, "either dest_id or dest_name is required");
+        return false;
+    }
+
+    // either target_id or stage and label are required
+    PXOPT_LOOKUP_S64(target_id, config->args,    "-target_id", false, false);
+    PXOPT_LOOKUP_STR(stage, config->args,        "-stage", false, false);
+    PXOPT_LOOKUP_STR(label, config->args,        "-label", false, false);
+    PXOPT_LOOKUP_STR(filter, config->args,       "-filter", false, false);
+    PXOPT_LOOKUP_BOOL(clean, config->args,       "-clean", false);
+
+    if (!target_id) {
+        bool error = false;
+        if (!stage) {
+            psError(PS_ERR_UNKNOWN, true, "stage is required if target_id is not supplied");
+            error = true;
+        }
+        if (!label) {
+            psError(PS_ERR_UNKNOWN, !error, "label is required if target_id is not supplied");
+            error = true;
+        }
+        if (error) {
+            return false;
+        }
+    }
 
     // optional
+    PXOPT_LOOKUP_S64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_STR(state, config->args,        "-set_state", false, false);
-
-    // XXX: should we insure that these names do not contatin any whitespace?
-
-    rcInterestRow *row = rcInterestRowAlloc(
-            0,          // int_id
-            dest_id,
-            target_id,
-            state ? state : "enabled"
-            );
-            
-    if (!row) {
-        psError(PS_ERR_UNKNOWN, false, "failed to allocate rcInterest object");
-        return false;
-    }
-   if (!rcInterestInsertObject(config->dbh, row)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(row);
-        return false;
-    }
-
-    // get the assigned target_id
-    row->int_id = psDBLastInsertID(config->dbh);
-
-    if (!rcInterestPrintObject(stdout, row, true)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print object");
-        psFree(row);
-        return false;
-    }
-
-    psFree(row);
+    if (state) {
+        if (strcmp(state, "enabled") && strcmp(state, "disabled")) {
+            psError(PS_ERR_PROGRAMMING, true, "state must be enabled or disabled");
+            return false;
+        }
+    } else {
+        // default state
+        state = "enabled";
+    }
+
+    // now that we've done all of our argument checking, copy the values to where
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-dest_name", "rcDestination.name", "==");
+    PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
+
+    psString query = pxDataGet("disttool_defineinterest.sql");
+
+    if (!psListLength(where->list)) {
+        // can't get here
+        psError(PS_ERR_PROGRAMMING, true, "search parameters are required");
+        psFree(where);
+        psFree(query);
+        return false;
+    }
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " AND %s", whereClause);
+    psFree(whereClause);
+    psFree(where);
+    if (clean) {
+        psStringAppend(&query, " AND (distTarget.clean)");
+    } else {
+        psStringAppend(&query, " AND (!distTarget.clean)");
+    }
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+    {
+        // psStringSubstitute fails unless the input is a psString which it determines by
+        // comparing the memory blocks free function to an expected value.
+        // pxDataGet uses psSlurp which leaves a different free function on the memory block.
+        // To work around this make a copy of the query before doing the substitution.
+        psString queryCopy = psStringCopy(query);
+        psFree(query);
+        query = queryCopy;
+    }
+    // change the @STATE@ in the sql file to our state
+    if (!psStringSubstitute(&query, state, "@STATE@")) {
+        psError(PS_ERR_UNKNOWN, false, "failed to substitute state string");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+    int numInserted = psDBAffectedRows(config->dbh);
+    printf("inserted %d rows into rcInterest\n", numInserted);
 
     return true;
@@ -1710 +1648 @@
     return true;
 }
-
+static bool listinterestsMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-int_id", "int_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-dest_id", "dest_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-dest_name", "name", "==");
+    PXOPT_COPY_S64(config->args, where, "-target_id", "target_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-filter", "filter", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-state", "state", "==");
+
+    PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
+    PXOPT_LOOKUP_BOOL(full, config->args, "-full", false);
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    if (clean && full) {
+        psError(PS_ERR_UNKNOWN, false, "can't select both -clean and -full");
+        return false;
+    }
+
+    psString query = pxDataGet("disttool_listinterests.sql");
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+        if (clean) {
+            psStringAppend(&query, " AND (clean)");
+        } else if (full) {
+            psStringAppend(&query, " AND (!clean)");
+        }
+    } else if (clean) {
+        psStringAppend(&query, " WHERE clean");
+    } else if (full) {
+        psStringAppend(&query, " WHERE !clean");
+    }
+    psFree(where);
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("disttool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    if (!ippdbPrintMetadatas(stdout, output, "rcInterest", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}

branches/eam_branches/20090820/ippTools/src/disttool.h

@@ -31 +31 @@
     DISTTOOL_MODE_PENDINGCOMPONENT,
     DISTTOOL_MODE_ADDPROCESSEDCOMPONENT,
+    DISTTOOL_MODE_REVERTCOMPONENT,
     DISTTOOL_MODE_PROCESSEDCOMPONENT,
     DISTTOOL_MODE_TOADVANCE,
@@ -42 +43 @@
     DISTTOOL_MODE_DEFINEDESTINATION,
     DISTTOOL_MODE_UPDATEDESTINATION,
-    DISTTOOL_MODE_DEFINEDSPRODUCT,
-    DISTTOOL_MODE_UPDATEDSPRODUCT,
     DISTTOOL_MODE_DEFINETARGET,
     DISTTOOL_MODE_UPDATETARGET,
@@ -49 +48 @@
     DISTTOOL_MODE_DEFINEINTEREST,
     DISTTOOL_MODE_UPDATEINTEREST,
+    DISTTOOL_MODE_LISTINTERESTS,
 } disttoolMode;
 

branches/eam_branches/20090820/ippTools/src/disttoolConfig.c

@@ -95 +95 @@
     // -pendingcomponent
     psMetadata *pendingcomponentArgs = psMetadataAlloc();
+    psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage",    0, "limit results to runs for stage (required)", NULL);
     psMetadataAddS64(pendingcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
-    psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-stage",    0, "limit results to runs for stage", NULL);
     psMetadataAddStr(pendingcomponentArgs, PS_LIST_TAIL, "-label",    PS_META_DUPLICATE_OK, "limit results to label", NULL);
     psMetadataAddU64(pendingcomponentArgs, PS_LIST_TAIL, "-limit",  0,  "limit result set to N items", 0);
@@ -111 +111 @@
     psMetadataAddS32(addprocessedcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
 
+    // -revertcomponent
+    psMetadata *revertcomponentArgs = psMetadataAlloc();
+    psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
+    psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-component", 0, "define component", NULL);
+    psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-stage",    0, "define stage", NULL);
+    psMetadataAddS64(revertcomponentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage_id", 0);
+    psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-state",    0, "define state", NULL);
+    psMetadataAddStr(revertcomponentArgs, PS_LIST_TAIL, "-label",    0, "define label", NULL);
+    psMetadataAddS16(revertcomponentArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
+    psMetadataAddBool(revertcomponentArgs, PS_LIST_TAIL, "-all",    0, "revert all faulted runs", NULL);
+
     // -processedcomponent
     psMetadata *processedcomponentArgs = psMetadataAlloc();
@@ -135 +146 @@
     psMetadata *addfilesetArgs = psMetadataAlloc();
     psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
-    psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define prod_id", 0);
+    psMetadataAddS64(addfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dest_id", 0);
     psMetadataAddStr(addfilesetArgs, PS_LIST_TAIL, "-name",    0, "define file name", NULL);
     psMetadataAddS32(addfilesetArgs, PS_LIST_TAIL, "-fault",   0, "define fault code", 0);
@@ -150 +161 @@
     psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-fs_id",   0, "define fs_id", 0);
     psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dist_id", 0, "define dist_id", 0);
-    psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-prod_id", 0, "define dist_id", 0);
+    psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-dest_id", 0, "define dist_id", 0);
     psMetadataAddStr(revertfilesetArgs, PS_LIST_TAIL, "-stage",   0, "define stage", NULL);
     psMetadataAddS64(revertfilesetArgs, PS_LIST_TAIL, "-stage_id",0, "define stage_id", 0);
@@ -162 +173 @@
     psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dist_id",   0, "define dist_id", 0);
     psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-dest_id",   0, "define dest_id", 0);
-    psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-prod_id",   0, "define prod_id", 0);
     psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0);
     psMetadataAddS64(queuercrunArgs, PS_LIST_TAIL, "-fs_id",     0, "define fs_id", 0);
@@ -189 +199 @@
     psMetadataAddBool(revertrcrunArgs, PS_LIST_TAIL, "-all",     0, "revert all faulted runs", NULL);
 
-    // -definedsproduct
-    psMetadata *definedsproductArgs = psMetadataAlloc();
-    psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-name",  0, "define product name", NULL);
-    psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL);
-    psMetadataAddStr(definedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL);
-
-    // -updatedsproduct
-    // does this mode make sense?
-    psMetadata *updatedsproductArgs = psMetadataAlloc();
-    psMetadataAddS64(updatedsproductArgs, PS_LIST_TAIL, "-prod_id",   0, "select by prod_id", 0);
-    // can't select by name because it isn't necssarily unique
-    psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name", NULL);
-    psMetadataAddStr(updatedsproductArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host", NULL);
-
     // -definedestination
     psMetadata *definedestinationArgs = psMetadataAlloc();
-    psMetadataAddS64(definedestinationArgs, PS_LIST_TAIL, "-prod_id",     0, "define prod_id (required)", 0);
     psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-name",        0, "define destination name (required)", NULL);
+    psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbname",0, "define data store database name (required)", NULL);
+    psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-ds_dbhost",0, "define data store database host (required)", NULL);
     psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-status_uri",  0, "define status_uri", NULL);
     psMetadataAddStr(definedestinationArgs, PS_LIST_TAIL, "-comment",     0, "define comment", NULL);
@@ -215 +212 @@
     psMetadata *updatedestinationArgs = psMetadataAlloc();
     psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-dest_id",     0, "define dest_id", 0);
-    psMetadataAddS64(updatedestinationArgs, PS_LIST_TAIL, "-prod_id",     0, "define prod_id", 0);
     psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-name",        0, "define destination name", NULL);
     psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-status_uri",  0, "define status_uri", NULL);
     psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-comment",     0, "define comment", NULL);
-//  last_fileset gets updated by -updatercrun
-//  psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_last_fileset",0, "define last_fileset", NULL);
     psMetadataAddStr(updatedestinationArgs, PS_LIST_TAIL, "-set_state",   0, "define state", NULL);
 
@@ -254 +248 @@
     // -defineinterest
     psMetadata *defineinterestArgs = psMetadataAlloc();
-    psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id",   0, "define dest_id (required)", 0);
-    psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id (required)", 0);
+    psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-dest_id",   0, "define dest_id", 0);
+    psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-dest_name", 0, "define destination name (LIKE comparison)", NULL);
+    psMetadataAddS64(defineinterestArgs, PS_LIST_TAIL, "-target_id", 0, "define target_id", 0);
+    psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-stage",     0, "define stage", NULL);
+    psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-label",     0, "define label", NULL);
+    psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-filter",    0, "define filter (LIKE comparison)", NULL);
+    psMetadataAddBool(defineinterestArgs, PS_LIST_TAIL,"-clean",     0, "list clean targets", false);
     psMetadataAddStr(defineinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state", NULL);
+    psMetadataAddU64(defineinterestArgs, PS_LIST_TAIL, "-limit",     0, "limit number of targets listed to N", 0);
 
     // -updateinterest
@@ -265 +265 @@
     psMetadataAddStr(updateinterestArgs, PS_LIST_TAIL, "-set_state", 0, "define state (required)", NULL);
 
+    // -listinterests
+    psMetadata *listinterestsArgs = psMetadataAlloc();
+    psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-target_id", 0, "list interests with target_id", 0);
+    psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-dest_id", 0, "list interests with dest_id", 0);
+    psMetadataAddS64(listinterestsArgs, PS_LIST_TAIL, "-int_id", 0, "list interests with int_id", 0);
+    psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-dest_name",  0, "list interests for destinationn name)", NULL);
+    psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-label",  0, "list interests for label (LIKE comparison)", NULL);
+    psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-filter",    0, "list interests by filter (LIKE comparison)", NULL);
+    psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-stage",     0, "list interests for stage", NULL);
+    psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-clean",     0, "list clean interests", false);
+    psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL,"-full",      0, "list full interests", false);
+    psMetadataAddStr(listinterestsArgs, PS_LIST_TAIL, "-state",     0, "list interests in state", NULL);
+    psMetadataAddU64(listinterestsArgs, PS_LIST_TAIL, "-limit",     0, "limit number of interests listed to N", 0);
+    psMetadataAddBool(listinterestsArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+
     psMetadata *argSets = psMetadataAlloc();
     psMetadata *modes = psMetadataAlloc();
@@ -272 +287 @@
     PXOPT_ADD_MODE("-updaterun",    "", DISTTOOL_MODE_UPDATERUN, updaterunArgs);
     PXOPT_ADD_MODE("-revertrun",    "", DISTTOOL_MODE_REVERTRUN, revertrunArgs);
-    PXOPT_ADD_MODE("-pendingcomponent",       "", DISTTOOL_MODE_PENDINGCOMPONENT,    pendingcomponentArgs);
-    PXOPT_ADD_MODE("-addprocessedcomponent",      "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs);
+    PXOPT_ADD_MODE("-pendingcomponent",   "", DISTTOOL_MODE_PENDINGCOMPONENT,    pendingcomponentArgs);
+    PXOPT_ADD_MODE("-addprocessedcomponent", "", DISTTOOL_MODE_ADDPROCESSEDCOMPONENT, addprocessedcomponentArgs);
+    PXOPT_ADD_MODE("-revertcomponent",    "", DISTTOOL_MODE_REVERTCOMPONENT, revertcomponentArgs);
     PXOPT_ADD_MODE("-processedcomponent", "", DISTTOOL_MODE_PROCESSEDCOMPONENT, processedcomponentArgs);
     PXOPT_ADD_MODE("-toadvance",          "", DISTTOOL_MODE_TOADVANCE, toadvanceArgs);
@@ -284 +300 @@
     PXOPT_ADD_MODE("-pendingdest",        "", DISTTOOL_MODE_PENDINGDEST, pendingdestArgs);
 
-    PXOPT_ADD_MODE("-definedsproduct",    "", DISTTOOL_MODE_DEFINEDSPRODUCT, definedsproductArgs);
-    PXOPT_ADD_MODE("-updatedsproduct",    "", DISTTOOL_MODE_UPDATEDSPRODUCT, updatedsproductArgs);
-//  PXOPT_ADD_MODE("-listdsproduct",      "", DISTTOOL_MODE_LISTDSPRODUCT, updatedsproductArgs);
-
     PXOPT_ADD_MODE("-definedestination",  "", DISTTOOL_MODE_DEFINEDESTINATION, definedestinationArgs);
     PXOPT_ADD_MODE("-updatedestination",  "", DISTTOOL_MODE_UPDATEDESTINATION, updatedestinationArgs);
@@ -298 +310 @@
     PXOPT_ADD_MODE("-defineinterest",     "", DISTTOOL_MODE_DEFINEINTEREST, defineinterestArgs);
     PXOPT_ADD_MODE("-updateinterest",     "", DISTTOOL_MODE_UPDATEINTEREST, updateinterestArgs);
-//  PXOPT_ADD_MODE("-listinterest",       "", DISTTOOL_MODE_LISTINTEREST, listinterestArgs);
+    PXOPT_ADD_MODE("-listinterests",       "", DISTTOOL_MODE_LISTINTERESTS, listinterestsArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/eam_branches/20090820/ippTools/src/faketool.c

@@ -52 +52 @@
 static bool tofullimfileMode(pxConfig *config);
 static bool topurgedimfileMode(pxConfig *config);
+static bool toscrubbedimfileMode(pxConfig *config);
 static bool exportrunMode(pxConfig *config);
 static bool importrunMode(pxConfig *config);
@@ -91 +92 @@
         MODECASE(FAKETOOL_MODE_TOFULLIMFILE,            tofullimfileMode);
         MODECASE(FAKETOOL_MODE_TOPURGEDIMFILE,          topurgedimfileMode);
+        MODECASE(FAKETOOL_MODE_TOSCRUBBEDIMFILE,        toscrubbedimfileMode);
         MODECASE(FAKETOOL_MODE_EXPORTRUN,               exportrunMode);
         MODECASE(FAKETOOL_MODE_IMPORTRUN,               importrunMode);
@@ -1251 +1253 @@
     return change_imfile_data_state(config, "purged", "goto_purged");
 }
+static bool toscrubbedimfileMode(pxConfig *config)
+{
+     return change_imfile_data_state(config, "scrubbed", "goto_scrubbed");
+}
 
 bool exportrunMode(pxConfig *config)

branches/eam_branches/20090820/ippTools/src/faketool.h

@@ -45 +45 @@
     FAKETOOL_MODE_TOFULLIMFILE,
     FAKETOOL_MODE_TOPURGEDIMFILE,
+    FAKETOOL_MODE_TOSCRUBBEDIMFILE,
     FAKETOOL_MODE_EXPORTRUN,
     FAKETOOL_MODE_IMPORTRUN

branches/eam_branches/20090820/ippTools/src/faketoolConfig.c

@@ -301 +301 @@
     psMetadataAddStr(topurgedimfileArgs, PS_LIST_TAIL, "-class_id",  0,        "class ID to update", NULL);
 
+    // -toscrubbedimfile
+    psMetadata *toscrubbedimfileArgs = psMetadataAlloc();
+    psMetadataAddS64(toscrubbedimfileArgs, PS_LIST_TAIL, "-fake_id", 0,         "fake ID to update", 0);
+    psMetadataAddStr(toscrubbedimfileArgs, PS_LIST_TAIL, "-class_id", 0,        "class ID to update", NULL);
+
     // -exportrun
     psMetadata *exportrunArgs = psMetadataAlloc();
@@ -337 +342 @@
     PXOPT_ADD_MODE("-tofullimfile",        "set imfile state to full",               FAKETOOL_MODE_TOFULLIMFILE,         tofullimfileArgs);
     PXOPT_ADD_MODE("-topurgedimfile",      "set imfile state to purged",             FAKETOOL_MODE_TOPURGEDIMFILE,       topurgedimfileArgs);
+    PXOPT_ADD_MODE("-toscrubbedimfile",    "set imfile state to scrubbed",           FAKETOOL_MODE_TOSCRUBBEDIMFILE,     toscrubbedimfileArgs);
     PXOPT_ADD_MODE("-exportrun",            "export run for import on other database", FAKETOOL_MODE_EXPORTRUN, exportrunArgs);
     PXOPT_ADD_MODE("-importrun",            "import run from metadata file",           FAKETOOL_MODE_IMPORTRUN, importrunArgs);

branches/eam_branches/20090820/ippTools/src/magicdstool.c

@@ -37 +37 @@
 static bool todestreakMode(pxConfig *config);
 static bool adddestreakedfileMode(pxConfig *config);
+static bool advancerunMode(pxConfig *config);
 static bool revertdestreakedfileMode(pxConfig *config);
 static bool getskycellsMode(pxConfig *config);
 static bool toremoveMode(pxConfig *config);
-static bool torestoreMode(pxConfig *config);
 static bool torevertMode(pxConfig *config);
+static bool completedrevertMode(pxConfig *config);
 
 static bool setmagicDSRunState(pxConfig *config, psS64 magic_id, const char *state);
-static bool magicDSRunComplete(pxConfig *config, bool setmagicked);
 static bool magicDSGetIDs(pxConfig *config, psString stage, psS64 magic_id, psS64 *stage_id, psS64 *cam_id);
 
@@ -70 +70 @@
         MODECASE(MAGICDSTOOL_MODE_TODESTREAK,          todestreakMode);
         MODECASE(MAGICDSTOOL_MODE_ADDDESTREAKEDFILE,   adddestreakedfileMode);
+        MODECASE(MAGICDSTOOL_MODE_ADVANCERUN,          advancerunMode);
         MODECASE(MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,revertdestreakedfileMode);
         MODECASE(MAGICDSTOOL_MODE_GETSKYCELLS,         getskycellsMode);
         MODECASE(MAGICDSTOOL_MODE_TOREMOVE,            toremoveMode);
-        MODECASE(MAGICDSTOOL_MODE_TORESTORE,           torestoreMode);
         MODECASE(MAGICDSTOOL_MODE_TOREVERT,            torevertMode);
+        MODECASE(MAGICDSTOOL_MODE_COMPLETEDREVERT,     completedrevertMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -105 +106 @@
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
     PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false);
-    PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false);
+    PXOPT_LOOKUP_BOOL(noreplace, config->args, "-noreplace", false);
     PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
     PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
@@ -119 +120 @@
     PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
     PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-magic_id","magic_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-magic_id","magicRun.magic_id", "==");
     PXOPT_COPY_S32(config->args, where, "-streaks_max","streaks", "<=");
 
@@ -268 +269 @@
                 outroot,
                 recoveryroot,
-                re_place,
+                noreplace ? 0 :1,   // re_place
                 0); // remove
 
@@ -396 +397 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
+
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "==");
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
-    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    // look for "inputs" that need to processed
-    psString query = pxDataGet("magicdstool_todestreak.sql");
+    psString sql_file = NULL;
+    psStringAppend(&sql_file, "magicdstool_todestreak_%s.sql", stage);
+
+    psString query = pxDataGet(sql_file);
     if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", sql_file);
+        psFree(sql_file);
+        return false;
+    }
+    psFree(sql_file);
 
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " WHERE %s", whereClause);
+        psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
@@ -656 +662 @@
     }
 
-    if (!magicDSRunComplete(config, setmagicked)) {
-            // rollback
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
     if (!psDBCommit(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -738 +735 @@
 }
 
-static bool magicDSRunComplete(pxConfig *config, bool setmagicked)
+static bool advancerunMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "magicDSRun.label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
 
     // look for completed magicDSRuns
@@ -749 +752 @@
     }
 
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -766 +775 @@
         return true;
     }
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
     for (long i = 0; i < psArrayLength(output); i++) {
         psMetadata *row = output->data[i];
@@ -771 +784 @@
         psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id");
 
-        // if requested, set stageRun.magicked
+        // if re_place, set stageRun.magicked
+        bool setmagicked = psMetadataLookupBool(NULL, row, "re_place");
         if (setmagicked && !setRunMagicked(config, magic_ds_id)) {
             psError(PS_ERR_UNKNOWN, false, "failed to change stageRun.magicked for magic_ds_id: %" PRId64,
                 magic_ds_id);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
         }
@@ -783 +800 @@
                 magic_ds_id);
             psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
             return false;
         }
     }
-
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
 
     return true;
@@ -795 +818 @@
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_BOOL(i_am_sure, config->args, "-i_am_sure", true);
+    if (!i_am_sure) {
+        psError(PS_ERR_UNKNOWN, true, "Reverting destreaked files must be done carefully. -i_am_sure is required.");
+        return false;
+    }
 
     psMetadata *where = psMetadataAlloc();
@@ -802 +831 @@
     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
 
-    psString query = psStringCopy("DELETE FROM magicDSFile USING magicDSFile, magicDSRun  WHERE (magicDSRun.magic_ds_id = magicDSFile.magic_ds_id) AND magicDSFile.fault != 0");
+    psString query = pxDataGet("magicdstool_revertdestreakedfile.sql");
 
     if (psListLength(where->list)) {
@@ -808 +837 @@
         psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
+    } else {
+        psError(PS_ERR_UNKNOWN, true, "search arguments are required");
+        return false;
     }
     psFree(where);
@@ -813 +845 @@
     if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "failed to revert");
-        return false;
-    }
-    return true;
-}
-
-static bool getskycellsMode(pxConfig *config)
-{
-    // required
-    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+    return true;
+}
+
+static bool completedrevertMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-class_id",    "warpSkyCellMap.class_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "warpSkyCellMap.skycell_id", "==");
-
-    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psString query = pxDataGet("magicdstool_getskycells.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-        return false;
-    }
-
+    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magicDSRun.magic_ds_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+
+    psString query = pxDataGet("magicdstool_completedrevert.sql");
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    psString whereString = NULL;
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
+        psStringAppend(&whereString, "\nAND %s", whereClause);
         psFree(whereClause);
     }
     psFree(where);
 
-    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
+    if (!p_psDBRunQueryF(config->dbh, query, whereString ? whereString : "")) {
+        psFree(whereString);
+        psError(PS_ERR_UNKNOWN, false, "failed to revert");
+        return false;
+    }
+    psFree(whereString);
     psFree(query);
-
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
@@ -868 +904 @@
         return true;
     }
-
-    if (psArrayLength(output)) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+    for (int i=0; i<psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+        psS64 magic_ds_id = psMetadataLookupS64(NULL, row, "magic_ds_id"); 
+        psString old_state = psMetadataLookupStr(NULL, row, "state"); 
+        psString new_state;
+        if (!strcmp(old_state, "goto_censored")) {
+            new_state = "censored";
+        } else if (!strcmp(old_state, "goto_restored")) {
+            new_state = "restored";
+        } else {
+            psError(PXTOOLS_ERR_PROG, true, "unexpected state found: %s", old_state);
             psFree(output);
             return false;
         }
-    }
-
+        char *query2 = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64;
+        if (!p_psDBRunQueryF(config->dbh, query2, new_state, magic_ds_id)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to set run magicDSRun.state to %s", new_state);
+            return false;
+        }
+    }
     psFree(output);
 
@@ -883 +929 @@
 }
 
-static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state)
-{
-    PS_ASSERT_PTR_NON_NULL(state, false);
-
-    // check that state is a valid string value
-    if (!(
-            (strncmp(state, "new", 4) == 0)
-            || (strncmp(state, "full", 5) == 0)
-        )
-    ) {
-        psError(PS_ERR_UNKNOWN, false,
-                "invalid magicDSRun state: %s", state);
-        return false;
-    }
-
-    char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64;
-    if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) {
-        psError(PS_ERR_UNKNOWN, false,
-                "failed to change state for magic_id %" PRId64, magic_ds_id);
-        return false;
-    }
-
-    return true;
-}
-
-static bool toremoveMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
+static bool getskycellsMode(pxConfig *config)
+{
+    // required
+    PXOPT_LOOKUP_S64(magic_ds_id, config->args, "-magic_ds_id", true, false);
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-magic_ds_id", "magic_ds_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
-
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_COPY_STR(config->args, where, "-class_id",    "warpSkyCellMap.class_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id",  "warpSkyCellMap.skycell_id", "==");
+
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    // look for "inputs" that need to processed
-    psString query = pxDataGet("magicdstool_toremove.sql");
+    psString query = pxDataGet("magicdstool_getskycells.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -929 +948 @@
     if (psListLength(where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " WHERE %s", whereClause);
+        psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
     }
     psFree(where);
 
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -970 +982 @@
     if (psArrayLength(output)) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "magicDiffSkyfile", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -981 +993 @@
     return true;
 }
-static bool torestoreMode(pxConfig *config)
+
+static bool setmagicDSRunState(pxConfig *config, psS64 magic_ds_id, const char *state)
+{
+    PS_ASSERT_PTR_NON_NULL(state, false);
+
+    // check that state is a valid string value
+    if (!((strcmp(state, "new") == 0) ||
+          (strcmp(state, "full") == 0) ||
+          (strcmp(state, "restored") == 0) ||
+          (strcmp(state, "censored") == 0) ||
+          (strcmp(state, "purged") == 0) ||
+          (strcmp(state, "goto_restored") == 0) ||
+          (strcmp(state, "goto_censored") == 0) ||
+          (strcmp(state, "goto_purged") == 0))
+        ) {
+        psError(PS_ERR_UNKNOWN, false,
+                "invalid magicDSRun state: %s", state);
+        return false;
+    }
+
+    char *query = "UPDATE magicDSRun SET state = '%s' WHERE magic_ds_id = %" PRId64;
+    if (!p_psDBRunQueryF(config->dbh, query, state, magic_ds_id)) {
+        psError(PS_ERR_UNKNOWN, false,
+                "failed to change state for magic_id %" PRId64, magic_ds_id);
+        return false;
+    }
+
+    return true;
+}
+
+static bool toremoveMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -994 +1036 @@
 
     // look for "inputs" that need to processed
-    psString query = pxDataGet("magicdstool_torestore.sql");
+    psString query = pxDataGet("magicdstool_toremove.sql");
     if (!query) {
         psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
@@ -1043 +1085 @@
     if (psArrayLength(output)) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "torestore", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "toremove", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -1054 +1096 @@
     return true;
 }
-
 
 static bool torevertMode(pxConfig *config)

branches/eam_branches/20090820/ippTools/src/magicdstool.h

@@ -30 +30 @@
     MAGICDSTOOL_MODE_TODESTREAK,
     MAGICDSTOOL_MODE_ADDDESTREAKEDFILE,
+    MAGICDSTOOL_MODE_ADVANCERUN,
     MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE,
     MAGICDSTOOL_MODE_GETSKYCELLS,
     MAGICDSTOOL_MODE_TOREMOVE,
-    MAGICDSTOOL_MODE_TORESTORE,
     MAGICDSTOOL_MODE_TOREVERT,
+    MAGICDSTOOL_MODE_COMPLETEDREVERT,
 } MAGICDStoolMode;
 

branches/eam_branches/20090820/ippTools/src/magicdstoolConfig.c

@@ -52 +52 @@
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir",     0, "define workdir", NULL);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-replace", 0, "replace input files with the destreaked versions", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace input files with the destreaked versions", false);
     psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",    0, "define label", NULL);
 
@@ -75 +75 @@
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL);
-    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-replace", 0, "use the simple output format", false);
-    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false);
+    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-noreplace", 0, "do not replace the input wit with the destreaked versions", false);
+//    psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-remove", 0, "use the simple output format", false);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-label", 0, "define label", NULL);
     psMetadataAddBool(definerunArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
@@ -93 +93 @@
     // -todestreak
     psMetadata *todestreakArgs = psMetadataAlloc();
+    psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage (required)", NULL);
     psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
     psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
     psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
-    psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL);
     psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
@@ -114 +114 @@
     psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-component", 0, "search by component", NULL);
     psMetadataAddS16(revertdestreakedfileArgs, PS_LIST_TAIL, "-fault", 0, "search by fault code", 0);
+    psMetadataAddBool(revertdestreakedfileArgs, PS_LIST_TAIL, "-i_am_sure", 0, "confirm that you know what you are doing", false);
     psMetadataAddStr(revertdestreakedfileArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
 
     // -getskycells
     psMetadata *getskycellsArgs = psMetadataAlloc();
-    psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID (required)", 0);
+    psMetadataAddS64(getskycellsArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "define magic de-streak ID", 0);
     psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-class_id", 0, "define class identifier", NULL);
     psMetadataAddStr(getskycellsArgs, PS_LIST_TAIL, "-skycell_id", 0, "define skycell identifier", NULL);
@@ -130 +131 @@
     psMetadataAddU64(toremoveArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     psMetadataAddBool(toremoveArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
-
-    // -torestore
-    psMetadata *torestoreArgs = psMetadataAlloc();
-    psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
-    psMetadataAddS64(torestoreArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
-    psMetadataAddStr(torestoreArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
-    psMetadataAddU64(torestoreArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
-    psMetadataAddBool(torestoreArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
     // -torevert
@@ -147 +140 @@
     psMetadataAddU64(torevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     psMetadataAddBool(torevertArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+
+    // -completedrevert
+    psMetadata *completedrevertArgs = psMetadataAlloc();
+    psMetadataAddS64(completedrevertArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
+    psMetadataAddStr(completedrevertArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
+    psMetadataAddU64(completedrevertArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+
+    // -advancerun
+    psMetadata *advancerunArgs = psMetadataAlloc();
+    psMetadataAddS64(advancerunArgs, PS_LIST_TAIL, "-magic_ds_id", 0, "search by magic Destreak ID", 0);
+    psMetadataAddStr(advancerunArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
+    psMetadataAddU64(advancerunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
 
     psFree(now);
@@ -163 +168 @@
     PXOPT_ADD_MODE("-adddestreakedfile",   "add a de-streaked file",
                     MAGICDSTOOL_MODE_ADDDESTREAKEDFILE, adddestreakedfileArgs);
+    PXOPT_ADD_MODE("-advancerun", "change state for runs that have finished destrreaking",
+                    MAGICDSTOOL_MODE_ADVANCERUN, advancerunArgs);
     PXOPT_ADD_MODE("-revertdestreakedfile", " revert a faulted de-streaked file",
                     MAGICDSTOOL_MODE_REVERTDESTREAKEDFILE, revertdestreakedfileArgs);
@@ -169 +176 @@
     PXOPT_ADD_MODE("-toremove", "backup images pending removal",
                     MAGICDSTOOL_MODE_TOREMOVE, toremoveArgs);
-    PXOPT_ADD_MODE("-torestore", "images pending restore of exicsed streak pixels",
-                    MAGICDSTOOL_MODE_TORESTORE, torestoreArgs);
-    PXOPT_ADD_MODE("-torevert", "faulted images to revert",
+    PXOPT_ADD_MODE("-torevert", "images to restore or revert",
                     MAGICDSTOOL_MODE_TOREVERT, torevertArgs);
+
+    PXOPT_ADD_MODE("-completedrevert", "change state for runs that have finished reverting",
+                    MAGICDSTOOL_MODE_COMPLETEDREVERT, completedrevertArgs);
+
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {

branches/eam_branches/20090820/ippTools/src/magictool.c

@@ -1393 +1393 @@
 }
 
-static bool censorStage(pxConfig *config, psString stage, psString whereClause)
-{
-    psString queryFile = NULL;
-    psStringAppend(&queryFile, "magictool_censor_%s.sql", stage);
-    psString query = pxDataGet(queryFile);
-    if (!query) {
-        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement from %s", queryFile);
-        psFree(queryFile);
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        return false;
-    }
-    psFree(queryFile);
-
-    psStringAppend(&query, " WHERE %s",  whereClause);
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        return false;
-    }
-    psFree(query);
-
-    return true;
-}
-
 static bool censorrunMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    psError(PS_ERR_PROGRAMMING, true, "-censorrun mode not ready yet");
-    return false;
-
     psMetadata *where = psMetadataAlloc();
+
+    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", false, false);
+    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
+
+    if (!magic_id) {
+        if (!exp_id && !label) {
+            psError(PS_ERR_UNKNOWN, true, "either -magic_id or exp_id and label is required");
+            return false;
+        }
+    }
 
     // at least one of these required
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
 
     if (!psListLength(where->list)) {
@@ -1470 +1449 @@
     // Now queue any destreaked files to be re-verted
 
-    // note: on failure censorStage issues the rollback
-    if (!censorStage(config, "raw", whereClause)) {
+    // note: on failure magicRestoreStage issues the rollback
+    if (!magicRestoreStage(config, "raw", whereClause, "goto_censored")) {
         psFree(whereClause);
         return false;
     }
-    if (!censorStage(config, "chip", whereClause)) {
+    if (!magicRestoreStage(config, "chip", whereClause, "goto_censored")) {
         psFree(whereClause);
         return false;
     }
-    if (!censorStage(config, "camera", whereClause)) {
+    if (!magicRestoreStage(config, "camera", whereClause, "goto_censored")) {
         psFree(whereClause);
         return false;
     }
-    if (!censorStage(config, "warp", whereClause)) {
+    if (!magicRestoreStage(config, "warp", whereClause, "goto_censored")) {
         psFree(whereClause);
         return false;
     }
-    if (!censorStage(config, "diff", whereClause)) {
+    if (!magicRestoreStage(config, "diff", whereClause, "goto_censored")) {
         psFree(whereClause);
         return false;

branches/eam_branches/20090820/ippTools/src/magictoolConfig.c

@@ -165 +165 @@
     psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magictool ID", 0);
     psMetadataAddS64(censorrunArgs, PS_LIST_TAIL, "-exp_id", 0, "define exposure ID", 0);
+    psMetadataAddStr(censorrunArgs, PS_LIST_TAIL, "-label",       0, "define label", NULL);
 
     psFree(now);

branches/eam_branches/20090820/ippTools/src/pstamptool.c

@@ -44 +44 @@
 static bool pendingjobMode(pxConfig *config);
 static bool updatejobMode(pxConfig *config);
+static bool revertjobMode(pxConfig *config);
 static bool addprojectMode(pxConfig *config);
 static bool projectMode(pxConfig *config);
@@ -80 +81 @@
         MODECASE(PSTAMPTOOL_MODE_PENDINGJOB, pendingjobMode);
         MODECASE(PSTAMPTOOL_MODE_UPDATEJOB, updatejobMode);
+        MODECASE(PSTAMPTOOL_MODE_REVERTJOB, revertjobMode);
         MODECASE(PSTAMPTOOL_MODE_ADDPROJECT, addprojectMode);
         MODECASE(PSTAMPTOOL_MODE_MODPROJECT, modprojectMode);
@@ -224 +226 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_STR(uri,         config->args, "-uri",           true, false);
-    // PXOPT_LOOKUP_STR(outFileset,  config->args, "-out_fileset",   true, false);
-    PXOPT_LOOKUP_S64(ds_id,       config->args, "-ds_id",         false, false);
-
-    char *query ="INSERT INTO pstampRequest"
-                     " (state, uri, ds_id, fault)"
-                     " VALUES( 'new', '%s', %" PRId64 ", 0 )";
-    if (!p_psDBRunQueryF(config->dbh, query, uri, ds_id)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    psU64 affected = psDBAffectedRows(config->dbh);
-    if (affected != 1) {
-        psError(PS_ERR_UNKNOWN, false, 
-            "should have affected one row but %" PRIu64 " rows were modified",
-            affected);
+    PXOPT_LOOKUP_STR(uri,         config->args, "-uri",   true, false);
+    PXOPT_LOOKUP_STR(name,        config->args, "-name",  false, false);
+    PXOPT_LOOKUP_STR(label,       config->args, "-label",  false, false);
+    PXOPT_LOOKUP_S64(ds_id,       config->args, "-ds_id", false, false);
+
+    if (!pstampRequestInsert(config->dbh,
+        0,      // req_id
+        ds_id,
+        "new",  //state
+        name,
+        NULL,   // reqType
+        label,
+        NULL,   // outProduct
+        uri,    
+        0       // fault 
+        )) {
+        psError(PS_ERR_UNKNOWN, false, "failed to insert request");
         return false;
     }
@@ -317 +319 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!=");
     PXOPT_COPY_STR(config->args, where, "-name", "name", "==");
 
@@ -381 +384 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = 
-        "SELECT * FROM pstampRequest WHERE state = 'run' AND "
-            "(SELECT count(*) FROM pstampJob WHERE pstampJob.req_id = pstampRequest.req_id "
-            " AND pstampJob.state != 'stop') = 0" ;
+    psString query = pxDataGet("pstamptool_completedreq.sql");
 
     // treat limit == 0 as "no limit"
@@ -491 +491 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S64(req_id,     config->args, "-req_id",     true, false);
-
-    // printf("Revert request %" PRId64 "\n", req_id);
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fault", "req_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-state", "pstampRequest.state", "==");
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
+        return false;
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psFree(where);
+
+    // delete any jobs that were queued by requests that didn't complete parsing (pstampRequest.state = 'new'
+    // If state =  'run' was supplied this will be a no-op
+    psString query = pxDataGet("pstamptool_revertreq_deletejobs.sql");
+    psStringAppend(&query, " AND %s", whereClause);
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(query);
+
+    // clear fault for requests
+    query = pxDataGet("pstamptool_revertreq.sql");
+    psStringAppend(&query, " AND %s", whereClause);
     
-    if (!p_psDBRunQueryF(config->dbh, "DELETE FROM pstampJob where req_id = %" PRId64, req_id)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!p_psDBRunQueryF(config->dbh, 
-        "UPDATE pstampRequest set state ='new', name=NULL, reqType=NULL, fault=0 where req_id = %" PRId64, req_id)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
+    psFree(whereClause);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+    psFree(query);
 
     return true;
@@ -521 +544 @@
     PXOPT_LOOKUP_S16(fault,       config->args, "-fault",      false, false);
     PXOPT_LOOKUP_S64(exp_id,      config->args, "-exp_id",     false, false);
+    PXOPT_LOOKUP_S64(options,     config->args, "-options",     false, false);
 
     // unless the job is being inserted with stop state require outputBase
@@ -545 +569 @@
             fault, 
             exp_id, 
-            outputBase
+            outputBase,
+            options
             )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -569 +594 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S64(req_id, config->args, "-req_id", false, false);
-    PXOPT_LOOKUP_S64(job_id, config->args, "-job_id", false, false);
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fault",  "fault", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = NULL;
-
-    if (!req_id && !job_id) {
-        fprintf(stderr, "either req_id or job_id must be specified\n");
+    if (!psListLength(where->list)) {
+        fprintf(stderr, "search arguments are required\n");
         exit (1);
     }
 
-    if (req_id) {
-        psStringAppend(&query,
-                "SELECT"
-                " pstampJob.*, pstampRequest.name, pstampRequest.outProduct"
-                " FROM pstampJob"
-                " JOIN pstampRequest USING(req_id)"
-                " WHERE req_id = %" PRId64, req_id
-            );
-    } else {
-        psStringAppend(&query,
-                "SELECT"
-                " pstampJob.*, pstampRequest.name, pstampRequest.outProduct"
-                " FROM pstampJob"
-                " JOIN pstampRequest USING(req_id)"
-                " WHERE job_id = %" PRId64, job_id
-            );
-    } 
+    psString query = pxDataGet("pstamptool_listjob.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    // use psDBGenerateWhereSQL because the SQL yields an intermediate table
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob");
+        psStringAppend(&query, " WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
 
     // treat limit == 0 as "no limit"
@@ -729 +750 @@
         psError(PS_ERR_UNKNOWN, false, "should have affected one row but %" 
                                         PRIu64 " rows were modified", affected);
+        return false;
+    }
+
+    return true;
+}
+
+static bool revertjobMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, where, "-job_id", "job_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-fault",  "fault", "==");
+    PXOPT_COPY_S64(config->args, where, "-req_id_min",  "req_id", ">=");
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+
+    psString query = pxDataGet("pstamptool_revertjob.sql");
+    if (!psListLength(where->list) && !all) {
+        psFree(where);
+        psError(PXTOOLS_ERR_DATA, false, "search parameters or -all are required");
+        return false;
+    }
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampJob");
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+    
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
     }

branches/eam_branches/20090820/ippTools/src/pstamptool.h

@@ -39 +39 @@
     PSTAMPTOOL_MODE_JOBRESULT,
     PSTAMPTOOL_MODE_UPDATEJOB,
+    PSTAMPTOOL_MODE_REVERTJOB,
     PSTAMPTOOL_MODE_ADDPROJECT,
     PSTAMPTOOL_MODE_MODPROJECT,

branches/eam_branches/20090820/ippTools/src/pstamptoolConfig.c

@@ -63 +63 @@
     // -addreq
     psMetadata *addreqArgs = psMetadataAlloc();
-    psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0,            "define request file uri (required)", NULL); 
-    psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0,            "define request ds_id", 0); 
-    // psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-out_fileset", 0,            "define request output_fileset", NULL); 
+    psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-uri", 0,    "define request file uri (required)", NULL); 
+    psMetadataAddS64(addreqArgs, PS_LIST_TAIL, "-ds_id", 0,  "define request ds_id", 0); 
+    psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-name", 0,   "define request name", NULL); 
+    psMetadataAddStr(addreqArgs, PS_LIST_TAIL, "-label", 0,  "define request label", NULL); 
 
     // -pendingreq
@@ -77 +78 @@
     psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-req_id", 0,            "list by req_id", 0); 
     psMetadataAddStr(listreqArgs, PS_LIST_TAIL, "-name", 0,              "list by name", NULL); 
+    psMetadataAddS64(listreqArgs, PS_LIST_TAIL, "-not_req_id", 0,        "req_id to not list", 0); 
     psMetadataAddU64(listreqArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
-    psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0,            "use the simple output format", false);
+    psMetadataAddBool(listreqArgs, PS_LIST_TAIL, "-simple", 0,           "use the simple output format", false);
 
     // -completedreq
@@ -97 +99 @@
     // -revertreq
     psMetadata *revertreqArgs = psMetadataAlloc();
-    psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0,            "req_id for which to revert", 0); 
+    psMetadataAddS64(revertreqArgs, PS_LIST_TAIL, "-req_id", 0,     "req_id to revert", 0); 
+    psMetadataAddS16(revertreqArgs, PS_LIST_TAIL, "-fault",  0,     "fault to revert", 0); 
+    psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0,      "state to revert", NULL); 
 
     // -addjob
     psMetadata *addjobArgs = psMetadataAlloc();
-    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0,            "define job req_id", 0); 
-    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0,            "define job rownum", NULL); 
-    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0,            "define job job_type", "stamp"); 
-    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0,            "define job outputBase", NULL); 
-    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0,            "new state", "run"); 
-    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0,           "exposure id", 0); 
-    psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0,            "new result", 0); 
+    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-req_id", 0,           "define job req_id", 0); 
+    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-rownum", 0,           "define job rownum", NULL); 
+    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-job_type", 0,         "define job job_type", "stamp"); 
+    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-outputBase", 0,       "define job outputBase", NULL); 
+    psMetadataAddStr(addjobArgs, PS_LIST_TAIL, "-state", 0,            "define state", "run"); 
+    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-exp_id", 0,           "define exposure id", 0); 
+    psMetadataAddS64(addjobArgs, PS_LIST_TAIL, "-options", 0,          "define options", 0); 
+    psMetadataAddS16(addjobArgs, PS_LIST_TAIL, "-fault", 0,            "define job result", 0); 
 
     // -listjob
     psMetadata *listjobArgs = psMetadataAlloc();
-    psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0,            "define request", 0); 
-    psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0,            "define job", 0); 
-    psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
-    psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0,            "use the simple output format", false);
+    psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-req_id", 0,          "select by request ID", 0); 
+    psMetadataAddS64(listjobArgs, PS_LIST_TAIL, "-job_id", 0,          "select by job ID", 0); 
+    psMetadataAddS16(listjobArgs, PS_LIST_TAIL, "-fault", 0,           "select by fault", 0); 
+    psMetadataAddU64(listjobArgs, PS_LIST_TAIL, "-limit",  0,          "limit result set to N items", 0);
+    psMetadataAddBool(listjobArgs, PS_LIST_TAIL, "-simple", 0,         "use the simple output format", false);
 
     // -pendingjob
@@ -128 +134 @@
     psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0,            "new state", NULL); 
     psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-fault", 0,            "new result", NULL); 
+
+    // -revertjob
+    psMetadata *revertjobArgs = psMetadataAlloc();
+    psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id", 0,     "req_id to revert", 0); 
+    psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-req_id_min", 0, "minimum req_id to revert", 0); 
+    psMetadataAddS64(revertjobArgs, PS_LIST_TAIL, "-job_id", 0,     "job_id to revert", 0); 
+    psMetadataAddS16(revertjobArgs, PS_LIST_TAIL, "-fault",  0,     "fault to revert", 0); 
+    psMetadataAddBool(revertjobArgs, PS_LIST_TAIL, "-all", 0,       "revert all faulted jobs", false);
+    psMetadataAddU64(revertjobArgs, PS_LIST_TAIL, "-limit", 0,      "limit result set to N items", 0);
 
     // -addproject
@@ -159 +174 @@
     PXOPT_ADD_MODE("-addreq",          "", PSTAMPTOOL_MODE_ADDREQ,       addreqArgs);
     PXOPT_ADD_MODE("-pendingreq",      "", PSTAMPTOOL_MODE_PENDINGREQ,   pendingreqArgs);
-    PXOPT_ADD_MODE("-updatereq",    "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs);
+    PXOPT_ADD_MODE("-updatereq",       "", PSTAMPTOOL_MODE_UPDATEREQ, updatereqArgs);
     PXOPT_ADD_MODE("-listreq",         "", PSTAMPTOOL_MODE_LISTREQ,      listreqArgs);
     PXOPT_ADD_MODE("-completedreq",    "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs);
@@ -167 +182 @@
     PXOPT_ADD_MODE("-listjob",         "", PSTAMPTOOL_MODE_LISTJOB,      listjobArgs);
     PXOPT_ADD_MODE("-pendingjob",      "", PSTAMPTOOL_MODE_PENDINGJOB,   pendingjobArgs);
-    PXOPT_ADD_MODE("-updatejob",    "", PSTAMPTOOL_MODE_UPDATEJOB, updatejobArgs);
+    PXOPT_ADD_MODE("-updatejob",       "", PSTAMPTOOL_MODE_UPDATEJOB,    updatejobArgs);
+    PXOPT_ADD_MODE("-revertjob",       "", PSTAMPTOOL_MODE_REVERTJOB,    revertjobArgs);
 
     PXOPT_ADD_MODE("-adddatastore",    "", PSTAMPTOOL_MODE_ADDDATASTORE, adddatastoreArgs);

branches/eam_branches/20090820/ippTools/src/pubtool.c

@@ -259 +259 @@
     // required
     PXOPT_LOOKUP_S64(pub_id, config->args, "-pub_id", true, false);
-    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base",  true, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
 
     // optional
+    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
+    PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
     PXOPT_LOOKUP_S32(fault, config->args, "-fault", false, false);
 
@@ -269 +271 @@
     }
 
-    if (!publishDoneInsert(config->dbh, pub_id, path_base, fault)) {
+    if (!publishDoneInsert(config->dbh, pub_id, path_base, hostname, dtime_script, fault)) {
         psError(PS_ERR_UNKNOWN, false, "Unable to add file");
         if (!psDBRollback(config->dbh)) {

branches/eam_branches/20090820/ippTools/src/pubtoolConfig.c

@@ -68 +68 @@
     psMetadataAddS64(addArgs, PS_LIST_TAIL, "-pub_id", 0, "define pub_id (required)", 0);
     psMetadataAddStr(addArgs, PS_LIST_TAIL, "-path_base", 0, "define path_base (required)", NULL);
+    psMetadataAddStr(addArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", NULL);
+    psMetadataAddF32(addArgs, PS_LIST_TAIL, "-dtime_script", 0, "define time for script", NAN);
     psMetadataAddS32(addArgs, PS_LIST_TAIL, "-fault", 0, "define fault code", 0);
 

branches/eam_branches/20090820/ippTools/src/pxinject.c

@@ -137 +137 @@
     PXOPT_LOOKUP_STR(tmp_class_id, config->args, "-tmp_class_id", true, false);
     PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false);
+    PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false);
+    PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false);
 
     // insert with error flag state set to 0 (no errors)
-    if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL)) {
+    if (!newImfileInsert(config->dbh, exp_id, tmp_class_id, uri, NULL, bytes, md5sum)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;

branches/eam_branches/20090820/ippTools/src/pxinjectConfig.c

@@ -61 +61 @@
     psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-tmp_class_id",  0,            "define the class ID (required)", NULL);
     psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-uri",  0,            "define the URI (required)", NULL);
+    psMetadataAddS32(newImfileArgs, PS_LIST_TAIL, "-bytes",  0,            "define the size of the file", 0);
+    psMetadataAddStr(newImfileArgs, PS_LIST_TAIL, "-md5sum",  0,            "define the size of the file", NULL);
 
     // -updatenewExp

branches/eam_branches/20090820/ippTools/src/pxtools.c

@@ -37 +37 @@
     if (!strcmp(state, "goto_cleaned")) return true;
     if (!strcmp(state, "error_cleaned")) return true;
+    if (!strcmp(state, "goto_purged")) return true;
+    if (!strcmp(state, "error_purged")) return true;
     if (!strcmp(state, "goto_scrubbed")) return true;
     if (!strcmp(state, "error_scrubbed")) return true;
@@ -42 +44 @@
     if (!strcmp(state, "update")) return true;
     if (!strcmp(state, "purged")) return true;
-    if (!strcmp(state, "goto_purged")) return true;
-    if (!strcmp(state, "error_purged")) return true;
-
+    if (!strcmp(state, "scrubbed")) return true;
     return false;
 }
 
-// 'scrubbed' is a virtual state equivalent to cleaned, but allows files to be removed
-// even if the config files is missing
+// 'scrubbed' is no longer a virtual state equivalent to cleaned, but allows files to be removed
+// even if the config files is missing.  This change was prompted as files that are cleaned can
+// be regenerated, but that is not certain after being scrubbed.
 
 

branches/eam_branches/20090820/ippTools/src/pxtools.h

@@ -37 +37 @@
 #include "pxtoolsErrorCodes.h"
 
+#include "pxadd.h"
 #include "pxcam.h"
 #include "pxchip.h"
@@ -45 +46 @@
 #include "pxtag.h"
 #include "pxtree.h"
+#include "pxmagic.h"
 
 # define MAX_ROWS 10e9

branches/eam_branches/20090820/ippTools/src/pztool.c

@@ -401 +401 @@
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
     PXOPT_LOOKUP_BOOL(row_lock, config->args, "-row_lock", false);
+    PXOPT_LOOKUP_S32(bytes, config->args, "-bytes", false, false);
+    PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false);
 
     // default values
@@ -457 +459 @@
             fault,
             NULL,    // epoch
-            hostname
+            hostname,
+            bytes,
+            md5sum
     )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -595 +599 @@
                 "       pzDownloadImfile.class_id," // tmp_class_id
                 "       pzDownloadImfile.uri,"      // uri
-                "       NULL"                       // epoch
+                "       NULL,"                       // epoch
+                "       pzDownloadImfile.bytes,"    // bytes
+                "       pzDownloadImfile.md5sum"    // md5sum
                 "   FROM pzDownloadImfile"
                 "   WHERE"

branches/eam_branches/20090820/ippTools/src/pztoolConfig.c

@@ -95 +95 @@
     psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-label",  0,        "define the label for the chip stage", NULL);
     psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-hostname",  0,     "define the host that copied the image", NULL);
+    psMetadataAddS32(copydoneArgs, PS_LIST_TAIL, "-bytes",  0,     "define the size in bytes for the copied image", 0);
+    psMetadataAddStr(copydoneArgs, PS_LIST_TAIL, "-md5sum",  0,     "define the md5 sum for the copied image", NULL);
     psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-fault",  0,            "set fault code", 0);
     psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0,     "lock pzDownImfile rows while advancing an exposure", false);

branches/eam_branches/20090820/ippTools/src/regtool.c

@@ -224 +224 @@
     PXOPT_LOOKUP_F32(teltemp_extra, config->args, "-teltemp_extra", false, false);
     PXOPT_LOOKUP_F32(pon_time, config->args, "-pon_time", false, false);
+    PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false);
     PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false);
     PXOPT_LOOKUP_F64(user_2, config->args, "-user_2", false, false);
@@ -238 +239 @@
     PXOPT_LOOKUP_TIME(dateobs, config->args, "-dateobs", false, false);
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
+    PXOPT_LOOKUP_S32(bytes, config->args,  "-bytes", false, false);
+    PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false);
 
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
@@ -307 +310 @@
         quality,
         NULL,
-        0
+        0,
+        bytes,
+        md5sum,
+        0   // burntool_state
     )) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -327 +333 @@
     PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs",  ">=");
     PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "dateobs",  "<=");
+    PXOPT_COPY_STR(config->args, where,  "-filter",        "filter", "==");
+    PXOPT_COPY_S64(config->args, where,  "-magicked",      "magicked", "==");
+
+    PXOPT_LOOKUP_S64(magicked, config->args, "-magicked", false, false);
+    PXOPT_LOOKUP_BOOL(destreaked, config->args,     "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(faulted, config->args, "-faulted", false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(ordered_by_date, config->args, "-ordered_by_date", false);
 
     psString query = pxDataGet("regtool_processedimfile.sql");
@@ -352 +365 @@
         // don't list faulted rows
         psStringAppend(&query, " %s", "AND rawImfile.fault = 0");
+    }
+
+    if (not_destreaked) {
+        if (destreaked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense");
+            return false;
+        }
+        if (magicked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense");
+            return false;
+        }
+        psStringAppend(&query, " AND rawImfile.magicked = 0");
+    }
+    if (destreaked) {
+        psStringAppend(&query, " AND rawImfile.magicked != 0");
+    }
+
+
+    // add the ORDER BY statement if desired
+    if (ordered_by_date) {
+        psStringAppend(&query, " ORDER BY dateobs");
     }
 
@@ -456 +490 @@
 
     PXOPT_LOOKUP_S16(fault, config->args, "-fault",   false, false);
-    PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false);
-
-    if ((fault == INT16_MAX) && !isfinite(user_1)) { 
-        psError(PS_ERR_UNKNOWN, false, "one of -fault or -user_1 must be selected");
-        return false;
-    }
-    if ((fault != INT16_MAX) && isfinite(user_1)) { 
-        psError(PS_ERR_UNKNOWN, false, "only one of -fault or -user_1 must be selected");
+/*     PXOPT_LOOKUP_F64(user_1, config->args, "-user_1", false, false); */
+    PXOPT_LOOKUP_S16(burntool_state, config->args, "-burntool_state", false, false);
+    
+    if ((fault == INT16_MAX) && !isfinite(burntool_state)) { 
+        psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state must be selected");
+        return false;
+    }
+    if ((fault != INT16_MAX) && isfinite(burntool_state)) { 
+        psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected");
         return false;
     }
@@ -480 +515 @@
     }
 
-    if (isfinite(user_1)) {
+    if (isfinite(burntool_state)) {
         psString query = pxDataGet("regtool_updateprocessedimfile.sql");
         if (!query) {
@@ -487 +522 @@
         }
 
-        if (!p_psDBRunQueryF(config->dbh, query, user_1, exp_id, class_id)) {
+        if (!p_psDBRunQueryF(config->dbh, query, burntool_state, exp_id, class_id)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(query);

branches/eam_branches/20090820/ippTools/src/regtoolConfig.c

@@ -125 +125 @@
     ADD_OPT(Time, addprocessedimfileArgs, "-dateobs",        "define observation time",         NULL);
     ADD_OPT(Str,  addprocessedimfileArgs, "-hostname",       "define host name",                NULL);
+    ADD_OPT(Str,  addprocessedimfileArgs, "-md5sum",         "define md5sum",                NULL);
+    ADD_OPT(S32,  addprocessedimfileArgs, "-bytes",          "define bytes",                0);
+    ADD_OPT(S16,  addprocessedimfileArgs, "-burntool_state",        "set burntool state", 0);
     ADD_OPT(S16,  addprocessedimfileArgs, "-fault",           "set fault code",                  0);
     ADD_OPT(S16,  addprocessedimfileArgs, "-quality",        "set quality flag", 0);
@@ -133 +136 @@
     ADD_OPT(Str,  processedimfileArgs, "-exp_name",  "search by exposure name",               NULL);
     ADD_OPT(Str,  processedimfileArgs, "-class_id",  "search by class ID",                    NULL);
+    ADD_OPT(Str,  processedimfileArgs, "-filter",  "search by filter",                        NULL);
     ADD_OPT(Time, processedimfileArgs, "-dateobs_begin", "search for exposures by time (>=)", NULL);
     ADD_OPT(Time, processedimfileArgs, "-dateobs_end", "search for exposures by time (<)", NULL);
+    ADD_OPT(S64,  processedimfileArgs, "-magicked",    "search by magicked value",            0);
+    ADD_OPT(Bool, processedimfileArgs, "-destreaked",   "only return imfiles that have been destreaked", false);
+    ADD_OPT(Bool, processedimfileArgs, "-not_destreaked", "only return imfiles that have not been destreaked", false);
     ADD_OPT(U64,  processedimfileArgs, "-limit",     "limit result set to N items",           0);
     ADD_OPT(Bool, processedimfileArgs, "-faulted",   "only return imfiles with a fault status set", false);
@@ -153 +160 @@
     ADD_OPT(S64, updateprocessedimfileArgs, "-exp_id",        "search by exposure ID", 0);
     ADD_OPT(Str, updateprocessedimfileArgs, "-class_id",      "search by class ID", NULL);
-    ADD_OPT(F64, updateprocessedimfileArgs, "-user_1",        "set user stat (1)", NAN);
+    ADD_OPT(S16, updateprocessedimfileArgs, "-burntool_state",        "set burntool state", 0);
     ADD_OPT(S16, updateprocessedimfileArgs, "-fault",          "set fault code", INT16_MAX);
 

branches/eam_branches/20090820/ippTools/src/stacktool.c

@@ -139 +139 @@
     PXOPT_COPY_F32(config->args,  where, "-select_fwhm_minor_min",     "camProcessedExp.fwhm_minor", ">=");
     PXOPT_COPY_F32(config->args,  where, "-select_fwhm_minor_max",     "camProcessedExp.fwhm_minor", "<=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m2_max",     "camProcessedExp.iq_m2", "<=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m2_min",     "camProcessedExp.iq_m2", ">=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m3_max",     "camProcessedExp.iq_m3", "<=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m4_min",     "camProcessedExp.iq_m4", ">=");
+    PXOPT_COPY_F32(config->args,  where, "-select_iq_m4_max",     "camProcessedExp.iq_m4", "<=");
+    
     PXOPT_COPY_STR(config->args,  where, "-select_exp_type",           "rawExp.exp_type", "==");
     PXOPT_COPY_F32(config->args,  where, "-select_good_frac_min",      "warpSkyfile.good_frac", ">=");
@@ -853 +859 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
     PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "==");
     PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE");
+    PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
+
+//  The following three selectors are incompatible with the sql so omit them
+//    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
+//     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
+//    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -873 +886 @@
         psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
-    }
+    } else if (!all) {
+        psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required");
+        return false;
+    }
+
     psFree(where);
 

branches/eam_branches/20090820/ippTools/src/stacktoolConfig.c

@@ -77 +77 @@
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_min", 0, "define min fwhm (minor axis)", NAN);
     psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_fwhm_minor_max", 0, "define max fwhm (minor axis)", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_min", 0, "define min iq_m2", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m2_max", 0, "define max iq_m2", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_min", 0, "define min iq_m3", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m3_max", 0, "define max iq_m3", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_min", 0, "define min iq_m4", NAN);
+    psMetadataAddF64(definebyqueryArgs, PS_LIST_TAIL, "-select_iq_m4_max", 0, "define max iq_m4", NAN);
     psMetadataAddS32(definebyqueryArgs, PS_LIST_TAIL, "-random", 0, "use this number of random elements", 0);
     psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-all", 0, "allow everything to be queued without search terms", false);
@@ -158 +164 @@
     psMetadata *sumskyfileArgs= psMetadataAlloc();
     psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-stack_id", 0,            "search by stack ID", 0);
-    psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warp ID", 0);
     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-tess_id", 0,            "search by tess ID", 0);
     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0,         "search by skycell ID", 0);
+#ifdef notdef
+    // These don't work so omit (for now) We probably should create a different mode for this type of search.
+    psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,            "search by warp ID", 0);
     psMetadataAddS64(sumskyfileArgs, PS_LIST_TAIL, "-exp_id", 0,            "search by exposure ID", 0);
     psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-exp_name", 0,          "search by exposure name", NULL);
+#endif
+    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-label", 0,             "search by stackRun.label", NULL);
+    psMetadataAddStr(sumskyfileArgs, PS_LIST_TAIL, "-filter", 0,            "search by filter (LIKE comparison)", NULL);
+    psMetadataAddS16(sumskyfileArgs, PS_LIST_TAIL, "-fault",  0,            "search by fault code", 0);
     psMetadataAddU64(sumskyfileArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);
+    psMetadataAddBool(sumskyfileArgs, PS_LIST_TAIL, "-all",  0,            "enable search without arguments", false);
 
     // -revertsumskyfile

branches/eam_branches/20090820/ippTools/src/warptool.c

@@ -53 +53 @@
 static bool tocleanedskyfileMode(pxConfig *config);
 static bool topurgedskyfileMode(pxConfig *config);
+static bool toscrubbedskyfileMode(pxConfig *config);
 static bool tofullskyfileMode(pxConfig *config);
 static bool updateskyfileMode(pxConfig *config);
@@ -102 +103 @@
         MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE,   tocleanedskyfileMode);
         MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE,    topurgedskyfileMode);
+        MODECASE(WARPTOOL_MODE_TOSCRUBBEDSKYFILE,  toscrubbedskyfileMode);
         MODECASE(WARPTOOL_MODE_TOFULLSKYFILE,      tofullskyfileMode);
         MODECASE(WARPTOOL_MODE_UPDATESKYFILE,      updateskyfileMode);
@@ -1196 +1198 @@
     PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
     PXOPT_COPY_S64(config->args, where, "-fake_id",    "fakeRun.fake_id", "==");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs",  ">=");
+    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawExp.dateobs",  "<=");
+    PXOPT_COPY_STR(config->args, where, "-filter",    "rawExp.filter", "==");
+    PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "==");
+    pxAddLabelSearchArgs (config, where, "-label",   "warpRun.label", "LIKE");
+
+    PXOPT_LOOKUP_U64(magicked, config->args, "-magicked", false, false);
+    PXOPT_LOOKUP_BOOL(destreaked, config->args,     "-destreaked", false);
+    PXOPT_LOOKUP_BOOL(not_destreaked, config->args, "-not_destreaked", false);
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -1211 +1224 @@
         psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
+    } else if (!all) {
+        psError(PXTOOLS_ERR_DATA, true, "search parameters or -all are required");
+        return false;
     }
     psFree(where);
+
+    if (not_destreaked) {
+        if (destreaked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -destreaked makes no sense");
+            return false;
+        }
+        if (magicked) {
+            psError(PXTOOLS_ERR_DATA, true, "providing -not_destreaked and -magicked makes no sense");
+            return false;
+        }
+        psStringAppend(&query, " AND warpSkyfile.magicked = 0");
+    }
+    if (destreaked) {
+        psStringAppend(&query, " AND warpSkyfile.magicked != 0");
+    }
 
     // treat limit == 0 as "no limit"
@@ -1668 +1699 @@
     return change_skyfile_data_state(config, "purged", "goto_purged");
 }
+static bool toscrubbedskyfileMode(pxConfig *config)
+{
+     return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed");
+}
 
 static bool updateskyfileMode(pxConfig *config)

branches/eam_branches/20090820/ippTools/src/warptool.h

@@ -49 +49 @@
     WARPTOOL_MODE_TOCLEANEDSKYFILE,
     WARPTOOL_MODE_TOPURGEDSKYFILE,
+    WARPTOOL_MODE_TOSCRUBBEDSKYFILE,
     WARPTOOL_MODE_TOFULLSKYFILE,
     WARPTOOL_MODE_UPDATESKYFILE,

branches/eam_branches/20090820/ippTools/src/warptoolConfig.c

@@ -215 +215 @@
     psMetadataAddStr(warpedArgs, PS_LIST_TAIL, "-exp_name", 0,          "search by exposure tag", 0);
     psMetadataAddS64(warpedArgs, PS_LIST_TAIL, "-fake_id", 0,           "search by phase 3 version of exposure tag", 0);
+    psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_begin", 0,    "search for exposures by time (>=)", NULL);
+    psMetadataAddTime(warpedArgs, PS_LIST_TAIL, "-dateobs_end", 0,      "search for exposures by time (<=)", NULL);
+    psMetadataAddStr(warpedArgs, PS_LIST_TAIL,  "-filter", 0,           "search for exposures by filter", NULL);
+    psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-destreaked",  0,      "search for destreaked images", false);
+    psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-not_destreaked",  0,  "search for images that have not been destreaked", false);
+    psMetadataAddS64(warpedArgs, PS_LIST_TAIL,  "-magicked", 0,         "search by magicked value", 0);
+    psMetadataAddS16(warpedArgs, PS_LIST_TAIL,  "-fault",  0,           "search by fault code", 0);
+    psMetadataAddStr(warpedArgs, PS_LIST_TAIL,  "-label",  PS_META_DUPLICATE_OK, "search by warpRun label", NULL);
+
+    psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-all",  0,             "search without arguments", false);
     psMetadataAddU64(warpedArgs, PS_LIST_TAIL, "-limit",  0,            "limit result set to N items", 0);
     psMetadataAddBool(warpedArgs, PS_LIST_TAIL, "-simple",  0,          "use the simple output format", false);
@@ -278 +288 @@
     psMetadataAddS64(topurgedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0,    "warptool ID to update", 0);
     psMetadataAddStr(topurgedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
+
+    // -toscrubbedskyfile
+    psMetadata *toscrubbedskyfileArgs = psMetadataAlloc();
+    psMetadataAddS64(toscrubbedskyfileArgs, PS_LIST_TAIL, "-warp_id", 0, "warptool ID to update", 0);
+    psMetadataAddStr(toscrubbedskyfileArgs, PS_LIST_TAIL, "-skycell_id", 0, "skycell ID to update", NULL);
 
     // -tofullskyfile
@@ -328 +343 @@
     PXOPT_ADD_MODE("-tocleanedskyfile", "set skyfile as cleaned", WARPTOOL_MODE_TOCLEANEDSKYFILE, tocleanedskyfileArgs);
     PXOPT_ADD_MODE("-topurgedskyfile", "set skyfile as purged", WARPTOOL_MODE_TOPURGEDSKYFILE, topurgedskyfileArgs);
+    PXOPT_ADD_MODE("-toscrubbedskyfile", "set skyfile as scrubbed", WARPTOOL_MODE_TOSCRUBBEDSKYFILE, toscrubbedskyfileArgs);
     PXOPT_ADD_MODE("-tofullskyfile", "set skyfile as full (updated)", WARPTOOL_MODE_TOFULLSKYFILE, tofullskyfileArgs);
     PXOPT_ADD_MODE("-updateskyfile", "update fault code for skyfile", WARPTOOL_MODE_UPDATESKYFILE, updateskyfileArgs);

@@ -34 +34 @@
 disttool
 receivetool
-
+addtool
 pubtool