Changeset 24882

Timestamp:

Jul 21, 2009, 12:57:48 PM (17 years ago)

Author:

bills

Message:

Added magicdstool -definebyquery and fixed a bug in magicdstool_torevert_diff.sql

Location:

trunk/ippTools

Files:

• share/Makefile.am (modified) (1 diff)
• share/magicdstool_definebyquery_camera.sql (added)
• share/magicdstool_definebyquery_chip.sql (added)
• share/magicdstool_definebyquery_diff.sql (added)
• share/magicdstool_definebyquery_raw.sql (added)
• share/magicdstool_definebyquery_warp.sql (added)
• share/magicdstool_getrunids.sql (modified) (1 diff)
• share/magicdstool_torevert_diff.sql (modified) (1 diff)
• src/magicdstool.c (modified) (11 diffs)
• src/magicdstoolConfig.c (modified) (2 diffs)

trunk/ippTools/share/Makefile.am

@@ -154 +154 @@
      magictool_revertnode.sql \
      magicdstool_completed_runs.sql \
+     magicdstool_definebyquery_raw.sql \
+     magicdstool_definebyquery_chip.sql \
+     magicdstool_definebyquery_camera.sql \
+     magicdstool_definebyquery_warp.sql \
+     magicdstool_definebyquery_diff.sql \
      magicdstool_getrunids.sql \
      magicdstool_getskycells.sql \

trunk/ippTools/share/magicdstool_getrunids.sql

@@ -5 +5 @@
     chip_id,
     cam_id,
-    warp_id 
+    warp_id,
+    diffSkyfile.diff_id
 FROM magicRun 
 JOIN magicInputSkyfile USING(magic_id)

trunk/ippTools/share/magicdstool_torevert_diff.sql

@@ -9 +9 @@
     outroot,
     diffSkyfile.path_base,
-    NULL AS cam_path_base,
+    "NULL" AS cam_path_base,
     0 AS bytes,
     0 AS md5sum

trunk/ippTools/src/magicdstool.c

@@ -2 +2 @@
  * magicdstool.c
  *
- * Copyright (C) 2006-2007  IfA
+ * Copyright (C) 2006-2009  IfA
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -27 +27 @@
 #include <math.h>
 #include <ippdb.h>
+#include <ippStages.h>
 
 #include "pxtools.h"
@@ -97 +98 @@
 static bool definebyqueryMode(pxConfig *config)
 {
-    PS_ASSERT_PTR_NON_NULL(config, false);
-    psError(PS_ERR_UNKNOWN, true, "definebyquery not implelmented yet");
-
-    return false;
-#ifdef notyet
 
     // Required
+    PXOPT_LOOKUP_STR(stage, config->args, "-stage", true, false);
+
+    // Optional
     PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
-
-    // Optional
-    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
-    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
-    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
+    PXOPT_LOOKUP_STR(recoveryroot, config->args, "-recoveryroot", false, false);
+    PXOPT_LOOKUP_BOOL(re_place, config->args, "-replace", false);
+    PXOPT_LOOKUP_STR(set_label, config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_BOOL(rerun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_BOOL(dry_run, config->args, "-dry_run", false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    // Create temporary table of the best diffs
-    {
-        psString query = pxDataGet("magictool_definebyquery_temp_create.sql");
-        if (!query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
-        }
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            return false;
-        }
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+
+    // search args
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-exp_id",  "exp_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-cam_id",  "cam_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-magic_id","magic_id", "==");
+
+    pxAddLabelSearchArgs (config, where, "-label", "magicRun.label", "==");
+
+    ippStage stageNum = ippStringToStage(stage);
+    
+    psString query = NULL;
+    switch (stageNum) {
+    case IPP_STAGE_RAW:
+        query = pxDataGet("magicdstool_definebyquery_raw.sql");
+        break;
+    case IPP_STAGE_CHIP:
+        query = pxDataGet("magicdstool_definebyquery_chip.sql");
+        break;
+    case IPP_STAGE_CAMERA:
+        query = pxDataGet("magicdstool_definebyquery_camera.sql");
+        break;
+    case IPP_STAGE_WARP:
+        query = pxDataGet("magicdstool_definebyquery_warp.sql");
+        break;
+    case IPP_STAGE_DIFF:
+        query = pxDataGet("magicdstool_definebyquery_diff.sql");
+        break;
+    case IPP_STAGE_FAKE:
+    case IPP_STAGE_STACK:
+        psError(PXTOOLS_ERR_DATA, true, "%sRuns do not need to be destreaked", stage);
+        return false;
+    case IPP_STAGE_NONE:
+        psError(PXTOOLS_ERR_DATA, true, "%s is not a valid stage", stage);
+        return false;
+    default:
+        psError(PXTOOLS_ERR_PROG, true, "ippStageToString returned %d for invalid stage %s",
+            stageNum, stage);
+        return false;
+    }
+
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "\n AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    dry_run = true;
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, rerun ? "\n1 " : "\n0 ")) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
-    }
-
-    // Insert list of best diffs into temporary table
-    {
-        psString query = pxDataGet("magictool_definebyquery_temp_insert.sql");
-        if (!query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
-        }
-
-        psMetadata *where = psMetadataAlloc();
-        PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-        PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
-        PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
-
-        psString whereClause = NULL;    // WHERE conditions
-        if (psListLength(where->list)) {
-            whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringPrepend(&whereClause, "\n AND ");
-        }
-        psFree(where);
-
-        if (!p_psDBRunQueryF(config->dbh, query, whereClause)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(whereClause);
-            psFree(query);
-            return false;
-        }
-        psFree(whereClause);
-        psFree(query);
-    }
-
-    // Get list of exposures ready to magic
-    {
-        psString query = pxDataGet("magictool_definebyquery_select.sql");
-        if (!query) {
-            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
-            return false;
-        }
-
-        psString magicSkyCellNumsWhere = NULL;    // WHERE conditions for magicSkyCellNums
-        {
-            psMetadata *where = psMetadataAlloc();
-            PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
-            PXOPT_COPY_STR(config->args, where, "-diff_label", "diffRun.label", "==");
-            PXOPT_COPY_F32(config->args, where, "-good_frac", "warpSkyfile.good_frac", ">=");
-
-            if (psListLength(where->list)) {
-                magicSkyCellNumsWhere = psDBGenerateWhereConditionSQL(where, NULL);
-                psStringPrepend(&magicSkyCellNumsWhere, "\n AND ");
-            }
-            psFree(where);
-        }
-
-        // "available" means only concern ourselves with exposures that have all diffs completed, unless we're
-        // told to only take what's available.
-        // "new" means we want a new run even if there's already a magic run defined
-        PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
-        PXOPT_LOOKUP_BOOL(new, config->args, "-new", false);
-
-        psString queryWhere = NULL;     // WHERE conditions for entire query
-        if (available) {
-            psStringAppend(&queryWhere, " WHERE num_done = num_todo");
-        }
-        if (new) {
-            const char *newWhere = " magic_id IS NULL"; // String to add
-            if (queryWhere) {
-                psStringAppend(&queryWhere, " AND %s", newWhere);
-            } else {
-                psStringAppend(&queryWhere, " WHERE %s", newWhere);
-            }
-        }
-        if (queryWhere) {
-            psStringAppend(&query, " %s", queryWhere);
-            psFree(queryWhere);
-        }
-
-
-        if (!p_psDBRunQueryF(config->dbh, query, magicSkyCellNumsWhere ? magicSkyCellNumsWhere : "")) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(magicSkyCellNumsWhere);
-            psFree(query);
-            return false;
-        }
-        psFree(magicSkyCellNumsWhere);
-        psFree(query);
-    }
+        return false;
+    }
+    psFree(query);
 
     psArray *output = p_psDBFetchResult(config->dbh);
@@ -233 +203 @@
     }
 
-    // Parse the list of exposures ready to magic
-
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    psString insert = pxDataGet("magictool_definebyquery_insert.sql"); // Insert query
+    // Parse the list of runs ready to be destreaked
+
+    if (!dry_run && !psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
 
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
     for (long i = 0; i < psArrayLength(output); i++) {
         psMetadata *row = output->data[i]; // Row of interest
-        psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id"); // Exposure identifier
+        psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
+        psS64 exp_id = psMetadataLookupS64(NULL, row, "exp_id");
+        psS64 magic_id = psMetadataLookupS64(NULL, row, "magic_id");
+        psS64 cam_id = psMetadataLookupS64(NULL, row, "cam_id");
+        psString magicRunLabel = psMetadataLookupStr(NULL, row, "label");
+        psString magicRunWorkdir = psMetadataLookupStr(NULL, row, "workdir");
+        
+        // if workdir is not supplied use the magicRun's
+        if (!workdir) {
+            workdir = magicRunWorkdir;
+        }
+
+        psString outroot = NULL;
+        // set outroot to workdir/exp_id/stage for example /somewhere/424242/chip
+        psStringAppend(&outroot, "%s/%" PRId64 "/%s", workdir, exp_id, stage);
 
         // create a new magicRun for this group
-        magicRunRow *run = magicRunRowAlloc(0, exp_id, "new", workdir, "dirty", label, dvodb, registered, 0);
+        magicDSRunRow *run = magicDSRunRowAlloc(
+                0, // magic_ds_id
+                magic_id,
+                "new",
+                stage,
+                stage_id,
+                cam_id,
+                set_label ? set_label : magicRunLabel,
+                outroot,
+                recoveryroot,
+                re_place,
+                0); // remove
+
+        psFree(outroot);
         if (!run) {
-            psAbort("failed to alloc magicRun object");
-        }
-
-        if (!magicRunInsertObject(config->dbh, run)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(run);
-            psFree(insert);
-            psFree(output);
-            psFree(list);
-            if (!psDBRollback(config->dbh)) {
+            psAbort("failed to alloc magicDSRun object");
+        }
+
+        if (!dry_run) {
+            if (!magicDSRunInsertObject(config->dbh, run)) {
                 psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(run);
+                psFree(output);
+                psFree(list);
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error");
+                }
+                return false;
+                psS64 magic_ds_id = psDBLastInsertID(config->dbh); // Assigned identifier
+                run->magic_ds_id = magic_ds_id;
             }
-            return false;
-        }
-
-        psS64 magic_id = psDBLastInsertID(config->dbh); // Assigned identifier
-        run->magic_id = magic_id;
+        }
 
         psArrayAdd(list, list->n, run);
         psFree(run);
-
-        // Create a suitable insertion query for this run
-        psString thisInsert = psStringCopy(insert);
-        {
-            psString idString = NULL;
-            psStringAppend(&idString, "%" PRId64, magic_id);
-            psStringSubstitute(&thisInsert, idString, "@MAGIC_ID@");
-            psFree(idString);
-        }
-        {
-            psString idString = NULL;
-            psStringAppend(&idString, "%" PRId64, exp_id);
-            psStringSubstitute(&thisInsert, idString, "@EXP_ID@");
-            psFree(idString);
-        }
-
-        if (!p_psDBRunQueryF(config->dbh, thisInsert, magic_id, exp_id)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(thisInsert);
-            psFree(insert);
-            psFree(output);
-            psFree(list);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psFree(thisInsert);
-    }
-
-    if (!psDBCommit(config->dbh)) {
+    }
+
+    if (!dry_run && !psDBCommit(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         return false;
@@ -306 +273 @@
     psFree(output);
 
-    if (!magicRunPrintObjects(stdout, list, !simple)) {
+    if (!magicDSRunPrintObjects(stdout, list, !simple)) {
         psError(PS_ERR_UNKNOWN, false, "failed to print object");
         psFree(list);
@@ -315 +282 @@
 
     return true;
-#endif // notyet
 }
 
@@ -405 +371 @@
     PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
     pxAddLabelSearchArgs (config, where, "-label", "label", "==");
+    PXOPT_COPY_STR(config->args, where, "-stage", "stage", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -501 +468 @@
     psMetadata *row = output->data[0];
 
-    psString stage = psMetadataLookupStr(NULL, row, "stage");
+    psString stage= psMetadataLookupStr(NULL, row, "stage");
     psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
 
-    if (!strcmp(stage, "camera")) {
-        // no there is no magicked column in camProcessedExp
+    ippStage stageNum = ippStringToStage(stage);
+
+    // chose the appropriate query based on the stage
+    switch (stageNum) {
+    case IPP_STAGE_RAW:
+        query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'";
+        break;
+    case IPP_STAGE_CHIP:
+        query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'";
+        break;
+    case IPP_STAGE_CAMERA:
+        // no there is no magicked column in camProcessedExp so we have nothing to do
         psFree(output);
         return true;
-    }
-
-    // chose the appropriate query based on the stage
-    if (!strcmp(stage, "raw")) {
-        query = "UPDATE rawImfile SET magicked = %" PRId64 " where exp_id = %" PRId64 " AND class_id = '%s'";
-    } else if (!strcmp(stage, "chip")) {
-        query = "UPDATE chipProcessedImfile SET magicked = %" PRId64 " where chip_id = %" PRId64 " AND class_id = '%s'";
-    } else if (!strcmp(stage, "warp")) {
+    case IPP_STAGE_WARP:
         query = "UPDATE warpSkyfile SET magicked = %" PRId64 " where warp_id = %" PRId64 " AND skycell_id = '%s'";
-    } else if (!strcmp(stage, "diff")) {
+        break;
+    case IPP_STAGE_DIFF:
         query = "UPDATE diffSkyfile SET magicked = %" PRId64 " where diff_id = %" PRId64 " AND skycell_id = '%s'";
-    } else {
+        break;
+    default:
         psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage);
         psFree(output);
         return false;
     }
+
     if (!p_psDBRunQueryF(config->dbh, query, magic_ds_id, stage_id, component)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -571 +544 @@
     psS64 stage_id = psMetadataLookupS64(NULL, row, "stage_id");
 
+    ippStage stageNum = ippStringToStage(stage);
 
     // chose the appropriate query based on the stage
-    if (!strcmp(stage, "raw")) {
+    switch (stageNum) {
+    case IPP_STAGE_RAW:
         query = "UPDATE rawExp SET magicked = %" PRId64 " where exp_id = %" PRId64;
-    } else if (!strcmp(stage, "chip")) {
+        break;
+    case IPP_STAGE_CHIP:
         query = "UPDATE chipRun SET magicked = %" PRId64 " where chip_id = %" PRId64;
-    } else if (!strcmp(stage, "camera")) {
+        break;
+    case IPP_STAGE_CAMERA:
         query = "UPDATE camRun SET magicked = %" PRId64 " where cam_id = %" PRId64;
-    } else if (!strcmp(stage, "warp")) {
+        break;
+    case IPP_STAGE_WARP:
         query = "UPDATE warpRun SET magicked = %" PRId64 " where warp_id = %" PRId64;
-    } else if (!strcmp(stage, "diff")) {
+        break;
+    case IPP_STAGE_DIFF:
         query = "UPDATE diffRun SET magicked = %" PRId64 " where diff_id = %" PRId64;
-    } else {
+        break;
+    default:
         psError(PS_ERR_UNKNOWN, true, "unexpected value for stage: %s found", stage);
         psFree(output);
@@ -671 +651 @@
     PS_ASSERT_PTR_NON_NULL(cam_id, false);
 
-    if (!strcmp(stage, "diff")) {
-        // don't need these ids for diff stage because diff_id is in the magicRun
-        *stage_id = 0;
-        *cam_id = 0;
-        return true;
-    }
-
-    int stageNum;
-    if (!strcmp(stage, "raw")) {
-        stageNum = 0;
-    } else if (!strcmp(stage, "chip")) {
-        stageNum = 1;
-    } else if (!strcmp(stage, "camera")) {
-        stageNum = 2;
-    } else if (!strcmp(stage, "warp")) {
-        stageNum = 3;
-    } else {
-        psError(PXTOOLS_ERR_DATA, true, "%s is not a valid value for stage", stage);
-        return false;
-    }
-
+    int stageNum = ippStringToStage(stage);;
+    if (stageNum == IPP_STAGE_NONE) {
+        psError(PXTOOLS_ERR_DATA, false, "%s is not a valid value for stage", stage);
+        return false;
+    }
 
     psString query = pxDataGet("magicdstool_getrunids.sql");
@@ -724 +688 @@
 
     *cam_id = psMetadataLookupS64(NULL, row, "cam_id");
-    if (stageNum == 0) {
+    switch (stageNum) {
+    case IPP_STAGE_RAW:
         *stage_id = psMetadataLookupS64(NULL, row, "exp_id");
-    } else if (stageNum == 1) {
+        break;
+    case IPP_STAGE_CHIP:
         *stage_id = psMetadataLookupS64(NULL, row, "chip_id");
-    } else if (stageNum == 2) {
+        break;
+    case IPP_STAGE_CAMERA:
         *stage_id = *cam_id;
-    } else if (stageNum == 3) {
+        return true;
+    case IPP_STAGE_WARP:
         *stage_id = psMetadataLookupS64(NULL, row, "warp_id");
+        break;
+    case IPP_STAGE_DIFF:
+        *stage_id = psMetadataLookupS64(NULL, row, "diff_id");
+        break;
     }
 

trunk/ippTools/src/magicdstoolConfig.c

@@ -49 +49 @@
     // -definebyquery
     psMetadata *definebyqueryArgs = psMetadataAlloc();
-#ifdef notyet
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir",     0, "define workdir (required)", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label",       0, "define label", NULL);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-dvodb",       0, "define dvodb", NULL);
-    psMetadataAddTime(definebyqueryArgs, PS_LIST_TAIL, "-registered", 0, "time detrend run was registered", now);
-    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id", 0, "search exp_id", 0);
-    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-diff_label", 0, "select diff label", NULL);
-    psMetadataAddF32(definebyqueryArgs, PS_LIST_TAIL, "-good_frac", 0, "limit good_frac", NAN);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-available", 0, "process what's immediately available?", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-new", 0, "generate new run even if existing?", false);
-    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
-#endif
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-stage",       0, "define stage (required)", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-workdir",     0, "define workdir", NULL);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-replace", 0, "replace input files with the destreaked versions", false);
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_label",    0, "define label", NULL);
+
+    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by magicRun.label", NULL);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-exp_id",   0, "search by exp_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-chip_id",  0, "search by chip_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-cam_id",  0, "search by cam_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-warp_id",  0, "search by warp_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-diff_id",  0, "search by diff_id", 0);
+    psMetadataAddS64(definebyqueryArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic_id", 0);
+
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-rerun",   0, "queue a new run even if one arleady exists", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-dry_run", 0, "don't queue runs just display what would be queued", false);
+    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple",  0, "use the simple output format", false);
+    psMetadataAddU64(definebyqueryArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
 
     // -definerun
     psMetadata *definerunArgs = psMetadataAlloc();
     psMetadataAddS64(definerunArgs, PS_LIST_TAIL, "-magic_id", 0, "define magic_id (required)", 0);
-    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define output directory (required)", NULL);
+    psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-stage", 0, "define stage for run (required)", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-outroot", 0, "define output directory (required)", NULL);
     psMetadataAddStr(definerunArgs, PS_LIST_TAIL, "-recoveryroot", 0, "define recovery directory", NULL);
@@ -89 +95 @@
     psMetadataAddS64(todestreakArgs, PS_LIST_TAIL, "-magic_id", 0, "search by magic ID", 0);
     psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "define label", NULL);
+    psMetadataAddStr(todestreakArgs, PS_LIST_TAIL, "-stage", 0, "limit query to stage", NULL);
     psMetadataAddU64(todestreakArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     psMetadataAddBool(todestreakArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);