Changeset 27874

Timestamp:

May 6, 2010, 1:55:20 PM (16 years ago)

Author:

bills

Message:

Various changes to the postage stamp server and associated tables.
Implemented cleanup. Added pstampRequest.outdir changed spelling of out_dir
to outdir. various other cleanups

Location:

trunk

Files:

• dbconfig/changes.txt (modified) (1 diff)
• dbconfig/pstamp.md (modified) (1 diff)
• ippTasks/pstamp.pro (modified) (16 diffs)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/pstamptool_pendingcleanup.sql (added)
• ippTools/share/pstamptool_pendingdependent.sql (modified) (1 diff)
• ippTools/share/pstamptool_pendingreq.sql (modified) (1 diff)
• ippTools/share/pstamptool_updatejob.sql (modified) (1 diff)
• ippTools/share/pxadmin_create_tables.sql (modified) (1 diff)
• ippTools/src/pstamptool.c (modified) (17 diffs)
• ippTools/src/pstamptool.h (modified) (1 diff)
• ippTools/src/pstamptoolConfig.c (modified) (6 diffs)
• pstamp/scripts/Makefile.am (modified) (1 diff)
• pstamp/scripts/dquery_finish.pl (modified) (8 diffs)
• pstamp/scripts/dqueryparse.pl (modified) (12 diffs)
• pstamp/scripts/pstamp_checkdependent.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_cleanup.pl (added)
• pstamp/scripts/pstamp_dorequest.pl (modified) (1 diff)
• pstamp/scripts/pstamp_finish.pl (modified) (12 diffs)
• pstamp/scripts/pstamp_get_image_job.pl (modified) (3 diffs)
• pstamp/scripts/pstamp_job_run.pl (modified) (2 diffs)
• pstamp/scripts/pstamp_parser_run.pl (modified) (10 diffs)
• pstamp/scripts/pstampparse.pl (modified) (7 diffs)
• pstamp/scripts/request_finish.pl (modified) (4 diffs)

trunk/dbconfig/changes.txt

@@ -1615 +1615 @@
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 
+ALTER TABLE pstampRequest ADD COLUMN outdir VARCHAR(64) after uri;

trunk/dbconfig/pstamp.md

@@ -30 +30 @@
     outProduct  STR         64
     uri         STR         255
+    outdir      STR         255
     fault       S32         0
 END

trunk/ippTasks/pstamp.pro

@@ -14 +14 @@
 $pstampRev_DB = 0
 $pstampDep_DB = 0
+$pstampCleanup_DB = 0
 
 # set PS_DBSERVER if postage stamp database host is not the same as the value for DBSERVER in site.config
@@ -30 +31 @@
     book init pstampFinish
     book init pstampDependent
+    book init pstampCleanup
 end
 
@@ -58 +60 @@
         active true
     end
-    task pstamp.job.revert
-        active true
-    end
     task pstamp.dependent.load
         active true
@@ -94 +93 @@
     end
     task pstamp.dependent.load
-        active true
+        active false
+    end
+    task pstamp.dependent.run
+        active false
     end
 end
@@ -105 +107 @@
 macro pstamp.revert.off
     task pstamp.job.revert
+        active false
+    end
+end
+macro pstamp.cleanup.on
+    task pstamp.cleanup.load
+        active true
+    end
+    task pstamp.cleanup.run
+        active true
+    end
+end
+macro pstamp.cleanup.off
+    task pstamp.cleanup.run
+        active false
+    end
+    task pstamp.cleanup.run
         active false
     end
@@ -157 +175 @@
 
     task.exec
-        stdout $LOGSUBDIR/pstamp.request.load.log
+        stdout NULL
         stderr $LOGSUBDIR/pstamp.request.load.log
         $run = pstamptool -pendingreq
@@ -200 +218 @@
     periods     -exec $RUNEXEC
     periods     -timeout 300
-    npending  6
-
-    task.exec
-        stdout $LOGSUBDIR/pstamp.request.run.log
+
+    task.exec
+        stdout NULL
         stderr $LOGSUBDIR/pstamp.request.run.log
         book npages pstampRequest -var N
@@ -216 +233 @@
         book getword pstampRequest $pageName uri -var URI
         book getword pstampRequest $pageName ds_outProduct -var PRODUCT
+        book getword pstampRequest $pageName outdir -var OUTDIR
         book getword pstampRequest $pageName label -var LABEL
 
         host anyhost
 
-        $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --label $LABEL --redirect-output
+        $run = pstamp_parser_run.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --outdir $OUTDIR --label $LABEL --redirect-output
 
         add_standard_args run
@@ -261 +279 @@
 
     task.exec
-        stdout $LOGSUBDIR/pstamp.finish.load.log
+        stdout NULL
         stderr $LOGSUBDIR/pstamp.finish.load.log
         $run = pstamptool  -completedreq
@@ -306 +324 @@
 
     task.exec
-        stdout $LOGSUBDIR/request.finish.run.log
+        stdout NULL
         stderr $LOGSUBDIR/request.finish.run.log
         book npages pstampFinish -var N
@@ -321 +339 @@
         book getword pstampFinish $pageName name -var REQ_NAME
         book getword pstampFinish $pageName outProduct -var PRODUCT
-
-        $run = request_finish.pl --req_id $REQ_ID --req_type $REQ_TYPE --req_file $URI --req_name $REQ_NAME --product $PRODUCT --redirect-output
+        book getword pstampFinish $pageName outdir -var OUTDIR
+
+        $run = request_finish.pl --req_id $REQ_ID --req_type $REQ_TYPE --req_file $URI --req_name $REQ_NAME --product $PRODUCT --outdir $OUTDIR --redirect-output
 
         add_standard_args run
@@ -363 +382 @@
 
     task.exec
-        stdout $LOGSUBDIR/pstamp.job.load.log
+        stdout NULL
         stderr $LOGSUBDIR/pstamp.job.load.log
         $run = pstamptool -pendingjob
@@ -411 +430 @@
 
     task.exec
-        stdout $LOGSUBDIR/pstamp.job.run.log
+        stdout NULL
         stderr $LOGSUBDIR/pstamp.job.run.log
         book npages pstampJob -var N
@@ -489 +508 @@
 
     task.exec
-        stdout $LOGSUBDIR/pstamp.job.revert.log
+        stdout NULL
         stderr $LOGSUBDIR/pstamp.job.revert.log
         $run = pstamptool -revertjob -all
@@ -532 +551 @@
 
     task.exec
-        stdout $LOGSUBDIR/pstamp.dependent.load.log
+        stdout NULL
         stderr $LOGSUBDIR/pstamp.dependent.load.log
         $run = pstamptool -pendingdependent
@@ -660 +679 @@
 end
 
+task pstamp.cleanup.load
+    host        local
+
+    periods     -poll $LOADPOLL
+    periods     -exec $LOADEXEC
+    periods     -timeout 300
+    npending    1
+
+    task.exec
+        stdout NULL
+        stderr $LOGSUBDIR/pstamp.cleanup.load.log
+        $run = pstamptool -pendingcleanup
+        if ($DB:n == 0)
+            option DEFAULT
+        else 
+            option $DB:$pstampCleanup_DB
+            $run = $run $PS_DBSERVER -dbname $DB:$pstampCleanup_DB
+            $pstampCleanup_DB ++
+            if ($pstampCleanup_DB >= $DB:n) set pstampCleanup_DB = 0
+        end
+        add_poll_args run
+        add_poll_labels run
+        command $run
+    end
+
+    task.exit $EXIT_SUCCESS
+        ipptool2book stdout pstampCleanup -key req_id -uniq -setword dbname $options:0 -setword pantaskState INIT
+        if ($VERBOSE > 2)
+            echo starting request
+            book listbook pstampCleanup
+        end
+
+        process_cleanup pstampCleanup
+    end
+
+    task.exit   default
+        showcommand failure
+    end
+
+    task.exit   crash
+        showcommand crash
+    end
+
+    task.exit   timeout
+        showcommand timeout
+    end
+end
+
+task pstamp.cleanup.run
+    periods     -poll $RUNPOLL
+    periods     -exec $RUNEXEC
+    periods     -timeout 300
+    # since everything is on one file system keep npending low to avoid
+    # overloading nfs
+
+    npending    10
+
+    task.exec
+        book npages pstampCleanup -var N
+        if ($N == 0) 
+            periods -exec $RUNEXEC
+            break
+        end
+        periods -exec 0.05
+        
+        book getpage pstampCleanup 0 -var pageName -key pantaskState INIT
+        if ("$pageName" == "NULL") break
+
+        book setword pstampCleanup $pageName pantaskState RUN
+        book getword pstampCleanup $pageName req_id -var REQ_ID
+        book getword pstampCleanup $pageName dbname -var DBNAME
+        book getword pstampCleanup $pageName name -var NAME
+        book getword pstampCleanup $pageName outdir -var OUTDIR
+        book getword pstampCleanup $pageName uri -var URI
+        book getword pstampCleanup $pageName reqType -var REQTYPE
+        book getword pstampCleanup $pageName outProduct -var PRODUCT
+
+        # XXX: have the script set this up this
+        $MYLOGFILE=/data/ippdb02.0/pstamp/work/logs/cleanup.$REQ_ID
+        stdout $MYLOGFILE
+        stderr $MYLOGFILE
+
+        host anyhost
+
+        $run = pstamp_cleanup.pl --req_id $REQ_ID --uri $URI --product $PRODUCT --name $NAME --outdir $OUTDIR --reqType $REQTYPE
+
+        add_standard_args run
+        options $pageName
+
+        if ($VERBOSE > 1) 
+            echo command $run
+        end
+        command $run
+    end
+
+
+    task.exit $EXIT_SUCCESS
+        process_exit pstampCleanup $options:0 $JOB_STATUS
+    end
+
+    task.exit default
+        showcommand failure
+        process_exit pstampCleanup $options:0 $JOB_STATUS
+    end
+
+    task.exit crash
+        showcommand crash
+        book setword pstampCleanup $options:0 pantaskState CRASH
+    end
+
+    task.exit timeout
+        showcommand timeout
+        book setword pstampCleanup $options:0 pantaskState TIMEOUT
+    end
+end
+

trunk/ippTools/share/Makefile.am

@@ -226 +226 @@
      pstamptool_getdependent.sql \
      pstamptool_listjob.sql \
+     pstamptool_pendingcleanup.sql \
      pstamptool_pendingdependent.sql \
      pstamptool_pendingjob.sql \

trunk/ippTools/share/pstamptool_pendingdependent.sql

@@ -1 @@
-SELECT DISTINCT pstampDependent.*,
+SELECT DISTINCT
+    pstampDependent.*,
     IFNULL(Label.priority, 10000) AS priority
 FROM pstampDependent

trunk/ippTools/share/pstamptool_pendingreq.sql

@@ -4 +4 @@
     pstampDataStore.outProduct AS ds_outProduct,
     IFNULL(Label.priority, 10000) AS priority
-    FROM pstampRequest
+FROM pstampRequest
     LEFT JOIN pstampDataStore USING(ds_id)
     LEFT JOIN Label ON pstampRequest.label = Label.label

trunk/ippTools/share/pstamptool_updatejob.sql

@@ -1 +1 @@
 UPDATE pstampJob LEFT JOIN pstampDependent USING(dep_id)
-SET pstampJob.state = '%s'
-    -- set state hook %s
+SET 

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -1303 +1303 @@
         outProduct VARCHAR(64),
         uri VARCHAR(255),
+        outdir     VARCHAR(255),
         fault SMALLINT,
         PRIMARY KEY(req_id),

trunk/ippTools/src/pstamptool.c

@@ -40 +40 @@
 static bool updatereqMode(pxConfig *config);
 static bool revertreqMode(pxConfig *config);
+static bool pendingcleanupMode(pxConfig *config);
 static bool addjobMode(pxConfig *config);
 static bool listjobMode(pxConfig *config);
@@ -81 +82 @@
         MODECASE(PSTAMPTOOL_MODE_UPDATEREQ, updatereqMode);
         MODECASE(PSTAMPTOOL_MODE_REVERTREQ, revertreqMode);
+        MODECASE(PSTAMPTOOL_MODE_PENDINGCLEANUP, pendingcleanupMode);
         MODECASE(PSTAMPTOOL_MODE_ADDJOB, addjobMode);
         MODECASE(PSTAMPTOOL_MODE_LISTJOB, listjobMode);
@@ -265 +267 @@
         NULL,   // outProduct
         uri,
+        NULL,   // outdir
         0       // fault
         )) {
@@ -341 +344 @@
 }
 
-static bool listreqMode(pxConfig *config)
+static bool pendingcleanupMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
@@ -347 +350 @@
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!=");
-    PXOPT_COPY_STR(config->args, where, "-name", "name", "==");
-
-    PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
+    pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    if (!psListLength(where->list)) {
-        psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied");
-        return false;
-    }
-
-    psString query = psStringCopy("SELECT * from pstampRequest");
-
-    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-    psStringAppend(&query, " WHERE %s", whereClause);
-    psFree(whereClause);
-    psFree(where);
+    psString query = pxDataGet("pstamptool_pendingcleanup.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, "pstampRequest");
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    psStringAppend(&query, " ORDER BY priority DESC, req_id");
 
     // treat limit == 0 as "no limit"
@@ -385 +390 @@
     }
     if (!psArrayLength(output)) {
-        psTrace("pstamptool", PS_LOG_INFO, "request not found");
-        // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for
-        psError(PXTOOLS_ERR_CONFIG, true, "request not found");
-        psFree(output);
-        // we return false so that the caller can determine that a request does not exist
-        return false;
+        psTrace("pstamptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
     }
 
@@ -405 +407 @@
 }
 
-static bool completedreqMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE");
-
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+static bool listreqMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-req_id", "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-not_req_id", "req_id", "!=");
+    PXOPT_COPY_STR(config->args, where, "-name", "name", "==");
+
+    PXOPT_LOOKUP_U64(limit,   config->args, "-limit",  false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = pxDataGet("pstamptool_completedreq.sql");
-
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
-        psFree(whereClause);
-    }
-    psFree(where);
-
-    psStringAppend(&query, " ORDER BY priority DESC, req_id");
+    if (!psListLength(where->list)) {
+        psError(PS_ERR_UNKNOWN, true, "-req_id or -name must be supplied");
+        return false;
+    }
+
+    psString query = psStringCopy("SELECT * from pstampRequest");
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, " WHERE %s", whereClause);
+    psFree(whereClause);
+    psFree(where);
 
     // treat limit == 0 as "no limit"
@@ -438 +443 @@
         return false;
     }
+    psFree(query);
 
     psArray *output = p_psDBFetchResult(config->dbh);
@@ -444 +450 @@
         return false;
     }
-
     if (!psArrayLength(output)) {
-        psTrace("pstamptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
+        psTrace("pstamptool", PS_LOG_INFO, "request not found");
+        // This causes main to exit with PS_EXIT_DATA_ERROR which the script is looking for
+        psError(PXTOOLS_ERR_CONFIG, true, "request not found");
+        psFree(output);
+        // we return false so that the caller can determine that a request does not exist
+        return false;
     }
 
@@ -463 +471 @@
 }
 
+static bool completedreqMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    pxAddLabelSearchArgs(config, where, "-label", "label", "LIKE");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    psString query = pxDataGet("pstamptool_completedreq.sql");
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    psStringAppend(&query, " ORDER BY priority DESC, req_id");
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!psArrayLength(output)) {
+        psTrace("pstamptool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "pstampRequest", !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
 static bool updatereqMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    PXOPT_LOOKUP_S64(req_id,     config->args, "-req_id",     true, false);
-    PXOPT_LOOKUP_STR(state,      config->args, "-state",      false, false);
-    PXOPT_LOOKUP_STR(outProduct, config->args, "-outProduct", false, false);
-    PXOPT_LOOKUP_S16(fault,      config->args, "-fault",      false, false);
-    PXOPT_LOOKUP_STR(uri,        config->args, "-uri",        false, false);
-    PXOPT_LOOKUP_STR(name,       config->args, "-name",       false, false);
-    PXOPT_LOOKUP_STR(reqType,    config->args, "-reqType",    false, false);
-
-    psString query = NULL;
-    psStringAppend(&query, "UPDATE pstampRequest SET");
-
+    // PXOPT_LOOKUP_S64(req_id,     config->args, "-req_id",   true, false);
+    PXOPT_LOOKUP_STR(state,      config->args, "-set_state",      false, false);
+    PXOPT_LOOKUP_STR(label,      config->args, "-set_label",      false, false);
+    PXOPT_LOOKUP_STR(outProduct, config->args, "-set_outProduct", false, false);
+    PXOPT_LOOKUP_S16(fault,      config->args, "-set_fault",          false, false);
+    PXOPT_LOOKUP_STR(uri,        config->args, "-set_uri",        false, false);
+    PXOPT_LOOKUP_STR(outdir,     config->args, "-set_outdir",        false, false);
+    PXOPT_LOOKUP_STR(name,       config->args, "-set_name",       false, false);
+    PXOPT_LOOKUP_STR(reqType,    config->args, "-set_reqType",    false, false);
+    PXOPT_LOOKUP_BOOL(clearfault,config->args, "-clearfault",    false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-req_id",     "req_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-req_id_max", "req_id", "<=");
+    PXOPT_COPY_S32(config->args, where, "-fault",      "fault", "==");
+    PXOPT_COPY_STR(config->args, where, "-state",      "state", "==");
+    pxAddLabelSearchArgs(config, where, "-label",      "pstampRequest.label", "LIKE");
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = psStringCopy("UPDATE pstampRequest SET");
+
+    psString stateCheck = NULL;
     char c = ' ';
     if (state) {
         psStringAppend(&query, "%c state = '%s'", c, state);
         c = ',';
+        if (!strcmp(state, "goto_cleaned")) {
+            psStringAppend(&stateCheck, " AND state != 'cleaned'");
+        }
+    }
+    if (label) {
+        psStringAppend(&query, "%c label = '%s'", c, label);
+        c = ',';
     }
     if (outProduct) {
@@ -487 +575 @@
         c = ',';
     }
-    if (fault) {
+    if (outdir) {
+        psStringAppend(&query, "%c outdir = '%s'", c, outdir);
+        c = ',';
+    }
+    if (clearfault) {
+        if (fault) {
+            psError(PXTOOLS_ERR_CONFIG, true, "only one of -fault and -clearfault is allowed");
+            return false;
+        }
+        psStringAppend(&query, "%c fault = 0", c);
+        c = ',';
+    } else if (fault) {
         psStringAppend(&query, "%c fault = %d", c, fault);
         c = ',';
@@ -503 +602 @@
         c = ',';
     }
-    if (!state && !outProduct && !fault && !uri && !name && !reqType) {
-        psError(PS_ERR_UNKNOWN, true, "at least one of state, outProduct, fault, uri, name, and reqType must be specified");
-        return false;
-    }
-
-    psStringAppend(&query, " WHERE req_id = %" PRId64, req_id);
+    if (c != ',') {
+        psError(PS_ERR_UNKNOWN, true, "at least one set option is required");
+        return false;
+    }
+
+    psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+    psStringAppend(&query, "\nWHERE %s", whereClause);
+    psFree(whereClause);
+    psFree(where);
+
+    if (stateCheck) {
+        psStringAppend(&query, stateCheck);
+    }
 
     if (!p_psDBRunQuery(config->dbh, query)) {
@@ -516 +622 @@
     }
 
-    psU64 affected = psDBAffectedRows(config->dbh);
-    // note zero is not an error
-    if (affected > 1) {
-        psError(PS_ERR_UNKNOWN, false, "should have affected one row but %"
-                                        PRIu64 " rows were modified", affected);
-        return false;
-    }
+    // psU64 affected = psDBAffectedRows(config->dbh);
+    // psLogMsg("pstamptool", PS_LOG_INFO, "Updated %" PRIu64 " pstampRequests", affected);
 
     return true;
@@ -778 +879 @@
         return false;
     }
-    PXOPT_LOOKUP_STR(state,     config->args, "-state",  true, false);
-    PXOPT_LOOKUP_S32(fault,     config->args, "-fault",  false, false);
+
+    PXOPT_LOOKUP_STR(state,     config->args, "-set_state",  false, false);
+    PXOPT_LOOKUP_S32(fault,     config->args, "-set_fault",  false, false);
+    
+    if (!state && !fault) {
+        psError(PS_ERR_UNKNOWN, true, "at least one of -set_state and -set_fault is required");
+        return false;
+    }
 
     psMetadata *where = psMetadataAlloc();
@@ -786 +893 @@
     PXOPT_COPY_S64(config->args, where, "-dep_id", "dep_id", "==");
 
-    psString faultStr = NULL;
-    if (!fault) {
-        faultStr = psStringCopy("");
-    } else {
-        psStringAppend(&faultStr, "\n, pstampJob.fault = %d", fault);
-    }
-
     psString query = pxDataGet("pstamptool_updatejob.sql");
 
+    char c = ' ';
+    if (state) {
+        psStringAppend(&query, "\n %c pstampJob.state = '%s'", c, state);
+        c = ',';
+    }
+    if (fault) {
+        psStringAppend(&query, "\n %c pstampJob.fault = %d", c, fault);
+        c = ',';
+    }
+    
     psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
     psStringAppend(&query, " WHERE %s", whereClause);
@@ -800 +910 @@
     psFree(where);
 
-    if (!p_psDBRunQueryF(config->dbh, query, state, faultStr)) {
+    if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
         return false;
     }
-    psFree(faultStr);
     psFree(query);
 
@@ -1159 +1268 @@
     PXOPT_LOOKUP_S64(dep_id,    config->args, "-dep_id", true, false);
     PXOPT_LOOKUP_STR(state,     config->args, "-set_state",  false, false);
-    PXOPT_LOOKUP_S16(fault,     config->args, "-fault",  false, false);
+    PXOPT_LOOKUP_S16(fault,     config->args, "-set_fault",  false, false);
 
     if (!fault && !state) {

trunk/ippTools/src/pstamptool.h

@@ -34 +34 @@
     PSTAMPTOOL_MODE_UPDATEREQ,
     PSTAMPTOOL_MODE_REVERTREQ,
+    PSTAMPTOOL_MODE_PENDINGCLEANUP,
     PSTAMPTOOL_MODE_ADDJOB,
     PSTAMPTOOL_MODE_LISTJOB,

trunk/ippTools/src/pstamptoolConfig.c

@@ -98 +98 @@
     // -updatereq
     psMetadata *updatereqArgs = psMetadataAlloc();
-    psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0,            "req_id for which to change state", 0);
-    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0,            "new state", NULL);
-    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-outProduct", 0,            "define request outProduct", NULL);
-    psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0,            "define request fault code", 0);
-    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-uri", 0,            "define the uri", NULL);
-    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-name", 0,            "define the name", NULL);
-    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-reqType", 0,            "define the reqType", NULL);
+    psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id", 0,       "req_id for which to update", 0);
+    psMetadataAddS64(updatereqArgs, PS_LIST_TAIL, "-req_id_max", 0,   "maximum req_id for which to update", 0);
+    psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-fault", 0,        "search by fault code", 0);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-state", 0,        "search by state", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampJob label (LIKE comparision)", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_state", 0,        "new state", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_label", 0,        "new label", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_outProduct", 0,   "new outProduct", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_outdir", 0,   "new outdir", NULL);
+    psMetadataAddS16(updatereqArgs, PS_LIST_TAIL, "-set_fault", 0,        "new fault code", 0);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_uri", 0,          "new uri", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_name", 0,         "new name", NULL);
+    psMetadataAddStr(updatereqArgs, PS_LIST_TAIL, "-set_reqType", 0,      "new reqType", NULL);
+    psMetadataAddBool(updatereqArgs, PS_LIST_TAIL, "-clearfault", 0,      "set fault to zero", false);
 
     // -revertreq
@@ -112 +119 @@
     psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-state", 0,      "state to revert", NULL);
     psMetadataAddStr(revertreqArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL);
+
+    // -pendingcleanup
+    psMetadata *pendingcleanupArgs = psMetadataAlloc();
+    psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-req_id", 0,   "define req_id", 0);
+    psMetadataAddS64(pendingcleanupArgs, PS_LIST_TAIL, "-ds_id", 0,    "define ds_id", 0);
+    psMetadataAddStr(pendingcleanupArgs, PS_LIST_TAIL, "-label", PS_META_DUPLICATE_OK, "search by pstampRequest label (LIKE comparision)", NULL);
+    psMetadataAddU64(pendingcleanupArgs, PS_LIST_TAIL, "-limit",  0,   "limit result set to N items", 0);
+    psMetadataAddBool(pendingcleanupArgs, PS_LIST_TAIL, "-simple", 0,  "use the simple output format", false);
 
     // -addjob
@@ -145 +160 @@
     psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-job_id", 0,            "req_id for which to change state", 0);
     psMetadataAddS64(updatejobArgs, PS_LIST_TAIL, "-dep_id", 0,            "dep_id for which to change state", 0);
-    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-state", 0,            "new state", NULL);
-    psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-fault", 0,            "new result", 0);
+    psMetadataAddStr(updatejobArgs, PS_LIST_TAIL, "-set_state", 0,            "new state", NULL);
+    psMetadataAddS16(updatejobArgs, PS_LIST_TAIL, "-set_fault", 0,            "new result", 0);
 
     // -revertjob
@@ -160 +175 @@
     // -getdependent
     psMetadata *getdependentArgs = psMetadataAlloc();
+    // arguabely all of these could be -set_ arguments, but since this mode is both a query and
+    // a create if doesn't exist I like it this way
     psMetadataAddS64(getdependentArgs, PS_LIST_TAIL, "-stage_id", 0, "define stage id for dependent (required)", 0);
     psMetadataAddStr(getdependentArgs, PS_LIST_TAIL, "-stage", 0,    "define stage for dependent (required)", NULL);
@@ -173 +190 @@
     psMetadataAddS64(updatedependentArgs, PS_LIST_TAIL, "-dep_id", 0, "define id for dependent (required)", 0);
     psMetadataAddStr(updatedependentArgs, PS_LIST_TAIL, "-set_state", 0, "new value for state", NULL);
-    psMetadataAddS16(updatedependentArgs, PS_LIST_TAIL, "-fault",  0,   "new value for fault", 0);
+    psMetadataAddS16(updatedependentArgs, PS_LIST_TAIL, "-set_fault",  0,   "new value for fault", 0);
 
     // -revertdependent
@@ -227 +244 @@
     PXOPT_ADD_MODE("-completedreq",    "", PSTAMPTOOL_MODE_COMPLETEDREQ, completedreqArgs);
     PXOPT_ADD_MODE("-revertreq",       "", PSTAMPTOOL_MODE_REVERTREQ,    revertreqArgs);
+    PXOPT_ADD_MODE("-pendingcleanup",  "", PSTAMPTOOL_MODE_PENDINGCLEANUP, pendingcleanupArgs);
 
     PXOPT_ADD_MODE("-addjob",          "", PSTAMPTOOL_MODE_ADDJOB,       addjobArgs);

trunk/pstamp/scripts/Makefile.am

@@ -3 +3 @@
 
 install_files = \
+        pstamp_cleanup.pl \
         pstamp_finish.pl \
         pstamp_insert_request.pl \

trunk/pstamp/scripts/dquery_finish.pl

@@ -20 +20 @@
 use PS::IPP::Config qw( :standard );
 
-my ( $req_id, $req_name, $req_file, $product, $out_dir, $dbname, $dbserver, $verbose, $save_temps );
+my ( $req_id, $req_name, $req_file, $product, $outdir, $dbname, $dbserver, $verbose, $save_temps );
 
 GetOptions(
@@ -27 +27 @@
            'req_file=s' => \$req_file,
            'product=s'  => \$product,
-           'out_dir=s'  => \$out_dir,
+           'outdir=s'   => \$outdir,
            'dbname=s'   => \$dbname,
            'dbserver=s' => \$dbserver,
@@ -41 +41 @@
 $err .= "--req_name is required\n" if !$req_name;
 $err .= "--product is required\n" if !$product;
-$err .= "--out_dir is required\n" if !$out_dir;
+$err .= "--outdir is required\n" if !$outdir;
 
 die "$err" if $err;
@@ -76 +76 @@
     die "product directory does not exist $prod_dir";
 }
-my $out_dir = "$prod_dir/$req_name";
+my $outdir = "$prod_dir/$req_name";
 }
-if (! -e $out_dir ) {
+if (! -e $outdir ) {
     # something must have gone wrong at the parse stage
-    print STDERR "output fileset directory $out_dir does not exist\n" if $verbose;
-    if (! mkdir $out_dir ) {
+    print STDERR "output fileset directory $outdir does not exist\n" if $verbose;
+    if (! mkdir $outdir ) {
         stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
-        die "cannot create output directory $out_dir";
+        die "cannot create output directory $outdir";
     }
-} elsif (! -d $out_dir) {
+} elsif (! -d $outdir) {
     stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
-    die "output fileset directory $out_dir exists but is not a directory";
+    die "output fileset directory $outdir exists but is not a directory";
 }
 
@@ -118 +118 @@
 
 # XXX: have the jobs produce the reglist as with postage stamp requests
-my ($REGLIST, $reg_list) = tempfile("$out_dir/reqlist.XXXX", UNLINK => !$save_temps);
+my ($REGLIST, $reg_list) = tempfile("$outdir/reqlist.XXXX", UNLINK => !$save_temps);
 
 foreach my $job (@jobs) {
     my $job_id = $job->{job_id};
     my $response_file = "response${job_id}.fits";
-    my $response_path = "$out_dir/$response_file";
+    my $response_path = "$outdir/$response_file";
 
     if (-e $response_path) {
@@ -131 +131 @@
         # do the same if we have an error file.  Should the parse data be uploaded as well?
         my $err_file = "parse_error.txt";
-        if (-e "$out_dir/$err_file") {
+        if (-e "$outdir/$err_file") {
             print $REGLIST "$err_file|||text|\n";
         }
@@ -143 +143 @@
 if (-s $reg_list) {
     my $command = "$dsreg --add $req_name --product $product --list $reg_list";
-    $command .= " --copy --datapath $out_dir";
+    $command .= " --copy --datapath $outdir";
     $command .= " --type MOPS_DETECTABILITY_RESPONSE";
     $command .= " --ps0 $req_id";
@@ -166 +166 @@
     my $verbose = shift;
     
-    my $command = "$pstamptool -updatereq -req_id $req_id -state stop";
-    $command   .= " -fault $fault" if $fault;
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop";
+    $command   .= " -set_fault $fault" if $fault;
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;

trunk/pstamp/scripts/dqueryparse.pl

@@ -26 +26 @@
 
 my ($no_update, $imagedb, $label);
-my ($req_file, $req_id, $out_dir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps);
+my ($req_file, $req_id, $outdir, $product, $mode, $dbname, $dbserver, $verbose, $save_temps);
 my ($job_id,$rownum); # stuff from the post-update world
 #
@@ -37 +37 @@
         'job_id=s'        =>      \$job_id,
         'rownum=s'        =>      \$rownum,
-        'out_dir=s'       =>      \$out_dir,
+        'outdir=s'       =>      \$outdir,
         'label=s'         =>      \$label,
         'product=s'       =>      \$product,
@@ -52 +52 @@
 if ($mode ne "list_uri") {
     die "req_id is required" if !$req_id;
-    die "out_dir is required" if !$out_dir;
+    die "outdir is required" if !$outdir;
     die "product is required" if !$product;
 }
@@ -80 +80 @@
 # Unless we're running as a job, write the parse arguments in case we need to rerun this parsing.
 if (!$job_id) {
-    my $argslist = "$out_dir/parse.args";
+    my $argslist = "$outdir/parse.args";
     open ARGSLIST, ">$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
-    print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --out_dir $out_dir --file $req_file\n";
+    print ARGSLIST "--label $label --mode $mode --req_id $req_id --product $product --outdir $outdir --file $req_file\n";
     close ARGSLIST;
 }
@@ -105 +105 @@
 
 # Set up the workdir for this query.
-if (! -e $out_dir ) {
-    mkdir $out_dir or my_die("cannot create output directory $out_dir", $PS_EXIT_PROG_ERROR);
-} elsif (! -d $out_dir ) {
-    my_die ("output fileset directory $out_dir exists but is not a directory", $PS_EXIT_PROG_ERROR);
+if (! -e $outdir ) {
+    mkdir $outdir or my_die("cannot create output directory $outdir", $PS_EXIT_PROG_ERROR);
+} elsif (! -d $outdir ) {
+    my_die ("output fileset directory $outdir exists but is not a directory", $PS_EXIT_PROG_ERROR);
 }
 
 
 # Pass along the request file to the response generator.
-my $response_file = "$out_dir/${req_name}.dresponse.${req_id}.fits";
+my $response_file = "$outdir/${req_name}.dresponse.${req_id}.fits";
 my $fault;
 my $data_to_update = '';
 {
-    my $command = "$detectresponse --input $req_file --output $response_file --workdir $out_dir";
+    my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir";
     $command .= " --save-temps" if $save_temps;
     $command .= " --verbose" if $verbose;
@@ -137 +137 @@
 # for the completed work, and move the response to a standardized name.
 if ($fault == 0) {
-    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $out_dir"; 
+    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
     $command .= " -job_type detect_query -state stop -fault 0";
     $command .= " -rownum 1";
@@ -147 +147 @@
         chomp $job_id;
         if ($job_id && -e $response_file) {
-            rename $response_file, "$out_dir/response${job_id}.fits";
+            rename $response_file, "$outdir/response${job_id}.fits";
         }
         $result = 0;
@@ -158 +158 @@
     # Failed to run correctly, which means that we need to queue a job and flag data for updating.
     # Get the dependency id for the data we're requesting be updated.
-    my $dep_id = queue_update_run($req_id,$job_id,$out_dir,$label,$data_to_update);
+    my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
 
     # Link this request to a job and link that job to any dependency
-    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $out_dir"; 
+    my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir"; 
     $command .= " -job_type detect_query -state run -fault 0";
     $command .= " -rownum 1";
@@ -174 +174 @@
         if ($job_id && -e $response_file) {
             # We shouldn't have a response file at this stage.
-            rename $response_file, "$out_dir/response${job_id}.fits";
+            rename $response_file, "$outdir/response${job_id}.fits";
         }
         $result = 0;
@@ -186 +186 @@
 # which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
 {
-    my $command = "$pstamptool -updatereq -req_id $req_id -name $req_name -outProduct $product";
-    $command .= " -fault $result" if $result;
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
+    $command .= " -set_fault $result" if $result;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -201 +201 @@
 # If we have to queue an update run, do so and create a new dependent
 sub queue_update_run {
-    my ($req_id, $job_id, $out_dir, $label, $data_to_update) = @_;
+    my ($req_id, $job_id, $outdir, $label, $data_to_update) = @_;
 
     my ($state, $stage, $stage_id, $component, $need_magic, $imagedb) = split /\s+/, $data_to_update;
@@ -211 +211 @@
     my $dep_id;
     my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component ";
-    $command .= " -outdir $out_dir";
+    $command .= " -outdir $outdir";
     $command .= " -need_magic" if $need_magic;
 

trunk/pstamp/scripts/pstamp_checkdependent.pl

@@ -575 +575 @@
     my ($state, $stage, $stage_id, $job_fault) = @_;
 
-    my $command = "$pstamptool -updatejob -state stop -fault $job_fault -dep_id $dep_id";
+    my $command = "$pstamptool -updatejob -set_state stop -set_fault $job_fault -dep_id $dep_id";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -642 +642 @@
     carp $msg;
 
-    my $command = "$pstamptool -updatedependent -fault $fault -dep_id $dep_id";
+    my $command = "$pstamptool -updatedependent -set_fault $fault -dep_id $dep_id";
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;

trunk/pstamp/scripts/pstamp_dorequest.pl

@@ -106 +106 @@
 {
     ## TODO: what about request status
-    my $command = "$pstamptool -updatereq -req_id $request_id -state stop"; 
+    my $command = "$pstamptool -updatereq -req_id $request_id -set_state stop"; 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbosity);

trunk/pstamp/scripts/pstamp_finish.pl

@@ -25 +25 @@
 use PS::IPP::PStamp::Job qw( :standard );
 
-my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output);
+my ( $req_id, $req_name, $req_file, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output);
 
 # the char to the right of the bar may be used as a single - alias for the longer name
@@ -34 +34 @@
            'req_file=s'     => \$req_file,
            'product=s'      => \$product,
-           'out_dir=s'      => \$out_dir,
+           'outdir=s'      => \$outdir,
            'dbname=s'       => \$dbname,
            'dbserver=s'     => \$dbserver,
@@ -44 +44 @@
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
 
-die "usage: --req_id id --req_name name --req_file file --product product --out_dir output_directory [--dbname dbname --verbose]\n"
-    if !$req_id or !$req_name or !$req_file or !$product or !$out_dir;
+die "usage: --req_id id --req_name name --req_file file --product product --outdir output_directory [--dbname dbname --verbose]\n"
+    if !$req_id or !$req_name or !$req_file or !$product or !$outdir;
 
 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
 if ($redirect_output) {
-    my $logDest = "$out_dir/psfinish.$req_id.log";
+    my $logDest = "$outdir/psfinish.$req_id.log";
     $ipprc->redirect_output($logDest);
 }
@@ -82 +82 @@
     my $fileset = $req_name;
 
-    print STDERR "product: $product  REQ_NAME: $req_name $out_dir\n" if $verbose;
-
-    if (!-e $out_dir) {
+    print STDERR "product: $product  REQ_NAME: $req_name $outdir\n" if $verbose;
+
+    if (!-e $outdir) {
         # something must have gone wrong parsing the request
-        print STDERR  "output directory $out_dir does not exist\n";
-
-        if (!mkdir $out_dir) {
-            print STDERR "cannot create output directory $out_dir";
+        print STDERR  "output directory $outdir does not exist\n";
+
+        if (!mkdir $outdir) {
+            print STDERR "cannot create output directory $outdir";
             stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
         }
 
 
-    } elsif (! -d $out_dir ) {
+    } elsif (! -d $outdir ) {
         # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the
         # request
-        print STDERR "output directory $out_dir exists but is not a directory";
+        print STDERR "output directory $outdir exists but is not a directory";
         stop_request_and_exit($req_id, $PS_EXIT_UNKNOWN_ERROR);
     }
@@ -122 +122 @@
     # the following is for a postage stamp request
 
-    my ($rlf, $reglist_name) = tempfile ("$out_dir/reglist.XXXX", UNLINK => !$save_temps);
+    my ($rlf, $reglist_name) = tempfile ("$outdir/reglist.XXXX", UNLINK => !$save_temps);
     print $rlf "results.fits|||table|\n";
     print $rlf "results.mdc|||text|\n";
 
     my $err_file = "parse_error.txt";
-    if (-e "$out_dir/$err_file" ) {
+    if (-e "$outdir/$err_file" ) {
         print $rlf "$err_file|||text|\n";
     }
 
-    my ($tdf, $table_def_name) = tempfile ("$out_dir/tabledef.XXXX", UNLINK => !$save_temps);
+    my ($tdf, $table_def_name) = tempfile ("$outdir/tabledef.XXXX", UNLINK => !$save_temps);
 
 
@@ -208 +208 @@
 
         if (($job_type eq "stamp") || ($job_type eq "get_image") || ($job_type eq "none")) {
-            my $jreglist = "$out_dir/reglist$job_id";
+            my $jreglist = "$outdir/reglist$job_id";
             if (open JRL, "<$jreglist") {;
                 # process the reglist file to get the list of files produced by this job
@@ -225 +225 @@
                     # If not found check the PHU. If that doesn't work just set them to zero.
                     # XXX do this more cleanly
-                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
+                    my (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields -x 0 RA_DEG DEC_DEG`;
                     if (!defined $ra_deg) {
-                        (undef, $ra_deg, $dec_deg) = split " ", `echo $out_dir/$img_name | fields RA_DEG DEC_DEG`;
+                        (undef, $ra_deg, $dec_deg) = split " ", `echo $outdir/$img_name | fields RA_DEG DEC_DEG`;
                     }
                     $ra_deg = 0.0 if (!$ra_deg);
@@ -258 +258 @@
     # make the results file
     {
-        my $command = "$pstamp_results --input $table_def_name --output $out_dir/results.fits";
+        my $command = "$pstamp_results --input $table_def_name --output $outdir/results.fits";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
             run(command => $command, verbose => $verbose);
@@ -266 +266 @@
         } else {
             # dump a textual representation
-            my $command = "$pstampdump $out_dir/results.fits > $out_dir/results.mdc";
+            my $command = "$pstampdump $outdir/results.fits > $outdir/results.mdc";
             my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
                 run(command => $command, verbose => $verbose);
@@ -278 +278 @@
         # register the fileset
         my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS";
-        $command .= " --link --datapath $out_dir --ps0 $req_id";
-        $command .= " --dbname $dbname" if $dbname;
+        $command .= " --link --datapath $outdir --ps0 $req_id";
+# XXX: let dsreg and config handle resolving dbname and dbserver
+#        $command .= " --dbname $dbname" if $dbname;
 
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -292 +293 @@
     # set the request's state to stop
     {
-        my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $request_fault";
+        my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $request_fault";
         $command   .= " -dbname $dbname" if $dbname;
         $command   .= " -dbserver $dbserver" if $dbserver;
@@ -308 +309 @@
     my $fault  = shift;
 
-    my $command = "$pstamptool -updatereq -req_id $req_id -state stop -fault $fault";
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop -set_fault $fault";
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;

trunk/pstamp/scripts/pstamp_get_image_job.pl

@@ -87 +87 @@
 }
 
-my $out_dir = dirname($output_base);
+my $outdir = dirname($output_base);
 my $prefix = basename($output_base) . "_";
 my $results_file = $output_base . ".bundle_results";
@@ -102 +102 @@
     my $command = "$dist_bundle --camera $camera --stage $stage --stage_id $stage_id";
     $command .= " --component $component";
-    $command .= " --path_base $path_base --outdir $out_dir --results_file $results_file";
+    $command .= " --path_base $path_base --outdir $outdir --results_file $results_file";
     $command .= " --prefix $prefix";
     $command .= " --magicked" if $magicked;
@@ -139 +139 @@
 }
 
-my $reglist = "$out_dir/reglist$job_id";
+my $reglist = "$outdir/reglist$job_id";
 if (! open(REGLIST, ">$reglist") ) {
     my_die("failed to open registration list: $reglist", $PS_EXIT_UNKNOWN_ERROR);

trunk/pstamp/scripts/pstamp_job_run.pl

@@ -198 +198 @@
 # mark the job stopped in the database
 {
-    my $command = "$pstamptool -updatejob -job_id $job_id -state stop"; 
-    $command .= " -fault $jobStatus" if $jobStatus;
+    my $command = "$pstamptool -updatejob -job_id $job_id -set_state stop"; 
+    $command .= " -set_fault $jobStatus" if $jobStatus;
     $command .= " -dbname $dbname" if $dbname;
     $command .= " -dbserver $dbserver" if $dbserver;
@@ -343 +343 @@
         my $command = "$pstamptool -updatejob";
         $command .= " -job_id $job_id";
-        $command .= " -fault $exit_code";
+        $command .= " -set_fault $exit_code";
         # XXX: fix pstamptool to not require -state when -fault with nonzero value is provided
-        $command .= " -state run";
+        $command .= " -set_state run";
         $command .= " -dbname $dbname" if defined $dbname;
         $command .= " -dbserver $dbserver" if defined $dbserver;

trunk/pstamp/scripts/pstamp_parser_run.pl

@@ -28 +28 @@
 my $redirect_output;
 my $product;
+my $outdir;
 my $label;
 my $verbose;
@@ -37 +38 @@
     'uri=s'             =>  \$uri,
     'product=s'         =>  \$product,
+    'outdir=s'          =>  \$outdir,
     'label=s'           =>  \$label,
     'redirect-output'   =>  \$redirect_output,
@@ -81 +83 @@
 }
 
-# workdir is where all of the files generated for this request are placed
-# NOTE: this location needs to be kept in sync with the web interface ( request.php )
-my $datestr = strftime "%Y%m%d", gmtime;
-my $datedir = "$pstamp_workdir/$datestr";
-if (! -e $datedir ) {
-    mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
+if (!$outdir or ($outdir eq "NULL")) {
+    # workdir is where all of the files generated for this request are placed
+    # NOTE: this location needs to be kept in sync with the web interface ( request.php )
+    my $datestr = strftime "%Y%m%d", gmtime;
+    my $datedir = "$pstamp_workdir/$datestr";
+    if (! -e $datedir ) {
+        mkdir $datedir or my_die( "failed to create working directory $datedir for request id $req_id", $req_id,
+            $PS_EXIT_CONFIG_ERROR);
+    }
+
+    $outdir = "$datedir/$req_id";
+}
+
+if (! -e $outdir ) {
+    mkdir $outdir or my_die("failed to create working directory $outdir for request id $req_id", $req_id,
         $PS_EXIT_CONFIG_ERROR);
 }
-
-my $workdir = "$datedir/$req_id";
-if (! -e $workdir ) {
-    mkdir $workdir or my_die("failed to create working directory $workdir for request id $req_id", $req_id,
-        $PS_EXIT_CONFIG_ERROR);
-}
+    
 
 if ($redirect_output) {
-    my $logDest = "$workdir/psparse.$req_id.log";
+    my $logDest = "$outdir/psparse.$req_id.log";
     $ipprc->redirect_output($logDest);
 }
@@ -108 +114 @@
 
 my $fn = basename($uri);
-my $new_uri = "$workdir/$fn";
+my $new_uri = "$outdir/$fn";
 if ($uri =~ /^http:/) {
     # if the uri is an http uri download the file 
@@ -123 +129 @@
     }
     if (! copy $uri, $new_uri) {
-        my_die ("failed to copy request file $uri to workdir $workdir", $req_id, $PS_EXIT_UNKNOWN_ERROR);
+        my_die ("failed to copy request file $uri to workdir $outdir", $req_id, $PS_EXIT_UNKNOWN_ERROR);
     }
 }
@@ -173 +179 @@
     # that they sent us a request file that we don't understand
 
-    my $command = "$pstamptool -updatereq -req_id $req_id -state run";
-    $command   .= " -reqType unknown";
-    $command   .= " -fault $PS_EXIT_DATA_ERROR";
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_state run";
+    $command   .= " -set_reqType unknown";
+    $command   .= " -set_fault $PS_EXIT_DATA_ERROR";
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;
@@ -186 +192 @@
 }
 
-$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --out_dir $workdir --file $uri";
+$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --outdir $outdir --file $uri";
 $parse_cmd .= " --dbname $dbname" if $dbname;
 $parse_cmd .= " --dbserver $dbserver" if $dbserver;
@@ -194 +200 @@
 my $fault;
 {
-    my $error_file_name = "$workdir/parse_error.txt";
+    my $error_file_name = "$outdir/parse_error.txt";
     # get rid of any error file from previous attempt to parse this request
     unlink $error_file_name if (-e $error_file_name);
@@ -227 +233 @@
 #
 {
-    my $command = "$pstamptool -updatereq -req_id $req_id -state $newState";
-    $command   .= " -reqType $reqType" if $reqType;
-    $command   .= " -uri $new_uri" if $new_uri;
-    $command   .= " -fault $fault" if $fault;
+    my $command = "$pstamptool -updatereq -req_id $req_id -set_state $newState";
+    $command   .= " -set_outdir $outdir";
+    $command   .= " -set_reqType $reqType" if $reqType;
+    $command   .= " -set_uri $new_uri" if $new_uri;
+    $command   .= " -set_fault $fault" if $fault;
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;
@@ -269 +276 @@
     carp($msg);
 
-    my $command = "$pstamptool -updatereq -req_id $req_id  -fault $fault";
+    my $command = "$pstamptool -updatereq -req_id $req_id  -set_fault $fault";
     $command   .= " -dbname $dbname" if $dbname;
     $command   .= " -dbserver $dbserver" if $dbserver;

trunk/pstamp/scripts/pstampparse.pl

@@ -25 +25 @@
 my $request_file_name;
 my $mode = "list_uri";
-my $out_dir;
+my $outdir;
 my $product;
 my $label;
@@ -34 +34 @@
     'file=s'    =>  \$request_file_name,
     'req_id=s'  =>  \$req_id,
-    'out_dir=s' =>  \$out_dir,
+    'outdir=s' =>  \$outdir,
     'product=s' =>  \$product,
     'label=s'   =>  \$label,
@@ -50 +50 @@
 if ($mode ne "list_uri") {
     die "req_id is required"   if !$req_id;
-    die "out_dir is required"  if !$out_dir;
+    die "outdir is required"  if !$outdir;
     die "product is required"  if !$product;
 }
@@ -129 +129 @@
     # update the database with the request name. This will be used as the
     # the output data store's product name
-    my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
-    $command .= " -outProduct $product";
+    my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
+    $command .= " -set_outProduct $product";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -522 +522 @@
     $base =~ s/.fits$//;
             
-    my $output_base = "$out_dir/${rownum}_${job_num}_${base}";
+    my $output_base = "$outdir/${rownum}_${job_num}_${base}";
     my $argslist = "${output_base}.args";
 
@@ -712 +712 @@
         my $exp_id = $image->{exp_id};
             
-        my $output_base = "$out_dir/${rownum}_${job_num}";
+        my $output_base = "$outdir/${rownum}_${job_num}";
 
         write_params($output_base, $image);
@@ -894 +894 @@
 
     my $dep_id;
-    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $out_dir";
+    my $command = "$pstamptool -getdependent -stage $stage -stage_id $stage_id -imagedb $imagedb -component $component -outdir $outdir";
     $command .= " -need_magic" if $need_magic;
 

trunk/pstamp/scripts/request_finish.pl

@@ -21 +21 @@
 use PS::IPP::Config qw( :standard );
 
-my ( $req_id, $req_name, $req_file, $req_type, $out_dir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output );
+my ( $req_id, $req_name, $req_file, $req_type, $outdir, $product, $dbname, $dbserver, $verbose, $save_temps, $redirect_output );
 
 GetOptions(
@@ -28 +28 @@
            'req_file=s' => \$req_file,
            'req_type=s' => \$req_type,
-           'out_dir=s'  => \$out_dir,
+           'outdir=s'  => \$outdir,
            'product=s'  => \$product,
            'dbname=s'   => \$dbname,
@@ -46 +46 @@
 $err .= "--req_name is required\n" if !$req_name;
 $err .= "--product is required\n" if !$product;
-# $err .= "--out_dir is required\n" if !$out_dir;
+# $err .= "--outdir is required\n" if !$outdir;
 
 die "$err" if $err;
 
 
-if (!$out_dir) {
-    $out_dir = dirname($req_file);
+if (!$outdir) {
+    $outdir = dirname($req_file);
 }
 
 if ($redirect_output) {
-    my $logDest = "$out_dir/reqfinish.$req_id.log";
+    my $logDest = "$outdir/reqfinish.$req_id.log";
     my $ipprc = PS::IPP::Config->new();
     $ipprc->redirect_output($logDest);
@@ -79 +79 @@
 }
 if ($finish_cmd) {
-    my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product --out_dir $out_dir";
+    my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product --outdir $outdir";
     $command   .= " --dbname $dbname" if $dbname;
     $command   .= " --dbserver $dbserver" if $dbserver;