Index: /trunk/ippScripts/scripts/summit_copy.pl =================================================================== --- /trunk/ippScripts/scripts/summit_copy.pl (revision 23496) +++ /trunk/ippScripts/scripts/summit_copy.pl (revision 23497) @@ -46,5 +46,5 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --uri --filename --exp_name --inst --telescope --class --class_id --workdir", +pod2usage( -msg => "Required options: --uri --filename --exp_name --inst --telescope --class --class_id", -exitval => 3) unless defined $uri @@ -54,6 +54,5 @@ and defined $telescope and defined $class - and defined $class_id - and defined $workdir; + and defined $class_id; # Look for programs we need @@ -107,7 +106,7 @@ $command .= " -class_id $class_id"; $command .= " -uri $filename"; -$command .= " -workdir $workdir"; +# $command .= " -workdir $workdir"; $command .= " -hostname $host"; -$command .= " -end_stage $end_stage" if defined $end_stage; +# $command .= " -end_stage $end_stage" if defined $end_stage; $command .= " -dbname $dbname" if defined $dbname; Index: /trunk/ippTasks/summit.copy.pro =================================================================== --- /trunk/ippTasks/summit.copy.pro (revision 23496) +++ /trunk/ippTasks/summit.copy.pro (revision 23497) @@ -27,4 +27,6 @@ # list of summit imfiles that need to be downloaded book init pzPendingImfile +# list of pzDownloadExps that have completed downloading +book init pzPendingAdvance macro copy.on @@ -50,4 +52,7 @@ active true end + task summit.toadvance + active true + end task summit.advance active true @@ -75,4 +80,7 @@ end task pztool.clearfault + active false + end + task summit.toadvance active false end @@ -86,4 +94,5 @@ $pztoolPendingExp_DB = 0 $pztoolPendingImfile_DB = 0 +$pztoolPendingAdvance_DB = 0 $pztoolClearFault_DB = 0 $pztoolAdvance_DB = 0; @@ -433,5 +442,5 @@ stderr $LOGDIR/summit.copy.log - $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --end_stage reg --workdir $workdir --dbname $DBNAME --timeout 120 --verbose --copies 2 + $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --dbname $DBNAME --timeout 120 --verbose --copies 2 if ($COMPRESS) $run = $run --compress @@ -510,48 +519,126 @@ end -# promote exposures for which all imfiles have been copied -task summit.advance - host local - - periods -poll $LOADPOLL - periods -exec $LOADEXEC - periods -timeout 30 - npending 1 - - stdout NULL - stderr $LOGDIR/summit.advance.log - - task.exec - $run = pztool -advance - if ($DB:n == 0) - option DEFAULT - else - # save the DB name for the exit tasks - option $DB:$pztoolAdvance_DB - $run = $run -dbname $DB:$pztoolAdvance_DB - $pztoolAdvance_DB ++ - if ($pztoolAdvance_DB >= $DB:n) set pztoolAdvance_DB = 0 - end - add_poll_args run - command $run - end - - # success - task.exit 0 - end - - # locked list - task.exit default - showcommand failure - end - - task.exit crash - showcommand crash - end - - # operation times out? - task.exit timeout - showcommand timeout - end -end - +# build a book of exposures that have completed downloading +task summit.toadvance + host local + + periods -exec 30 + periods -poll 1 + periods -timeout 120 + # trage 16:00 23:59 + # trage 00:00 04:00 + npending 1 + + # select entries from the current DB; cycle to the next DB, if it exists + # iff the DB list is not set, use the value defined in .ipprc + task.exec + if ($DB:n == 0) + option DEFAULT + command pztool -toadvance -limit 40 + else + # save the DB name for the exit tasks + option $DB:$pztoolPendingAdvance_DB + command pztool -toadvance -limit 60 -dbname $DB:$pztoolPendingAdvance_DB + $pztoolPendingAdvance_DB ++ + if ($pztoolPendingAdvance_DB >= $DB:n) set pztoolPendingAdvance_DB = 0 + end + end + + # success + task.exit 0 + # convert 'stdout' to book format + ipptool2book stdout pzPendingAdvance -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT + book shuffle pzPendingAdvance + + # delete existing entries in the appropriate pantaskStates + process_cleanup pzPendingAdvance + end + + task.exit default + showcommand failure + end + task.exit crash + showcommand crash + end + task.exit timeout + showcommand timeout + end +end + +task summit.advance + periods -exec 5 + periods -poll 0.05 + periods -timeout 650 + # trage 16:00 23:59 + # trage 00:00 04:00 + + task.exec + if ($NETWORK == 0) break + + # if we are waiting on data, make the interval long + book npages pzPendingAdvance -var N + if ($N == 0) + periods -exec 20 + break + end + periods -exec 0.05 + + # find an exp that needs imfiles fetched + book getpage pzPendingAdvance 0 -var pageName -key pantaskState INIT + if ("$pageName" == "NULL") break + + # set that exp to run + book setword pzPendingAdvance $pageName pantaskState RUN + + book getword pzPendingAdvance $pageName exp_name -var EXP_NAME + book getword pzPendingAdvance $pageName camera -var CAMERA + book getword pzPendingAdvance $pageName telescope -var TELESCOPE + book getword pzPendingAdvance $pageName dbname -var DBNAME + + # 2007-08-30T05:09:59Z + substr $DATEOBS 0 4 YEAR + substr $DATEOBS 5 2 MONTH + substr $DATEOBS 8 2 DAY + + # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship + # this function uses workdir_template, default_host, volume_template, volume_default, + # it sets workdir and volume + set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base + + $workdir = $workdir_template/$CAMERA/$YEAR\$MONTH\$DAY + + # workdir examples: + # file://data/@HOST@.0/gpc1/20080130 + # neb://@HOST@.0/gpc1/20080130 + + stdout $LOGDIR/summit.advance.log + stderr $LOGDIR/summit.advance.log + + $run = pztool -advance -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -end_stage reg -workdir $workdir -dbname $DBNAME + + # store the pageName for future reference below + options $pageName + + # create the command line + if ($VERBOSE > 1) + echo command $run + end + command $run + end + + # default exit status + task.exit default + process_exit pzPendingAdvance $options:0 $JOB_STATUS + end + + task.exit crash + showcommand crash + book setword pzPendingAdvance $options:0 pantaskState CRASH + end + + # operation timed out? + task.exit timeout + showcommand timeout + book setword pzPendingAdvance $options:0 pantaskState TIMEOUT + end +end Index: /trunk/ippTools/share/pztool_find_completed_exp.sql =================================================================== --- /trunk/ippTools/share/pztool_find_completed_exp.sql (revision 23496) +++ /trunk/ippTools/share/pztool_find_completed_exp.sql (revision 23497) @@ -1,8 +1,7 @@ SELECT DISTINCT - exp_name, -- return should match pzDownloadExp + exp_name, camera, telescope, - state, - NULL as epoch -- epoch + state FROM ( SELECT Index: /trunk/ippTools/src/pztool.c =================================================================== --- /trunk/ippTools/src/pztool.c (revision 23496) +++ /trunk/ippTools/src/pztool.c (revision 23497) @@ -44,7 +44,8 @@ static bool clearcommonfaultsMode(pxConfig *config); +static bool toadvanceMode(pxConfig *config); static bool advanceMode(pxConfig *config); -static bool copydoneCompleteExp(pxConfig *config); +// static bool copydoneCompleteExp(pxConfig *config); static psArray *pzGetPendingCameras(pxConfig *config); static psArray *pzArrayZip(psArray *arraySet, psS64 limit); @@ -80,4 +81,5 @@ MODECASE(PZTOOL_MODE_REVERTCOPIED, revertcopiedMode); MODECASE(PZTOOL_MODE_CLEARCOMMONFAULTS, clearcommonfaultsMode); + MODECASE(PZTOOL_MODE_TOADVANCE, toadvanceMode); MODECASE(PZTOOL_MODE_ADVANCE, advanceMode); default: @@ -403,6 +405,4 @@ PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); - // NOTE : the rest of the command-line args are parsed in copydoneCompleteExp - // start a transaction early so it will contain any row level locks if (!psDBTransaction(config->dbh)) { @@ -463,15 +463,4 @@ } -#ifdef notdef - // we've changed to use -advance instead - if (!copydoneCompleteExp(config)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed"); - return false; - } -#endif // point of no return if (!psDBCommit(config->dbh)) { @@ -487,21 +476,10 @@ } -static bool copydoneCompleteExp(pxConfig *config) -{ - // THIS FUNCTION MUST BE INVOKED FROM INSIDE A TRANSACTION!!! - - PS_ASSERT_PTR_NON_NULL(config, false); - - // XXX this is an ugly hack! - // we are passing exp level info to a imfile level mode (-copydone) - // these options are thrown away unless we just -copydone'd the last imfile - // in an exp. - - // optional - PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); - PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); - PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); - PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); - PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); +static bool toadvanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // optional args + PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); @@ -517,5 +495,6 @@ PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); - PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); + PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); + PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); if (psListLength(where->list)) { @@ -525,4 +504,5 @@ } psFree(where); + // treat limit == 0 as "no limit" @@ -552,34 +532,54 @@ } - for (long i = 0; i < psArrayLength(output); i++) { - psMetadata *row = output->data[i]; - - pzDownloadExpRow *doneExp = pzDownloadExpObjectFromMetadata(row); - if (!doneExp) { - psError(PS_ERR_UNKNOWN, false, "pzDownloadExpObjectFromMetadata() failed"); - psFree(doneExp); - psFree(output); - return false; - } - - if (!newExpInsert(config->dbh, - 0x0, // exp_id - doneExp->exp_name, // tmp_exp_name - doneExp->camera, // tmp_camera - doneExp->telescope, // tmp_telescope - "run", // state - workdir, // workdir - "dirty", // workdir state - NULL, // reduction class - dvodb, // dvodb - tess_id, // tess_id - end_stage, // end_stage - label, - NULL // epoch - ) + if (!ippdbPrintMetadatas(stdout, output, "toadvance", !simple)) { + psError(PS_ERR_UNKNOWN, false, "failed to print array"); + psFree(output); + return false; + } + + psFree(output); + + return true; +} + +static bool advanceMode(pxConfig *config) +{ + PS_ASSERT_PTR_NON_NULL(config, false); + + // required + PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); + PXOPT_LOOKUP_STR(inst, config->args, "-inst", true, false); + PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); + PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); + + // optional + PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); + PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); + PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); + PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); + + // start a transaction so it's all rows or nothing + if (!psDBTransaction(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + + if (!newExpInsert(config->dbh, + 0x0, // exp_id + exp_name, // tmp_exp_name + inst, // tmp_camera + telescope, // tmp_telescope + "run", // state + workdir, // workdir + "dirty", // workdir state + NULL, // reduction class + dvodb, // dvodb + tess_id, // tess_id + end_stage, // end_stage + label, // label + NULL // epoch + ) ) { psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(doneExp); - psFree(output); return false; } @@ -592,8 +592,8 @@ "INSERT INTO newImfile" " SElECT" - " %" PRId64 "," // exp_id + " %" PRId64 "," // exp_id " pzDownloadImfile.class_id," // tmp_class_id - " pzDownloadImfile.uri," // uri - " NULL" // epoch + " pzDownloadImfile.uri," // uri + " NULL" // epoch " FROM pzDownloadImfile" " WHERE" @@ -602,33 +602,33 @@ " AND pzDownloadImfile.telescope = '%s'"; - if (!p_psDBRunQueryF(config->dbh, query, exp_id, doneExp->exp_name, doneExp->camera, doneExp->telescope)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - psFree(doneExp); - psFree(output); - return false; - } - - // sanity check: we should have inserted at least one row - psU64 affected = psDBAffectedRows(config->dbh); - if (psDBAffectedRows(config->dbh) < 1) { - psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected); - psFree(doneExp); - psFree(output); - return false; - } - } - - // set pzDownloadExp.state to 'stop' - if (!pzDownloadExpSetState(config, doneExp->exp_name, doneExp->camera, doneExp->telescope, "stop")) { - psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %s:%s:%s", doneExp->exp_name, doneExp->camera, doneExp->telescope); - psFree(doneExp); - psFree(output); + if (!p_psDBRunQueryF(config->dbh, query, exp_id, exp_name, inst, telescope)) { + psError(PS_ERR_UNKNOWN, false, "database error"); return false; - } - - psFree(doneExp); - } - - psFree(output); + } + + // sanity check: we should have inserted at least one row + psU64 affected = psDBAffectedRows(config->dbh); + if (psDBAffectedRows(config->dbh) < 1) { + psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected); + return false; + } + } + + // set pzDownloadExp.state to 'stop' + if (!pzDownloadExpSetState(config, exp_name, inst, telescope, "stop")) { + psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %s:%s:%s", exp_name, inst, telescope); + return false; + } + + // point of no return + if (!psDBCommit(config->dbh)) { + // rollback + if (!psDBRollback(config->dbh)) { + psError(PS_ERR_UNKNOWN, false, "database error"); + } + psError(PS_ERR_UNKNOWN, false, "database error"); + return false; + } + return true; @@ -872,39 +872,4 @@ } -static bool advanceMode(pxConfig *config) -{ - PS_ASSERT_PTR_NON_NULL(config, false); - - // NOTE : the command-line args are parsed in copydoneCompleteExp - - // start a transaction so it's all rows or nothing - if (!psDBTransaction(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - if (!copydoneCompleteExp(config)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed"); - return false; - } - - // point of no return - if (!psDBCommit(config->dbh)) { - // rollback - if (!psDBRollback(config->dbh)) { - psError(PS_ERR_UNKNOWN, false, "database error"); - } - psError(PS_ERR_UNKNOWN, false, "database error"); - return false; - } - - - return true; -} - static bool pzDownloadExpSetState(pxConfig *config, const char *exp_name, const char *camera, const char *telescope, const char *state) Index: /trunk/ippTools/src/pztool.h =================================================================== --- /trunk/ippTools/src/pztool.h (revision 23496) +++ /trunk/ippTools/src/pztool.h (revision 23497) @@ -35,4 +35,5 @@ PZTOOL_MODE_REVERTCOPIED, PZTOOL_MODE_CLEARCOMMONFAULTS, + PZTOOL_MODE_TOADVANCE, PZTOOL_MODE_ADVANCE } pztoolMode; Index: /trunk/ippTools/src/pztoolConfig.c =================================================================== --- /trunk/ippTools/src/pztoolConfig.c (revision 23496) +++ /trunk/ippTools/src/pztoolConfig.c (revision 23497) @@ -97,4 +97,6 @@ psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0, "lock pzDownImfile rows while advancing an exposure", false); + // XXX: remove this once advance is fixed + psMetadataAddU64(copydoneArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -copied @@ -129,16 +131,24 @@ // -clearcommonfaults psMetadata *clearcommonfaultsArgs = psMetadataAlloc(); + // + // -toadvance + psMetadata *toadvanceArgs = psMetadataAlloc(); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); + psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); + psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); + psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); // -advance psMetadata *advanceArgs = psMetadataAlloc(); - psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); - psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); - psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); - psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-workdir", 0, "define the \"default\" workdir for this exposure", NULL); - psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); - psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); - psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); - psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); - psMetadataAddU64(advanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID (required)", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-workdir", 0, "define the \"default\" workdir for this exposure (required)", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); + psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); psMetadata *argSets = psMetadataAlloc(); @@ -155,4 +165,5 @@ PXOPT_ADD_MODE("-revertcopied", "", PZTOOL_MODE_REVERTCOPIED,revertcopiedArgs); PXOPT_ADD_MODE("-clearcommonfaults","", PZTOOL_MODE_CLEARCOMMONFAULTS,clearcommonfaultsArgs); + PXOPT_ADD_MODE("-toadvance", "", PZTOOL_MODE_TOADVANCE, toadvanceArgs); PXOPT_ADD_MODE("-advance", "", PZTOOL_MODE_ADVANCE, advanceArgs);