Changeset 23497
- Timestamp:
- Mar 24, 2009, 10:18:19 AM (17 years ago)
- Location:
- trunk
- Files:
-
- 6 edited
-
ippScripts/scripts/summit_copy.pl (modified) (3 diffs)
-
ippTasks/summit.copy.pro (modified) (6 diffs)
-
ippTools/share/pztool_find_completed_exp.sql (modified) (1 diff)
-
ippTools/src/pztool.c (modified) (11 diffs)
-
ippTools/src/pztool.h (modified) (1 diff)
-
ippTools/src/pztoolConfig.c (modified) (3 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/summit_copy.pl
r23295 r23497 46 46 47 47 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; 48 pod2usage( -msg => "Required options: --uri --filename --exp_name --inst --telescope --class --class_id --workdir",48 pod2usage( -msg => "Required options: --uri --filename --exp_name --inst --telescope --class --class_id", 49 49 -exitval => 3) 50 50 unless defined $uri … … 54 54 and defined $telescope 55 55 and defined $class 56 and defined $class_id 57 and defined $workdir; 56 and defined $class_id; 58 57 59 58 # Look for programs we need … … 107 106 $command .= " -class_id $class_id"; 108 107 $command .= " -uri $filename"; 109 $command .= " -workdir $workdir";108 # $command .= " -workdir $workdir"; 110 109 $command .= " -hostname $host"; 111 $command .= " -end_stage $end_stage" if defined $end_stage;110 # $command .= " -end_stage $end_stage" if defined $end_stage; 112 111 $command .= " -dbname $dbname" if defined $dbname; 113 112 -
trunk/ippTasks/summit.copy.pro
r23232 r23497 27 27 # list of summit imfiles that need to be downloaded 28 28 book init pzPendingImfile 29 # list of pzDownloadExps that have completed downloading 30 book init pzPendingAdvance 29 31 30 32 macro copy.on … … 50 52 active true 51 53 end 54 task summit.toadvance 55 active true 56 end 52 57 task summit.advance 53 58 active true … … 75 80 end 76 81 task pztool.clearfault 82 active false 83 end 84 task summit.toadvance 77 85 active false 78 86 end … … 86 94 $pztoolPendingExp_DB = 0 87 95 $pztoolPendingImfile_DB = 0 96 $pztoolPendingAdvance_DB = 0 88 97 $pztoolClearFault_DB = 0 89 98 $pztoolAdvance_DB = 0; … … 433 442 stderr $LOGDIR/summit.copy.log 434 443 435 $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM -- end_stage reg --workdir $workdir --dbname $DBNAME --timeout 120 --verbose --copies 2444 $run = summit_copy.pl --uri $URI --filename $FILENAME --exp_name $EXP_NAME --inst $CAMERA --telescope $TELESCOPE --class $CLASS --class_id $CLASS_ID --bytes $BYTES --md5 $MD5SUM --dbname $DBNAME --timeout 120 --verbose --copies 2 436 445 if ($COMPRESS) 437 446 $run = $run --compress … … 510 519 end 511 520 512 # promote exposures for which all imfiles have been copied 513 task summit.advance 514 host local 515 516 periods -poll $LOADPOLL 517 periods -exec $LOADEXEC 518 periods -timeout 30 519 npending 1 520 521 stdout NULL 522 stderr $LOGDIR/summit.advance.log 523 524 task.exec 525 $run = pztool -advance 526 if ($DB:n == 0) 527 option DEFAULT 528 else 529 # save the DB name for the exit tasks 530 option $DB:$pztoolAdvance_DB 531 $run = $run -dbname $DB:$pztoolAdvance_DB 532 $pztoolAdvance_DB ++ 533 if ($pztoolAdvance_DB >= $DB:n) set pztoolAdvance_DB = 0 534 end 535 add_poll_args run 536 command $run 537 end 538 539 # success 540 task.exit 0 541 end 542 543 # locked list 544 task.exit default 545 showcommand failure 546 end 547 548 task.exit crash 549 showcommand crash 550 end 551 552 # operation times out? 553 task.exit timeout 554 showcommand timeout 555 end 556 end 557 521 # build a book of exposures that have completed downloading 522 task summit.toadvance 523 host local 524 525 periods -exec 30 526 periods -poll 1 527 periods -timeout 120 528 # trage 16:00 23:59 529 # trage 00:00 04:00 530 npending 1 531 532 # select entries from the current DB; cycle to the next DB, if it exists 533 # iff the DB list is not set, use the value defined in .ipprc 534 task.exec 535 if ($DB:n == 0) 536 option DEFAULT 537 command pztool -toadvance -limit 40 538 else 539 # save the DB name for the exit tasks 540 option $DB:$pztoolPendingAdvance_DB 541 command pztool -toadvance -limit 60 -dbname $DB:$pztoolPendingAdvance_DB 542 $pztoolPendingAdvance_DB ++ 543 if ($pztoolPendingAdvance_DB >= $DB:n) set pztoolPendingAdvance_DB = 0 544 end 545 end 546 547 # success 548 task.exit 0 549 # convert 'stdout' to book format 550 ipptool2book stdout pzPendingAdvance -key exp_name:camera:telescope -uniq -setword dbname $options:0 -setword pantaskState INIT 551 book shuffle pzPendingAdvance 552 553 # delete existing entries in the appropriate pantaskStates 554 process_cleanup pzPendingAdvance 555 end 556 557 task.exit default 558 showcommand failure 559 end 560 task.exit crash 561 showcommand crash 562 end 563 task.exit timeout 564 showcommand timeout 565 end 566 end 567 568 task summit.advance 569 periods -exec 5 570 periods -poll 0.05 571 periods -timeout 650 572 # trage 16:00 23:59 573 # trage 00:00 04:00 574 575 task.exec 576 if ($NETWORK == 0) break 577 578 # if we are waiting on data, make the interval long 579 book npages pzPendingAdvance -var N 580 if ($N == 0) 581 periods -exec 20 582 break 583 end 584 periods -exec 0.05 585 586 # find an exp that needs imfiles fetched 587 book getpage pzPendingAdvance 0 -var pageName -key pantaskState INIT 588 if ("$pageName" == "NULL") break 589 590 # set that exp to run 591 book setword pzPendingAdvance $pageName pantaskState RUN 592 593 book getword pzPendingAdvance $pageName exp_name -var EXP_NAME 594 book getword pzPendingAdvance $pageName camera -var CAMERA 595 book getword pzPendingAdvance $pageName telescope -var TELESCOPE 596 book getword pzPendingAdvance $pageName dbname -var DBNAME 597 598 # 2007-08-30T05:09:59Z 599 substr $DATEOBS 0 4 YEAR 600 substr $DATEOBS 5 2 MONTH 601 substr $DATEOBS 8 2 DAY 602 603 # we need to set the workdir based on 1) nebulous or not? 2) chip/host relationship 604 # this function uses workdir_template, default_host, volume_template, volume_default, 605 # it sets workdir and volume 606 set.workdir.by.camera $CAMERA $CLASS_ID $workdir_template $default_host workdir_base 607 608 $workdir = $workdir_template/$CAMERA/$YEAR\$MONTH\$DAY 609 610 # workdir examples: 611 # file://data/@HOST@.0/gpc1/20080130 612 # neb://@HOST@.0/gpc1/20080130 613 614 stdout $LOGDIR/summit.advance.log 615 stderr $LOGDIR/summit.advance.log 616 617 $run = pztool -advance -exp_name $EXP_NAME -inst $CAMERA -telescope $TELESCOPE -end_stage reg -workdir $workdir -dbname $DBNAME 618 619 # store the pageName for future reference below 620 options $pageName 621 622 # create the command line 623 if ($VERBOSE > 1) 624 echo command $run 625 end 626 command $run 627 end 628 629 # default exit status 630 task.exit default 631 process_exit pzPendingAdvance $options:0 $JOB_STATUS 632 end 633 634 task.exit crash 635 showcommand crash 636 book setword pzPendingAdvance $options:0 pantaskState CRASH 637 end 638 639 # operation timed out? 640 task.exit timeout 641 showcommand timeout 642 book setword pzPendingAdvance $options:0 pantaskState TIMEOUT 643 end 644 end -
trunk/ippTools/share/pztool_find_completed_exp.sql
r17925 r23497 1 1 SELECT DISTINCT 2 exp_name, -- return should match pzDownloadExp2 exp_name, 3 3 camera, 4 4 telescope, 5 state, 6 NULL as epoch -- epoch 5 state 7 6 FROM ( 8 7 SELECT -
trunk/ippTools/src/pztool.c
r21479 r23497 44 44 45 45 static bool clearcommonfaultsMode(pxConfig *config); 46 static bool toadvanceMode(pxConfig *config); 46 47 static bool advanceMode(pxConfig *config); 47 48 48 static bool copydoneCompleteExp(pxConfig *config);49 // static bool copydoneCompleteExp(pxConfig *config); 49 50 static psArray *pzGetPendingCameras(pxConfig *config); 50 51 static psArray *pzArrayZip(psArray *arraySet, psS64 limit); … … 80 81 MODECASE(PZTOOL_MODE_REVERTCOPIED, revertcopiedMode); 81 82 MODECASE(PZTOOL_MODE_CLEARCOMMONFAULTS, clearcommonfaultsMode); 83 MODECASE(PZTOOL_MODE_TOADVANCE, toadvanceMode); 82 84 MODECASE(PZTOOL_MODE_ADVANCE, advanceMode); 83 85 default: … … 403 405 PXOPT_LOOKUP_S16(code, config->args, "-code", false, false); 404 406 405 // NOTE : the rest of the command-line args are parsed in copydoneCompleteExp406 407 407 // start a transaction early so it will contain any row level locks 408 408 if (!psDBTransaction(config->dbh)) { … … 463 463 } 464 464 465 #ifdef notdef466 // we've changed to use -advance instead467 if (!copydoneCompleteExp(config)) {468 // rollback469 if (!psDBRollback(config->dbh)) {470 psError(PS_ERR_UNKNOWN, false, "database error");471 }472 psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed");473 return false;474 }475 #endif476 465 // point of no return 477 466 if (!psDBCommit(config->dbh)) { … … 487 476 } 488 477 489 static bool copydoneCompleteExp(pxConfig *config) 490 { 491 // THIS FUNCTION MUST BE INVOKED FROM INSIDE A TRANSACTION!!! 492 493 PS_ASSERT_PTR_NON_NULL(config, false); 494 495 // XXX this is an ugly hack! 496 // we are passing exp level info to a imfile level mode (-copydone) 497 // these options are thrown away unless we just -copydone'd the last imfile 498 // in an exp. 499 500 // optional 501 PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false); 502 PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); 503 PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); 504 PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); 505 PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); 478 static bool toadvanceMode(pxConfig *config) 479 { 480 PS_ASSERT_PTR_NON_NULL(config, false); 481 482 // optional args 483 PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false); 506 484 PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false); 507 485 … … 517 495 PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "=="); 518 496 PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); 519 PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "=="); 497 PXOPT_COPY_STR(config->args, where, "-inst", "camera", "=="); 498 PXOPT_COPY_STR(config->args, where, "-label", "label", "=="); 520 499 521 500 if (psListLength(where->list)) { … … 525 504 } 526 505 psFree(where); 506 527 507 528 508 // treat limit == 0 as "no limit" … … 552 532 } 553 533 554 for (long i = 0; i < psArrayLength(output); i++) { 555 psMetadata *row = output->data[i]; 556 557 pzDownloadExpRow *doneExp = pzDownloadExpObjectFromMetadata(row); 558 if (!doneExp) { 559 psError(PS_ERR_UNKNOWN, false, "pzDownloadExpObjectFromMetadata() failed"); 560 psFree(doneExp); 561 psFree(output); 562 return false; 563 } 564 565 if (!newExpInsert(config->dbh, 566 0x0, // exp_id 567 doneExp->exp_name, // tmp_exp_name 568 doneExp->camera, // tmp_camera 569 doneExp->telescope, // tmp_telescope 570 "run", // state 571 workdir, // workdir 572 "dirty", // workdir state 573 NULL, // reduction class 574 dvodb, // dvodb 575 tess_id, // tess_id 576 end_stage, // end_stage 577 label, 578 NULL // epoch 579 ) 534 if (!ippdbPrintMetadatas(stdout, output, "toadvance", !simple)) { 535 psError(PS_ERR_UNKNOWN, false, "failed to print array"); 536 psFree(output); 537 return false; 538 } 539 540 psFree(output); 541 542 return true; 543 } 544 545 static bool advanceMode(pxConfig *config) 546 { 547 PS_ASSERT_PTR_NON_NULL(config, false); 548 549 // required 550 PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false); 551 PXOPT_LOOKUP_STR(inst, config->args, "-inst", true, false); 552 PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false); 553 PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); 554 555 // optional 556 PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false); 557 PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false); 558 PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false); 559 PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); 560 561 // start a transaction so it's all rows or nothing 562 if (!psDBTransaction(config->dbh)) { 563 psError(PS_ERR_UNKNOWN, false, "database error"); 564 return false; 565 } 566 567 if (!newExpInsert(config->dbh, 568 0x0, // exp_id 569 exp_name, // tmp_exp_name 570 inst, // tmp_camera 571 telescope, // tmp_telescope 572 "run", // state 573 workdir, // workdir 574 "dirty", // workdir state 575 NULL, // reduction class 576 dvodb, // dvodb 577 tess_id, // tess_id 578 end_stage, // end_stage 579 label, // label 580 NULL // epoch 581 ) 580 582 ) { 581 583 psError(PS_ERR_UNKNOWN, false, "database error"); 582 psFree(doneExp);583 psFree(output);584 584 return false; 585 585 } … … 592 592 "INSERT INTO newImfile" 593 593 " SElECT" 594 " %" PRId64 "," // exp_id594 " %" PRId64 "," // exp_id 595 595 " pzDownloadImfile.class_id," // tmp_class_id 596 " pzDownloadImfile.uri," // uri597 " NULL" // epoch596 " pzDownloadImfile.uri," // uri 597 " NULL" // epoch 598 598 " FROM pzDownloadImfile" 599 599 " WHERE" … … 602 602 " AND pzDownloadImfile.telescope = '%s'"; 603 603 604 if (!p_psDBRunQueryF(config->dbh, query, exp_id, doneExp->exp_name, doneExp->camera, doneExp->telescope)) { 605 psError(PS_ERR_UNKNOWN, false, "database error"); 606 psFree(doneExp); 607 psFree(output); 608 return false; 609 } 610 611 // sanity check: we should have inserted at least one row 612 psU64 affected = psDBAffectedRows(config->dbh); 613 if (psDBAffectedRows(config->dbh) < 1) { 614 psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected); 615 psFree(doneExp); 616 psFree(output); 617 return false; 618 } 619 } 620 621 // set pzDownloadExp.state to 'stop' 622 if (!pzDownloadExpSetState(config, doneExp->exp_name, doneExp->camera, doneExp->telescope, "stop")) { 623 psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %s:%s:%s", doneExp->exp_name, doneExp->camera, doneExp->telescope); 624 psFree(doneExp); 625 psFree(output); 604 if (!p_psDBRunQueryF(config->dbh, query, exp_id, exp_name, inst, telescope)) { 605 psError(PS_ERR_UNKNOWN, false, "database error"); 626 606 return false; 627 } 628 629 psFree(doneExp); 630 } 631 632 psFree(output); 607 } 608 609 // sanity check: we should have inserted at least one row 610 psU64 affected = psDBAffectedRows(config->dbh); 611 if (psDBAffectedRows(config->dbh) < 1) { 612 psError(PS_ERR_UNKNOWN, false, "should have affected at least 1 row but %" PRIu64 " rows were modified", affected); 613 return false; 614 } 615 } 616 617 // set pzDownloadExp.state to 'stop' 618 if (!pzDownloadExpSetState(config, exp_name, inst, telescope, "stop")) { 619 psError(PS_ERR_UNKNOWN, false, "failed to change pzDownloadExp.state for %s:%s:%s", exp_name, inst, telescope); 620 return false; 621 } 622 623 // point of no return 624 if (!psDBCommit(config->dbh)) { 625 // rollback 626 if (!psDBRollback(config->dbh)) { 627 psError(PS_ERR_UNKNOWN, false, "database error"); 628 } 629 psError(PS_ERR_UNKNOWN, false, "database error"); 630 return false; 631 } 632 633 633 634 634 return true; … … 872 872 } 873 873 874 static bool advanceMode(pxConfig *config)875 {876 PS_ASSERT_PTR_NON_NULL(config, false);877 878 // NOTE : the command-line args are parsed in copydoneCompleteExp879 880 // start a transaction so it's all rows or nothing881 if (!psDBTransaction(config->dbh)) {882 psError(PS_ERR_UNKNOWN, false, "database error");883 return false;884 }885 886 if (!copydoneCompleteExp(config)) {887 // rollback888 if (!psDBRollback(config->dbh)) {889 psError(PS_ERR_UNKNOWN, false, "database error");890 }891 psError(PS_ERR_UNKNOWN, false, "copydoneCompleteExp() failed");892 return false;893 }894 895 // point of no return896 if (!psDBCommit(config->dbh)) {897 // rollback898 if (!psDBRollback(config->dbh)) {899 psError(PS_ERR_UNKNOWN, false, "database error");900 }901 psError(PS_ERR_UNKNOWN, false, "database error");902 return false;903 }904 905 906 return true;907 }908 909 874 910 875 static bool pzDownloadExpSetState(pxConfig *config, const char *exp_name, const char *camera, const char *telescope, const char *state) -
trunk/ippTools/src/pztool.h
r18314 r23497 35 35 PZTOOL_MODE_REVERTCOPIED, 36 36 PZTOOL_MODE_CLEARCOMMONFAULTS, 37 PZTOOL_MODE_TOADVANCE, 37 38 PZTOOL_MODE_ADVANCE 38 39 } pztoolMode; -
trunk/ippTools/src/pztoolConfig.c
r21479 r23497 97 97 psMetadataAddS16(copydoneArgs, PS_LIST_TAIL, "-code", 0, "set fault code", 0); 98 98 psMetadataAddBool(copydoneArgs, PS_LIST_TAIL, "-row_lock", 0, "lock pzDownImfile rows while advancing an exposure", false); 99 // XXX: remove this once advance is fixed 100 psMetadataAddU64(copydoneArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); 99 101 100 102 // -copied … … 129 131 // -clearcommonfaults 130 132 psMetadata *clearcommonfaultsArgs = psMetadataAlloc(); 133 // 134 // -toadvance 135 psMetadata *toadvanceArgs = psMetadataAlloc(); 136 psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); 137 psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); 138 psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); 139 psMetadataAddStr(toadvanceArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); 140 psMetadataAddBool(toadvanceArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false); 141 psMetadataAddU64(toadvanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); 131 142 132 143 // -advance 133 144 psMetadata *advanceArgs = psMetadataAlloc(); 134 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID", NULL); 135 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID", NULL); 136 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID", NULL); 137 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-workdir", 0, "define the \"default\" workdir for this exposure", NULL); 138 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); 139 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); 140 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); 141 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); 142 psMetadataAddU64(advanceArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0); 145 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-exp_name", 0, "define exposure ID (required)", NULL); 146 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-inst", 0, "define camera ID (required)", NULL); 147 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-telescope", 0, "define telescope ID (required)", NULL); 148 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-workdir", 0, "define the \"default\" workdir for this exposure (required)", NULL); 149 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-dvodb", 0, "define the dvodb for the next processing step", NULL); 150 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-tess_id", 0, "define the tess_id for the next processing step", NULL); 151 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-end_stage", 0, "define the end goal processing step", NULL); 152 psMetadataAddStr(advanceArgs, PS_LIST_TAIL, "-label", 0, "define the label for the chip stage", NULL); 143 153 144 154 psMetadata *argSets = psMetadataAlloc(); … … 155 165 PXOPT_ADD_MODE("-revertcopied", "", PZTOOL_MODE_REVERTCOPIED,revertcopiedArgs); 156 166 PXOPT_ADD_MODE("-clearcommonfaults","", PZTOOL_MODE_CLEARCOMMONFAULTS,clearcommonfaultsArgs); 167 PXOPT_ADD_MODE("-toadvance", "", PZTOOL_MODE_TOADVANCE, toadvanceArgs); 157 168 PXOPT_ADD_MODE("-advance", "", PZTOOL_MODE_ADVANCE, advanceArgs); 158 169
Note:
See TracChangeset
for help on using the changeset viewer.
