Changeset 28169
- Timestamp:
- May 28, 2010, 5:40:05 PM (16 years ago)
- Location:
- branches/haf_branches/ipp.20100512/ippTasks
- Files:
-
- 2 edited
-
addstar.pro (modified) (1 diff)
-
minidvodb.pro (modified) (17 diffs)
Legend:
- Unmodified
- Added
- Removed
-
branches/haf_branches/ipp.20100512/ippTasks/addstar.pro
r28060 r28169 254 254 end 255 255 end 256 257 task minidvodb.create258 host local259 260 periods -poll 5.0261 periods -exec 60.0262 periods -timeout 1200263 npending 1264 active false265 266 stdout NULL267 stderr $LOGDIR/revert.log268 269 task.exec270 if ($LABEL:n == 0) break271 $run = addtool -revertprocessedexp272 if ($DB:n == 0)273 option DEFAULT274 else275 # save the DB name for the exit tasks276 option $DB:$addstar_revert_DB277 $run = $run -dbname $DB:$addstar_revert_DB278 $addstar_revert_DB ++279 if ($addstar_revert_DB >= $DB:n) set addstar_revert_DB = 0280 end281 add_poll_labels run282 command $run283 end284 285 # success286 task.exit 0287 end288 289 # locked list290 task.exit default291 showcommand failure292 end293 294 task.exit crash295 showcommand crash296 end297 298 # operation times out?299 task.exit timeout300 showcommand timeout301 end302 end303 -
branches/haf_branches/ipp.20100512/ippTasks/minidvodb.pro
r28101 r28169 6 6 check.globals 7 7 8 9 #There is a book for each task, because I don't use labels. 10 8 11 if (not($?haveBooks)) 9 12 book create MINIDVODB 13 book create MINIDVODB_MERGE 14 book create MINIDVODB_CREATE 15 book create MINIDVODB_ACTIVE 10 16 $haveBooks = TRUE 11 17 end 12 if (not($?haveSurveyBooks)) 13 book create SURVEY_MINIDVODB 14 $haveSurveyBooks = TRUE 15 end 18 16 19 $MINIDVODB_DB = 0 17 20 18 21 $MINIDVODB_WAIT_DB = 0 22 $MINIDVODB_MERGE_DB = 0 23 $MINIDVODB_CREATE_DB = 0 24 $MINIDVODB_ACTIVE_DB = 0 19 25 20 26 book init minidvodbWaitlist 27 book init minidvodbMergelist 28 book init minidvodbCreatelist 29 book init minidvodbActivelist 30 31 macro minidvodb.create.status 32 book listbook minidvodbCreatelist 33 end 34 35 macro minidvodb.create.reset 36 book init minidvodbCreatelist 37 end 38 39 macro minidvodb.active.status 40 book listbook minidvodbActivelist 41 end 42 43 macro minidvodb.active.reset 44 book init minidvodbActivelist 45 end 46 21 47 22 48 macro minidvodb.wait.status … … 28 54 end 29 55 30 31 56 macro minidvodb.merge.status 57 book listbook minidvodbMergelist 58 end 59 60 macro minidvodb.merge.reset 61 book init minidvodbMergelist 62 end 63 64 #this is the create task, there is no load, it just runs 32 65 macro minidvodb.create.on 33 task minidvodbcreate66 task minidvodb.create 34 67 active true 35 68 end 36 69 end 37 70 macro minidvodb.create.off 38 task minidvodbcreate71 task minidvodb.create 39 72 active false 40 73 end 41 74 end 42 75 ##these are the tasks that flip from new -> active 76 macro minidvodb.active.on 77 task minidvodb.active.load 78 active true 79 end 80 task minidvodb.active.run 81 active true 82 end 83 end 84 macro minidvodb.active.off 85 task minidvodb.active.load 86 active false 87 end 88 task minidvodb.active.run 89 active false 90 end 91 end 43 92 44 93 ##these are the tasks that check to see if addRun processing is finished … … 61 110 end 62 111 63 64 ## at some point, I need to add/separate the ones for the survey task 112 ##these merge the dbs 113 114 macro minidvodb.merge.on 115 task minidvodb.merge.load 116 active true 117 end 118 task minidvodb.merge.run 119 active true 120 end 121 end 122 macro minidvodb.merge.off 123 task minidvodb.merge.load 124 active false 125 end 126 task minidvodb.merge.run 127 active false 128 end 129 end 130 131 ## you get no choice - you add all of them in at the same time. you can always turn off the tasks you don't want to run. 65 132 macro add.minidvodb 66 if ($0 != 3)67 echo "USAGE: add.minidvodb (minidvodb_group) (dvodb) "133 if ($0 != 5) 134 echo "USAGE: add.minidvodb (minidvodb_group) (dvodb) (interval days) (camera)" 68 135 break 69 136 end 137 138 #wait - shoudl be renamed MINIDVODB_WAIT 70 139 book newpage MINIDVODB $1 71 140 book setword MINIDVODB $1 MINIDVODB_GROUP $1 72 141 book setword MINIDVODB $1 DVODB $2 73 142 book setword MINIDVODB $1 STATE PENDING 143 #merge 144 book newpage MINIDVODB_MERGE $1 145 book setword MINIDVODB_MERGE $1 MINIDVODB_GROUP $1 146 book setword MINIDVODB_MERGE $1 DVODB $2 147 book setword MINIDVODB_MERGE $1 STATE PENDING 148 #active 149 book newpage MINIDVODB_ACTIVE $1 150 book setword MINIDVODB_ACTIVE $1 MINIDVODB_GROUP $1 151 book setword MINIDVODB_ACTIVE $1 DVODB $2 152 book setword MINIDVODB_ACTIVE $1 STATE PENDING 153 154 #create note that camera should be GPC1 for it to work. I couldn't figure out how to easily get this out. 155 book newpage MINIDVODB_CREATE $1 156 book setword MINIDVODB_CREATE $1 MINIDVODB_GROUP $1 157 book setword MINIDVODB_CREATE $1 DVODB $2 158 book setword MINIDVODB_CREATE $1 DVODB_DAYS $3 159 book setword MINIDVODB_CREATE $1 CAMERA $4 160 book setword MINIDVODB_CREATE $1 STATE PENDING 74 161 end 75 162 … … 80 167 end 81 168 book delpage MINIDVODB $1 169 book delpage MINIDVODB_MERGE $1 170 book delpage MINIDVODB_CREATE $1 171 book delpage MINIDVODB_ACTIVE $1 82 172 end 83 173 … … 87 177 break 88 178 end 179 echo "minidvodb wait" 89 180 book listbook MINIDVODB 90 end 181 echo "minidvodb merge" 182 book listbook MINIDVODB_MERGE 183 echo "minidvodb create" 184 book listbook MINIDVODB_CREATE 185 echo "minidvodb_active" 186 book listbook MINIDVODB_ACTIVE 187 end 188 189 190 191 192 193 91 194 92 195 … … 100 203 101 204 stdout NULL 102 stderr $LOGDIR/minidvodb.wait. run.log205 stderr $LOGDIR/minidvodb.wait.load.log 103 206 104 207 task.exec … … 123 226 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 124 227 end 125 126 127 128 129 228 #using check as opposed to list because it sees if it is done with the addRun state yet. 130 229 $run = addtool -checkminidvodbrunaddrun -state waiting 131 230 $run = $run -minidvodb_group $minidvodb_group … … 172 271 173 272 174 # run the chip_imfile.pl script on pending images 273 175 274 task minidvodb.wait.run 176 275 periods -poll $RUNPOLL … … 184 283 book npages minidvodbWaitlist -var N 185 284 if ($N == 0) break 186 if ($NETWORK == 0) break 187 if ($BURNTOOLING == 1) break 188 285 189 286 # look for new images in minidvodbWaitlist (pantaskState == INIT) 190 287 book getpage minidvodbWaitlist 0 -var pageName -key pantaskState INIT … … 196 293 stdout $LOGDIR/minidvodb.wait.run.log 197 294 stderr $LOGDIR/minidvodb.wait.run.log 198 295 199 296 $run = addtool -updateminidvodbrun -minidvodb_id $MINIDVODB_ID -set_state to_be_merged 200 297 … … 209 306 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 210 307 end 211 # save the pageName for future reference below308 # save the pageName for future reference below 212 309 options $pageName 213 214 215 310 216 311 # create the command line … … 222 317 command $run 223 318 end 224 # default exit status319 # default exit status 225 320 task.exit default 226 321 process_exit minidvodbWaitlist $options:0 $JOB_STATUS … … 229 324 # locked list 230 325 task.exit crash 231 ### Getting a lot of chip crashes (no idea why), so remove verbosity for now232 #showcommand crash233 326 echo "hostname: $JOB_HOSTNAME" 234 235 # Set a fault code in the database236 237 327 process_exit minidvodbWaitlist $options:0 $EXIT_CRASH_ERR 238 328 end … … 247 337 248 338 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 task minidvodbcreate 269 host local 270 271 periods -poll 10 272 periods -exec 12000000000000 339 task minidvodb.merge.load 340 host local 341 342 periods -poll $LOADPOLL 343 periods -exec $LOADEXEC 273 344 periods -timeout 30 274 275 npending 1 276 task.exec 277 active false 278 279 stdout $LOGDIR/minidvodb.create.log 280 stderr $LOGDIR/minidvodb.create.log 281 282 # generate diff warp-warp runs 283 #task.exec 284 book npages MINIDVODB -var N 345 npending 1 346 347 stdout NULL 348 stderr $LOGDIR/minidvodb.merge.load.log 349 350 task.exec 351 book npages MINIDVODB_MERGE -var N 285 352 if ($N == 0) 286 # echo "No minidvodbs for processing"353 # echo "No labels for processing" 287 354 break 288 355 endif 289 356 290 book getpage MINIDVODB 0 -var minidvodb-key STATE NEW291 if ("$minidvodb " == "NULL")357 book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW 358 if ("$minidvodb_group" == "NULL") 292 359 # All labels have been done --- reset 293 360 # echo "Resetting labels" 294 361 for i 0 $N 295 book getpage MINIDVODB $i -var label296 book setword MINIDVODB $minidvodbSTATE NEW362 book getpage MINIDVODB_MERGE $i -var minidvodb_group 363 book setword MINIDVODB_MERGE $minidvodb_group STATE NEW 297 364 end 298 book getpage MINIDVODB 0 -var minidvodb-key STATE NEW365 book getpage MINIDVODB_MERGE 0 -var minidvodb_group -key STATE NEW 299 366 300 367 # Select different database … … 302 369 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 303 370 end 304 305 book setword MINIDVODB $minidvodb STATE DONE 306 book getword MINIDVODB $minidvodb DVODB -var dvodb 307 308 $run = minidvodb_createdb.pl 309 $run = $run --camera GPC1 310 $run = $run --dvodb $dvodb 311 $run = $run --minidvodb_group $minidvodb_group 312 $run = $run --outroot $dvodb 313 $run = $run --redirect-output 371 #finds the minidvodbs in a state of 'to_be_merged' 372 $run = addtool -listminidvodbrun -state to_be_merged 373 $run = $run -minidvodb_group $minidvodb_group 314 374 if ($DB:n == 0) 315 375 option DEFAULT 316 376 else 317 $run = $run --dbname $DB:$MINIDVODB_DB377 # save the DB name for the exit tasks 318 378 option $DB:$MINIDVODB_DB 319 end 320 321 # echo $run 379 $run = $run -dbname $DB:$MINIDVODB_DB 380 $MINIDVODB_DB ++ 381 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 382 end 383 add_poll_args run 384 322 385 command $run 323 386 end … … 325 388 # success 326 389 task.exit 0 327 # echo "Success" 328 end 390 # convert 'stdout' to book format 391 ipptool2book stdout minidvodbMergelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT 392 if ($VERBOSE > 2) 393 book listbook minidvodbMergelist 394 end 395 # delete existing entries in the appropriate pantaskStates 396 process_cleanup minidvodbMergelist 397 end 329 398 330 399 # locked list … … 344 413 345 414 415 416 417 task minidvodb.merge.run 418 periods -poll $RUNPOLL 419 periods -exec $RUNEXEC 420 periods -timeout 60 421 422 task.exec 423 # if we are unable to run the 'exec', use a long retry time 424 periods -exec $RUNEXEC 425 426 book npages minidvodbMergelist -var N 427 if ($N == 0) break 428 429 # look for new images in minidvodbWaitlist (pantaskState == INIT) 430 book getpage minidvodbMergelist 0 -var pageName -key pantaskState INIT 431 if ("$pageName" == "NULL") break 432 433 book setword minidvodbMergelist $pageName pantaskState RUN 434 book getword minidvodbMergelist $pageName minidvodb_id -var MINIDVODB_ID 435 book getword minidvodbMergelist $pageName state -var STATE 436 stdout $LOGDIR/minidvodb.merge.run.log 437 stderr $LOGDIR/minidvodb.merge.run.log 438 439 $run = addtool -updateminidvodbrun -minidvodb_id $MINIDVODB_ID -set_state merging 440 441 if ($DB:n == 0) 442 option DEFAULT 443 else 444 # save the DB name for the exit tasks 445 option $DB:$MINIDVODB_DB 446 $run = $run -dbname $DB:$MINIDVODB_DB 447 $MINIDVODB_DB ++ 448 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 449 end 450 # save the pageName for future reference below 451 options $pageName 452 453 # create the command line 454 if ($VERBOSE > 1) 455 echo command $run 456 end 457 # if we are unable to run the 'exec', use a long retry time 458 periods -exec 0.05 459 command $run 460 end 461 # default exit status 462 task.exit default 463 process_exit minidvodbMergelist $options:0 $JOB_STATUS 464 end 465 466 # locked list 467 task.exit crash 468 469 echo "hostname: $JOB_HOSTNAME" 470 471 # Set a fault code in the database 472 473 process_exit minidvodbMergelist $options:0 $EXIT_CRASH_ERR 474 end 475 476 # operation timed out? 477 task.exit timeout 478 showcommand timeout 479 book setword minidvodbMergelist $options:0 pantaskState TIMEOUT 480 end 481 end 482 483 484 #this is the complicated script - it creates the minidvodbRuns 485 #since none may exist by default, I decided not to have a load.task 486 #it calls a perl script, which creates a new one if: 487 #there is none in an active state (true if there is none at all) 488 #the current one is older than the dvodb_age 489 #there are none in new (we have a task that moves it to 'active') 490 #the current active one has > 30000 add_ids in it 491 492 #also confusing: it succeeds if it doesn't create it (if it doesn't want to create one) 493 #and it succeeds if it creates one. (otherwise there would be a lot of false failures, 494 # like in replication) 495 #it fails if it has problems with one of the addtool commands in the script. 496 497 task minidvodb.create 498 host local 499 500 periods -poll $LOADPOLL 501 periods -exec $LOADEXEC 502 periods -timeout 240 503 npending 1 504 505 stdout NULL 506 stderr $LOGDIR/minidvodb.create.log 507 508 task.exec 509 510 periods -poll 60 511 #wait a bit before trying again 512 513 book npages MINIDVODB_CREATE -var N 514 if ($N == 0) 515 # echo "No labels for processing" 516 break 517 endif 518 519 book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW 520 if ("$minidvodb_group" == "NULL") 521 # All labels have been done --- reset 522 # echo "Resetting labels" 523 for i 0 $N 524 book getpage MINIDVODB_CREATE $i -var minidvodb_group 525 book setword MINIDVODB_CREATE $minidvodb_group STATE NEW 526 end 527 book getpage MINIDVODB_CREATE 0 -var minidvodb_group -key STATE NEW 528 # Select different database 529 $MINIDVODB_DB ++ 530 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 531 end 532 533 book setword MINIDVODB_CREATE $minidvodb_group STATE DONE 534 book getword MINIDVODB_CREATE $minidvodb_group DVODB -var dvodb 535 book getword MINIDVODB_CREATE $minidvodb_group DVODB_DAYS -var dvodb_days 536 book getword MINIDVODB_CREATE $minidvodb_group CAMERA -var camera 537 $run = minidvodb_createdb.pl --camera $camera --outroot $dvodb --dvodb $dvodb --minidvodb_group $minidvodb_group --interval $dvodb_days 538 539 if ($DB:n == 0) 540 option DEFAULT 541 else 542 # save the DB name for the exit tasks 543 option $DB:$MINIDVODB_DB 544 $run = $run --dbname $DB:$MINIDVODB_DB 545 $MINIDVODB_DB ++ 546 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 547 end 548 command $run 549 end 550 551 # success 552 task.exit 0 553 if ($VERBOSE > 2) 554 showcommand 555 end 556 end 557 558 # locked list 559 560 # locked list 561 task.exit default 562 showcommand failure 563 end 564 565 task.exit crash 566 showcommand crash 567 end 568 569 # operation times out? 570 task.exit timeout 571 showcommand timeout 572 end 573 end 574 575 576 577 task minidvodb.active.load 578 host local 579 580 periods -poll $LOADPOLL 581 periods -exec $LOADEXEC 582 periods -timeout 30 583 npending 1 584 585 stdout NULL 586 stderr $LOGDIR/minidvodb.active.load.log 587 588 task.exec 589 book npages MINIDVODB_ACTIVE -var N 590 if ($N == 0) 591 # echo "No labels for processing" 592 break 593 endif 594 595 book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW 596 if ("$minidvodb_group" == "NULL") 597 # All labels have been done --- reset 598 # echo "Resetting labels" 599 for i 0 $N 600 book getpage MINIDVODB_ACTIVE $i -var minidvodb_group 601 book setword MINIDVODB_ACTIVE $minidvodb_group STATE NEW 602 end 603 book getpage MINIDVODB_ACTIVE 0 -var minidvodb_group -key STATE NEW 604 605 # Select different database 606 $MINIDVODB_DB ++ 607 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 608 end 609 610 611 $run = addtool -listminidvodbrun -state new 612 $run = $run -minidvodb_group $minidvodb_group 613 if ($DB:n == 0) 614 option DEFAULT 615 else 616 # save the DB name for the exit tasks 617 option $DB:$MINIDVODB_DB 618 $run = $run -dbname $DB:$MINIDVODB_DB 619 $MINIDVODB_DB ++ 620 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 621 end 622 add_poll_args run 623 624 command $run 625 end 626 627 # success 628 task.exit 0 629 # convert 'stdout' to book format 630 ipptool2book stdout minidvodbActivelist -key minidvodb_id -uniq -setword dbname $options:0 -setword pantaskState INIT 631 if ($VERBOSE > 2) 632 book listbook minidvodbActivelist 633 end 634 # delete existing entries in the appropriate pantaskStates 635 process_cleanup minidvodbActivelist 636 end 637 638 # locked list 639 task.exit default 640 showcommand failure 641 end 642 643 task.exit crash 644 showcommand crash 645 end 646 647 # operation times out? 648 task.exit timeout 649 showcommand timeout 650 end 651 end 652 653 654 655 656 task minidvodb.active.run 657 periods -poll $RUNPOLL 658 periods -exec $RUNEXEC 659 periods -timeout 60 660 661 task.exec 662 # if we are unable to run the 'exec', use a long retry time 663 periods -exec $RUNEXEC 664 665 book npages minidvodbActivelist -var N 666 if ($N == 0) break 667 668 # look for new images in minidvodbActivelist (pantaskState == INIT) 669 book getpage minidvodbActivelist 0 -var pageName -key pantaskState INIT 670 if ("$pageName" == "NULL") break 671 672 book setword minidvodbActivelist $pageName pantaskState RUN 673 book getword minidvodbActivelist $pageName minidvodb_group -var MINIDVODB_GROUP 674 stdout $LOGDIR/minidvodb.active.run.log 675 stderr $LOGDIR/minidvodb.active.run.log 676 677 $run = addtool -flipminidvodbrun -minidvodb_group $MINIDVODB_GROUP 678 679 680 if ($DB:n == 0) 681 option DEFAULT 682 else 683 # save the DB name for the exit tasks 684 option $DB:$MINIDVODB_DB 685 $run = $run -dbname $DB:$MINIDVODB_DB 686 $MINIDVODB_DB ++ 687 if ($MINIDVODB_DB >= $DB:n) set MINIDVODB_DB = 0 688 end 689 # save the pageName for future reference below 690 options $pageName 691 692 693 694 # create the command line 695 if ($VERBOSE > 1) 696 echo command $run 697 end 698 # if we are unable to run the 'exec', use a long retry time 699 periods -exec 0.05 700 command $run 701 end 702 # default exit status 703 task.exit default 704 process_exit minidvodbActivelist $options:0 $JOB_STATUS 705 end 706 707 # locked list 708 task.exit crash 709 #showcommand crash 710 echo "hostname: $JOB_HOSTNAME" 711 # Set a fault code in the database 712 713 process_exit minidvodbActivelist $options:0 $EXIT_CRASH_ERR 714 end 715 716 # operation timed out? 717 task.exit timeout 718 showcommand timeout 719 book setword minidvodbActivelist $options:0 pantaskState TIMEOUT 720 end 721 end 722
Note:
See TracChangeset
for help on using the changeset viewer.
