- Timestamp:
- Sep 6, 2011, 11:00:22 AM (15 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20110710/ippScripts/scripts/lap_science.pl
r31795 r32337 1 1 #!/usr/bin/env perl 2 3 2 4 3 use warnings; … … 29 28 my ( $lap_id ); 30 29 my ( $chip_mode, $monitor_mode, $cleanup_mode); 30 31 # 32 # Global configuration constants that probably should be read from elsewhere. 33 my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it. 34 my $minimum_stack_inputs = 6; # We can avoid magicking stack inputs if we have more than this number. 31 35 32 36 GetOptions( … … 255 259 $exposure->{pair_id} = 9223372036854775807; 256 260 update_this_exposure($exposure); 261 $exposure = remake_this_exposure($exposure); 257 262 $counter++; 258 263 next; 259 264 } 260 265 } 261 262 266 263 267 if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id … … 280 284 $counter++; 281 285 } 282 print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; 286 if ($verbose) { 287 print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; 288 } 283 289 } 284 290 … … 319 325 # Scan all exposures, and ensure that pairwise and private are set correctly 320 326 foreach my $exposure (@$exposures) { 321 print "YY: $exposure\n"; 327 if ($verbose) { 328 print "YY: $exposure\n"; 329 } 322 330 if ($exposure->{pairwise} && !($exposure->{pair_id})) { 323 331 $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. … … 448 456 449 457 # Things I want to know before I'm through 450 my $needs_qstack = 0;458 # my $needs_qstack = 0; 451 459 my $needs_something_remade = 0; 452 460 my $needs_something_private = 0; 453 my $can_qstack = 0;454 my $have_diff = 0;455 my $can_diff = 0;456 my $can_fstack = 0;461 # my $can_qstack = 0; 462 my $have_diff = 0; 463 my $can_diff = 0; 464 # my $can_fstack = 0; 457 465 my $total_exposures = 0; 458 my $lonely_exposures = 0; 466 my @lonely_exposures = (); 467 468 my $are_warped = 0; 469 my $are_magicked = 0; 470 459 471 foreach my $exposure (@$exposures) { 460 472 $total_exposures++; … … 472 484 } 473 485 } 486 487 if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. 488 $are_warped++; 489 # $can_qstack ++; 490 $can_diff ++; 491 $have_diff ++; 492 $are_magicked ++; 493 # $can_fstack ++; 494 next; 495 } 474 496 475 497 if ($exposure->{private}) { # I've declared this exposure private to this lapRun. 476 $needs_qstack = 1; 498 # $needs_qstack++; 499 push @lonely_exposures, $exposure; 477 500 # $lonely_exposures++; 478 501 } … … 512 535 unless ((defined($exposure->{diffRun_state}))&& 513 536 ($exposure->{diffRun_state} eq 'full')) { 514 $needs_qstack = 1; 537 $are_warped++; 538 # $needs_qstack++; 515 539 $needs_something_private = 1; 516 540 if ($companion) { … … 518 542 $companion->{pairwise} = 0; 519 543 &update_this_exposure($companion); 544 push @lonely_exposures, $companion; 520 545 # $lonely_exposures++; 521 546 } … … 525 550 $exposure->{data_state} = 'drop'; 526 551 &update_this_exposure($exposure); 527 528 552 } 529 553 530 if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through.531 $can_qstack ++;532 $can_diff ++;533 $have_diff ++;534 $can_fstack ++;535 next;536 }537 554 538 555 if (($exposure->{warpRun_state})&& # This exposure has a warp 539 556 ($exposure->{warpRun_state} eq 'full')) { # This exposure's warp is done. 540 $can_qstack ++; 557 $are_warped++; 558 # $can_qstack ++; 541 559 $can_diff ++; 542 560 } 543 561 if (($exposure->{magicked}&& 544 562 &S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. 545 $can_fstack ++; 563 $are_magicked++; 564 # $can_fstack ++; 546 565 } 547 566 if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { … … 556 575 print "STATUS: HAVE_FSTACK: $have_fstack\n"; 557 576 558 print "STATUS: NEEDS_QSTACK: $needs_qstack\n";577 # print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; 559 578 print "STATUS: NEEDS_REMADE: $needs_something_remade\n"; 560 579 print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n"; 561 print "STATUS: CAN_QSTACK: $can_qstack\n"; 580 print "STATUS: ARE WARPED: $are_warped\n"; 581 print "STATUS: ARE MAGICKED: $are_magicked\n"; 582 # print "STATUS: CAN_QSTACK: $can_qstack\n"; 562 583 print "STATUS: CAN_DIFF: $can_diff\n"; 563 584 print "STATUS: HAVE_DIFF: $have_diff\n"; 564 print "STATUS: CAN_FSTACK: $can_fstack\n";585 # print "STATUS: CAN_FSTACK: $can_fstack\n"; 565 586 566 587 print "STATUS: TOTAL_EXPOSURES: $total_exposures\n"; … … 569 590 exit(0); 570 591 } 571 if (($needs_qstack == 1)&& # Do we need the quick stack? 572 ($defined_qstack == 0)&& # Have we not made it already? 573 ($can_qstack == $total_exposures)&& # Are all warps done? 574 ($needs_something_remade == 0) # Do we need to wait for a stolen exposure? 575 ) { 576 print "STATUS: Will now queue quickstacks\n"; 577 &queue_quickstack($exposures); 578 } 579 580 if (($can_diff == $total_exposures)&& # Are all warps done? 581 ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? 582 ($have_diff < $can_diff)&& # We haven't already done them all. 583 (($needs_qstack == 0)|| # Are we just doing pairwise diffs? 584 ($needs_qstack == 1)&&($have_qstack == 1)) ) { # If we need a quickstack, do we have it? 585 print "STATUS: Will now queue diffs\n"; 586 &queue_diffs($exposures); 587 } 588 589 if (($can_fstack == $total_exposures)&& # Are all warps done? 590 ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? 591 ($defined_fstack == 0)) { # Have we not made it already? 592 print "STATUS: Will now queue final stacks\n"; 593 &queue_finalstack($exposures); 594 } 595 596 if (($have_fstack == 1)) { # Are we all done making the final product? 597 print "STATUS: Will now deactivate exposures for this complete lapRun.\n"; 598 &deactivate_exposures($exposures); 599 $command = "$laptool -updaterun -lap_id $lap_id"; 600 $command .= " -dbname $dbname " if defined $dbname; 601 $command .= " -set_state full "; 602 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 603 run(command => $command, verbose => $verbose); 604 unless ($success) { 605 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 606 &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); 607 } 608 } 592 593 # Do the decision making based on what we've learned 594 if ($are_warped == $total_exposures) { # Are all the warps done? 595 if ($defined_fstack == 0) { # We have not yet made any stacks. 596 print "STATUS: We can attempt to queue deep final stacks.\n"; 597 my $Nstacks_made = &queue_muggle_finalstack($exposures); 598 print "STATUS: Made $Nstacks_made final stacks.\n"; 599 exit($Nstacks_made); 600 # If this doesn't do anything, we may end up stuck here. Therefore, we need a more robust test. 601 } 602 else { # We have made a stack before for this lapRun. 603 if ($have_fstack == $defined_fstack) { # We have made all the stacks we have asked for (so far) 604 if ($are_magicked < $total_exposures) { # But we have not yet made diffs 605 if (($can_diff == $total_exposures)&& # And we have exposures that should be diffed. 606 ($have_diff < $can_diff)) { # and we haven't made them all yet, either. 607 print "STATUS: Dropping unpaired exposures and making pairwise diffs.\n"; 608 # foreach my $exposure (@lonely_exposures) { 609 # $exposure->{data_state} = 'drop'; 610 # &update_this_exposure($exposure); 611 # } 612 &queue_diffs($exposures); 613 exit(0); 614 } 615 else { 616 # This is apparently the case where we've queued diffs to run, but they have not yet 617 # completed though the destreak phase. 618 print "STATUS: Waiting for diffs/magic/destreak to complete.\n"; 619 exit(0); 620 } 621 } 622 elsif ($are_magicked == $total_exposures) { # We have made all of the diffs, and propagated through magic 623 print "STATUS: We can create all remaining stacks possible.\n"; 624 my $Nstacks_made = &queue_magic_finalstack($exposures); 625 if ($Nstacks_made == 0) { # We have made all the stacks that we will ever make. 626 print "STATUS: No more stacks generated. Deactivating exposures for this complete lapRun.\n"; 627 &deactivate_exposures($exposures); 628 $command = "$laptool -updaterun -lap_id $lap_id"; 629 $command .= " -dbname $dbname " if defined $dbname; 630 $command .= " -set_state full "; 631 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 632 run(command => $command, verbose => $verbose); 633 unless ($success) { 634 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 635 &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); 636 } 637 exit(0); 638 } 639 else { 640 print "STATUS: $Nstacks_made stacks made.\n"; 641 exit(0); 642 } 643 } 644 else { # What? 645 print "STATUS: More exposures magicked than exist. This is a mistake.\n"; 646 exit(0); 647 } 648 } 649 else { # We have not yet made all the stacks we've asked for. 650 print "STATUS: Defined $defined_fstack, completed $have_fstack. Waiting for stacking to finish.\n"; 651 exit(0); 652 } 653 } 654 } 655 else { # Not all warps finished. 656 # do nothing 657 print "STATUS: Not all warps are finished, waiting for that to finish.\n"; 658 exit(0); 659 } 660 609 661 print "\n"; 610 662 return(0); … … 626 678 my $warps = ''; 627 679 foreach $exposure (@$exposures) { 628 if (($exposure->{data_state} !='drop')&&680 if (($exposure->{data_state} ne 'drop')&& 629 681 (S64_IS_NOT_NULL($exposure->{warp_id}))) { 630 682 $warps .= " -warp_id $exposure->{warp_id} "; … … 638 690 my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); 639 691 my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; 640 my $data_group = "${label}.${proj_cell}.quick.${ date}";692 my $data_group = "${label}.${proj_cell}.quick.${lap_id}"; 641 693 642 694 my $command = "$stacktool "; 643 695 $command .= " -pretend " if defined $debug; 644 696 $command .= " -dbname $dbname " if defined $dbname; 645 $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; 697 $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; 698 # $command .= " -select_label $label "; # Removed to allow exposure sharing to work 646 699 $command .= " -set_label ${label} -set_data_group $data_group "; 647 700 $command .= " -set_workdir $workdir -set_dist_group NODIST "; … … 655 708 &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); 656 709 } 710 my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or 711 &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); 657 712 658 713 $command = "$stacktool "; … … 688 743 } 689 744 690 691 } 692 sub queue_ finalstack {745 return($#{ $stacks_made } + 1); 746 } 747 sub queue_muggle_finalstack { 693 748 my $exposures = shift; # reference to exposure array; 694 749 my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information … … 705 760 my $warps = ''; 706 761 foreach $exposure (@$exposures) { 707 if (($exposure->{data_state} !='drop')&&708 (S64_IS_NOT_NULL($exposure->{magicked}))&&762 if (($exposure->{data_state} ne 'drop')&& 763 # (S64_IS_NOT_NULL($exposure->{magicked}))&& 709 764 (S64_IS_NOT_NULL($exposure->{warp_id}))) { 710 765 $warps .= " -warp_id $exposure->{warp_id} "; … … 718 773 my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); 719 774 my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; 720 my $data_group = "${label}.${proj_cell}.final.${ date}";775 my $data_group = "${label}.${proj_cell}.final.${lap_id}"; 721 776 722 777 my $command = "$stacktool "; 723 778 $command .= " -pretend " if defined $debug; 724 779 $command .= " -dbname $dbname " if defined $dbname; 725 $command .= " -definebyquery -select_label $label -select_skycell_id ${proj_cell}.% -select_filter $filter "; 780 $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; 781 # $command .= " -select_label $label "; # Removed to allow exposure sharing to work 726 782 $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; 727 $command .= " -min_num 2-set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} ";783 $command .= " -min_num ${minimum_stack_inputs} -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; 728 784 $command .= " $warps "; 729 785 … … 734 790 &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); 735 791 } 736 792 my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or 793 &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); 737 794 738 795 $command = "$stacktool "; … … 767 824 } 768 825 826 return($#{ $stacks_made } + 1); 827 } 828 sub queue_magic_finalstack { 829 my $exposures = shift; # reference to exposure array; 830 my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information 831 832 my $lap_id = $exposure->{lap_id}; 833 my $label = $exposure->{label}; 834 my $filter = $exposure->{filter}; 835 my $proj_cell = $exposure->{projection_cell}; 836 837 unless (defined($label) && defined($filter) && defined($proj_cell)) { 838 &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); 839 } 840 841 my $warps = ''; 842 foreach $exposure (@$exposures) { 843 if (($exposure->{data_state} ne 'drop')&& 844 (S64_IS_NOT_NULL($exposure->{magicked}))&& 845 (S64_IS_NOT_NULL($exposure->{warp_id}))) { 846 $warps .= " -warp_id $exposure->{warp_id} "; 847 } 848 } 849 850 my @utctime = gmtime(); 851 $utctime[5] += 1900; 852 $utctime[4] += 1; 853 my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); 854 my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); 855 my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; 856 my $data_group = "${label}.${proj_cell}.final.${lap_id}"; 857 858 my $command = "$stacktool "; 859 $command .= " -pretend " if defined $debug; 860 $command .= " -dbname $dbname " if defined $dbname; 861 $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; 862 # $command .= " -select_label $label "; # Removed to allow exposure sharing to work 863 $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; 864 $command .= " -min_num 2 -min_new 2 -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; 865 $command .= " $warps "; 866 867 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 868 run(command => $command, verbose => $verbose); 869 unless ($success) { 870 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 871 &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); 872 } 873 my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or 874 &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); 875 876 $command = "$stacktool "; 877 $command .= " -dbname $dbname " if defined $dbname; 878 $command .= " -sassskyfile -data_group $data_group "; 879 $command .= " -filter $filter -projection_cell ${proj_cell} "; 880 881 ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 882 run(command => $command, verbose => $verbose); 883 unless ($success) { 884 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 885 &my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id); 886 } 887 888 my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or 889 &my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, ""); 890 891 my $stack = ${ $stacks }[0]; 892 my $sass_id = $stack->{sass_id}; 893 unless (defined($sass_id)) { 894 &my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, ""); 895 } 896 print "FINAL_SASS_ID: $sass_id\n"; 897 $command = "$laptool -updaterun -lap_id $lap_id -set_final_sass_id $sass_id"; 898 $command .= " -dbname $dbname " if defined $dbname; 899 900 ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 901 run(command => $command, verbose => $verbose); 902 unless ($success) { 903 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 904 &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); 905 } 906 return($#{ $stacks_made } + 1); 769 907 } 770 908 sub queue_diffs { … … 817 955 my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; 818 956 $command .= " -definewarpwarp "; 819 $command .= "-input_label $label -template_label $label ";957 # $command .= "-input_label $label -template_label $label "; 820 958 $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} "; 821 959 $retry_command = $command; … … 826 964 } 827 965 else { # warp-qstack 828 $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 "; 829 $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; 830 $already_queued{$exposure->{warp_id}} = 1; 966 # next; 967 # $command .= '-pretend'; 968 $command .= " -definewarpstack -available -good_frac 0.2 "; 969 $command .= " -warp_id $exposure->{warp_id} -stack_label ${label} "; 970 $already_queued{$exposure->{warp_id}} = 1; 831 971 } 832 972 … … 1060 1200 $command .= " -nopairwise "; 1061 1201 } 1062 if ($exposure->{active}) { 1202 # if ($exposure->{active}) { 1203 unless ($exposure->{active} == 0) { 1063 1204 $command .= " -active "; 1064 1205 }
Note:
See TracChangeset
for help on using the changeset viewer.
