Changeset 31945 for trunk/ippScripts/scripts/lap_science.pl
- Timestamp:
- Jul 27, 2011, 6:10:23 PM (15 years ago)
- File:
-
- 1 edited
-
trunk/ippScripts/scripts/lap_science.pl (modified) (21 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/lap_science.pl
r31939 r31945 1 1 #!/usr/bin/env perl 2 3 2 4 3 use warnings; … … 30 29 my ( $chip_mode, $monitor_mode, $cleanup_mode); 31 30 32 # Global configuration constants that probably should be read from elsewhere. 33 my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it. 31 # 32 # Global configuration constants that probably should be read from elsewhere. 33 my $qstack_threshold = 0.05; # Only make a quickstack if more than 5% of the exposures require it. 34 my $minimum_stack_inputs = 6; # We can avoid magicking stack inputs if we have more than this number. 34 35 35 36 GetOptions( … … 263 264 } 264 265 265 266 266 if (S64_IS_NOT_NULL($chip_id)) { # We already have a defined chip_id 267 267 if (($pairwise) && !($pair_id)) { … … 283 283 $counter++; 284 284 } 285 print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; 285 if ($verbose) { 286 print "ZZ: $exp_id $object $comment $matching{$object}{$comment}\n"; 287 } 286 288 } 287 289 … … 322 324 # Scan all exposures, and ensure that pairwise and private are set correctly 323 325 foreach my $exposure (@$exposures) { 324 print "YY: $exposure\n"; 326 if ($verbose) { 327 print "YY: $exposure\n"; 328 } 325 329 if ($exposure->{pairwise} && !($exposure->{pair_id})) { 326 330 $exposure->{pairwise} = 0; # We marked it for pairwise diffs, but didn't match it. Probably an error. … … 451 455 452 456 # Things I want to know before I'm through 453 my $needs_qstack = 0;457 # my $needs_qstack = 0; 454 458 my $needs_something_remade = 0; 455 459 my $needs_something_private = 0; 456 my $can_qstack = 0;457 my $have_diff = 0;458 my $can_diff = 0;459 my $can_fstack = 0;460 # my $can_qstack = 0; 461 my $have_diff = 0; 462 my $can_diff = 0; 463 # my $can_fstack = 0; 460 464 my $total_exposures = 0; 461 465 my @lonely_exposures = (); 466 467 my $are_warped = 0; 468 my $are_magicked = 0; 469 462 470 foreach my $exposure (@$exposures) { 463 471 $total_exposures++; … … 477 485 478 486 if ($exposure->{private}) { # I've declared this exposure private to this lapRun. 479 $needs_qstack++;487 # $needs_qstack++; 480 488 push @lonely_exposures, $exposure; 481 489 # $lonely_exposures++; … … 516 524 unless ((defined($exposure->{diffRun_state}))&& 517 525 ($exposure->{diffRun_state} eq 'full')) { 518 $needs_qstack++; 526 $are_warped++; 527 # $needs_qstack++; 519 528 $needs_something_private = 1; 520 529 if ($companion) { … … 530 539 $exposure->{data_state} = 'drop'; 531 540 &update_this_exposure($exposure); 532 533 541 } 534 542 535 543 if ($exposure->{data_state} eq 'drop') { # This exposure is impossible, so fudge the counts so we get through. 536 $can_qstack ++; 544 $are_warped++; 545 # $can_qstack ++; 537 546 $can_diff ++; 538 547 $have_diff ++; 539 $can_fstack ++; 548 $are_magicked ++; 549 # $can_fstack ++; 540 550 next; 541 551 } … … 543 553 if (($exposure->{warpRun_state})&& # This exposure has a warp 544 554 ($exposure->{warpRun_state} eq 'full')) { # This exposure's warp is done. 545 $can_qstack ++; 555 $are_warped++; 556 # $can_qstack ++; 546 557 $can_diff ++; 547 558 } 548 559 if (($exposure->{magicked}&& 549 560 &S64_IS_NOT_NULL($exposure->{magicked}))) { # This exposure has been magicked, so it is through with diff. 550 $can_fstack ++; 561 $are_magicked++; 562 # $can_fstack ++; 551 563 } 552 564 if (($exposure->{diff_id})&&(&S64_IS_NOT_NULL($exposure->{diff_id}))) { … … 561 573 print "STATUS: HAVE_FSTACK: $have_fstack\n"; 562 574 563 print "STATUS: NEEDS_QSTACK: $needs_qstack\n";575 # print "STATUS: NEEDS_QSTACK: $needs_qstack\n"; 564 576 print "STATUS: NEEDS_REMADE: $needs_something_remade\n"; 565 577 print "STATUS: NEEDS PRIVATIZE: $needs_something_private\n"; 566 print "STATUS: CAN_QSTACK: $can_qstack\n"; 578 print "STATUS: ARE WARPED: $are_warped\n"; 579 print "STATUS: ARE MAGICKED: $are_magicked\n"; 580 # print "STATUS: CAN_QSTACK: $can_qstack\n"; 567 581 print "STATUS: CAN_DIFF: $can_diff\n"; 568 582 print "STATUS: HAVE_DIFF: $have_diff\n"; 569 print "STATUS: CAN_FSTACK: $can_fstack\n";583 # print "STATUS: CAN_FSTACK: $can_fstack\n"; 570 584 571 585 print "STATUS: TOTAL_EXPOSURES: $total_exposures\n"; … … 574 588 exit(0); 575 589 } 576 if (($needs_qstack > 0)&& # Do we need the quick stack? 577 ($defined_qstack == 0)&& # Have we not made it already? 578 ($can_qstack == $total_exposures)&& # Are all warps done? 579 ($needs_something_remade == 0) # Do we need to wait for a stolen exposure? 580 ) { 581 if ($needs_qstack < $total_exposures * $qstack_threshold) { # Drop any exposures that are requiring the qstack 582 print "STATUS: Insufficient unpaired exposures to justify quickstack. Dropping unpaired exposures\n"; 583 foreach my $exposure (@lonely_exposures) { 584 $exposure->{data_state} = 'drop'; 585 &update_this_exposure($exposure); 586 } 587 } 588 else { 589 print "STATUS: Will now queue quickstacks\n"; 590 &queue_quickstack($exposures); 591 } 592 } 593 594 if (($can_diff == $total_exposures)&& # Are all warps done? 595 ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? 596 ($have_diff < $can_diff)&& # We haven't already done them all. 597 (($needs_qstack == 0)|| # Are we just doing pairwise diffs? 598 ($needs_qstack == 1)&&($have_qstack == 1)) ) { # If we need a quickstack, do we have it? 599 print "STATUS: Will now queue diffs\n"; 600 &queue_diffs($exposures); 601 } 602 603 if (($can_fstack == $total_exposures)&& # Are all warps done? 604 ($needs_something_remade == 0)&& # Do we need to wait for a stolen exposure? 605 ($defined_fstack == 0)) { # Have we not made it already? 606 print "STATUS: Will now queue final stacks\n"; 607 &queue_finalstack($exposures); 608 } 609 610 if (($have_fstack == 1)) { # Are we all done making the final product? 611 print "STATUS: Will now deactivate exposures for this complete lapRun.\n"; 612 &deactivate_exposures($exposures); 613 $command = "$laptool -updaterun -lap_id $lap_id"; 614 $command .= " -dbname $dbname " if defined $dbname; 615 $command .= " -set_state full "; 616 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 617 run(command => $command, verbose => $verbose); 618 unless ($success) { 619 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 620 &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); 621 } 622 } 590 591 # Do the decision making based on what we've learned 592 if ($are_warped == $total_exposures) { # Are all the warps done? 593 if ($defined_fstack == 0) { # We have not yet made any stacks. 594 print "STATUS: We can attempt to queue deep final stacks.\n"; 595 my $Nstacks_made = &queue_muggle_finalstack($exposures); 596 print "STATUS: Made $Nstacks_made final stacks.\n"; 597 exit($Nstacks_made); 598 # If this doesn't do anything, we may end up stuck here. Therefore, we need a more robust test. 599 } 600 else { # We have made a stack before for this lapRun. 601 if ($have_fstack == $defined_fstack) { # We have made all the stacks we have asked for (so far) 602 if ($are_magicked < $total_exposures) { # But we have not yet made diffs 603 if (($can_diff == $total_exposures)&& # And we have exposures that should be diffed. 604 ($have_diff < $can_diff)) { # and we haven't made them all yet, either. 605 print "STATUS: Dropping unpaired exposures and making pairwise diffs.\n"; 606 foreach my $exposure (@lonely_exposures) { 607 $exposure->{data_state} = 'drop'; 608 &update_this_exposure($exposure); 609 } 610 &queue_diffs($exposures); 611 exit(0); 612 } 613 else { 614 # This is apparently the case where we've queued diffs to run, but they have not yet 615 # completed though the destreak phase. 616 print "STATUS: Waiting for diffs/magic/destreak to complete.\n"; 617 exit(0); 618 } 619 } 620 elsif ($are_magicked == $total_exposures) { # We have made all of the diffs, and propagated through magic 621 print "STATUS: We can create all remaining stacks possible.\n"; 622 my $Nstacks_made = &queue_magic_finalstack($exposures); 623 if ($Nstacks_made == 0) { # We have made all the stacks that we will ever make. 624 print "STATUS: No more stacks generated. Deactivating exposures for this complete lapRun.\n"; 625 &deactivate_exposures($exposures); 626 $command = "$laptool -updaterun -lap_id $lap_id"; 627 $command .= " -dbname $dbname " if defined $dbname; 628 $command .= " -set_state full "; 629 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 630 run(command => $command, verbose => $verbose); 631 unless ($success) { 632 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 633 &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); 634 } 635 exit(0); 636 } 637 else { 638 print "STATUS: $Nstacks_made stacks made.\n"; 639 exit(0); 640 } 641 } 642 else { # What? 643 print "STATUS: More exposures magicked than exist. This is a mistake.\n"; 644 exit(0); 645 } 646 } 647 else { # We have not yet made all the stacks we've asked for. 648 print "STATUS: Defined $defined_fstack, completed $have_fstack. Waiting for stacking to finish.\n"; 649 exit(0); 650 } 651 } 652 } 653 else { # Not all warps finished. 654 # do nothing 655 print "STATUS: Not all warps are finished, waiting for that to finish.\n"; 656 exit(0); 657 } 658 623 659 print "\n"; 624 660 return(0); … … 657 693 $command .= " -pretend " if defined $debug; 658 694 $command .= " -dbname $dbname " if defined $dbname; 659 $command .= " -definebyquery -select_label ${label}% -select_skycell_id ${proj_cell}.% -select_filter $filter "; 695 $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; 696 # $command .= " -select_label $label "; # Removed to allow exposure sharing to work 660 697 $command .= " -set_label ${label} -set_data_group $data_group "; 661 698 $command .= " -set_workdir $workdir -set_dist_group NODIST "; … … 669 706 &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); 670 707 } 708 my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or 709 &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); 671 710 672 711 $command = "$stacktool "; … … 702 741 } 703 742 704 705 } 706 sub queue_ finalstack {743 return($#{ $stacks_made } + 1); 744 } 745 sub queue_muggle_finalstack { 707 746 my $exposures = shift; # reference to exposure array; 708 747 my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information … … 720 759 foreach $exposure (@$exposures) { 721 760 if (($exposure->{data_state} ne 'drop')&& 722 (S64_IS_NOT_NULL($exposure->{magicked}))&&761 # (S64_IS_NOT_NULL($exposure->{magicked}))&& 723 762 (S64_IS_NOT_NULL($exposure->{warp_id}))) { 724 763 $warps .= " -warp_id $exposure->{warp_id} "; … … 737 776 $command .= " -pretend " if defined $debug; 738 777 $command .= " -dbname $dbname " if defined $dbname; 739 $command .= " -definebyquery -select_label ${label}% -select_skycell_id ${proj_cell}.% -select_filter $filter "; 778 $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; 779 # $command .= " -select_label $label "; # Removed to allow exposure sharing to work 740 780 $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; 741 $command .= " -min_num 2-set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} ";781 $command .= " -min_num ${minimum_stack_inputs} -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; 742 782 $command .= " $warps "; 743 783 … … 748 788 &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); 749 789 } 750 790 my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or 791 &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); 751 792 752 793 $command = "$stacktool "; … … 781 822 } 782 823 824 return($#{ $stacks_made } + 1); 825 } 826 sub queue_magic_finalstack { 827 my $exposures = shift; # reference to exposure array; 828 my $exposure = ${ $exposures }[0]; # reference to the first exposure to get run level information 829 830 my $lap_id = $exposure->{lap_id}; 831 my $label = $exposure->{label}; 832 my $filter = $exposure->{filter}; 833 my $proj_cell = $exposure->{projection_cell}; 834 835 unless (defined($label) && defined($filter) && defined($proj_cell)) { 836 &my_die("Unable to perform stacktool. Insufficient information.", $lap_id); 837 } 838 839 my $warps = ''; 840 foreach $exposure (@$exposures) { 841 if (($exposure->{data_state} ne 'drop')&& 842 (S64_IS_NOT_NULL($exposure->{magicked}))&& 843 (S64_IS_NOT_NULL($exposure->{warp_id}))) { 844 $warps .= " -warp_id $exposure->{warp_id} "; 845 } 846 } 847 848 my @utctime = gmtime(); 849 $utctime[5] += 1900; 850 $utctime[4] += 1; 851 my $date = sprintf("%4d%02d%02d",$utctime[5],$utctime[4],$utctime[3]); 852 my $workdir_date = sprintf("%4d/%02d/%02d",$utctime[5],$utctime[4],$utctime[3]); 853 my $workdir = "neb://\@HOST\@.0/${dbname}/${label}/${workdir_date}"; 854 my $data_group = "${label}.${proj_cell}.final.${date}"; 855 856 my $command = "$stacktool "; 857 $command .= " -pretend " if defined $debug; 858 $command .= " -dbname $dbname " if defined $dbname; 859 $command .= " -definebyquery -select_skycell_id ${proj_cell}.% -select_filter $filter "; 860 # $command .= " -select_label $label "; # Removed to allow exposure sharing to work 861 $command .= " -set_label ${label} -set_workdir $workdir -set_data_group $data_group "; 862 $command .= " -min_num 2 -min_new 2 -set_reduction THREEPI_STACK -set_dist_group $exposure->{dist_group} "; 863 $command .= " $warps "; 864 865 my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 866 run(command => $command, verbose => $verbose); 867 unless ($success) { 868 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 869 &my_die("Unable to perform stacktool -definebyquery: $error_code", $lap_id); 870 } 871 my $stacks_made = $mdcParser->parse_list(join "", @$stdout_buf) or 872 &my_die("unable to parse metadata from stacktool -definebyquery", $lap_id, ""); 873 874 $command = "$stacktool "; 875 $command .= " -dbname $dbname " if defined $dbname; 876 $command .= " -sassskyfile -data_group $data_group "; 877 $command .= " -filter $filter -projection_cell ${proj_cell} "; 878 879 ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 880 run(command => $command, verbose => $verbose); 881 unless ($success) { 882 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 883 &my_die("Unable to perform stacktool -sassskyfile: $error_code", $lap_id); 884 } 885 886 my $stacks = $mdcParser->parse_list(join "", @$stdout_buf) or 887 &my_die("Unable to parse metadata from stacktool -sassskyfile", $lap_id, ""); 888 889 my $stack = ${ $stacks }[0]; 890 my $sass_id = $stack->{sass_id}; 891 unless (defined($sass_id)) { 892 &my_die("Unable to parse metadata from stacktool for sass_id", $lap_id, ""); 893 } 894 print "FINAL_SASS_ID: $sass_id\n"; 895 $command = "$laptool -updaterun -lap_id $lap_id -set_final_sass_id $sass_id"; 896 $command .= " -dbname $dbname " if defined $dbname; 897 898 ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 899 run(command => $command, verbose => $verbose); 900 unless ($success) { 901 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 902 &my_die("Unable to perform laptool -updaterun: $error_code", $lap_id); 903 } 904 return($#{ $stacks_made } + 1); 783 905 } 784 906 sub queue_diffs { … … 831 953 my $companion = ${ $exposures }[$match_hash{$exposure->{chip_id}}]; 832 954 $command .= " -definewarpwarp "; 833 $command .= "-input_label $label -template_label $label ";955 # $command .= "-input_label $label -template_label $label "; 834 956 $command .= "-warp_id $exposure->{warp_id} -template_warp_id $companion->{warp_id} "; 835 957 $retry_command = $command; … … 840 962 } 841 963 else { # warp-qstack 842 $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 "; 843 $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; 844 $already_queued{$exposure->{warp_id}} = 1; 964 next; # We no longer wish to make these, and should never get here. 965 $command .= '-pretend'; 966 # $command .= " -definewarpstack -set_reduction NOCONVDIFF -available -good_frac 0.2 "; 967 # $command .= " -warp_id $exposure->{warp_id} -stack_label ${label}.quick "; 968 # $already_queued{$exposure->{warp_id}} = 1; 845 969 } 846 970
Note:
See TracChangeset
for help on using the changeset viewer.
