Changeset 26831
- Timestamp:
- Feb 9, 2010, 5:40:49 PM (16 years ago)
- Location:
- trunk
- Files:
-
- 1 added
- 4 edited
-
ippScripts/scripts/automate_stacks.pl (modified) (22 diffs)
-
ippTasks/nightly_stacks.pro (modified) (1 diff)
-
ippconfig/recipes/Makefile.am (modified) (1 diff)
-
ippconfig/recipes/nightly_science.config (added)
-
ippconfig/system.config (modified) (1 diff)
Legend:
- Unmodified
- Added
- Removed
-
trunk/ippScripts/scripts/automate_stacks.pl
r26670 r26831 9 9 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt ); 10 10 use Pod::Usage qw( pod2usage ); 11 12 11 use DateTime; 12 13 # 13 14 # Set up 15 ################################################################################ 16 14 17 my $missing_tools = 0; 15 18 my $chiptool = can_run('chiptool') or (warn "Can't find chiptool" and $missing_tools = 1); 19 my $warptool = can_run('warptool') or (warn "Can't find warptool" and $missing_tools = 1); 16 20 my $stacktool= can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1); 21 my $difftool = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1); 17 22 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); 18 23 my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1); … … 23 28 } 24 29 25 # global variables. 26 my @target_list = ('M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10'); 27 my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000'); 30 # my @filter_list = ('g.00000','r.00000','i.00000','z.00000','y.00000'); 31 # my @target_list = ('CMB','M31','MD01','MD02','MD03','MD04','MD05','MD06','MD07','MD08','MD09','MD10', 32 # 'STS','SVS','SweetSpot','ThreePi'); 33 # my %tessID_list = ('CMB' => 'RINGS.V0', 'M31' => 'M31', 'MD01' => 'MD01', 'MD02' => 'MD02', 34 # 'MD03' => 'MD03', 'MD04' => 'MD04', 'MD05' => 'MD05', 'MD06' => 'MD06', 35 # 'MD07' => 'MD07', 'MD08' => 'MD08', 'MD09' => 'MD09', 'MD10' => 'MD10', 36 # 'STS' => 'STS', 'SVS' => 'RINGS.V0', 'SweetSpot' => 'RINGS.V0', 'ThreePi' => 'RINGS.V0'); 37 # my %comment_list = ('CMB' => 'CMB_Cold%', 'M31' => 'M31%', 'MD01' => 'MD01%', 'MD02' => 'MD02%', 38 # 'MD03' => 'MD03%', 'MD04' => 'MD04%', 'MD05' => 'MD05%', 'MD06' => 'MD06%', 39 # 'MD07' => 'MD07%', 'MD08' => 'MD08%', 'MD09' => 'MD09%', 'MD10' => 'MD10%', 40 # 'STS' => 'Stellar Transit%', 'SVS' => 'SVS%', 'SweetSpot' => 'Sweetspot%', 'ThreePi' => 'ThreePi%'); 41 # my %stackable_list = ('CMB' => 0, 'M31' => 1, 'MD01' => 1, 'MD02' => 1, 42 # 'MD03' => 1, 'MD04' => 1, 'MD05' => 1, 'MD06' => 1, 43 # 'MD07' => 1, 'MD08' => 1, 'MD09' => 1, 'MD10' => 1, 44 # 'STS' => 1, 'SVS' => 0, 'SweetSpot' => 0, 'ThreePi' => 0); 45 # my $retention_time = 9000; # days. 46 47 28 48 my $db; 29 49 my $debug = 0; … … 33 53 chomp($loghead = `date`); 34 54 print STDERR 'Starting: ' . $loghead . ' ' . $0 . ' ' . (join ' ', @ARGV) . "\n"; 55 35 56 # Grab options 36 57 my ( $date, $camera, $dbname, $logfile, $verbose); 37 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips, $check_stacks, $queue_stacks); 38 my ( $help, $this_target_only, $this_filter_only, $isburning, $force_stack_count); 58 my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only); 59 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); 60 my ( $check_stacks, $queue_stacks, $check_diffs, $queue_diffs, $clean_old); 61 39 62 GetOptions( 40 63 'help|h' => \$help, … … 44 67 'logfile=s' => \$logfile, 45 68 'verbose' => \$verbose, 69 'isburning' => \$isburning, 70 'force_stack_count' => \$force_stack_count, 71 'test_mode' => \$test_mode, 72 'this_target_only=s' => \$this_target_only, 73 'this_filter_only=s' => \$this_filter_only, 74 'check_registration' => \$check_registration, 75 'define_burntool' => \$define_burntool, 76 'queue_burntool' => \$queue_burntool, 46 77 'check_chips' => \$check_chips, 47 78 'queue_chips' => \$queue_chips, 48 79 'check_stacks' => \$check_stacks, 49 80 'queue_stacks' => \$queue_stacks, 50 'queue_burntool' => \$queue_burntool, 51 'define_burntool' => \$define_burntool, 52 'check_registration|R' => \$check_registration, 53 'this_target_only=s' => \$this_target_only, 54 'this_filter_only=s' => \$this_filter_only, 55 'isburning' => \$isburning, 56 'force_stack_count' => \$force_stack_count, 81 'check_diffs' => \$check_diffs, 82 'queue_diffs' => \$queue_diffs, 83 'clean_old' => \$clean_old, 57 84 ) or pod2usage ( 2 ); 58 85 pod2usage( -msg => … … 64 91 --dbname <db> Default gpc1. 65 92 --verbose 93 --isburning Signal that we are currently burntooling. 94 --force_stack_count Force the chip/warp counts. 66 95 --this_target_only Process only a single target. 67 96 --this_filter_only Process only a single filter. 68 --isburning Signal that we are currently burntooling.69 --force_stack_count Force the chip/warp counts.70 97 Modes: 71 98 --check_registration Confirm the data downloaded correctly. … … 75 102 --queue_chips Issue chiptool commands to queue chips. 76 103 --check_stacks Confirm that stacks can be built. 77 --queue_stacks Issue stacktool commands to queue stacks.\n", 104 --queue_stacks Issue stacktool commands to queue stacks. 105 --check_diffs Confirm that diffs can be done. 106 --queue_diffs Issue difftool commands to queue diffs. 107 --clean_old Mark old data 'goto_cleanup'.\n", 78 108 -exitval => 2, ) if (defined($help)); 79 109 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; … … 84 114 defined $check_registration or defined $define_burntool or defined $queue_burntool or 85 115 defined $queue_chips or defined $queue_stacks or 86 defined $check_chips or defined $check_stacks; 116 defined $check_chips or defined $check_stacks or 117 defined $test_mode or defined $clean_old; 118 119 # Configurable parameters from our config file. 120 my @target_list = (); 121 my @filter_list = (); 122 my %tessID_list = (); 123 my %comment_list= (); 124 my %stackable_list = (); 125 my $retention_time = 9000; 126 127 # Grab the configuration data. 128 my $conf_cmd = "$ppConfigDump -dump-recipe NIGHTLY_SCIENCE -"; 129 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 130 run(command => $conf_cmd, verbose => $verbose); 131 unless ($success) { 132 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 133 &my_die("Unable to perform ppConfigDUmp: $error_code", $date, $PS_EXIT_SYS_ERROR); 134 } 135 136 my $mdcParser = PS::IPP::Metadata::Config->new; 137 my $metadata = $mdcParser->parse(join "", @$stdout_buf); 138 foreach my $entry (@{ $metadata }) { 139 if (${ $entry }{name} eq 'RETENTION_TIME') { 140 $retention_time = ${ $entry }{value}; 141 } 142 elsif (${ $entry }{name} eq 'FILTERS') { 143 push @filter_list, ${ $entry }{value}; 144 } 145 elsif (${ $entry }{name} eq 'TARGETS') { 146 my @target_data = @{ ${ $entry }{value} }; 147 my $this_target = ''; 148 foreach my $tentry (@target_data) { 149 if (${ $tentry }{name} eq 'NAME') { 150 $this_target = ${ $tentry }{value}; 151 push @target_list, $this_target; 152 } 153 elsif (${ $tentry }{name} eq 'TESS') { 154 $tessID_list{$this_target} = ${ $tentry }{value}; 155 } 156 elsif (${ $tentry }{name} eq 'COMMENT') { 157 $comment_list{$this_target} = ${ $tentry }{value}; 158 } 159 elsif (${ $tentry }{name} eq 'STACKABLE') { 160 $stackable_list{$this_target} = ${ $tentry }{value}; 161 } 162 } 163 } 164 } 165 166 87 167 88 168 … … 120 200 } 121 201 122 if (defined($check_registration)) { 202 # 203 # Mode selection 204 ################################################################################ 205 if (defined($test_mode)) { 206 $debug = 1; 207 } 208 209 if (defined($check_registration) || defined($test_mode)) { 210 $metadata_out{nsState} = 'NEW'; 123 211 my ($Nsummit_exp,$Nfaults) = check_summit_copy($date); 124 125 if ($Nsummit_exp == 0) {126 print STDERR "No exposures were found on the summit for $date.\n";127 $metadata_out{nsState} = 'DROP';128 }129 elsif ($Nfaults != 0) {130 print STDERR "There were faults while downloading the exposures for $date.\n";131 $metadata_out{nsState} = 'NEW';132 }133 else {134 print STDERR "Summit copy and Registration have succeeded for $date.\n";135 $metadata_out{nsState} = 'REGISTERED';136 }137 212 return_metadata($date); 138 exit(0);139 } 140 if (defined($define_burntool) ) {213 unless (defined($test_mode)) { exit(0); } 214 } 215 if (defined($define_burntool) || defined($test_mode)) { 141 216 $metadata_out{nsState} = 'QUEUEBURNING'; 142 217 &find_burntool_ranges($date); 143 218 return_metadata($date); 144 exit(0);145 } 146 if (defined($queue_burntool) ) {219 unless (defined($test_mode)) { exit(0); } 220 } 221 if (defined($queue_burntool) || defined($test_mode)) { 147 222 $metadata_out{nsState} = 'BURNING'; 148 223 return_metadata($date); 149 exit(0);150 } 151 if (defined($check_chips) ) {224 unless (defined($test_mode)) { exit(0); } 225 } 226 if (defined($check_chips) || defined($test_mode)) { 152 227 $metadata_out{nsState} = 'QUEUECHIPS'; 153 228 &execute_chips($date,"pretend"); … … 156 231 } 157 232 return_metadata($date); 158 exit(0); 159 } 160 if (defined($check_stacks)) { 161 $metadata_out{nsState} = 'TOWARP'; 162 &execute_stacks($date,"pretend"); 163 if ($metadata_out{nsState} eq 'FORCETOWARP') { 164 $metadata_out{nsState} = 'TOWARP'; 165 } 166 return_metadata($date); 167 exit(0); 233 unless (defined($test_mode)) { exit(0); } 168 234 } 169 235 if (defined($queue_chips)) { … … 173 239 exit(0); 174 240 } 241 if (defined($check_stacks) || defined($test_mode)) { 242 $metadata_out{nsState} = 'TOWARP'; 243 &execute_stacks($date,"pretend"); 244 if ($metadata_out{nsState} eq 'FORCETOWARP') { 245 $metadata_out{nsState} = 'TOWARP'; 246 } 247 return_metadata($date); 248 unless (defined($test_mode)) { exit(0); } 249 } 175 250 if (defined($queue_stacks)) { 176 251 $metadata_out{nsState} = 'STACKING'; … … 179 254 exit(0); 180 255 } 181 256 if (defined($check_diffs) || defined($queue_diffs)) { 257 die("Diffs are currently not implemented."); 258 } 259 if (defined($clean_old) || defined($test_mode)) { 260 if (defined($test_mode)) { 261 &execute_cleans($date,"pretend"); 262 } 263 else { 264 &execute_cleans($date); 265 } 266 unless (defined($test_mode)) { exit(0); } 267 } 268 exit(0); 269 # 182 270 # Registration 271 ################################################################################ 183 272 184 273 sub check_summit_copy { … … 240 329 } 241 330 } 242 243 return($Nsummit_exps,$Nsummit_faults + $Ndownload_faults + $Nregister_faults); 331 332 my $Nfaults = $Nsummit_faults + $Ndownload_faults + $Nregister_faults; 333 if ($Nsummit_exps == 0) { 334 print STDERR "No exposures were found on the summit for $date.\n"; 335 $metadata_out{nsState} = 'DROP'; 336 } 337 elsif ($Nfaults != 0) { 338 print STDERR "There were faults while downloading the exposures for $date.\n"; 339 $metadata_out{nsState} = 'NEW'; 340 } 341 else { 342 print STDERR "Summit copy and Registration have succeeded for $date.\n"; 343 $metadata_out{nsState} = 'REGISTERED'; 344 } 345 346 return($Nsummit_exps,$Nfaults); 244 347 } 245 348 349 # 246 350 # Burntool 351 ################################################################################ 247 352 248 353 sub get_goodBTvalue { … … 325 430 } 326 431 327 432 # 328 433 # Chips 434 ################################################################################ 329 435 330 436 sub construct_chiptool_cmd { 331 437 my $date = shift; 332 438 my $target = shift; 439 440 my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); 333 441 334 442 my $select = "-dateobs_begin ${date}T00:00:00 -dateobs_end ${date}T23:59:59 "; … … 337 445 my $cmd = "$chiptool"; 338 446 $cmd .= ' -simple -dbname gpc1 -definebyquery -set_end_stage warp '; 339 $cmd .= " -set_label $ {target}.nightlystack";340 $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target";341 $cmd .= " -set_tess_id $t arget -comment ${target}% -set_data_group ${target}.${date}";447 $cmd .= " -set_label $label "; 448 $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; 449 $cmd .= " -set_tess_id $tess_id -comment '$comment' -set_data_group $data_group "; 342 450 $cmd .= " $select "; 343 451 if ($debug == 1) { … … 355 463 my $db = init_gpc_db(); 356 464 $date =~ s/-//g; 357 my $data_group = "${target}.${date}";465 my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); 358 466 359 467 my $sth = "SELECT exp_id from chipRun where data_group = '$data_group' AND exp_id = $exp_id"; … … 431 539 } 432 540 541 # 433 542 # Stacking 543 ################################################################################ 434 544 435 545 sub construct_stacktool_cmd { … … 437 547 my $target = shift; 438 548 my $filter = shift; 549 550 my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); 439 551 440 552 my $select = "-select_dateobs_begin ${date}T00:00:00 -select_dateobs_end ${date}T23:59:59 "; … … 444 556 # $cmd .= ' -pretend -simple -dbname gpc1 -definebyquery -min_new 4 '; # Probably silly, but I want to be safe and not overwrite 445 557 $cmd .= ' -simple -dbname gpc1 -definebyquery '; 446 $cmd .= " -set_label $ {target}.nightlystack -select_label ${target}.nightlystack";447 $cmd .= " -set_workdir neb://\@HOST\@.0/gpc1/${target}.nightlystack -set_dist_group $target";448 $cmd .= " -select_filter $filter -set_data_group $ {target}.${date}";558 $cmd .= " -set_label $label -select_label $label "; 559 $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; 560 $cmd .= " -select_filter $filter -set_data_group $data_group "; 449 561 $cmd .= " $select "; 450 562 if ($debug == 1) { … … 463 575 my $db = init_gpc_db(); 464 576 $date =~ s/-//g; 465 my $data_group = "${target}.${date}";577 my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); 466 578 467 579 my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter'"; … … 476 588 my $filter = shift; 477 589 590 my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($date,$target); 478 591 # check warp stage == chip stage 479 592 my $db = init_gpc_db(); … … 481 594 my $trunc_date = $date; $trunc_date =~ s/-//g; 482 595 483 my $where = " label = '$ {target}.nightlystack' AND data_group = '${target}.${trunc_date}' ";596 my $where = " label = '$label' AND data_group = '$data_group' "; 484 597 my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where "; 485 598 my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where "; … … 528 641 my $pretend = shift; 529 642 foreach my $target (@target_list) { 530 foreach my $filter (@filter_list) { 531 my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter); 532 if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :( 533 print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; 534 $metadata_out{nsState} = 'FORCETOWARP'; 535 next; 536 } 537 if ($Nexposures == 0) { 538 print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; 539 next; 540 } 541 if ($Nalready != 0) { 542 print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; 543 next; 544 } 545 unless ($metadata_out{nsState} eq 'FORCETOWARP') { 546 $metadata_out{nsState} = 'QUEUESTACKS'; 547 } 548 unless(defined($pretend)) { 549 print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; 550 stack_queue($date,$target,$filter); 551 } 552 } 553 } 554 555 } 556 643 if ($stackable_list{$target} == 1) { 644 foreach my $filter (@filter_list) { 645 my ($Nexposures,$NprocChips,$NprocWarps,$Nalready) = pre_stack_queue($date,$target,$filter); 646 if ((!defined($force_stack_count))&&($NprocChips != $NprocWarps)) { # This makes me sad. :( 647 print STDERR "execute_stacks: Target $target on $date is not fully processed. ($NprocChips $NprocWarps)\n"; 648 $metadata_out{nsState} = 'FORCETOWARP'; 649 next; 650 } 651 if ($Nexposures == 0) { 652 print STDERR "execute_stacks: Target $target in filter $filter on $date has no exposures.\n"; 653 next; 654 } 655 if ($Nalready != 0) { 656 print STDERR "execute_stacks: Not queueing $target in filter $filter on $date due to already existing stacks.\n"; 657 next; 658 } 659 unless ($metadata_out{nsState} eq 'FORCETOWARP') { 660 $metadata_out{nsState} = 'QUEUESTACKS'; 661 } 662 unless(defined($pretend)) { 663 print STDERR "execute_stacks: Target $target in filter $filter on $date has exposures and will be queued.\n"; 664 stack_queue($date,$target,$filter); 665 } 666 } 667 } 668 else { 669 print STDERR "execute_stacks: Target $target is not auto-stackable.\n"; 670 } 671 } 672 673 } 674 675 # 676 # Auto-Clean 677 ################################################################################ 678 679 sub construct_cleantool_args { 680 my $date = shift; 681 my $target = shift; 682 683 my ($year,$month,$day) = split /-/,$date; 684 my $dt = DateTime->new(year => $year, month => $month, day => $day, 685 hour => 0, minute => 0, second => 0, nanosecond => 0, 686 time_zone => 'Pacific/Honolulu'); 687 $dt->subtract(days => $retention_time); 688 my $cleaning_date = $dt->ymd; 689 690 my ($label,$workdir,$comment,$tess_id,$dist_group,$data_group) = get_tool_parameters($cleaning_date,$target); 691 692 my $args .= " -dbname gpc1 -updaterun -set_state goto_cleaned -state full -data_group $data_group "; 693 if ($debug == 1) { 694 $args .= ' -pretend '; 695 } 696 return($cleaning_date,$args); 697 } 698 699 sub execute_cleans { 700 my $date = shift; 701 my $pretend = shift; 702 703 foreach my $target (@target_list) { 704 my ($cleaning_date,$args) = construct_cleantool_args($date,$target); 705 706 my $command = $chiptool . $args; 707 print STDERR "$command\n"; 708 if (!(defined($pretend) || $debug == 1)) { 709 # print STDERR "BEAR IS DRIVING!?\n"; 710 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 711 run ( command => $command, verbose => $verbose ); 712 unless ($success) { 713 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 714 &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 715 } 716 } 717 $command = $warptool . $args; 718 print STDERR "$command\n"; 719 if (!(defined($pretend) || $debug == 1)) { 720 # print STDERR "BEAR IS DRIVING $pretend $debug!?\n"; 721 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 722 run ( command => $command, verbose => $verbose ); 723 unless ($success) { 724 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 725 &my_die("Unable to perform stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 726 } 727 } 728 } 729 return(0); 730 } 731 732 733 734 735 # 557 736 # Utilities 737 ################################################################################ 738 739 sub get_tool_parameters { 740 my $date = shift; 741 my $target = shift; 742 my $workdir_date = $date; $workdir_date =~ s%-%/%g; 743 my $trunc_date = $date; $trunc_date =~ s/-//g; 744 745 my $label = "${target}.nightlyscience"; 746 my $workdir = "neb://\@HOST\@.0/gpc1/${target}.nt/${workdir_date}"; 747 my $comment = $comment_list{$target}; 748 my $dist_group = $target; 749 my $data_group = "${target}.${trunc_date}"; 750 my $tess_id = $tessID_list{$target}; 751 return($label,$workdir,$comment,$tess_id,$dist_group,$data_group); 752 } 753 558 754 559 755 sub init_gpc_db { -
trunk/ippTasks/nightly_stacks.pro
r26710 r26831 151 151 152 152 command echo $today 153 # command automate_stacks.pl --clean_old --date $today 153 154 end 154 155 -
trunk/ippconfig/recipes/Makefile.am
r26368 r26831 29 29 ppNoiseMap.config \ 30 30 ppSkycell.config \ 31 ppVizPSF.mdc 31 ppVizPSF.mdc \ 32 nightly_science.config 32 33 33 34 install_DATA = $(install_files) -
trunk/ippconfig/system.config
r26368 r26831 58 58 PPSMOOTH STR recipes/ppSmooth.config # stand-alone image smoothing 59 59 PPVIZPSF STR recipes/ppVizPSF.mdc # PSF visualisation 60 NIGHTLY_SCIENCE STR recipes/nightly_science.config # Nightly Science 60 61 END
Note:
See TracChangeset
for help on using the changeset viewer.
