- Timestamp:
- Aug 19, 2010, 6:52:05 PM (16 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/ipp-20100621/ippScripts/scripts/automate_stacks.pl
r28794 r28981 25 25 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1); 26 26 my $mkBTpcontrol = can_run('make_burntool_pcontrol.pl') or (warn "Can't find make_burntool_pcontrol.pl" and $missing_tools = 1); 27 my $moondata = can_run('moondata') or (warn "Can't find moondata" and $missing_tools = 1); 27 28 28 29 if ($missing_tools) { … … 41 42 # Grab options 42 43 my ( $date, $camera, $dbname, $logfile, $verbose, $manual); 43 my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode );44 my ( $help, $isburning, $force_stack_count, $test_mode, $this_target_only, $this_filter_only, $this_mode_only, $check_mode, $burntool_stats); 44 45 my ( $check_registration, $define_burntool, $queue_burntool, $check_chips, $queue_chips); 45 my ( $check_stacks, $queue_stacks, $check_ diffs, $queue_diffs, $clean_old);46 my ( $check_stacks, $queue_stacks, $check_sweetspot, $queue_sweetspot, $check_diffs, $queue_diffs, $clean_old); 46 47 my ( $check_detrends, $queue_detrends, $check_dqstats, $queue_dqstats); 47 48 … … 58 59 'check' => \$check_mode, 59 60 'test_mode' => \$test_mode, 61 'burntool_stats' => \$burntool_stats, 60 62 'this_target_only=s' => \$this_target_only, 61 63 'this_filter_only=s' => \$this_filter_only, … … 68 70 'check_stacks' => \$check_stacks, 69 71 'queue_stacks' => \$queue_stacks, 72 'check_sweetspot' => \$check_sweetspot, 73 'queue_sweetspot' => \$queue_sweetspot, 70 74 'check_detrends' => \$check_detrends, 71 75 'queue_detrends' => \$queue_detrends, … … 89 93 --this_filter_only Process only a single filter. 90 94 --this_mode_only Process only a single clean mode. 95 --burntool_stats Display Nexp Nimfile Nburntooled Nqueued for check_chips. 91 96 Modes: 92 97 --check_registration Confirm the data downloaded correctly. … … 97 102 --check_stacks Confirm that stacks can be built. 98 103 --queue_stacks Issue stacktool commands to queue stacks. 104 --check_sweetspot See if we should queue SweetSpot stacks. 105 --queue_sweetspot Issue stacktool commands to queue SweetSpot stacks. 99 106 --check_detrends Confirm that detrend verify runs can be built. 100 107 --queue_detrends Issue dettool commands to queue detrend verify runs. … … 111 118 ) unless 112 119 defined $check_registration or defined $define_burntool or defined $queue_burntool or 113 defined $queue_chips or defined $queue_stacks or $queue_ detrends or $queue_dqstats or114 defined $check_chips or defined $check_stacks or $check_ detrends or $check_dqstats or120 defined $queue_chips or defined $queue_stacks or $queue_sweetspot or $queue_detrends or $queue_dqstats or 121 defined $check_chips or defined $check_stacks or $check_sweetspot or $check_detrends or $check_dqstats or 115 122 defined $test_mode or defined $clean_old or defined $check_mode; 116 123 … … 125 132 my %stackable_list = (); 126 133 my %reduction_class = (); 134 my @unrecoverable_quality = (); 127 135 my @detrend_list = (); 128 136 my %dettype_list = (); … … 170 178 elsif (${ $entry }{name} eq 'FILTERS') { 171 179 push @filter_list, ${ $entry }{value}; 180 } 181 elsif (${ $entry }{name} eq 'UNRECOVERABLE_QUALITY') { 182 push @unrecoverable_quality, ${ $entry }{value}; 172 183 } 173 184 elsif (${ $entry }{name} eq 'TARGETS') { … … 304 315 unless (defined($test_mode) || defined($check_mode)) { exit(0); } 305 316 } 317 306 318 if (defined($check_dqstats) || defined($test_mode)) { 307 319 $metadata_out{nsState} = 'CHECKDQSTATS'; … … 316 328 unless (defined($test_mode)) { exit(0); } 317 329 } 330 318 331 if (defined($check_detrends) || defined($test_mode) || defined($check_mode)) { 319 332 $metadata_out{nsState} = 'CHECKDETRENDS'; … … 328 341 exit(0); 329 342 } 343 330 344 if (defined($define_burntool) || defined($test_mode)) { 331 345 $metadata_out{nsState} = 'QUEUEBURNING'; … … 339 353 unless (defined($test_mode)) { exit(0); } 340 354 } 355 341 356 if (defined($check_chips) || defined($test_mode) || defined($check_mode)) { 342 357 $metadata_out{nsState} = 'QUEUECHIPS'; … … 354 369 exit(0); 355 370 } 371 356 372 if (defined($check_stacks) || defined($test_mode) || defined($check_mode)) { 357 373 $metadata_out{nsState} = 'TOWARP'; … … 372 388 exit(0); 373 389 } 390 391 if (defined($check_sweetspot) || defined($test_mode) || defined($check_mode)) { 392 $metadata_out{nsState} = 'CHECKSWEETSPOT'; 393 &execute_sweetspot($date,"pretend"); 394 return_metadata($date); 395 unless (defined($test_mode) || defined($check_mode)) { exit(0); } 396 } 397 if (defined($queue_sweetspot)) { 398 $metadata_out{nsState} = 'CHECKSWEETSPOT'; 399 &execute_sweetspot($date); 400 return_metadata($date); 401 exit(0); 402 } 403 374 404 if (defined($check_diffs) || defined($queue_diffs)) { 375 405 die("Diffs are currently not implemented."); 376 406 } 407 377 408 if (defined($clean_old) || defined($test_mode)) { 378 409 if (defined($test_mode)) { … … 656 687 my $pretend = shift; 657 688 my $exposures = 0; 689 658 690 foreach my $target (@target_list) { 659 691 my ($Nexposures,$Nimfiles,$Nburntooled,$Nalready) = pre_chip_queue($date,$target); 692 if (defined($burntool_stats)) { 693 print "$Nexposures $Nimfiles $Nburntooled $Nalready\n"; 694 } 695 660 696 if ($Nexposures == 0) { 661 697 print STDERR "execute_chips: Target $target on $date had no exposures.\n"; … … 861 897 862 898 # 899 # SweetSpot Stacking 900 ################################################################################ 901 sub construct_sweetspot_cmd { 902 my $date = shift; 903 my $target = 'SweetSpot'; 904 my $filter = 'w.00000'; 905 906 my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); 907 908 # Dateobs begin end? 909 my ($dateobs_begin,$dateobs_end) = get_lunation_extent($date); 910 my $select = "-select_dateobs_begin ${dateobs_begin}T00:00:00 -select_dateobs_end ${dateobs_end}T00:00:00"; 911 my $cmd = "$stacktool"; 912 $cmd .= " -simple -dbname $dbname -definebyquery "; 913 $cmd .= " -set_label SweetSpot.refstack -select_label $label "; 914 $cmd .= " -set_workdir $workdir -set_dist_group $dist_group "; 915 $cmd .= " -select_filter $filter -set_data_group $data_group "; 916 $cmd .= " -select_good_frac_min 0.1 -select_fwhm_major_max 8.0 "; 917 $cmd .= " -min_num 7 -min_new 4"; 918 $cmd .= " $select "; 919 if ($debug == 1) { 920 $cmd .= ' -pretend '; 921 } 922 print STDERR "$cmd\n"; 923 return($cmd); 924 } 925 926 sub get_lunation_extent { 927 my $date = shift; 928 my ($year,$month,$day) = split /-/,$date; 929 my $dateobs_begin; 930 my $dateobs_end; 931 932 my $dt = DateTime->new(year => $year, month => $month, day => $day, 933 hour => 0, minute => 0, second => 0, nanosecond => 0, 934 time_zone => 'Pacific/Honolulu'); 935 do { 936 $dt->subtract(days => 1); 937 my $ymd = $dt->ymd; 938 my $md_cmd = "moondata $ymd 0 0"; 939 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 940 run ( command => $md_cmd, verbose => $verbose); 941 unless ($success) { 942 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 943 &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); 944 } 945 my @result = split /\s+/,(join "\n", @$stdout_buf); 946 if (abs($result[6]) <= 0.5) { 947 $dateobs_end = $ymd; 948 } 949 } while (!defined($dateobs_end)); 950 951 do { 952 $dt->subtract(days => 1); 953 my $ymd = $dt->ymd; 954 my $md_cmd = "moondata $ymd 0 0"; 955 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 956 run ( command => $md_cmd, verbose => $verbose); 957 unless ($success) { 958 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 959 &my_die("Unable to perform moondata: $error_code", 0,0, $date, $PS_EXIT_SYS_ERROR); 960 } 961 my @result = split /\s+/,(join "", @$stdout_buf); 962 if (abs($result[6]) <= 0.5) { 963 $dateobs_begin = $ymd; 964 } 965 } while (!defined($dateobs_begin)); 966 967 return($dateobs_begin,$dateobs_end); 968 } 969 970 sub pre_sweetspot_queue { 971 my $date = shift; 972 my $command = construct_sweetspot_cmd($date) . ' -pretend '; 973 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 974 run ( command => $command, verbose => $verbose ); 975 unless ($success) { 976 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 977 &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 978 } 979 my @stacks = split /\n/, (join '', @$stdout_buf); 980 my $Nstacks = $#stacks + 1; 981 982 return(1,$Nstacks); 983 } 984 985 sub sweetspot_queue { 986 my $date = shift; 987 my $command = construct_sweetspot_cmd($date); 988 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 989 run ( command => $command, verbose => $verbose ); 990 unless ($success) { 991 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 992 &my_die("Unable to perform sweetspot stacktool: $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 993 } 994 my @stacks = split /\n/, (join '', @$stdout_buf); 995 my $Nstacks = $#stacks + 1; 996 997 return(1,$Nstacks); 998 } 999 1000 sub execute_sweetspot { 1001 my $date = shift; 1002 my $pretend = shift; 1003 1004 my ($is_lunation_date,$Nstacks) = pre_sweetspot_queue($date); 1005 if ($Nstacks == 0) { 1006 print STDERR "execute_sweetspot: No new stacks to make ($Nstacks)\n"; 1007 $metadata_out{nsState} = 'SS_EMPTY'; 1008 return(); 1009 } 1010 if ($Nstacks < 10) { 1011 print STDERR "execute_sweetspot: Too few new stacks to make ($Nstacks)\n"; 1012 $metadata_out{nsState} = 'SS_FEW'; 1013 return(); 1014 } 1015 if ($is_lunation_date == 0) { 1016 print STDERR "execute_sweetspot: Invalid lunation date. ($date $is_lunation_date)\n"; 1017 $metadata_out{nsState} = 'SS_ERROR'; 1018 return(); 1019 } 1020 $metadata_out{nsState} = 'SS_QUEUE'; 1021 $metadata_out{nsSweetSpot} = $Nstacks; 1022 unless(defined($pretend)) { 1023 $metadata_out{nsState} = 'SS_DONE'; 1024 sweetspot_queue($date); 1025 } 1026 } 1027 1028 # 863 1029 # Stacking 864 1030 ################################################################################ … … 900 1066 my ($label,$workdir,$obs_mode,$object,$comment,$tess_id,$dist_group,$data_group,$reduction) = get_tool_parameters($date,$target); 901 1067 902 my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter' ";1068 my $sth = "SELECT skycell_id from stackRun where data_group = '$data_group' AND skycell_id = '$skycell' AND filter = '$filter' AND tess_id = '$tess_id'"; 903 1069 my $data_ref = $db->selectall_arrayref( $sth ); 904 1070 … … 918 1084 919 1085 my $where = " label = '$label' AND data_group = '$data_group' "; 1086 1087 my $where_possibly_faulted = $where . " AND ( " ; 1088 foreach my $acceptable_quality (@unrecoverable_quality) { 1089 $where_possibly_faulted .= " quality = $acceptable_quality OR "; 1090 } 1091 $where_possibly_faulted .= " 0 )"; 1092 920 1093 my $chip_sth = "SELECT * from chipRun WHERE (state = 'full' OR state = 'new') AND $where "; 1094 my $cam_sth = "SELECT * from camRun JOIN camProcessedExp USING(cam_id) WHERE state = 'full' AND $where_possibly_faulted "; 921 1095 my $warp_sth = "SELECT * from warpRun WHERE state = 'full' AND $where "; 922 1096 923 1097 my $chip_ref = $db->selectall_arrayref( $chip_sth ); 1098 my $cam_ref = $db->selectall_arrayref( $cam_sth ); 924 1099 my $warp_ref = $db->selectall_arrayref( $warp_sth ); 925 1100 … … 942 1117 } 943 1118 944 return($#input_exposures + 1, $#{ $chip_ref } + 1, $#{ $ warp_ref } + 1, $Nalready);1119 return($#input_exposures + 1, $#{ $chip_ref } + 1, $#{ $cam_ref } + $#{ $warp_ref } + 2, $Nalready); 945 1120 } 946 1121 … … 1039 1214 my $args = $command; 1040 1215 if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) { 1041 $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned - time_stamp_end $cleaning_date ";1216 $args .= " -dbname $dbname -updaterun -set_state goto_cleaned -full -set_label goto_cleaned -label $label -time_stamp_end $cleaning_date "; 1042 1217 } 1043 1218 elsif ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'B')) { … … 1058 1233 1059 1234 foreach my $mode (@mode_list) { 1060 if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) {1061 my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode);1062 if ($cleaning_date eq 'no clean') {1063 next;1064 }1065 print STDERR "$command\n";1066 if (!(defined($pretend) || $debug == 1)) {1067 # print STDERR "BEAR IS DRIVING!?\n";1068 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =1069 run ( command => $command, verbose => $verbose );1070 unless ($success) {1071 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);1072 &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR);1073 }1074 }1075 }1076 else {1235 # if ((exists($clean_alternate{$mode})) && ($clean_alternate{$mode} eq 'A')) { 1236 # my ($cleaning_date,$command) = construct_cleantool_args($date,"",$mode); 1237 # if ($cleaning_date eq 'no clean') { 1238 # next; 1239 # } 1240 # print STDERR "$command\n"; 1241 # if (!(defined($pretend) || $debug == 1)) { 1242 # # print STDERR "BEAR IS DRIVING!?\n"; 1243 # my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 1244 # run ( command => $command, verbose => $verbose ); 1245 # unless ($success) { 1246 # $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR); 1247 # &my_die("Unable to perform cleantool ($command): $error_code", 0,0,$date, $PS_EXIT_SYS_ERROR); 1248 # } 1249 # } 1250 # } 1251 # else { 1077 1252 foreach my $target (@target_list) { 1078 1253 if (exists($noclean_list{$target})) { … … 1094 1269 } 1095 1270 } 1096 }1271 # } 1097 1272 } 1098 1273 return(0);
Note:
See TracChangeset
for help on using the changeset viewer.
