Changeset 26852
- Timestamp:
- Feb 10, 2010, 3:39:59 PM (16 years ago)
- Location:
- branches/eam_branches/20091201/tools
- Files:
-
- 5 edited
- 2 copied
-
dvo2mass.sh (copied) (copied from trunk/tools/dvo2mass.sh )
-
examine_burntool_pcontrol.pl (modified) (3 diffs)
-
gpc1_find_images.pl (modified) (17 diffs)
-
make_burntool_pcontrol.pl (modified) (10 diffs)
-
neb-ds9.pl (modified) (2 diffs)
-
warp_outputs.pl (modified) (3 diffs)
-
wiki_nightly_stacks_table.pl (copied) (copied from trunk/tools/wiki_nightly_stacks_table.pl )
Legend:
- Unmodified
- Added
- Removed
-
branches/eam_branches/20091201/tools/examine_burntool_pcontrol.pl
r25562 r26852 17 17 print STDERR " status of the processing. On the OTA grid:\n"; 18 18 print STDERR " _ Blank chip\n"; 19 print STDERR " N Null/zero value for burntool_state. No burntool attempted.\n"; 20 print STDERR " O -1 value for burntool_state. Burntool is running.\n"; 21 print STDERR " E -2 value for burntool_state. Error!\n"; 19 print STDERR " O Null/zero value for burntool_state. No burntool attempted (yet).\n"; 20 print STDERR " P -1 value for burntool_state. Burntool is running.\n"; 21 print STDERR " E -2 value for burntool_state. Error! (modified raw pixels)\n"; 22 print STDERR " X -3 value for burntool_state. Burntool segfaulted on this chip.\n"; 22 23 print STDERR " B Current version value for burntool_state. Burntool succeeded.\n"; 23 24 print STDERR " b Old version value for burntool_state. Burntool needs to be rerun.\n"; … … 51 52 $burncount++; 52 53 } 53 elsif ($burntool_state == -1) { 54 elsif ($burntool_state == -1) { # processing 54 55 $char = '[01;33mP[m'; 55 56 } 56 elsif ($burntool_state == 0) { 57 elsif ($burntool_state == 0) { # has not been processed yet 57 58 $char = '[31mO[m'; 58 59 } 59 elsif ($burntool_state == -2) { 60 elsif ($burntool_state == -2) { # modified pixel data 60 61 $char = '[35;01;44;05mE[m'; 61 62 } 62 elsif ($burntool_state == -3) { 63 elsif ($burntool_state == -3) { # previous burntool segfaulted. 63 64 $char = '[34mX[m'; 64 65 } 65 else { 66 else { # old version 66 67 $char = '[32mb[m'; 67 68 } … … 128 129 129 130 $data_ref = $db->selectall_arrayref( $sth ); 130 131 131 132 132 # Reset -
branches/eam_branches/20091201/tools/gpc1_find_images.pl
r26098 r26852 3 3 use DBI; 4 4 use Getopt::Std; 5 getopts('h FARCPp:BLMDWIvx:n:c:w:s:O:S:N:',\%opt);5 getopts('hEFARCPp:BLMDWIJVYTvx:n:c:w:s:d:e:O:S:N:',\%opt); 6 6 #getopt('xncwOS:hARCW',\%opt); 7 7 unless (exists($opt{O})) { … … 34 34 } 35 35 36 # abcdefghijklmnopqrstuvwxyz 37 #used: 38 # ABCDEF LMNOP RST W Y 39 # c h n s vwx 36 40 if (exists($opt{h})) { 37 41 print "USAGE find_images.pl ... \n"; 38 42 print " -h This help\n"; 39 print " -R Return raw data.\n"; 40 print " -C Return chip stage data.\n"; 41 print " -W Return warp stage data.\n"; 43 print " -R Return Raw data.\n"; 44 print " -C Return Chip stage data.\n"; 45 print " -E Return camEra stage data.\n"; 46 print " -W Return Warp stage data.\n"; 47 print " -T Get the sTacks for a given input.\n"; 48 print " -J Get the dIff stage data (warps only for now).\n"; 42 49 print " -A Return not just fits images.\n"; 43 50 print " -F Convert to real filenames.\n"; … … 45 52 print " -B Return burntool table.\n"; 46 53 print " -M Include masks in results.\n"; 54 print " -L Return the log files from this stage.\n"; 47 55 print " -D Return only pre-destreaked images.\n"; 56 print " -Y Yes, I want the raw, even if it's not been processed.\n"; 57 print " -V Return the cmf or smf files.\n"; 48 58 print " -v Be verbose on output.\n"; 49 59 print " \n"; 50 60 print " -x <exp_id> Specify a exp_id.\n"; 51 61 print " -c <chip_id> ...\n"; 62 print " -e <cam_id> ...\n"; 52 63 print " -w <warp_id> ...\n"; 53 64 print " -s <stack_id> ...\n"; 65 print " -d <diff_id> ...\n"; 54 66 print " -n <exp_name> ...\n"; 55 67 print " \n"; … … 64 76 $dbname = 'gpc1'; 65 77 $dbserver = 'ippdb01'; 66 $dbuser = 'ipp ';67 $dbpass = 'ipp ';78 $dbuser = 'ippuser'; 79 $dbpass = 'ippuser'; 68 80 $db = DBI->connect("DBI:mysql:database=${dbname};host=${dbserver};" . 69 81 "mysql_socket=" . DB_SOCKET(), … … 72 84 ) or die "Unable to connect to database $DBI::errstr\n"; 73 85 74 $sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack' from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING (exp_id)"; 86 if ($opt{Y}) { 87 $sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack',camRun.cam_id,camRun.workdir,0,'nodiff' from warpRun RIGHT OUTER JOIN fakeRun USING(fake_id) RIGHT OUTER JOIN camRun USING(cam_id) RIGHT OUTER JOIN chipRun USING(chip_id) RIGHT OUTER JOIN rawExp USING (exp_id)"; 88 } 89 else { 90 $sth_base = "SELECT warp_id,warpRun.workdir,chipRun.chip_id,chipRun.workdir,rawExp.exp_name,rawExp.exp_id,rawExp.workdir,rawExp.dateobs,rawExp.pon_time,0,'nostack',camRun.cam_id,camRun.workdir,0,'nodiff' from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING (exp_id)"; 91 } 75 92 76 93 # What do we have? … … 89 106 push @sth_adds, "warpRun.warp_id = $opt{w}"; 90 107 } 91 if (exists($opt{s})) { 92 if ($opt{s} != 0) { 108 if (exists($opt{e})) { 109 push @sth_adds, "camRun.cam_id = $opt{e}"; 110 } 111 if (exists($opt{s})||exists($opt{T})) { 112 # print "$opt{s}!\n"; 113 if (exists($opt{s})&&($opt{s} != 0)) { 93 114 push @sth_adds, "stackRun.stack_id = $opt{s}"; 94 115 } 95 $sth_base =~ s/0,"nostack"/stack_id,stackSumSkyfile.uri/; 116 $sth_base =~ s/0,'nostack'/stack_id,stackSumSkyfile.uri/; 117 if (exists($opt{T})) { 118 $sth_base .= ' RIGHT OUTER '; 119 } 96 120 $sth_base .= ' JOIN stackInputSkyfile USING (warp_id) JOIN stackSumSkyfile USING (stack_id) JOIN stackRun USING (stack_id)'; 121 # print "$sth_base\n"; 122 } 123 if (exists($opt{J})||(exists($opt{d}))) { 124 if (exists($opt{d})&&($opt{d} != 0)) { 125 push @sth_adds, "diffRun.diff_id = $opt{d}"; 126 } 127 $sth_base =~ s/0,'nodiff'/diffRun.diff_id,diffSkyfile.path_base/; 128 if (exists($opt{J})) { 129 # $sth_base .= ' RIGHT OUTER '; 130 } 131 $sth_base .= ' JOIN diffInputSkyfile ON (warpRun.warp_id = diffInputSkyfile.warp1 OR warpRun.warp_id = diffInputSkyfile.warp2) JOIN diffRun USING (diff_id) JOIN diffSkyfile USING (diff_id)'; 97 132 } 98 133 if ($#sth_adds != -1) { … … 112 147 ($warp_id, $warp_workdir, $chip_id, $chip_workdir, 113 148 $exp_name, $exp_id, $exp_workdir, $exp_dateobs, $exp_pontime, 114 $stack_id, $stack_sumfile_uri) = @{ $rr }; 149 $stack_id, $stack_sumfile_uri, 150 $cam_id, $cam_workdir, 151 $diff_id, $diff_workdir 152 ) = @{ $rr }; 115 153 if ($stack_id == 0) { 116 154 $stack_sumfile_uri = ''; 117 155 } 118 # print (join "\t", @{ $rr }) . "\n" ;156 # print (join "\t", @{ $rr }) . "\n" . "\n"; 119 157 # $exp_pontime = 0; # To stop warnings, but I want this around. 120 158 if (exists($opt{I})) { 121 159 $stack_id = 0; # Ditto. 160 $cam_id = 0; 161 $diff_id = 0; 122 162 } 123 163 if (exists($opt{N})) { … … 136 176 if (exists($opt{B})) { $previous_cmd .= ' -B '; } 137 177 if (exists($opt{v})) { $previous_cmd .= ' -v '; } 178 if (exists($opt{F})) { $previous_cmd .= ' -F '; } 138 179 if ($opt{A}) { $previous_cmd .= ' -A '; } 139 180 if ($opt{M}) { $previous_cmd .= ' -M '; } … … 156 197 (exists($opt{W})&&(exists($opt{O})))|| 157 198 exists($opt{I})) { 199 158 200 $sth_mapping = "SELECT skycell_id,class_id from warpSkyCellMap WHERE warp_id = $warp_id"; 159 201 $mr = $db->selectall_arrayref( $sth_mapping ); … … 166 208 push @{ $mapping{$class} }, $skycell; 167 209 push @{ $mapping{$raw_class} }, $skycell; 210 # print STDERR "$raw_class $class $skycell\n"; 168 211 } 169 212 } … … 217 260 close(NLS); 218 261 } 262 if (exists($opt{E})) { 263 open(NLS,"neb-ls $cam_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls"; 264 while (<NLS>) { 265 chomp; 266 if (is_valid_cam($_)) { 267 push @scan, $_; 268 $found++; 269 } 270 } 271 close(NLS); 272 } 219 273 if (exists($opt{W})) { 220 274 open(NLS,"neb-ls $warp_workdir/${exp_name}.${exp_id}/ |") || die "can't neb-ls"; … … 227 281 } 228 282 } 229 if (exists($opt{s})) { 230 push @scan, $stack_sumfile_uri; 231 $found++; 283 if (exists($opt{J})) { 284 @diff_tmp = split /\//, $diff_workdir; 285 $diff_prefix = pop(@diff_tmp); 286 $diff_true_dir = join '/',@diff_tmp; 287 288 open(NLS,"neb-ls $diff_true_dir |") || die "can't neb-ls"; 289 while (<NLS>) { 290 chomp; 291 if (is_valid_diff($_)) { 292 push @scan, $_; 293 $found++; 294 } 295 } 296 } 297 if (exists($opt{s})||exists($opt{T})) { 298 if (defined($stack_sumfile_uri)) { 299 push @scan, $stack_sumfile_uri; 300 $found++; 301 } 232 302 } 233 303 … … 277 347 elsif ($opt{B}) { 278 348 if ($key =~ /burn.tbl/) { 349 if (($opt{O} ne '')&&($key !~ /$opt{O}/)) { 350 return(0); # Not the requested OTA 351 } 352 if (($opt{S} ne '')&& !(is_mapped($key,$opt{S}))) { 353 return(0); # Not the remapped skycell requested 354 } 279 355 return(1); 280 356 } … … 332 408 } 333 409 } 410 elsif ($opt{V} && ($key =~ /cmf/)) { 411 return(1); 412 } 334 413 elsif ($key !~ /fits/) { 335 414 return(0); # Not a fits … … 353 432 } 354 433 434 sub is_valid_cam { 435 my $key = shift; 436 if ($key =~ /\.cm\./) { 437 if ($opt{A}) { 438 return(1); 439 } 440 elsif ($opt{L} && ($key =~ /log/)) { 441 return(1); 442 } 443 elsif ($opt{V} && ($key =~ /smf/)) { 444 return(1); 445 } 446 } 447 return(0); 448 } 449 450 sub is_valid_diff { 451 my $key = shift; 452 if ($key =~ /$diff_prefix/) { 453 if ($opt{D} && ($key !~ /SR/)) { 454 return(0); # Not destreaked when required 455 } 456 if (($opt{S} ne '')&&($key !~ /$opt{S}/)) { 457 return(0); # Not the requested skycell 458 } 459 if (($opt{O} ne '')&& !(is_mapped($key,$opt{O}))) { 460 return(0); # Not the remapped OTA requested 461 } 462 463 if ($opt{A}) { 464 return(1); # You asked for all, 465 } 466 elsif ($opt{L}) { 467 if ($key =~ /log/) { 468 return(1); 469 } 470 else { 471 return(0); 472 } 473 } 474 elsif ($opt{V} && ($key =~ /cmf/)) { 475 return(1); 476 } 477 elsif ($key !~ /fits/) { 478 return(0); # Not a fits 479 } 480 elsif ($key =~ /wt.fits/) { 481 return(0); # No handler for weight right now. 482 } 483 elsif ($key =~ /mask/) { 484 if ($opt{M}) { 485 return(1); # Requested mask 486 } 487 else { 488 return(0); 489 } 490 } 491 else { 492 return(1); # Should be just the warp. 493 } 494 } 495 496 return(0); 497 } 355 498 sub is_valid_warp { 356 499 my $key = shift; … … 380 523 } 381 524 } 525 elsif ($opt{V} && ($key =~ /cmf/)) { 526 return(1); 527 } 382 528 elsif ($key !~ /fits/) { 383 529 return(0); # Not a fits … … 399 545 return(0); 400 546 } 401 402 403 404 405 -
branches/eam_branches/20091201/tools/make_burntool_pcontrol.pl
r25952 r26852 35 35 ) or die "Unable to connect to database $DBI::errstr\n"; 36 36 37 # Determine what the value of "BURNTOOL.STATE.GOOD" currently is: 38 $burntoolStateGood = 999; 39 open(LAZY,"ppConfigDump -camera GPC1 -dump-camera - |") || die "Can't run ppConfigDump\n"; 40 while(<LAZY>) { 41 chomp; 42 if ($_ =~ /BURNTOOL.STATE.GOOD/) { 43 @line = split /\s+/; 44 $burntoolStateGood = $line[2]; 45 } 46 } 47 close(LAZY); 48 #unless( $burntoolStateGood == 999) { 49 # print STDERR "GOOD == $burntoolStateGood\n"; 50 #} 51 52 37 53 # List of which obs_modes are "science" and which are something else. 38 54 %science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1, … … 41 57 $science{Unknown} = 1; 42 58 } 43 # Zero the arrays. 59 60 # Zero the arrays and counters 44 61 @dates_min = (); 45 62 @dates_max = (); 46 63 $N_burntooled_images = 0; 64 $N_total_images = 0; 65 $N_ranges = 0; 47 66 # Parse SQL query and fill any valid dates to the arrays 48 67 if (exists($opt{Q})) { … … 66 85 } 67 86 } 87 68 88 # Read a single date, and calculate the end and insert 69 89 elsif (exists($opt{d})) { … … 73 93 push @dates_max, sprintf("%04d-%02d-%02d",$year,$mon,$date); 74 94 } 95 75 96 # Just shift the supplied values into the arrays. 76 97 else { … … 92 113 print "##query: SELECT dateobs FROM rawExp WHERE $opt{Q}\n"; 93 114 } 94 #for ($d = 0; $d <= $#dates_min; $d++) { 115 95 116 while ($#dates_min > -1) { 96 117 $date_min = shift(@dates_min); … … 138 159 $science{$obs_mode} = 0; 139 160 } 140 #%science = ('MD' => 1, '3PI' => 1, 'STS' => 1, 'CAL' => 1, 'M31' => 1, 'SS' => 1,141 # 'ENGINEERING' => 0, 'NULL' => 0, 'Unknown' => 0, ' ' => 0);142 # I don't like this bit, but we need to work around the bad values of obs_mode that periodically crop up.143 # This isn't the most robust solution (which would of course be "having trustworthy obs_mode values"), but144 # it'll probably work for now.145 161 if ((($science{$obs_mode} == 1)&&($comment !~ /Daytime/))|| 146 162 (($science{$obs_mode} == 0)&&( 147 163 ($comment =~ /MD/)||($comment =~ /ThreePi/)|| 148 164 ($comment =~ /STS/)||($comment =~ /M31/)|| 149 ($comment =~ /CAL/)||($comment =~ /SS/)||($comment =~ /SVS/)|| ($comment =~ /Sweetspot/)|| ($comment =~ /cal/)|| ($comment =~ /virgo/)|| ($comment =~ /Virgo/)|| ($comment =~ /kepler/)|| 165 ($comment =~ /CAL/i)||($comment =~ /SS/)||($comment =~ /SVS/)|| 166 ($comment =~ /Sweetspot/)||($comment =~ /virgo/i)||($comment =~ /kepler/)|| 150 167 ($comment =~ /sdss/)))) 151 168 { … … 174 191 next; 175 192 } 193 176 194 # Scan again and count how many exposures are between windows 177 195 foreach $row_ref (@{ $data_ref }) { … … 212 230 } 213 231 } 214 # print ">>>$i $start_s[$i] $skips[$i]\n"; 215 } 216 # print ">>> $dateobs\n"; 232 } 217 233 } 218 234 … … 227 243 } 228 244 # Print out the end points in the format burntool wants. 245 229 246 for ($i = 0; $i <= $#start_s; $i++) { 230 247 @start_t = localtime($start_s[$i]); 231 248 @end_t = localtime($end_s[$i]); 232 printf(" burntool %04d-%02d-%02dT%02d:%02d:%02d %04d-%02d-%02dT%02d:%02d:%02d\n", 233 $start_t[5] + 1900,$start_t[4] + 1,$start_t[3], 234 $start_t[2],$start_t[1],$start_t[0], 235 $end_t[5] + 1900,$end_t[4] + 1,$end_t[3], 236 $end_t[2],$end_t[1],$end_t[0]); 237 } 238 print "\n"; 239 } 240 241 unless(exists($opt{b})) { 249 250 my $date_min = sprintf("%04d-%02d-%02dT%02d:%02d:%02d", 251 $start_t[5] + 1900,$start_t[4] + 1,$start_t[3], 252 $start_t[2],$start_t[1],$start_t[0]); 253 my $date_max = sprintf("%04d-%02d-%02dT%02d:%02d:%02d", 254 $end_t[5] + 1900,$end_t[4] + 1,$end_t[3], 255 $end_t[2],$end_t[1],$end_t[0]); 256 257 my $total_images_sth = "SELECT count(exp_id) from rawImfile where dateobs >= '$date_min' AND dateobs <= '$date_max'"; 258 my $im_ref = $db->selectall_arrayref( $total_images_sth ); 259 $N_total_images += ${ $im_ref }[0][0]; 260 261 my $burntooled_images_sth = "SELECT count(exp_id) from rawImfile where dateobs >= '$date_min' AND dateobs <= '$date_max' AND abs(burntool_state) = $burntoolStateGood"; 262 $im_ref = $db->selectall_arrayref( $burntooled_images_sth ); 263 $N_burntooled_images += ${ $im_ref }[0][0]; 264 265 266 printf(" burntool %s %s\n",$date_min,$date_max); 267 } 268 unless(exists($opt{d})) { 269 print "\n"; 270 } 271 $N_ranges += ($#start_s + 1); 272 } 273 274 if ($N_ranges == 0) { 275 print_short_epilogue(); 276 } 277 278 unless(exists($opt{b}) || ($N_ranges == 0)) { 242 279 print_epilogue(); 243 280 } … … 275 312 return(0); 276 313 } 314 sub print_short_epilogue { 315 if (exists($opt{b})) { 316 print STDERR "There were no science exposures to process. Sorry, try again tomorrow.\n"; 317 return(); 318 } 319 print << 'END_SHORT_EPILOGUE'; 320 321 echo "There were no science exposures to process. Sorry, try again tomorrow." 322 323 end 324 END_SHORT_EPILOGUE 325 } 277 326 278 327 sub print_epilogue { 328 print " echo 'There were $N_total_images total images, of which $N_burntooled_images were already burntooled.'\n"; 329 print "### BTSTAT $N_total_images $N_burntooled_images\n"; 279 330 print << 'END_EPILOGUE'; 280 331 -
branches/eam_branches/20091201/tools/neb-ds9.pl
r26276 r26852 4 4 getopts('AM',\%opt); 5 5 6 $cmd = 'ds9 -scale mode zscale ';6 $cmd = 'ds9 -scale mode zscale '; 7 7 8 8 if (exists($opt{A})) { … … 21 21 } 22 22 } 23 #$cmd .= " -zoom to fit -regions format xy -regions system wcs -regions load targets.reg "; 23 24 system("$cmd"); -
branches/eam_branches/20091201/tools/warp_outputs.pl
r25859 r26852 24 24 my ($input); # Input list 25 25 my ($products); # Products of interest 26 my ($muggle); # No magicked inputs? 26 27 27 28 GetOptions( … … 33 34 'skycell_id=s' => \$skycell_id, # Skycell identifier 34 35 'products=s' => \$products, # Products of interest 35 ) or die "Unable to parse arguments.\n"; 36 'muggle' => \$muggle, # No magicked inputs? 37 ) or 38 die "Unable to parse arguments.\n"; 36 39 die "Unknown option: @ARGV\n" if @ARGV; 37 40 die "Required options: --dbhost --dbname --dbuser --dbpass --warp_id\n" … … 80 83 81 84 my $file = "$path$ext"; # File 85 86 if (defined $muggle and $ext ne ".psf") { 87 my @file = split /\//, $file; 88 my $last = pop @file; 89 push @file, "SR_$last"; 90 $file = join '/', @file; 91 } 92 82 93 my $source = `ipp_datapath.pl $file` or die "Unable to locate $file\n"; # Actual file 83 94 chomp $source;
Note:
See TracChangeset
for help on using the changeset viewer.
