Changeset 27859 for trunk/pstamp/scripts/detectability_respond.pl
- Timestamp:
- May 4, 2010, 4:10:59 PM (16 years ago)
- File:
-
- 1 edited
-
trunk/pstamp/scripts/detectability_respond.pl (modified) (21 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/pstamp/scripts/detectability_respond.pl
r27788 r27859 41 41 my $EXTVER = 1.0; 42 42 my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE'; 43 my ($req_id,$req_name,$product,$need_magic,$missing_tools );44 my ($request_file,$output,$ dbname,$verbose,$save_temps);43 my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project); 44 my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps); 45 45 GetOptions( 46 'input=s' =>\$request_file,46 'input=s' => \$request_file, 47 47 'output=s' => \$output, 48 'workdir=s' => \$workdir, 48 49 'dbname=s' => \$dbname, 50 'dbserver=s' => \$dbserver, 49 51 'verbose' => \$verbose, 50 52 'save-temps' => \$save_temps, … … 55 57 -exitval => 3, 56 58 ) unless 57 defined $request_file and defined $output and defined $ dbname;59 defined $request_file and defined $output and defined $workdir and defined $dbname; 58 60 59 61 my $detect_query_read = can_run('detect_query_read') or (warn "Can't find detect_query_read" and $missing_tools = 1); … … 71 73 72 74 my $ipprc = PS::IPP::Config->new(); 73 #my $tmp_dir = "/data/${host}.0/tmp/"; 74 75 my $project = resolve_project($ipprc,"project_name",$dbname); 75 76 if (!$dbserver) { 77 $dbserver = metadataLookupStr($ipprc->{_siteConfig}, 'PS_DBSERVER'); 78 } 79 # This is hardcoded in for the moment. 80 $project = resolve_project($ipprc,"gpc1",$dbname,$dbserver); 81 my $imagedb = $project->{dbname}; 82 if (!$imagedb) { 83 carp("failed to find imagedb for project: $project"); 84 } 85 76 86 # 77 87 # Parse input request file using detect_query_read (as it's already written). 78 88 # 79 80 89 my $dqr_command = "$detect_query_read --input $request_file"; 81 90 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 82 91 run(command => $dqr_command, verbose => $verbose); 83 92 unless ($success) { 84 my_die("Unable to perform $dqr_command error code: $error_code",-1,-1,-1,-1,-1,-1); 93 # This is a problem, because I'm not sure how we handle a failure to read something. 94 # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it, 95 # we can't send that response back. 96 die("Unable to perform $dqr_command error code: $error_code"); 85 97 } 86 98 my %query = (); … … 120 132 my %image_list_hash; 121 133 for (my $i = 1; $i < $Nrows; $i++) { 122 # print "$i $Nrows $query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]\n"; 134 # This could use the fact that locate_images now accepts position arrays, but 135 # I'll save that for after I get the majority of things working. 123 136 my $image_set_tmp = find_image_set($query{HEADER}{FPA_ID}[0],$query{HEADER}{STAGE}[0], 124 137 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0], … … 126 139 $query{CONTENT}{ROWNUM}[$i],$verbose); 127 140 unless (%$image_set_tmp) { 128 my_die("No images were returned.",$query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0], 129 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0], 130 $query{HEADER}{STAGE}[0]); 131 } 132 print "=== $image_set_tmp->{IMAGE}\n $image_set_tmp->{PSF}\n $image_set_tmp->{MASK}\n $image_set_tmp->{WEIGHT}\n $image_set_tmp->{SKY_COORDINATES}\n $image_set_tmp->{ROWNUM}\n"; 133 # This appends, assuming that if we get an image, we also get the identical psf/mask/weight/etc. 141 # No images were returned, so create a dummy entry that 142 $image_list_hash{'no_image'}{IMAGE} = 'no_image'; 143 $image_list_hash{'no_image'}{PSF} = 'no_psf'; 144 $image_list_hash{'no_image'}{MASK} = 'no_mask'; 145 $image_list_hash{'no_image'}{WEIGHT} = 'no_weight'; 146 $image_list_hash{'no_image'}{CATALOG} = 'no_catalog'; 147 $image_list_hash{'no_image'}{CLASS_ID} = 'no_class'; 148 $image_list_hash{'no_image'}{ERROR} = $PSTAMP_NO_IMAGE_MATCH; 149 push @{ $image_list_hash{'no_image'}{SKY_COORDINATES} }, "$query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]"; 150 push @{ $image_list_hash{'no_image'}{ROWNUM} }, $query{CONTENT}{ROWNUM}[$i]; 151 next; 152 } 153 # print "=== $image_set_tmp->{IMAGE}\n $image_set_tmp->{PSF}\n"; 154 # print " $image_set_tmp->{MASK}\n $image_set_tmp->{WEIGHT}\n"; 155 # print " $image_set_tmp->{SKY_COORDINATES}\n $image_set_tmp->{ROWNUM}\n"; 156 157 # This indexes the results for identical images into the same hash. 134 158 $image_list_hash{$image_set_tmp->{IMAGE}}{IMAGE} = $image_set_tmp->{IMAGE}; 135 159 $image_list_hash{$image_set_tmp->{IMAGE}}{PSF} = $image_set_tmp->{PSF}; … … 138 162 $image_list_hash{$image_set_tmp->{IMAGE}}{CATALOG} = $image_set_tmp->{CATALOG}; 139 163 $image_list_hash{$image_set_tmp->{IMAGE}}{CLASS_ID} = $image_set_tmp->{CLASS_ID}; 164 $image_list_hash{$image_set_tmp->{IMAGE}}{ERROR} = $image_set_tmp->{ERROR}; 140 165 push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{SKY_COORDINATES} }, $image_set_tmp->{SKY_COORDINATES}; 141 166 push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{ROWNUM} }, $image_set_tmp->{ROWNUM}; 142 167 } 168 143 169 my $i = 0; 144 145 146 170 foreach my $k (keys %image_list_hash) { 171 # If we errored out on finding an image, we need to not try to run psphot here. 172 if ($image_list_hash{$k}{ERROR} != 0) { 173 next; 174 } 147 175 # Write coordinates of the requested targets to a file. 148 my ($coordfile,$coordname) = tempfile(" /tmp/detect.coords.$i.XXXX",176 my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$i.XXXX", 149 177 UNLINK => !$save_temps); 150 my ($targetfile,$targetname) = tempfile(" /tmp/detect.targets.$i.XXXX",178 my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$i.XXXX", 151 179 UNLINK => !$save_temps); 152 180 … … 154 182 print $coordfile "$image_list_hash{$k}{SKY_COORDINATES}[$j]\n"; 155 183 } 156 print "$k\n";184 # print "$k\n"; 157 185 # Convert the sky coordinates to image coordinates with ppCoord. 158 186 my $command = "ppCoord -astrom $image_list_hash{$k}{CATALOG} -radec $coordname"; … … 172 200 } 173 201 174 print "psphot $image_list_hash{$k}{PSF}\n";202 # print "psphot $image_list_hash{$k}{PSF}\n"; 175 203 # Run psphotForced on the target list. 176 my $tmpdir = tempdir("detect.$i.XXXX", DIR => " /tmp/", CLEANUP => !$save_temps);204 my $tmpdir = tempdir("detect.$i.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps); 177 205 $image_list_hash{$k}{OUTROOT} = "$tmpdir/detectability.$query{HEADER}{STAGE}[0].$query{HEADER}{FPA_ID}[0]"; 178 206 … … 186 214 run(command => $psphot_cmd, verbose => $verbose); 187 215 unless ($success) { 188 my_die("Unable to perform $psphot_cmd. Error_code: $error_code", 189 $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0], 190 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0], 191 $query{HEADER}{STAGE}[0],$error_code); 216 $image_list_hash{$k}{ERROR} = $PSTAMP_SYSTEM_ERROR; 192 217 } 193 218 } … … 197 222 # 198 223 my @rownums = (); 224 my @out_errors = (); 199 225 my @psphot_Npix = (); 200 226 my @psphot_Qfact= (); 201 227 my @psphot_flux = (); 228 202 229 foreach my $k (keys %image_list_hash) { 203 my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf"; 204 205 my ($tmp_Npix,$tmp_Qfact,$tmp_flux) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE}); 206 207 push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; 208 push @psphot_Npix, @{ $tmp_Npix }; 209 push @psphot_Qfact, @{ $tmp_Qfact }; 210 push @psphot_flux, @{ $tmp_flux }; 230 if ($image_list_hash{$k}{ERROR} == 0) { 231 my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf"; 232 233 my ($tmp_Npix,$tmp_Qfact,$tmp_flux) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE}); 234 235 push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; 236 push @out_errors, (map { $image_list_hash{$k}{ERROR} } @{ $image_list_hash{$k}{ROWNUM} }); 237 push @psphot_Npix, @{ $tmp_Npix }; 238 push @psphot_Qfact, @{ $tmp_Qfact }; 239 push @psphot_flux, @{ $tmp_flux }; 240 } 241 else { 242 push @rownums, @{ $image_list_hash{$k}{ROWNUM} }; 243 push @out_errors, (map { $image_list_hash{$k}{ERROR} } @{ $image_list_hash{$k}{ROWNUM} }); 244 push @psphot_Npix, (map { 0 } @{ $image_list_hash{$k}{ROWNUM} }); 245 push @psphot_Qfact, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); 246 push @psphot_flux, (map { 0.0 } @{ $image_list_hash{$k}{ROWNUM} }); 247 } 211 248 } 212 249 … … 215 252 $query{HEADER}{MJD_OBS}[0],$query{HEADER}{filter}[0], 216 253 $query{HEADER}{obscode}[0], 217 \@rownums, \@ psphot_Npix, \@psphot_Qfact, \@psphot_flux);218 print "Wrote response file $output\n";254 \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux); 255 # print "Wrote response file $output\n"; 219 256 # 220 257 # Add to datastore 221 258 # 222 # my $finish_command = "dquery_finish.pl --req_id $req_id --req_name $req_name --product $product"; 223 # ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = 224 # run(command => $finish_command, verbose => $verbose); 225 # unless ($success) { 226 # warn ("Unable to perform $finish_command error code: $error_code"); 227 # } 228 259 # Files are added to the datastore by dquery_finish.pl 229 260 # 230 261 # Cleanup 231 262 # 232 263 # Since everything is written to temporary files, there should be nothing to cleanup. 264 233 265 # 234 266 # Utilities … … 243 275 my $index = shift; 244 276 my $verbose = shift; 245 277 278 # This is the set of things that we need in order to run psphotForced 246 279 my $option_mask |= 1; 247 280 $option_mask |= $PSTAMP_SELECT_IMAGE; … … 253 286 my $mjd_max = $mjd + 1; 254 287 255 # Construct a row list: 256 288 # Construct a row list. 257 289 my @rowList; 258 290 $rowList[0]->{CENTER_X} = $ra; … … 261 293 $rowList[0]->{STAGE} = $stage; 262 294 $rowList[0]->{COORD_MASK} = 0; 263 print "$stage\n"; 295 296 # print "$stage\n"; 264 297 # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter. 265 my @images = locate_images($ipprc,$ dbname,298 my @images = locate_images($ipprc,$imagedb, 266 299 \@rowList, 267 300 "bycoord",$stage, … … 284 317 } 285 318 elsif ($stage eq 'stack') { 286 # Stacks hide the exposure name very well, so I'm 287 # choosing the stack_id as the FPA_ID to match. This 288 # probably needs to be approved somehow. 319 # Stacks hide the exposure name very well, so we can only match against stage_id 289 320 if (${ $j }{stage_id} ne $FPA_ID) { 290 321 next; … … 299 330 } 300 331 # Debug prints of all the components of this image 301 foreach my $k (keys %{ $j }) { 302 print "$i $j $k ${ $j }{$k}\n"; 303 } 304 332 # foreach my $k (keys %{ $j }) { 333 # if ($k eq 'row_index') { 334 # print "$i $j $k @{${ $j }{$k} }\n"; 335 # } 336 # print "$i $j $k ${ $j }{$k}\n"; 337 # } 338 339 # Check for existance of the images. Drawn mostly from pstampparse.pl 340 my $run_state = ${ $j }{state}; 341 my $data_state = ${ $j }{data_state}; 342 $data_state = $run_state if $stage eq 'stack'; 343 my $fault = 0; 344 if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or 345 ($run_state eq 'drop') or 346 ($run_state eq 'error_cleaned') or 347 ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) { 348 # image is gone and it's not coming back 349 $fault = $PSTAMP_GONE; 350 } 351 elsif (($data_state ne 'full') or ($need_magic and (${ $j }{magicked} < 0))) { 352 if (($stage eq 'stack')||($stage eq 'diff')) { 353 # updating stacks and diffs isn't implemented 354 $fault = $PSTAMP_NOT_IMPLEMENTED; 355 } 356 if ($stage eq 'chip') { 357 my $burntool_state = ${ $j }{burntool_state}; 358 if ($burntool_state and (abs($burntool_state) < 14)) { 359 $fault = $PSTAMP_NOT_AVAILABLE; 360 } 361 } 362 363 if ($fault == 0) { 364 # This bombs us out to dqueryparse, which will then flag a job for this run to be updated. 365 my_die_for_update($data_state,$query{HEADER}{STAGE}[0], 366 ${ $j }{stage_id},${ $j }{class_id} || ${ $j }{skycell_id}, 367 $need_magic,$imagedb,$PSTAMP_NOT_AVAILABLE); 368 } 369 } 370 305 371 # This image matches, so we want to save the information into our output structure 306 372 $image_info{ROWNUM} = $index; … … 309 375 $image_info{MASK} = ${ $j }{mask}; 310 376 $image_info{WEIGHT} = ${ $j }{weight}; 377 $image_info{ERROR} = $fault; 311 378 $image_info{SKY_COORDINATES} = "$ra $dec"; 312 379 # To do sky->image coordinate transformations, we need to use the cmf/smf file. If … … 419 486 my $obscode = shift; 420 487 my $rownum_ref = shift; 488 my $out_err_ref = shift; 421 489 my $psphot_Npix_ref = shift; 422 490 my $psphot_Qfact_ref = shift; … … 427 495 my $columns = [ 428 496 # matching rownum from detectability original request 429 { name => 'ROWNUM', type => '20A', writetype => TSTRING }, 497 { name => 'ROWNUM', type => 'V', writetype => TULONG }, 498 # any errors that occurred during processing 499 { name => 'ERROR_CODE', type => 'V', writetype => TULONG }, 430 500 # number of pixels used in hypothetical PSF for the query detection 431 501 { name => 'DETECT_N', type => 'V', writetype => TULONG }, … … 469 539 for (my $i = 0; $i < $numRows; $i++) { 470 540 push @{$colData{'ROWNUM'}}, ${ $rownum_ref }[$i]; 541 push @{$colData{'ERROR_CODE'}}, ${ $out_err_ref }[$i]; 471 542 push @{$colData{'DETECT_N'}}, ${ $psphot_Npix_ref }[$i]; 472 543 push @{$colData{'DETECT_F'}}, ${ $psphot_Qfact_ref }[$i]; … … 543 614 544 615 } 616 617 sub my_die_for_update { 618 my $state = shift; 619 my $stage = shift; 620 my $stage_id = shift; 621 my $component = shift; 622 my $need_magic = shift; 623 my $imagedb = shift; 624 my $exit_code = shift; 625 626 print "$state $stage $stage_id $component $need_magic $imagedb\n"; 627 print STDERR "$state $stage $stage_id $component $need_magic $imagedb\n"; 628 exit($exit_code); 629 }
Note:
See TracChangeset
for help on using the changeset viewer.
