IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jul 29, 2010, 1:50:08 PM (16 years ago)
Author:
watersc1
Message:

functional, not 100% tested version of updated detectability server. Will test on the test pstamp server this afternoon

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/pstamp/scripts/detectability_respond.pl

    r28652 r28777  
    4242my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
    4343my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project);
    44 my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps);
     44my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom);
    4545GetOptions(
    4646    'input=s'         =>      \$request_file,
    … …  
    5151    'verbose'         =>      \$verbose,
    5252    'save-temps'      =>      \$save_temps,
     53    'ignore-wisdom'   =>      \$ignore_wisdom,
    5354    ) or pod2usage(2);
    5455
    … …  
    8485}
    8586
     87my %query = ();
     88my %image_list_hash = ();
     89my $wisdom_file = "${workdir}/wisdom.dat";
     90if ((-e $wisdom_file)&&!($ignore_wisdom)) {
     91    print "Reading wisdom file $wisdom_file instead of parsing...\n";
     92    open(WISDOM,"$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
     93    my $i = 0;
     94    while(<WISDOM>) {
     95        chomp;
     96        my ($fpa_id,@key_values) = split /\s+/;
     97        while ($#key_values > -1) {
     98            my $key = shift(@key_values);
     99            my $val = shift(@key_values);
     100            $query{$fpa_id}{$key}[$i] = $val;
     101        }
     102        $i++;
     103    }
     104    close(WISDOM);
     105} # End reading wisdom.
     106else {
    86107#
    87108# Parse input request file using detect_query_read (as it's already written).
    88109#
    89 my $dqr_command = "$detect_query_read --input $request_file";
    90 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    91     run(command => $dqr_command, verbose => $verbose);
    92 unless ($success) {
    93     # This is a problem, because I'm not sure how we handle a failure to read something.
    94     # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it,
    95     # we can't send that response back.
    96     die("Unable to perform $dqr_command error code: $error_code");
    97 }
    98 my %query = ();
    99 my $Nrows = 0;
    100 {
    101     my @column_names = ();
    102     my $section = '';
    103     foreach my $entry (split /\n/, (join "", @$stdout_buf)) {
    104         if ($entry =~ /^#/) {
    105             @column_names = split /\s+/, $entry;
    106             shift(@column_names);  # Dump the hash sign.
    107             if ($section eq 'HEADER') {
    108                 $section = 'CONTENT';
     110    my $dqr_command = "$detect_query_read --input $request_file";
     111    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     112        run(command => $dqr_command, verbose => $verbose);
     113    unless ($success) {
     114        # This is a problem, because I'm not sure how we handle a failure to read something.
     115        # We need to return a $PSTAMP_INVALID_REQUEST, I think, but if we can't read it,
     116        # we can't send that response back.
     117        die("Unable to perform $dqr_command error code: $error_code");
     118    }
     119    my $Nrows = 0;
     120    {
     121        my @column_names = ();
     122        foreach my $entry (split /\n/, (join "", @$stdout_buf)) {
     123            if ($entry =~ /^#/) {
     124                @column_names = split /\s+/, $entry;
     125                shift(@column_names);  # Dump the hash sign.
    109126            }
    110127            else {
    111                 $section = 'HEADER';
     128                # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG QUERY_ID FPA_ID MJD-OBS FILTER OBSCODE STAGE
     129                my %row_data;
     130                @row_data{@column_names} = (split /\s+/, $entry);
     131                for (my $i = 0; $i <= $#column_names; $i++) {
     132                push @{ $query{$row_data{"FPA_ID"}}{$column_names[$i]} }, $row_data{$column_names[$i]};
     133                $Nrows = scalar(keys(%query));
     134#               print "$row_data{'FPA_ID'} $Nrows $i $column_names[$i] $row_data{$column_names[$i]}\n";
    112135            }
    113         }
     136            }
     137        }
     138    }
     139#
     140# Identify target images.  This should properly collate targets on a single imfile.
     141#
     142    foreach my $fpa_id (keys %query) {
     143        my %temp_hash;
     144        my $query_style = 'byexp';
     145        my $stage;
     146        my $filter;
     147        my $mjd;
     148        # Determine the query style for this fpa_id
     149        if ($fpa_id =~ /o.*g.*o/) {
     150        $query_style = 'byexp';
     151    }
     152        elsif ($fpa_id =~ /\d+/) {
     153        $query_style = 'byid';
     154    }
    114155        else {
    115             # HEADER:
    116             # QUERY_ID FPA_ID MJD_OBS FILTER OBSCODE STAGE
    117             # CONTENT:
    118             # ROWNUM RA1_DEG DEC1_DEG RA2_DEG DEC2_DEG MAG
    119             my @columns = split /\s+/, $entry;
    120             for (my $i = 0; $i <= $#columns; $i++) {
    121                 $query{$section}{$column_names[$i]}[$Nrows] = $columns[$i];
    122 #               print "$section $column_names[$i] $Nrows $columns[$i]\n";
    123             }
    124             $Nrows++;
    125         }
    126     }
    127 }
    128 
    129 #
    130 # Identify target images.  This should properly collate targets on a single imfile.
    131 #
    132 my %image_list_hash;
    133 for (my $i = 1; $i < $Nrows; $i++) {
    134     # This could use the fact that locate_images now accepts position arrays, but
    135     # I'll save that for after I get the majority of things working.
    136     my $image_set_tmp  = find_image_set($query{HEADER}{FPA_ID}[0],$query{HEADER}{STAGE}[0],
    137                                         $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],
    138                                         $query{CONTENT}{RA1_DEG}[$i],$query{CONTENT}{DEC1_DEG}[$i],
    139                                         $query{CONTENT}{ROWNUM}[$i],$verbose);
    140     unless (%$image_set_tmp) {
    141         # No images were returned, so create a dummy entry that
    142         $image_list_hash{'no_image'}{IMAGE}    = 'no_image';
    143         $image_list_hash{'no_image'}{PSF}      = 'no_psf';
    144         $image_list_hash{'no_image'}{MASK}     = 'no_mask';
    145         $image_list_hash{'no_image'}{WEIGHT}   = 'no_weight';
    146         $image_list_hash{'no_image'}{CATALOG}  = 'no_catalog';
    147         $image_list_hash{'no_image'}{CLASS_ID} = 'no_class';
    148         $image_list_hash{'no_image'}{ERROR}    = $PSTAMP_NO_IMAGE_MATCH;
    149         push @{ $image_list_hash{'no_image'}{SKY_COORDINATES} }, "$query{CONTENT}{RA1_DEG}[$i] $query{CONTENT}{DEC1_DEG}[$i]";
    150         push @{ $image_list_hash{'no_image'}{ROWNUM} }, $query{CONTENT}{ROWNUM}[$i];
    151         next;
    152     }
    153 #     print "=== $image_set_tmp->{IMAGE}\n    $image_set_tmp->{PSF}\n";
    154 #     print "    $image_set_tmp->{MASK}\n    $image_set_tmp->{WEIGHT}\n";
    155 #     print "    $image_set_tmp->{SKY_COORDINATES}\n    $image_set_tmp->{ROWNUM}\n";
    156 
    157     # This indexes the results for identical images into the same hash.
    158     $image_list_hash{$image_set_tmp->{IMAGE}}{IMAGE}    = $image_set_tmp->{IMAGE};
    159     $image_list_hash{$image_set_tmp->{IMAGE}}{PSF}      = $image_set_tmp->{PSF};
    160     $image_list_hash{$image_set_tmp->{IMAGE}}{MASK}     = $image_set_tmp->{MASK};
    161     $image_list_hash{$image_set_tmp->{IMAGE}}{WEIGHT}   = $image_set_tmp->{WEIGHT};
    162     $image_list_hash{$image_set_tmp->{IMAGE}}{CATALOG}  = $image_set_tmp->{CATALOG};
    163     $image_list_hash{$image_set_tmp->{IMAGE}}{CLASS_ID} = $image_set_tmp->{CLASS_ID};
    164     $image_list_hash{$image_set_tmp->{IMAGE}}{ERROR}    = $image_set_tmp->{ERROR};
    165     push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{SKY_COORDINATES} }, $image_set_tmp->{SKY_COORDINATES};
    166     push @{ $image_list_hash{$image_set_tmp->{IMAGE}}{ROWNUM} }, $image_set_tmp->{ROWNUM};
    167 }
    168 
    169 my $i = 0;
    170 foreach my $k (keys %image_list_hash) {
    171     # If we errored out on finding an image, we need to not try to run psphot here.
    172     if ($image_list_hash{$k}{ERROR} != 0) {
    173         next;
    174     }
    175     # Write coordinates of the requested targets to a file.
    176     my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$i.XXXX",
    177                                             UNLINK => !$save_temps);
    178     my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$i.XXXX",
    179                                             UNLINK => !$save_temps);
    180 
    181     for (my $j = 0; $j <= $#{ $image_list_hash{$k}{SKY_COORDINATES} }; $j++) {
    182         print $coordfile "$image_list_hash{$k}{SKY_COORDINATES}[$j]\n";
    183     }
    184 #    print "$k\n";
    185     # Convert the sky coordinates to image coordinates with ppCoord.
    186     my $command = "ppCoord -astrom $image_list_hash{$k}{CATALOG} -radec $coordname";
    187     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    188         run(command => $command, verbose => $verbose);
    189     unless ($success) {
    190         my_die("Unable to perform $command. Error_code: $error_code",
    191                $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0],
    192                $query{HEADER}{MJD_OBS}[0],$query{HEADER}{FILTER}[0],$query{HEADER}{OBSCODE}[0],
    193                $query{HEADER}{STAGE}[0],$error_code);
    194     }
    195     my @response = split /\n/, (join "", @$stdout_buf);
    196     foreach my $line (@response) {
    197         my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line;
    198         print $targetfile "$r_x $r_y\n";
    199         $image_list_hash{$k}{EXTENSION_BASE} = $r_chip;
    200     }
    201 
    202 #     print "psphot $image_list_hash{$k}{PSF}\n";
    203     # Run psphotForced on the target list.
    204     my $tmpdir  = tempdir("detect.$i.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps);
    205     $image_list_hash{$k}{OUTROOT} = "$tmpdir/detectability.$query{HEADER}{STAGE}[0].$query{HEADER}{FPA_ID}[0]";
    206    
    207     my $psphot_cmd = "$psphotForced -psf $image_list_hash{$k}{PSF} ";
    208     $psphot_cmd .= "-file $image_list_hash{$k}{IMAGE} ";
    209     $psphot_cmd .= "-mask $image_list_hash{$k}{MASK} ";
    210     $psphot_cmd .= "-variance $image_list_hash{$k}{WEIGHT} ";
    211     $psphot_cmd .= "-srctext $targetname $image_list_hash{$k}{OUTROOT} ";
    212     $psphot_cmd .= "-D OUTPUT.FORMAT PS1_DV1 ";
    213 
    214     ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    215         run(command => $psphot_cmd, verbose => $verbose);
    216     unless ($success) {
    217         $image_list_hash{$k}{ERROR} = $PSTAMP_SYSTEM_ERROR;
    218     }
    219 }
    220 
    221 #
    222 # Convert psphot output to response
    223 #
    224 my @rownums = ();
    225 my @out_errors = ();
    226 my @psphot_Npix = ();
    227 my @psphot_Qfact= ();
    228 my @psphot_flux = ();
    229 my @psphot_error = ();
    230 
    231 foreach my $k (keys %image_list_hash) {
    232     if ($image_list_hash{$k}{ERROR} == 0) {
    233         my $cmf = "$image_list_hash{$k}{OUTROOT}.$image_list_hash{$k}{CLASS_ID}.cmf";
    234        
    235         my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$image_list_hash{$k}{EXTENSION_BASE});
    236        
    237         push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
    238         push @out_errors,     (map { $image_list_hash{$k}{ERROR} }  @{ $image_list_hash{$k}{ROWNUM} });
    239         push @psphot_Npix,    @{ $tmp_Npix };
    240         push @psphot_Qfact,   @{ $tmp_Qfact };
    241         push @psphot_flux,    @{ $tmp_flux };
    242         push @psphot_error,   @{ $tmp_flux_error };
    243     }
    244     else {
    245         push @rownums,        @{ $image_list_hash{$k}{ROWNUM} };
    246         push @out_errors,     (map { $image_list_hash{$k}{ERROR} }  @{ $image_list_hash{$k}{ROWNUM} });
    247         push @psphot_Npix,    (map { 0 }  @{ $image_list_hash{$k}{ROWNUM} });
    248         push @psphot_Qfact,   (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
    249         push @psphot_flux,    (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
    250         push @psphot_error,    (map { 0.0 }  @{ $image_list_hash{$k}{ROWNUM} });
    251     }   
    252 }
    253 
    254 write_response_file($output,
    255                     $query{HEADER}{QUERY_ID}[0],$query{HEADER}{FPA_ID}[0],
    256                     $query{HEADER}{MJD_OBS}[0],$query{HEADER}{filter}[0],
    257                     $query{HEADER}{obscode}[0],
    258                     \@rownums, \@out_errors, \@psphot_Npix, \@psphot_Qfact, \@psphot_flux, \@psphot_error);
    259 # print "Wrote response file $output\n";
    260 #
    261 # Add to datastore
    262 #
    263 # Files are added to the datastore by dquery_finish.pl
    264 #
    265 # Cleanup
    266 #
    267 # Since everything is written to temporary files, there should be nothing to cleanup.
    268 
    269 #
    270 # Utilities
    271 #
    272 sub find_image_set {
    273     my $FPA_ID = shift;
    274     my $stage  = lc(shift);
    275     my $mjd    = shift;
    276     my $filter = shift;
    277     my $ra     = shift;
    278     my $dec    = shift;
    279     my $index  = shift;
    280     my $verbose = shift;
    281 
    282     # This is the set of things that we need in order to run psphotForced
    283     my $option_mask |= 1;
    284     $option_mask |= $PSTAMP_SELECT_IMAGE;
    285     $option_mask |= $PSTAMP_SELECT_MASK;
    286     $option_mask |= $PSTAMP_SELECT_VARIANCE;
    287     $option_mask |= $PSTAMP_SELECT_PSF;
    288     my $need_magic = 1;
    289     my $mjd_min = $mjd;
    290     my $mjd_max = $mjd + 1;
    291 
    292     # Construct a row list.
    293     my @rowList;
    294     $rowList[0]->{CENTER_X} = $ra;
    295     $rowList[0]->{CENTER_Y} = $dec;
    296     $rowList[0]->{ID} = 1;
    297     $rowList[0]->{STAGE} = $stage;
    298     $rowList[0]->{COORD_MASK} = 0;
    299 
    300     #    print "$stage\n";
    301     # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
    302     my @images = locate_images($ipprc,$imagedb,
    303                                 \@rowList,
    304                                "bycoord",$stage,
    305                                undef,undef,undef,
    306                                $option_mask,$need_magic,
    307                                # $ra,$dec,
    308                                $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose); 
    309 
    310     my %image_info  = ();
    311     foreach my $i (@images) {           # Scan over each result
    312         foreach my $j (@{ $i }) {       # Scan over each image in the result
    313             # We only care about an image if it matches the FPA_ID in the request
    314             if ($stage eq 'diff') {
    315                 # Diffs match if either exposure name is defined and matches the FPA_ID
    316                 unless ((defined(${ $j }{exp_name_1}) && (${ $j }{exp_name_1} eq $FPA_ID))||
    317                         (defined(${ $j }{exp_name_2}) && (${ $j }{exp_name_2} eq $FPA_ID))||
    318                         (${ $j }{exp_id_1} eq $FPA_ID)||(${ $j }{exp_id_2} eq $FPA_ID)) {
    319                     next;
     156        exit_with_failure(21,"Parse error in request file");
     157    }
     158        # Confirm that we only have one stage/filter/mjd
     159        for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
     160        $temp_hash{STAGE}{$query{$fpa_id}{STAGE}[$i]} = 1;
     161        $temp_hash{FILTER}{$query{$fpa_id}{FILTER}[$i]} = 1;
     162        $temp_hash{'MJD-OBS'}{$query{$fpa_id}{'MJD-OBS'}[$i]} = 1;
     163    }
     164        if (scalar(keys(%{ $temp_hash{STAGE} })) == 1) {
     165            $stage = (keys(%{ $temp_hash{STAGE} }))[0];
     166        }
     167        else {
     168        exit_with_failure(21,"Too many STAGEs specified");
     169    }
     170        if (scalar(keys(%{ $temp_hash{FILTER} })) == 1) {
     171        $filter = (keys(%{ $temp_hash{FILTER} }))[0];
     172    }
     173        else {
     174        exit_with_failure(21,"Too many FILTERs specified");
     175    }
     176        if (scalar(keys(%{ $temp_hash{'MJD-OBS'} })) == 1) {
     177        $mjd = (keys(%{ $temp_hash{'MJD-OBS'} }))[0];
     178    }
     179        else {
     180        exit_with_failure(21,"Too many MJD-OBS specified");
     181    }
     182        # Set common request components
     183        my $option_mask |= 1;
     184        $option_mask |= $PSTAMP_SELECT_IMAGE;
     185        $option_mask |= $PSTAMP_SELECT_MASK;
     186        $option_mask |= $PSTAMP_SELECT_VARIANCE;
     187        $option_mask |= $PSTAMP_SELECT_PSF;
     188        my $need_magic = 1;
     189        if ($stage eq 'stack') {
     190            $need_magic = 0;
     191        }
     192        my $mjd_min = $mjd;
     193        my $mjd_max = $mjd + 1;
     194       
     195        # Construct a row list.
     196        my @rowList;
     197        for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
     198            $rowList[$i]->{CENTER_X} = $query{$fpa_id}{RA1_DEG}[$i];
     199            $rowList[$i]->{CENTER_Y} = $query{$fpa_id}{DEC1_DEG}[$i];
     200            $rowList[$i]->{ID} = $query{$fpa_id}{ROWNUM}[$i];
     201            $rowList[$i]->{COORD_MASK} = 0;
     202        }
     203       
     204        # Call the PStamp code to find the images that contain the target on the given MJD in the specified filter.
     205        my $pstamp_images_ref = locate_images($ipprc,$imagedb,
     206                                          \@rowList,
     207                                          $query_style,$stage,
     208                                          $fpa_id,undef,undef,
     209                                          $option_mask,$need_magic,
     210                                          # $ra,$dec,
     211                                          $mjd_min,$mjd_max,$filter . ".00000",undef,$verbose); 
     212       
     213        foreach my $this_image_ref (@{ $pstamp_images_ref }) {
     214            foreach my $key (sort (keys %{ $this_image_ref } )) {
     215                my $value = ${ $this_image_ref }{$key};
     216                if ($key eq 'row_index') {
     217                    $value = join ' ', @{ $this_image_ref->{$key} };
     218                }
     219#               print "$this_image_ref $key $value\n";
     220                foreach my $valid_index (@{ $this_image_ref->{row_index} }) {
     221                    $query{$fpa_id}{IMAGE}[$valid_index] = $this_image_ref->{image};
     222                    $query{$fpa_id}{MASK}[$valid_index] = $this_image_ref->{mask};
     223                    $query{$fpa_id}{WEIGHT}[$valid_index] = $this_image_ref->{weight};
     224                    $query{$fpa_id}{PSF}[$valid_index] = $this_image_ref->{psf};
     225                    $query{$fpa_id}{STAGE_ID}[$valid_index] = $this_image_ref->{stage_id};
     226                    $query{$fpa_id}{IMAGE_DB}[$valid_index] = $this_image_ref->{imagedb};
     227                    $query{$fpa_id}{NEED_MAGIC}[$valid_index] = $need_magic;
     228                   
     229                    if (exists($this_image_ref->{astrom})) {
     230                        $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{astrom};
     231                    }
     232                    else {
     233                    $query{$fpa_id}{CATALOG}[$valid_index] = $this_image_ref->{cmf};
     234                }
     235                    if (exists($this_image_ref->{class_id})) {
     236                        $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{class_id};
     237                        $query{$fpa_id}{CLASS_ID}[$valid_index] = $this_image_ref->{class_id};
     238                       
     239                    }
     240                    else {
     241                        $query{$fpa_id}{COMPONENT_ID}[$valid_index] = $this_image_ref->{skycell_id};
     242                        $query{$fpa_id}{CLASS_ID}[$valid_index] = 'fpa';
     243                    }
     244                   
     245                    $query{$fpa_id}{STATE}[$valid_index] = $this_image_ref->{state};
     246                    if (exists($this_image_ref->{data_state})) {
     247                        $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{data_state};
     248                    }
     249                    else {
     250                        $query{$fpa_id}{DATA_STATE}[$valid_index] = $this_image_ref->{state};
     251                    }
     252                    $query{$fpa_id}{FAULT}[$valid_index] = 0;
     253                    $query{$fpa_id}{MAGICKED}[$valid_index] = $this_image_ref->{magicked};
     254                    if ($stage eq 'chip') {
     255                        $query{$fpa_id}{BURNTOOL_STATE}[$valid_index] = $this_image_ref->{burntool_state};
     256                    }
     257                   
     258                    # Determine if the data exists.
     259                    if (($query{$fpa_id}{STATE}[$valid_index] eq 'goto_purged') or
     260                        ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'purged') or
     261                        ($query{$fpa_id}{STATE}[$valid_index] eq 'drop') or
     262                        ($query{$fpa_id}{STATE}[$valid_index] eq 'error_cleaned') or
     263                        ($query{$fpa_id}{STATE}[$valid_index] eq 'goto_scrubbed') or
     264                        ($query{$fpa_id}{DATA_STATE}[$valid_index] eq 'scrubbed')) {
     265                       
     266                        # image is gone and it's not coming back
     267                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
     268                    }
     269                    elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] = 0)) {
     270                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
     271                    }
     272                    elsif (($query{$fpa_id}{DATA_STATE}[$valid_index] ne 'full')) {                   
     273                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_AVAILABLE;
     274                       
     275                        # updating stacks isn't implemented
     276                        if (($stage eq 'stack')) {
     277                            $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
     278                        }
     279                        # updating old burntool data isn't implemented
     280                        elsif ($stage eq 'chip') {
     281                            if ($query{$fpa_id}{BURNTOOL_STATE}[$valid_index] and
     282                                (abs($query{$fpa_id}{BURNTOOL_STATE}[$valid_index]) < 14)) {
     283                                $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_IMPLEMENTED;
     284                            }
     285                        }
     286                    } # End determining error faults.
    320287                }
    321288            }
    322             elsif ($stage eq 'stack') {
    323                 # Stacks hide the exposure name very well, so we can only match against stage_id
    324                 if (${ $j }{stage_id} ne $FPA_ID) {
    325                     next;
    326                 }
     289        }
     290    }
     291} # End calculating wisdom
     292my %update_request;
     293my %processing_request;
     294
     295open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
     296foreach my $fpa_id (keys %query) {
     297    for (my $i = 0; $i <= $#{ $query{$fpa_id}{STAGE} }; $i++) {
     298        print WISDOM "$fpa_id\t";
     299        foreach my $key (keys %{ $query{$fpa_id} }) {
     300            print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
     301        }
     302        print WISDOM "\n";
     303        @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } =
     304            ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
     305             $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
     306        push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
     307    }
     308}
     309close(WISDOM);
     310my $exit_code = 0;
     311my $update_request_file = "${workdir}/update_request.dat";
     312open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
     313foreach my $images (keys %update_request) {
     314    foreach my $fault (keys %{ $update_request{$images} }) {
     315        if ($fault == 25) {
     316            $exit_code = 25;
     317        }
     318        my $update_request = join ' ', @{ $update_request{$images}{$fault} };
     319        print UPDATE_REQUEST "$update_request\n";
     320    }
     321}
     322close(UPDATE_REQUEST);
     323# if ($exit_code != 0) {
     324#     exit($exit_code);
     325# }
     326
     327# This duplicates stuff returned by PSTAMP, but my thought is to convert that ---^ into a conditional, in which I read
     328# from the wisdom.dat file.  This means only one pass on all that potentially slow stuff.  If that's the case, then
     329# I can recalculate the processing request.
     330
     331foreach my $fpa_id (keys %processing_request) {
     332    foreach my $image (keys %{ $processing_request{$fpa_id} }) {
     333        print "$fpa_id $image\t";
     334        foreach my $i (@{ $processing_request{$fpa_id}{$image} }) {
     335            print "$i ";
     336        }
     337        print "\n";
     338    }
     339}
     340#exit(10);
     341
     342# run ppCoord and psphotForced to calculate the required data.
     343foreach my $fpa_id (keys %processing_request) {
     344    foreach my $image (keys %{ $processing_request{$fpa_id} }) {
     345        # Get this image specific data from the first entry. That entry is now the king of this set.
     346        my $index = $processing_request{$fpa_id}{$image}[0];
     347        my $fault = $query{$fpa_id}{FAULT}[$index];
     348        my $catalog = $query{$fpa_id}{CATALOG}[$index];
     349        my $psf   = $query{$fpa_id}{PSF}[$index];
     350        my $mask  = $query{$fpa_id}{MASK}[$index];
     351        my $weight= $query{$fpa_id}{WEIGHT}[$index];
     352        my $stage = $query{$fpa_id}{STAGE}[$index];
     353        # if there's a fault, then we can't process this image.
     354        if ($fault != 0) {
     355            next;
     356        }
     357
     358        # Create coordinate file to convert to positions.
     359        my ($coordfile,$coordname) = tempfile("${workdir}/detect.coords.$index.XXXX",
     360                                              UNLINK => !$save_temps);
     361        my ($targetfile,$targetname) = tempfile("${workdir}/detect.targets.$index.XXXX",
     362                                                UNLINK => !$save_temps);
     363        foreach my $i (@{ $processing_request{$fpa_id}{$image} }) {
     364            print $coordfile "$query{$fpa_id}{RA1_DEG}[$i] $query{$fpa_id}{DEC1_DEG}[$i]\n";
     365        }
     366       
     367        # Convert the sky coordinates to image coordinates with ppCoord.
     368        my $command = "ppCoord -astrom $catalog -radec $coordname";
     369        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     370            run(command => $command, verbose => $verbose);
     371        unless ($success) {
     372            my_die("Unable to perform $command. Error_code: $error_code",
     373                   $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index],
     374                   $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index],
     375                   $error_code);
     376        }
     377        my @response = split /\n/, (join "", @$stdout_buf);
     378        my $i = 0;
     379        foreach my $line (@response) {
     380            my ($r_ra,$r_dec,$trash,$r_x,$r_y,$r_chip) = split /\s+/, $line;
     381            print $targetfile "$r_x $r_y\n";
     382            if (($r_ra == $query{$fpa_id}{RA1_DEG}[$processing_request{$fpa_id}{$image}[$i]])&&
     383                ($r_dec == $query{$fpa_id}{DEC1_DEG}[$processing_request{$fpa_id}{$image}[$i]])) {
     384                $query{$fpa_id}{X_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_x;
     385                $query{$fpa_id}{Y_PXL}[$processing_request{$fpa_id}{$image}[$i]] = $r_y;
     386                $query{$fpa_id}{EXTENSION_BASE}[$processing_request{$fpa_id}{$image}[$i]] = $r_chip;
    327387            }
    328388            else {
    329                 # For all the other stages (warp and chip are the ones I've tested), we can simply
    330                 # directly match the exposure name to the FPA_ID
    331                 unless ((${ $j }{exp_name} eq $FPA_ID)||(${ $j }{exp_id} eq $FPA_ID)) {
    332                     next;
    333                 }
     389                $error_code = $PS_EXIT_PROG_ERROR;
     390                my_die("Unable to match input RA/DEC with output RA/DEC: ($query{$fpa_id}{RA1_DEG}[$i],$query{$fpa_id}{DEC1_DEG}[$i]) -> ($r_ra,$r_dec) i. $error_code",
     391                   $query{$fpa_id}{QUERY_ID}[$index],$fpa_id,$query{$fpa_id}{'MJD-OBS'}[$index],
     392                   $query{$fpa_id}{FILTER}[$index],$query{$fpa_id}{OBSCODE}[$index],$query{$fpa_id}{STAGE}[$index],
     393                   $error_code);
    334394            }
    335             # Debug prints of all the components of this image
    336 #           foreach my $k (keys %{ $j }) {
    337 #               if ($k eq 'row_index') {
    338 #                   print "$i $j $k @{${ $j }{$k} }\n";
    339 #               }
    340 #               print "$i $j $k ${ $j }{$k}\n";
    341 #           }
    342 
    343             # Check for existance of the images. Drawn mostly from pstampparse.pl
    344             my $run_state = ${ $j }{state};
    345             my $data_state = ${ $j }{data_state};
    346             $data_state = $run_state if $stage eq 'stack';
    347             my $fault = 0;
    348             if (($run_state eq 'goto_purged') or ($data_state eq 'purged') or
    349                 ($run_state eq 'drop') or
    350                 ($run_state eq 'error_cleaned') or
    351                 ($run_state eq 'goto_scrubbed') or ($data_state eq 'scrubbed')) {
    352                 # image is gone and it's not coming back
    353                 $fault = $PSTAMP_GONE;
     395            $i++;
     396        }
     397
     398        # Run psphotForced on the target list.
     399        my $tmpdir  = tempdir("detect.$index.XXXX", DIR => "${workdir}/", CLEANUP => !$save_temps);
     400        my $outroot = "$tmpdir/detectability.${stage}.${fpa_id}.${index}";
     401        $query{$fpa_id}{PROC_ERROR}[$index] = 0;
     402        my $psphot_cmd = "$psphotForced -psf $psf ";
     403        $psphot_cmd .= "-file $image ";
     404        $psphot_cmd .= "-mask $mask ";
     405        $psphot_cmd .= "-variance $weight ";
     406        $psphot_cmd .= "-srctext $targetname $outroot ";
     407        $psphot_cmd .= "-D OUTPUT.FORMAT PS1_DV1 ";
     408       
     409        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     410            run(command => $psphot_cmd, verbose => $verbose);
     411        unless ($success) {
     412            $query{$fpa_id}{PROC_ERROR}[$index] = $PSTAMP_SYSTEM_ERROR;
     413        }
     414       
     415        # Why not parse out results here?
     416# Convert psphot output to response
     417        if ($query{$fpa_id}{PROC_ERROR}[$index] == 0) {
     418            my $class_id = $query{$fpa_id}{CLASS_ID}[$index];
     419            my $cmf = "${outroot}.${class_id}.cmf";
     420            my ($tmp_Npix,$tmp_Qfact,$tmp_flux,$tmp_flux_error) = read_cmf_file($cmf,$query{$fpa_id}{EXTENSION_BASE}[$index]);
     421            foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) {
     422                my $result_index = $processing_request{$fpa_id}{$image}[$i];
     423                $query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index];
     424
     425                $query{$fpa_id}{NPIX}[$result_index] = ${ $tmp_Npix }[$i];
     426                $query{$fpa_id}{QFACTOR}[$result_index] = ${ $tmp_Qfact }[$i];
     427                $query{$fpa_id}{FLUX}[$result_index] = ${ $tmp_flux }[$i];
     428                $query{$fpa_id}{FLUX_SIG}[$result_index] = ${ $tmp_flux_error }[$i];
     429               
     430#               print "$fpa_id $image $#{ $processing_request{$fpa_id}{$image} } $result_index $i ${ $tmp_Npix }[$i]\n";
    354431            }
    355             elsif  (($data_state ne 'full') or ($need_magic and (${ $j }{magicked} < 0))) {
    356                 if (($stage eq 'stack')||($stage eq 'diff')) {
    357                     # updating stacks and diffs isn't implemented
    358                     $fault = $PSTAMP_NOT_IMPLEMENTED;
    359                 }
    360                 if ($stage eq 'chip') {
    361                     my $burntool_state = ${ $j }{burntool_state};
    362                     if ($burntool_state and (abs($burntool_state) < 14)) {
    363                         $fault = $PSTAMP_NOT_AVAILABLE;
    364                     }
    365                 }
    366                
    367                 if ($fault == 0) {
    368                     # This bombs us out to dqueryparse, which will then flag a job for this run to be updated.
    369                     my_die_for_update($data_state,$query{HEADER}{STAGE}[0],
    370                                       ${ $j }{stage_id},${ $j }{class_id} || ${ $j }{skycell_id},
    371                                       $need_magic,$imagedb,$PSTAMP_NOT_AVAILABLE);
    372                 }
     432        }
     433        else {
     434            foreach ($i = 0; $i <= $#{ $processing_request{$fpa_id}{$image} }; $i++) {
     435                my $result_index = $processing_request{$fpa_id}{$image}[$i];
     436                $query{$fpa_id}{PROC_ERROR}[$result_index] = $query{$fpa_id}{PROC_ERROR}[$index];
     437
     438                $query{$fpa_id}{NPIX}[$result_index] = 0;
     439                $query{$fpa_id}{QFACTOR}[$result_index] = 0.0;
     440                $query{$fpa_id}{FLUX}[$result_index] = 0.0;
     441                $query{$fpa_id}{FLUX_SIG}[$result_index] = 0.0;
    373442            }
    374 
    375             # This image matches, so we want to save the information into our output structure
    376             $image_info{ROWNUM} = $index;
    377             $image_info{IMAGE}  = ${ $j }{image};
    378             $image_info{PSF}    = ${ $j }{psf};
    379             $image_info{MASK}   = ${ $j }{mask};
    380             $image_info{WEIGHT} = ${ $j }{weight};
    381             $image_info{ERROR}  = $fault;
    382             $image_info{SKY_COORDINATES} = "$ra $dec";
    383             # To do sky->image coordinate transformations, we need to use the cmf/smf file. If
    384             # an astrom reference (the camera stage smf file) exists, then use that, as we're dealing with
    385             # with the chip stage. Otherwise, use the stage-dependent cmf (and set the class_id to fpa).
    386             # The EXTENSION_BASE stores the basename of the extension that will be generated by psphotForced.
    387             if (exists(${ $j }{astrom})) {
    388                 $image_info{CATALOG} = ${ $j }{astrom};
    389                 $image_info{CLASS_ID} = ${ $j }{class_id};
    390 
    391             }
    392             else {
    393                 $image_info{CATALOG} = ${ $j }{cmf};
    394                 $image_info{CLASS_ID} = 'fpa';
    395 
    396             }
    397 
    398         }
    399     }
    400     return(\%image_info);
    401 }
     443        }
     444
     445               
     446    }
     447}
     448
     449write_response_file($output,\%query);
     450
     451#
     452# Utilities
     453#
    402454
    403455# Taken largely from detect_query_read
    … …  
    503555sub write_response_file {
    504556    my $outfile = shift;
    505     my $query_id = shift;
    506     my $FPA_ID = shift;
    507     my $MJD_OBS = shift;
    508     my $filter = shift;
    509     my $obscode = shift;
    510     my $rownum_ref = shift;
    511     my $out_err_ref = shift;
    512     my $psphot_Npix_ref = shift;
    513     my $psphot_Qfact_ref = shift;
    514     my $psphot_flux_ref = shift;
    515     my $psphot_error_ref = shift;
    516     my $status = 0;
    517 
    518     # Specification of columns to write
    519     my $columns = [
    520         # matching rownum from detectability original request
    521         { name => 'ROWNUM',   type => 'V', writetype => TULONG },
    522         # any errors that occurred during processing
    523         { name => 'ERROR_CODE',   type => 'V', writetype => TULONG },
    524         # number of pixels used in hypothetical PSF for the query detection
    525         { name => 'DETECT_N', type => 'V',   writetype => TULONG },
    526         # detectibility, indicating the fraction of PSF pixels detetable by IPP
    527         { name => 'DETECT_F', type => 'D',   writetype => TDOUBLE },
    528         # flux of the target source
    529         { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
    530         # error in the flux of the target source
    531         { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
    532         ];
    533    
    534     # Header translation table
    535     my $headers = {
    536         'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING,
    537                         comment => 'MOPS Query ID for this batch query' },
    538         'FPA_ID' => { name => 'FPA_ID',   type => TSTRING,
    539                       comment => 'original FPA_ID used at ingest' },
    540     };
     557    my $query_ref = shift;
     558
     559    my %query = %{ $query_ref };
     560
     561    my $columns;
     562    my $headers;
     563
     564    my $EXTVER_IS_1 = (scalar(keys(%query)) == 1);
     565#    print "EXTVER: $EXTVER_IS_1\n";
     566    my ($query_id,$FPA_ID,$MJD_OBS,$filter,$obscode,$status);
     567    if ($EXTVER_IS_1 == 1) {
     568        # Specification of columns to write
     569        $columns = [
     570            # matching rownum from detectability original request
     571            { name => 'ROWNUM',   type => 'V', writetype => TULONG },
     572            # any errors that occurred during processing
     573            { name => 'ERROR_CODE',   type => 'V', writetype => TULONG },
     574            # number of pixels used in hypothetical PSF for the query detection
     575            { name => 'DETECT_N', type => 'V',   writetype => TULONG },
     576            # detectibility, indicating the fraction of PSF pixels detetable by IPP
     577            { name => 'DETECT_F', type => 'D',   writetype => TDOUBLE },
     578            # flux of the target source
     579            { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
     580            # error in the flux of the target source
     581            { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
     582            ];
     583       
     584        # Header translation table
     585        $headers = {
     586            'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING,
     587                            comment => 'MOPS Query ID for this batch query' },
     588            'FPA_ID' => { name => 'FPA_ID',   type => TSTRING,
     589                          comment => 'original FPA_ID used at ingest' },
     590#           'MJD-OBS' => { name => 'FPA_ID',   type => TSTRING,
     591#                         comment => 'original FPA_ID used at ingest' },
     592#           'FILTER' => { name => 'FPA_ID',   type => TSTRING,
     593#                         comment => 'original FPA_ID used at ingest' },
     594#           'OBSCODE' => { name => 'FPA_ID',   type => TSTRING,
     595#                         comment => 'original FPA_ID used at ingest' },
     596        };
     597    }
     598    else {
     599        # Specification of columns to write
     600        $columns = [
     601            # matching rownum from detectability original request
     602            { name => 'ROWNUM',   type => 'V', writetype => TULONG },
     603            # any errors that occurred during processing
     604            { name => 'ERROR_CODE',   type => 'V', writetype => TULONG },
     605            # number of pixels used in hypothetical PSF for the query detection
     606            { name => 'DETECT_N', type => 'V',   writetype => TULONG },
     607            # detectibility, indicating the fraction of PSF pixels detetable by IPP
     608            { name => 'DETECT_F', type => 'D',   writetype => TDOUBLE },
     609            # flux of the target source
     610            { name => 'TARGET_FLUX', type => 'D', writetype => TDOUBLE },
     611            # error in the flux of the target source
     612            { name => 'TARGET_FLUX_SIG', type => 'D', writetype => TDOUBLE },
     613            # The FPA That would be in the header if it were to be there.
     614            { name => 'FPA_ID',   type => '20A', writetype => TSTRING },
     615            ];
     616       
     617        # Header translation table
     618        $headers = {
     619            'QUERY_ID' => { name => 'QUERY_ID', type => TSTRING,
     620                            comment => 'MOPS Query ID for this batch query' },
     621        };
     622    }   
    541623
    542624    # Parse the list of columns
    … …  
    552634    }
    553635
    554     my $numRows = $#{ $rownum_ref } + 1;
    555636    my $inHeader = { };
    556637
    557638    # Hack to force the data to match the detect_response_create formats
     639
    558640    $inHeader->{QUERY_ID}->{value} = $query_id;
    559     $inHeader->{FPA_ID}->{value} = $FPA_ID;
    560     $inHeader->{MJD_OBS}->{value} = $MJD_OBS;
    561     $inHeader->{FILTER}->{value} = $filter;
    562     $inHeader->{OBSCODE}->{value} = $obscode;
     641    if ($EXTVER_IS_1 == 1) {
     642        my $fpa_id = (keys(%query))[0];
     643        $inHeader->{FPA_ID}->{value} = $fpa_id;
     644    }
    563645   
    564646    # Fill the table columns with the data, making sure the flux is defined
    565     for (my $i = 0; $i < $numRows; $i++) {
    566         push @{$colData{'ROWNUM'}},      ${ $rownum_ref }[$i];
    567         push @{$colData{'ERROR_CODE'}},  ${ $out_err_ref }[$i];
    568         push @{$colData{'DETECT_N'}},    ${ $psphot_Npix_ref }[$i];
    569         push @{$colData{'DETECT_F'}},    ${ $psphot_Qfact_ref }[$i];
    570         push @{$colData{'TARGET_FLUX'}}, ${ $psphot_flux_ref }[$i];
    571         push @{$colData{'TARGET_FLUX_SIG'}}, ${ $psphot_error_ref }[$i];
    572         unless (defined(${ $colData{'TARGET_FLUX'}}[-1])) {
    573             $colData{'TARGET_FLUX'}[-1] = 0.0;
    574         }
    575         unless (defined(${ $colData{'TARGET_FLUX_SIG'}}[-1])) {
    576             $colData{'TARGET_FLUX_SIG'}[-1] = 0.0;
    577         }
    578     }
    579 
     647    foreach my $fpa_id (keys %query) {
     648        $inHeader->{QUERY_ID}->{value} = $query{$fpa_id}{QUERY_ID}[0];
     649        if ($EXTVER_IS_1 == 1) {
     650            my $fpa_id = (keys(%query))[0];
     651            $inHeader->{FPA_ID}->{value} = $fpa_id;
     652        }
     653
     654        push @{$colData{'ROWNUM'}}, @{ $query{$fpa_id}{ROWNUM} };
     655        push @{$colData{'ERROR_CODE'}}, @{ $query{$fpa_id}{PROC_ERROR} };
     656        push @{$colData{'DETECT_N'}}, @{ $query{$fpa_id}{NPIX} };
     657        push @{$colData{'DETECT_F'}}, @{ $query{$fpa_id}{QFACTOR} };
     658        push @{$colData{'TARGET_FLUX'}}, @{ $query{$fpa_id}{FLUX} };
     659        push @{$colData{'TARGET_FLUX_SIG'}}, @{ $query{$fpa_id}{FLUX_SIG} };
     660        if ($EXTVER_IS_1 != 1) {
     661            push @{$colData{'FPA_ID'}}, (map {$fpa_id} @{ $query{$fpa_id}{ROWNUM} });
     662        }
     663    }
     664    my $numRows = 0;
    580665    # Back to detect_response_create:
     666    $status = 0;
     667#    print "$output\n";
    581668    my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
    582669    check_fitsio( $status );
    … …  
    584671    check_fitsio( $status );
    585672    my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
     673   
    586674    $outFits->create_tbl( BINARY_TBL(), $numRows, scalar @colNames, \@colNames, \@colTypes, undef, $EXTNAME, $status);
    587675    check_fitsio( $status );
    … …  
    605693        check_fitsio( $status );
    606694    }
     695
    607696    for (my $i = 0; $i < scalar @colNames; $i++) {
    608697        my $colName = $colNames[$i];# Column name
    609698        my $writeType = $colWriteType[$i];
     699        my $numRows = scalar(@{$colData{$colName}});
    610700        unless(defined($writeType)) {
    611701            print "write type undefined for $colName\n";
    … …  
    662752}
    663753
    664 sub my_die_for_update {
    665     my $state = shift;
    666     my $stage = shift;
    667     my $stage_id = shift;
    668     my $component = shift;
    669     my $need_magic = shift;
    670     my $imagedb = shift;
    671     my $exit_code = shift;
    672 
    673     print "$state $stage $stage_id $component $need_magic $imagedb\n";
    674     print STDERR "$state $stage $stage_id $component $need_magic $imagedb\n";
    675     exit($exit_code);
    676 }
     754sub exit_with_failure {
     755    my $status = shift;
     756    my $message = shift;
     757    carp("$message");
     758    exit($status);
     759}
     760
Note: See TracChangeset for help on using the changeset viewer.