IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 25058


Ignore:
Timestamp:
Aug 12, 2009, 3:08:50 PM (17 years ago)
Author:
bills
Message:

better error reporting of parse stage errors to the client

Location:
trunk/pstamp/scripts
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/pstamp/scripts/pstamp_parser_run.pl

    r24961 r25058  
    200200    unlink $error_file_name if (-e $error_file_name);
    201201
     202    # Run the parser
     203
    202204    my $command = "$parse_cmd";
    203205    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

  • trunk/pstamp/scripts/pstampparse.pl

    r25052 r25058  
    9191# make sure the file contains what we are expecting
    9292
    93 die "$request_file_name is not a PS1_PS_REQEST" if $extname ne "PS1_PS_REQUEST";
    94 die "REQ_NAME not found in $request_file_name"  if (!$req_name);
    95 die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
     93# we shouldn't get here if this is the case so just die. No need to notify the client
     94my_die("$request_file_name is not a PS1_PS_REQEST", $PS_EXIT_PROG_ERROR) if $extname ne "PS1_PS_REQUEST";
     95my_die("REQ_NAME not found in $request_file_name", $PS_EXIT_PROG_ERROR)  if (!$req_name);
     96my_die("wrong EXTVER $extver found in $request_file_name", $PS_EXIT_PROG_ERROR) if ($extver ne "1");
    9697
    9798if ($req_id and !$no_update) {
     99    # update the database with the request name. This will be used as the
     100    # the output data store's product name
    98101    my $command = "$pstamptool -updatereq -req_id $req_id  -name $req_name";
    99102    $command .= " -outProduct $product";
    … …  
    103106        run(command => $command, verbose => $verbose);
    104107    unless ($success) {
    105         die "$command failed";
     108        my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
    106109    }
    107110}
    … …  
    120123    }
    121124    my $table =  $mdcParser->parse(join "", @$stdout_buf) or
    122         die("Unable to parse metdata config doc");
     125        my_die("Unable to parse metdata config doc", $PS_EXIT_UNKNOWN_ERROR);
    123126
    124127    $rows = parse_md_list($table);
    … …  
    154157    my $option_mask= $row->{OPTION_MASK};
    155158
    156     die "valid region of interest is required to make postage stamps"
    157         if (($job_type eq "stamp") and ! validROI($row));
    158 
    159159    my $skycenter = $row->{skycenter} = ! ($row->{COORD_MASK} & $PSTAMP_CENTER_IN_PIXELS);
    160     $component = "" if (defined($component) and ($component eq "null" or $component eq "all"));
     160
     161    my $search_component = (!defined($component) or ($component eq "null")) ? "" : $component;
    161162   
    162     die "job_type is list_uri but mode is $mode" if ($job_type eq "list_uri") and ($mode ne "list_uri");
     163    if (!$skycenter and !$search_component) {
     164        print STDERR "COMPONENT must be specified for pixel coordinate ROI\n" if $verbose;
     165        insertFakeJobForRow($row, 1, 45);
     166        $num_jobs++;
     167        next;
     168    }
     169
     170    $search_component = "" if $search_component eq "all";
     171
     172    if (($job_type eq "stamp") and ! validROI($row)) {
     173        print STDERR "invalid ROI for row $rownum\n" if $verbose;
     174        insertFakeJobForRow($row, 1, 46);
     175        $num_jobs++;
     176        next;
     177    }
     178
     179   
     180    # $mode list_uri is a debugging mode (it may used by the http interface)
     181    # if this happens just croak
     182    my_die("job_type is list_uri but mode is $mode", $PS_EXIT_PROG_ERROR) if ($job_type eq "list_uri") and ($mode ne "list_uri");
    163183
    164184    # note: resolve_project avoids running pstamptool every time by remembering the
    165185    # last project resolved
    166186    my $proj_hash = resolve_project($ipprc, $project, $dbname, $dbserver);
    167     die "project $project not found\n" unless $proj_hash;
     187    if (!$proj_hash) {
     188        print STDERR "project $project not found\n"  if $verbose;
     189        insertFakeJobForRow($row, 1, 46);
     190        $num_jobs++;
     191        next;
     192    }
    168193
    169194    my $image_db   = $proj_hash->{dbname};
    … …  
    175200    if (@rowList) {
    176201        my $firstRow = $rowList[0];
    177         # XXX: the collecting might work with !$skycenter but I need to think about it
    178         if ($skycenter and same_images_of_interest($row, $firstRow)) {
     202#        if ($skycenter and same_images_of_interest($row, $firstRow)) {
     203        if (same_images_of_interest($row, $firstRow)) {
    179204            push @rowList, $row;
    180205            # On to the next row
    … …  
    188213
    189214    if ($req_type eq "bycoord") {
    190         die "center must be specified in sky coordintes for bycoord" if ( ! $skycenter);
    191         die "lookup bycoord is not yet implemented";
     215        if (!$skycenter) {
     216            print STDERR "center must be specified in sky coordintes for bycoord" if $verbose;
     217            insertFakeJobForRow($row, 1, 46);
     218            $num_jobs++;
     219            next;
     220        }
     221        print STDERR "lookup bycoord is not yet implemented" if $verbose;
     222        insertFakeJobForRow($row, 1, 47);
     223        $num_jobs++;
     224        next;
    192225    } else {
    193 
    194         # Call PS::IPP::PStamp::Job's locate_images routine to get the parameters for this
    195         # request specification. An array reference is returned
    196         my ($x, $y);
    197         $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $component, $skycenter,
    198                 $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
    199     }
    200 
    201     if (!$imageList or !@$imageList) {
    202         print STDERR "no matching images found for row $rownum\n";
    203     }
    204     $row->{need_magic} = $need_magic;
    205     push @rowList, $row;
    206 }
    207 
    208 if (@rowList) {
    209     $num_jobs += queueJobs($mode, \@rowList, $imageList);
    210 }
    211 
    212 if ($mode eq "queue_jobs") {
    213     if ($num_jobs) {
    214         exit 0;
    215     } else {
    216         # no jobs queued return error to get the request stopped
    217         # TODO: need to define meaningful error codes
    218         exit $PS_EXIT_UNKNOWN_ERROR;
    219     }
    220 } else {
    221     exit 0;
    222 }
     226            # Call PS::IPP::PStamp::Job's locate_images subroutine to get the parameters for this
     227            # request specification. An array reference is returned.
     228            my ($x, $y);
     229            $imageList = locate_images($ipprc, $image_db, $req_type, $stage, $id, $search_component,
     230                $skycenter, $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
     231        }
     232
     233        if (!$imageList or !@$imageList) {
     234            print STDERR "no matching images found for row $rownum\n" if $verbose;
     235            # note in this case queueJobs inserts the fake job for these rows
     236        }
     237        $row->{need_magic} = $need_magic;
     238        push @rowList, $row;
     239    }
     240
     241    if (@rowList) {
     242        $num_jobs += queueJobs($mode, \@rowList, $imageList);
     243}
     244
     245if (($mode eq "queue_jobs") and ($num_jobs eq 0)) {
     246    print STDERR "no jobs created for $req_name\n" if $verbose;
     247    insertFakeJobForRow(undef, 1, 41);
     248}
     249exit 0;
    223250
    224251
    … …  
    230257    my $have_skycells = shift;
    231258    my $need_magic = shift;
     259
     260    my $rownum = $row->{ROWNUM};
     261
     262    my $components = $row->{components};
     263    my $numComponents = scalar keys %$components;
     264    if ( $numComponents == 0 ) {
     265        print STDERR "no jobs for row $rownum\n" if $verbose;
     266        insertFakeJobForRow($row, 1, 44);
     267        return 1;
     268    }
     269
     270    my $roi_string;
     271
     272    # note values were insured to be numbers in validROI()
     273    my $x = $row->{CENTER_X};
     274    my $y = $row->{CENTER_Y};
     275    my $w = $row->{WIDTH};
     276    my $h = $row->{HEIGHT};
     277    my $coord_mask = $row->{COORD_MASK};
     278    if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
     279        if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
     280            $roi_string = "-pixcenter $x $y";
     281        } else {
     282            $roi_string = "-skycenter $x $y";
     283        }
     284        if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
     285            $roi_string .= " -pixrange $w $h";
     286        } else {
     287            $roi_string .= " -arcrange $w $h";
     288        }
     289    }
    232290
    233291    # loop over images
    … …  
    242300
    243301        # skip this component if it is not in the list for this row
    244         next if !$row->{components}->{$component};
    245 
    246         my $rownum = $row->{ROWNUM};
    247 
    248         my $roi_string;
    249 
    250         # note values were insured to be numbers in validROI()
    251         my $x = $row->{CENTER_X};
    252         my $y = $row->{CENTER_Y};
    253         my $w = $row->{WIDTH};
    254         my $h = $row->{HEIGHT};
    255         my $coord_mask = $row->{COORD_MASK};
    256         if ($x && ($x ne "null") && $y && ($y ne "null") && $w && ($w ne "null") && $h && ($h ne "null")) {
    257             if ($coord_mask & $PSTAMP_CENTER_IN_PIXELS) {
    258                 $roi_string = "-pixcenter $x $y";
    259             } else {
    260                 $roi_string = "-skycenter $x $y";
    261             }
    262             if ($coord_mask & $PSTAMP_RANGE_IN_PIXELS) {
    263                 $roi_string .= " -pixrange $w $h";
    264             } else {
    265                 $roi_string .= " -arcrange $w $h";
    266             }
    267         }
     302        next if ! $components->{$component};
    268303
    269304        $job_num++;
    … …  
    271306        my $imagefile = $image->{image};
    272307        if (($stage ne "stack") and ($need_magic and !$image->{magicked})) {
    273             # XXX: should we add a faulted job so the client can know what happened?
    274             # print STDERR "skippping non-magicked image $imagefile\n" if $verbose;
     308            # XXX: should we add a faulted job so the client can know what happened if no images come back?
     309            print STDERR "skippping non-magicked image $imagefile\n" if $verbose;
     310
     311            # for now assume yes.
    275312
    276313            insertFakeJobForRow($row, $job_num, 43);
    … …  
    303340
    304341        # copy the argument list to a file
    305         open ARGSLIST, ">$argslist" or die "failed to open $argslist";
     342        open ARGSLIST, ">$argslist" or my_die("failed to open $argslist", $PS_EXIT_UNKNOWN_ERROR);
    306343        print ARGSLIST "$args\n";
    307         close ARGSLIST or die "failed to close $argslist";
    308 
    309         # XXX: TODO: here is where we need to check whether or not the source inputs still exist
    310         # and if not, queue an update job and set the job state appropriately.
     344        close ARGSLIST or my_die("failed to close $argslist", $PS_EXIT_UNKNOWN_ERROR);
    311345
    312346        my $newState = "run";
     347        my $fault = 0;
     348        if ($image->{data_state} ne "full") {
     349            # XXX here is where we need to queue an update job
     350            # for now just say that the image is not available
     351            $newState = 'stop';
     352            $fault = 49;
     353        }
    313354
    314355        $num_jobs++;
    315356        my $command = "$pstamptool -addjob  -req_id $req_id -job_type $row->{JOB_TYPE}"
    316357                        . " -outputBase $output_base -rownum $rownum -state $newState";
     358        $command .= " -fault $fault" if $fault;
    317359        $command .= " -exp_id $exp_id" if $exp_id;
    318360        $command .= " -dbname $dbname" if $dbname;
    … …  
    332374                # should we keep going for other uris? If so how do we report that some
    333375                # of the work that the request wanted isn't going to get done
    334                 die "failed to queue job for request $req_id";
     376                my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
    335377            }
    336378        } else {
    … …  
    341383}
    342384
     385# queue jobs for a collection of request specifications that have the same Images of Interest
    343386sub queueJobs
    344387{
    … …  
    359402        }
    360403    } elsif ($job_type eq "get_image") {
    361         die "get_image jobs not implemented yet";
     404        print STDERR "get_image jobs not implemented yet" if $verbose;
     405        foreach my $row (@$rowList) {
     406            insertFakeJobForRow($row, 1, 47);
     407            $num_jobs++;
     408        }
    362409    } else {
    363410        if (!$imageList or (scalar @$imageList eq 0)) {
    … …  
    370417            return $num_jobs;
    371418        }
    372         my $thisRun;
    373 
    374         my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
    375         foreach my $row (@$rowList) {
    376             print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
    377             $row->{components} = {};
    378         }
    379         close $pointsList;
    380419
    381420        my $have_skycells;
    … …  
    386425        }
    387426       
     427        my $thisRun;
     428
     429        my ($pointsList, $pointsListName) = tempfile ("/tmp/pointsList.XXXX", UNLINK => !$save_temps);
     430        my $npoints = 0;
     431        foreach my $row (@$rowList) {
     432            $row->{components} = {};
     433            if ($row->{skycenter}) {
     434                print $pointsList "$row->{ROWNUM} $row->{CENTER_X} $row->{CENTER_Y}\n";
     435                $npoints++;
     436            } else {
     437                # this row's center is in pixel coordinates add all images to the component list for this row
     438                foreach my $i (@$imageList) {
     439                    my $component = $have_skycells ? $i->{skycell_id} : $i->{class_id};
     440                    my_die( "image found with no value for component", $PS_EXIT_UNKNOWN_ERROR) if !$component;
     441                    $row->{components}->{$component} = 1;
     442                }
     443            }
     444        }
     445        close $pointsList;
     446
    388447        my $tess_dir_abs;
    389448        my $last_tess_id = "";
    390449        while ($thisRun = getOneRun($stage, $imageList)) {
    391             {
     450            if ($npoints) {
     451                # we collected a set of sky coordintates above filter the images so that only
     452                # those tat contain the centers are processed
    392453                my $command = "$dvoImagesAtCoords $pointsListName";
    393454                if ($have_skycells) {
    … …  
    401462                } else {
    402463                    my $astrom = $thisRun->[0]->{astrom};
    403                     die "no astrometry file found" if !$astrom;
     464                    my_die( "no astrometry file found", $PS_EXIT_UNKNOWN_ERROR) if !$astrom;
    404465                    my $astrom_resolved = $ipprc->file_resolve($astrom);
    405466                    $command .= " -astrom $astrom_resolved";
    … …  
    410471                    print STDERR @$stderr_buf;
    411472                    my $rc = $error_code >> 8;
    412                     die "dvoImagesAtCoords failed: $rc";
     473                    my_die( "dvoImagesAtCoords failed: $rc", $PS_EXIT_UNKNOWN_ERROR);
    413474                }
    414475                # now we have a list of row numbers and components
    … …  
    457518            # should we keep going for other uris? If so how do we report that some
    458519            # of the work that the request wanted isn't going to get done
    459             die "failed to queue job for request $req_id";
     520            my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
    460521        }
    461522    } else {
    … …  
    480541        return $image->{diff_id};
    481542    } else {
    482         die "unenexpected stage: $stage found";
     543        my_die("unenexpected stage: $stage found", $PS_EXIT_PROG_ERROR);
    483544    }
    484545}
    … …  
    554615    return undef;
    555616}
     617
     618sub my_die
     619{
     620    my $msg = shift;
     621    my $fault = shift;
     622
     623    carp $msg;
     624
     625    # we don't fault the request here pstamp_parser_run.pl handles that if necessary
     626
     627    return $fault;
     628}
Note: See TracChangeset for help on using the changeset viewer.