IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 33174


Ignore:
Timestamp:
Jan 27, 2012, 2:48:47 PM (14 years ago)
Author:
bills
Message:

Changes to make detectability queries better behaved in the
case of inputs needing to be updated and errors

Location:
tags/ipp-20111222
Files:
9 edited

Legend:

Unmodified
Added
Removed

  • tags/ipp-20111222/PS-IPP-PStamp

  • tags/ipp-20111222/PS-IPP-PStamp/lib/PS/IPP/PStamp/Job.pm

    r32731 r33174  
    487487    my $command = "$difftool -dbname $imagedb -pstamp_order";
    488488   
    489     my $listrun = 0;
     489    my $choose_components = 0;
    490490    if ($byid) {
    491491        if ($skycell_id and ($skycell_id ne 'all')) {
    492492            $command .= " -diffskyfile -diff_id $id -skycell_id $skycell_id";
    493493        } else {
    494             $listrun = 1;
     494            $choose_components = 1;
    495495            $command .= " -listrun -diff_id $id";
    496496            # the following is a work around for the problem reported in ticket #1394
    … …  
    512512
    513513    my $images;
    514     if ($listrun) {
     514    if ($choose_components) {
    515515        $images = selectComponents($ipprc, $imagedb, 'byid', 'diff', $rowList, $output, $verbose);
    516516    } else {
    … …  
    603603            my @imageList = ($image);
    604604
    605             setRowRefs($rowList, \@imageList);
     605            setRowRefs($rowList, \@imageList) unless $choose_components;
    606606            # the $image is going to be returned directly in this case so we need to duplicate
    607607            # some of processing that lookup does for other img_types

  • tags/ipp-20111222/pstamp

  • tags/ipp-20111222/pstamp/scripts

    • Property svn:mergeinfo deleted

  • tags/ipp-20111222/pstamp/scripts/detectability_respond.pl

    r29616 r33174  
    4141my $EXTVER = 1.0;
    4242my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
    43 my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project);
     43my ($req_id,$job_id,$req_name,$product,$need_magic,$missing_tools,$project);
    4444my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom);
    4545GetOptions(
    … …  
    4747    'output=s'        =>      \$output,
    4848    'workdir=s'       =>      \$workdir,
     49    'job_id=s'        =>      \$job_id,
    4950    'dbname=s'        =>      \$dbname,
    5051    'dbserver=s'      =>      \$dbserver,
    … …  
    298299                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
    299300                    }
    300                     elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] = 0)) {
     301                    elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) {
    301302                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
    302303                    }
    … …  
    324325my %processing_request;
    325326
    326 open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
    327 foreach my $fpa_id (keys %query) {
    328     for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
    329         print WISDOM "$fpa_id\t";
    330         foreach my $key (keys %{ $query{$fpa_id} }) {
    331             print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
    332         }
    333         print WISDOM "\n";
    334         if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
    335             @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } =
    336                 ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
    337                  $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
    338             push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
    339         }
    340     }
    341 }
    342 close(WISDOM);
    343 
    344 # If there is anything that needs to be updated, create the update request list, and then exit the program.
    345 my $exit_code = 0;
    346 my $update_request_file = "${workdir}/update_request.dat";
    347 open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
    348 foreach my $images (keys %update_request) {
    349     foreach my $fault (keys %{ $update_request{$images} }) {
    350         if ($fault == 25) {
    351             $exit_code = 25;
    352         }
    353         elsif ($fault != 0) {
    354             $exit_code = 21;
    355         }
    356         my $update_request = join ' ', @{ $update_request{$images}{$fault} };
    357         print UPDATE_REQUEST "$update_request\n";
    358     }
    359 }
    360 close(UPDATE_REQUEST);
    361 if ($exit_code != 0) {
    362     exit($exit_code);
     327if (!$job_id) {
     328    open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
     329    foreach my $fpa_id (keys %query) {
     330        for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
     331            print WISDOM "$fpa_id\t";
     332            foreach my $key (keys %{ $query{$fpa_id} }) {
     333                print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
     334            }
     335            print WISDOM "\n";
     336            my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
     337            if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
     338                if ($data_state ne 'full') {
     339                   
     340                    @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } =
     341                        ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
     342                         $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
     343                }
     344                push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
     345            }
     346        }
     347    }
     348    close(WISDOM);
     349
     350    # If there is anything that needs to be updated, create the update request list, and then exit the program.
     351    my $exit_code = 0;
     352    my $update_request_file = "${workdir}/update_request.dat";
     353    open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
     354    foreach my $images (keys %update_request) {
     355        foreach my $fault (keys %{ $update_request{$images} }) {
     356            if ($fault == 25) {
     357                $exit_code = 25;
     358            }
     359            elsif ($fault != 0) {
     360                $exit_code = 21;
     361            }
     362            my $update_request = join ' ', @{ $update_request{$images}{$fault} };
     363            print UPDATE_REQUEST "$update_request\n";
     364        }
     365    }
     366    close(UPDATE_REQUEST);
     367    if ($exit_code != 0) {
     368        exit($exit_code);
     369    }
    363370}
    364371
    … …  
    710717    $status = 0;
    711718#    print "$output\n";
     719    if (-e $output) {
     720        unlink $output or die "failed to unlink existing response file $output\n";
     721    }
    712722    my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
    713723    check_fitsio( $status );

  • tags/ipp-20111222/pstamp/scripts/dquery_finish.pl

    r27874 r33174  
    6666if ($product eq "NULL") {
    6767    # parsing failed just with fault = 0 (this leaves previously set fault in place
    68     stop_request($req_id, 0, $verbose);
     68    update_request($req_id, 0, $verbose);
    6969    exit 0;
    7070}
    … …  
    7373if (! -e $prod_dir ) {
    7474    # something must have gone wrong at the parse stage
    75     stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
     75    update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
    7676    die "product directory does not exist $prod_dir";
    7777}
    … …  
    8282    print STDERR "output fileset directory $outdir does not exist\n" if $verbose;
    8383    if (! mkdir $outdir ) {
    84         stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
     84        update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
    8585        die "cannot create output directory $outdir";
    8686    }
    8787} elsif (! -d $outdir) {
    88     stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
     88    update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
    8989    die "output fileset directory $outdir exists but is not a directory";
    9090}
    … …  
    121121
    122122foreach my $job (@jobs) {
     123    next if $job->{parent_id};
    123124    my $job_id = $job->{job_id};
    124     my $response_file = "response${job_id}.fits";
     125    my $response_file = "response.${job_id}.fits";
    125126    my $response_path = "$outdir/$response_file";
    126127
    … …  
    157158}
    158159
    159 stop_request($req_id, $request_fault, $verbose);
     160update_request($req_id, $request_fault, $verbose);
    160161
    161162exit 0;
    162163
    163 sub stop_request {
     164sub update_request {
    164165    my $req_id = shift;
    165166    my $fault = shift;
    166167    my $verbose = shift;
    167168   
    168     my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop";
    169     $command   .= " -set_fault $fault" if $fault;
     169    my $command = "$pstamptool -updatereq -req_id $req_id";
     170    if ($fault) {
     171        $command .= " -set_fault $fault";
     172    } else {
     173        $command .= " -set_state stop";
     174    }
    170175    $command   .= " -dbname $dbname" if $dbname;
    171176    $command   .= " -dbserver $dbserver" if $dbserver;

  • tags/ipp-20111222/pstamp/scripts/dqueryparse.pl

    r28818 r33174  
    115115my $response_file = "$outdir/${req_name}.dresponse.${req_id}.fits";
    116116my $fault;
    117 my $data_to_update = '';
     117# my $data_to_update = '';
    118118{
    119119    my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir";
     120    if ($job_id) {
     121        $command .= " --job_id $job_id";
     122    } else {
     123        $command .= " --ignore-wisdom";
     124    }
    120125    $command .= " --save-temps" if $save_temps;
    121126    $command .= " --verbose" if $verbose;
    … …  
    132137            my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
    133138        }
     139        warn("Some inputs are not available and must be updated.");
    134140    }   
    135141   
    … …  
    137143
    138144my $result;
    139 # If we returned correctly with a valid response file, get a job ID
    140 # for the completed work, and move the response to a standardized name.
    141 if ($fault == 0) {
    142     my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir";
    143     $command .= " -job_type detect_query -state stop -fault 0";
    144     $command .= " -rownum 1";
    145 
    146     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    147         run(command => $command, verbose => $verbose);
    148     if ($success) {
    149         $job_id = join "", @$stdout_buf;
    150         chomp $job_id;
    151         if ($job_id && -e $response_file) {
    152             rename $response_file, "$outdir/response${job_id}.fits";
     145unless ($job_id) {
     146    # We are running as a parse job
     147    # If we returned correctly with a valid response file, get a job ID
     148    # for the completed work, and move the response to a standardized name.
     149    if ($fault == 0) {
     150        my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir";
     151        $command .= " -job_type detect_query -state stop -fault 0";
     152        $command .= " -rownum 1";
     153
     154        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     155            run(command => $command, verbose => $verbose);
     156        if ($success) {
     157            $job_id = join "", @$stdout_buf;
     158            chomp $job_id;
     159            if ($job_id && -e $response_file) {
     160                rename $response_file, "$outdir/response.${job_id}.fits";
     161            }
     162            $result = 0;
     163        } else {
     164            warn("Unable to perform $command error code: $error_code");
     165            $result = $error_code >> 8;
    153166        }
    154         $result = 0;
    155     } else {
    156         warn("Unable to perform $command error code: $error_code");
    157         $result = $error_code >> 8;
    158     }
    159 }
    160 elsif ($fault == $PSTAMP_NOT_AVAILABLE) {
    161     # Failed to run correctly, which means that we need to queue a job and flag data for updating.
    162     # Get the dependency id for the data we're requesting be updated.
    163     open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
    164     while (<UPDATE_REQUEST>) {
    165         my $data_to_update = $_;
    166         chomp($data_to_update);
    167         my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
    168 
    169         # Link this request to a job and link that job to any dependency
    170         my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir";
    171         $command .= " -job_type detect_query -state run -fault 0";
    172         $command .= " -rownum 1";
    173         $command .= " -dep_id $dep_id" if $dep_id;
    174 
    175         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    176             run(command => $command, verbose => $verbose);
    177 
    178         if ($success) {
    179             $job_id = join "", @$stdout_buf;
    180             chomp $job_id;
    181             if ($job_id && -e $response_file) {
    182                 # We shouldn't have a response file at this stage.
    183                 rename $response_file, "$outdir/response${job_id}.fits";
    184             }
    185             $result = 0;
    186         } else {
    187             warn("Unable to perform $command error code: $error_code");
    188             $result = $error_code >> 8;
    189         }
    190     }
    191     close(UPDATE_REQUEST);
    192 }
    193 else {
    194     my_die ("Parse fault!!", $PS_EXIT_UNKNOWN_ERROR);
    195 }
    196 
    197 # This does not set the request state to stop.  That will happen with the request_finish.pl script,
    198 # which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
    199 {
    200     my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
    201     $command .= " -set_fault $result" if $result;
    202 
    203     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    204         run(command => $command, verbose => $verbose);
    205     unless ($success) {
    206         my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
    207     }
    208 }
     167    }
     168    elsif ($fault == $PSTAMP_NOT_AVAILABLE) {
     169        # Failed to run correctly, which means that we need to queue a job and flag data for updating.
     170        # first create a parent job for the actual detectabilty query
     171        my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/1_";
     172        $command .= " -job_type detect_query -state run -fault 0 -is_parent";
     173        $command .= " -rownum 1";
     174
     175        my $parent_job_id;
     176        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     177            run(command => $command, verbose => $verbose);
     178        if ($success) {
     179            # pstamptool returns the job_id of the new job
     180            $parent_job_id = join "", @$stdout_buf;
     181            chomp $parent_job_id;
     182            if ($parent_job_id) {
     183                $result = 0;
     184            } else {
     185                print STDERR "pstamptool did not return a valid job_id for parent job\n";
     186                $result = $PS_EXIT_UNKNOWN_ERROR;
     187            }
     188        } else {
     189            warn("Unable to perform $command error code: $error_code");
     190            $result = $error_code >> 8;
     191        }
     192        # now create child jobs for each dependent. All these jobs do is finish once the dependency
     193        # is satisfied
     194        if (!$result) {
     195            # Get the dependency id for the data we need to have updated.
     196            my $job_num = 2;
     197            open(UPDATE_REQUEST,"$outdir/update_request.dat") || my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
     198            while (<UPDATE_REQUEST>) {
     199                my $data_to_update = $_;
     200                chomp($data_to_update);
     201                my $dep_id = queue_update_run($req_id,$job_id,$outdir,$label,$data_to_update);
     202
     203                # Link this request to a job and link that job to the dependent
     204                my $command = "$pstamptool -addjob -req_id $req_id -outputBase $outdir/${job_num}_";
     205                $command .= " -job_type child -state run -fault 0 -parent_id $parent_job_id";
     206                $command .= " -rownum 1"; # XXX: we should choose a correct rownum
     207                $command .= " -dep_id $dep_id" if $dep_id;
     208
     209                $job_num++;
     210
     211                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     212                    run(command => $command, verbose => $verbose);
     213
     214                if ($success) {
     215                    $job_id = join "", @$stdout_buf;
     216                    chomp $job_id;
     217                    $result = 0;
     218                } else {
     219                    warn("Unable to perform $command error code: $error_code");
     220                    $result = $error_code >> 8;
     221                    last;
     222                }
     223            }
     224            close(UPDATE_REQUEST);
     225        }
     226    }
     227    else {
     228        my_die ("Parse fault!!", $fault);
     229    }
     230    # This does not set the request state to stop.  That will happen with the request_finish.pl script,
     231    # which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
     232    {
     233        my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
     234        $command .= " -set_fault $result" if $result;
     235
     236        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     237            run(command => $command, verbose => $verbose);
     238        unless ($success) {
     239            my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
     240        }
     241    }
     242} else {
     243    # We are running as a job presumably because an input needed
     244    # to be regenerated
     245    if ($fault) {
     246        # in some cases we will need to finish off the request
     247        my_die ("Run fault!!", $fault);
     248    }
     249    if (-e $response_file) {
     250        rename $response_file, "$outdir/response.${job_id}.fits";
     251    }
     252}
     253
    209254 
    210255exit 0;

  • tags/ipp-20111222/pstamp/scripts/pstamp_job_run.pl

    r33057 r33174  
    9090my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    9191
    92 my $params = read_params_file($outputBase);
    9392
    9493my $jobStatus;
    9594if ($jobType eq "stamp") {
     95    my $params = read_params_file($outputBase);
    9696
    9797    my $argString;
    … …  
    283283
    284284    my $pstamp_bundle_root = metadataLookupStr($ipprc->{_siteConfig}, "PSTAMP_BUNDLE_ROOT");
     285
     286    my $params = read_params_file($outputBase);
    285287    my $imagedb = $params->{imagedb};
    286288
    … …  
    304306    # Load the argument list that dqueryparse should have created the first time it ran, so we know how to
    305307    # run it again the same way.
    306     my $argslist = "$outputBase/parse.args";
     308    my $outdir = dirname($outputBase);
     309    my $argslist = "$outdir/parse.args";
    307310    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
    308311    my $argString = <ARGSLIST>;
    … …  
    317320    $command .= " --dbserver $dbserver" if $dbserver;
    318321    $command .= " --verbose" if $verbose;
     322    $command .= " --save-temps" if $save_temps;
    319323    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    320324        run(command => $command, verbose => $verbose);
    … …  
    327331        my_die("dqueryparse.pl failed with error code: $jobStatus", $job_id, $jobStatus);
    328332    }
     333} elsif ($jobType eq "child") {
     334    # the only thing jobs of jobType child is to finish
     335    $jobStatus = 0;
    329336} else {
    330337    my_die("unknown jobType $jobType found", $job_id, $PS_EXIT_PROG_ERROR);
    … …  
    611618
    612619    $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code;
     620    if ($exit_code > 100) {
     621        carp ("invalid exit code: $exit_code changing to $PS_EXIT_UNKNOWN_ERROR");
     622        $exit_code = $PS_EXIT_UNKNOWN_ERROR;
     623    }
    613624    $job_state = 'run' unless $job_state;
    614625

  • tags/ipp-20111222/pstamp/scripts/pstamp_parser_run.pl

    r33057 r33174  
    294294    carp($msg);
    295295
     296    if (!$req_id) {
     297        exit $PS_EXIT_CONFIG_ERROR;
     298    }
     299
    296300    my $command = "$pstamptool -updatereq -req_id $req_id  -set_fault $fault";
    297301    $command   .= " -dbname $dbname" if $dbname;
Note: See TracChangeset for help on using the changeset viewer.