IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 21410


Ignore:
Timestamp:
Feb 6, 2009, 5:44:55 PM (17 years ago)
Author:
bills
Message:

Create stamps in workdir area instead of the data store product directory
various changes to improve logging

Location:
trunk/pstamp/scripts
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/pstamp/scripts/pstamp_finish.pl

    r21190 r21410  
    1414use File::Temp qw( tempfile );
    1515use File::Copy;
     16use File::Basename qw(dirname);
    1617
    1718use PS::IPP::Metadata::Config;
    … …  
    2324use PStamp::Job qw( :standard );
    2425
    25 my ( $req_id, $req_name, $req_file, $product, $dbname, $verbose, $save_temps );
     26my ( $req_id, $req_name, $req_file, $out_dir, $product, $dbname, $verbose, $save_temps, $redirect_output);
    2627
    2728# the char to the right of the bar may be used as a single - alias for the longer name
    2829# for example --input and -i are equivalent
    2930GetOptions(
    30            'req_id=s'   => \$req_id,
    31            'req_name=s' => \$req_name,
    32            'req_file=s' => \$req_file,
    33            'product=s'  => \$product,
    34            'dbname=s'   => \$dbname,
    35            'verbose'    => \$verbose,
    36            'save-temps' => \$save_temps,
     31           'req_id=s'       => \$req_id,
     32           'req_name=s'     => \$req_name,
     33           'req_file=s'     => \$req_file,
     34           'product=s'      => \$product,
     35           'out_dir=s'      => \$out_dir,
     36           'dbname=s'       => \$dbname,
     37           'verbose'        => \$verbose,
     38           'save-temps'     => \$save_temps,
     39           'redirect-output' => \$redirect_output,
    3740) or pod2usage( 2 );
    3841
    3942pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
    4043
    41 die "usage: --req_id id --req_name name --req_file file --product product [--dbname dbname --verbose]\n"
    42     if !$req_id or !$req_name or !$req_file or !$product;
     44die "usage: --req_id id --req_name name --req_file file --product product --out_dir output_directory [--dbname dbname --verbose]\n"
     45    if !$req_id or !$req_name or !$req_file or !$product or !$out_dir;
     46
     47my $ipprc = PS::IPP::Config->new(); # IPP Configuration
     48
     49if ($redirect_output) {
     50    my $logDest = "$out_dir/psfinish.$req_id.log";
     51    $ipprc->redirect_output($logDest);
     52}
    4353
    4454my $missing_tools;
    … …  
    5363}
    5464
    55 my $ipprc = PS::IPP::Config->new(); # IPP Configuration
    5665
    5766my $outputDataStoreRoot = metadataLookupStr($ipprc->{_siteConfig}, 'DATA_STORE_ROOT');
    … …  
    6473    my $fileset = $req_name;
    6574
     75
    6676    # Here we invoke the assumption that the output for the request is placed in the
    6777    # fileset directory directly
    68     my $out_dir = "$outputDataStoreRoot/$product/$fileset";
     78#    my $out_dir = "$outputDataStoreRoot/$product/$fileset";
     79
     80    # now we are assuming that the output directory is the dirname of the request file
     81    # XXX: put this in the database
    6982
    7083    print STDERR "product: $product  REQ_NAME: $req_name $out_dir\n" if $verbose;
    … …  
    7285    if (!-e $out_dir) {
    7386        # something must have gone wrong parsing the request
    74         print STDERR  "output fileset directory $out_dir does not exist\n";
     87        print STDERR  "output directory $out_dir does not exist\n";
    7588
    7689        if (!mkdir $out_dir) {
    … …  
    8396        # XXX TODO: fault the request so we pstamp_finish doesn't keep trying to process the
    8497        # request
    85         print STDERR "output fileset directory $out_dir exists but is not a directory";
     98        print STDERR "output directory $out_dir exists but is not a directory";
    8699        stop_request($req_id, $PS_EXIT_UNKNOWN_ERROR, $dbname);
    87100    }
    … …  
    111124    print $rlf "results.fits|||table|\n";
    112125
    113     # XXX: this file is in the request's workdir not the out_dir
    114     my $err_file = "$out_dir/parse_error.txt";
    115     if (-e $err_file ) {
     126    my $err_file = "parse_error.txt";
     127    if (-e "$out_dir/$err_file" ) {
    116128        print $rlf "$err_file|||text|\n";
    117129    }
    … …  
    222234    if (!$request_fault) {
    223235        # register the fileset
    224         # Note that we are assuming that the fileset's files are already in the fileset directory.
    225         # We could put them somewhere else and copy them in but for now we set the output directory
    226         # to the fileset directory at parse time
    227236        my $command = "$dsreg --list $reglist_name --add $fileset --product $product --type PSRESULTS";
     237        $command .= " --link --datapath $out_dir";
    228238        $command .= " --dbname $dbname" if $dbname;
    229239

  • trunk/pstamp/scripts/pstamp_job_run.pl

    r20152 r21410  
    1818my $dbname;
    1919my $job_id;
     20my $redirect_output;
     21my $output_base;
    2022
    2123my $host = hostname();
    … …  
    2325
    2426GetOptions(
    25     'job_id=s'  =>  \$job_id,
    26     'dbname=s'  =>  \$dbname,
    27     'verbose'   =>  \$verbose,
     27    'job_id=s'      =>  \$job_id,
     28    'output_base=s' =>  \$output_base,
     29    'redirect-output' => \$redirect_output,
     30    'dbname=s'      =>  \$dbname,
     31    'verbose'       =>  \$verbose,
    2832);
    2933
    … …  
    3539
    3640die "job_id is required" if !$job_id;
     41die "output_base is required" if !$output_base;
    3742
    3843use IPC::Cmd 0.36 qw( can_run run );
    … …  
    5358                       caturi
    5459                       );
     60my $ipprc = PS::IPP::Config->new(); # IPP Configuration
     61
     62if ($redirect_output) {
     63    my $logDest = "$output_base.log";
     64    $ipprc->redirect_output($logDest);
     65}
    5566
    5667my $missing_tools;

  • trunk/pstamp/scripts/pstamp_listjobs.pl

    r18545 r21410  
    101101    my $add_fits = 0;
    102102    if ($job->{jobType} ne "get_image") {
    103         $add_fits = 1;
     103#        $add_fits = 1;
    104104    }
    105105
    … …  
    115115    }
    116116
    117     print "$job->{job_id} $job->{state} $uri\n";
     117    print "$job->{job_id} $job->{state} $job->{fault} $job->{name} $job->{outProduct} $uri\n";
    118118}
    119119

  • trunk/pstamp/scripts/pstamp_parser_run.pl

    r21385 r21410  
    1212use Sys::Hostname;
    1313use Getopt::Long qw( GetOptions );
    14 
     14use File::Basename qw( basename dirname);
     15
     16my $req_id;
     17my $uri;
     18my $redirect_output;
     19my $product;
    1520my $verbose;
    1621my $dbname;
    17 my $req_id;
    1822
    1923GetOptions(
    20     'verbose'   =>  \$verbose,
    21     'dbname=s'  =>  \$dbname,
    22     'req_id=s'  =>  \$req_id,
     24    'req_id=s'          =>  \$req_id,
     25    'uri=s'             =>  \$uri,
     26    'product=s'         =>  \$product,
     27    'redirect-output'   =>  \$redirect_output,
     28    'verbose'           =>  \$verbose,
     29    'dbname=s'          =>  \$dbname,
    2330);
    2431
    … …  
    2936}
    3037
    31 die "req_id is required" if !defined($req_id);
     38die "--req_id --uri --product are required"
     39    if !defined($req_id) or
     40       !defined($uri) or
     41       !defined($product);
    3242
    3343use IPC::Cmd 0.36 qw( can_run run );
    … …  
    6676}
    6777
     78if ($redirect_output) {
     79    my $logDest = "$workdir/psparse.$req_id.log";
     80    $ipprc->redirect_output($logDest);
     81}
     82
    6883my $defaultDSProduct = metadataLookupStr($ipprc->{_siteConfig}, 'PSTAMP_DATA_STORE_PRODUCT');
    6984exit ($PS_EXIT_CONFIG_ERROR) unless defined $defaultDSProduct;
    … …  
    8398my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    8499
    85 my $psrequest;
    86 #Look up the uri for the given request
    87 {
    88     my $command = "$pstamptool -pendingreq -req_id $req_id";
    89     $command .= " -dbname $dbname" if $dbname;
     100
     101my $fn = basename($uri);
     102my $new_uri = "$workdir/$fn";
     103if ($uri =~ /^http:/) {
     104    # if the uri is an http uri download the file
     105    my $command = "$dsget --uri $uri --filename $new_uri";
    90106    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    91107        run(command => $command, verbose => $verbose);
    … …  
    93109        die("Unable to perform $command error code: $error_code");
    94110    }
    95 
    96     if (@$stdout_buf == 0) {
    97         print STDERR "pending pstamp request id $req_id not found\n";
    98         exit 1;
    99     }
    100     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    101         die("Unable to parse metdata config doc");
    102 
    103     my $requests = parse_md_list($metadata);
    104 
    105     my $num = @$requests;
    106     die "unexpected number of requests $num found" if $num > 1;
    107 
    108     $psrequest = $requests->[0];
    109 }
    110 
    111 if (!$psrequest) {
    112     print STDERR "pending postage stamp request $req_id not found\n";
    113     exit 1;
    114 }
    115 
    116 my $uri = $psrequest->{uri};
    117 
    118 # if the uri is from a data store, download it
    119 
    120 my $ds_id = $psrequest->{ds_id};
    121 
    122 my $new_uri;
    123 if ($ds_id && ($uri =~ /^http:/)) {
    124     {
    125         $new_uri = "$workdir/request.fits";
    126         my $command = "$dsget --uri $uri --filename $new_uri";
    127         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    128             run(command => $command, verbose => $verbose);
    129         unless ($success) {
    130             die("Unable to perform $command error code: $error_code");
    131         }
    132 
    133         $uri = $new_uri;
    134     }
    135 }
    136 
    137 #  find the name of the output directory
    138 my $product;
    139 if ($ds_id) {
    140     my $command = "$pstamptool -datastore -ds_id $ds_id";
    141     $command .= " -dbname $dbname" if $dbname;
    142     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    143         run(command => $command, verbose => $verbose);
    144     unless ($success) {
    145         die("Unable to perform $command error code: $error_code");
    146     }
    147 
    148     if (@$stdout_buf == 0) {
    149         print STDERR "data store with data store id $ds_id not found\n";
    150         exit 1;
    151     }
    152     my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
    153         die("Unable to parse metdata config doc");
    154 
    155     my $dataStores = parse_md_list($metadata);
    156 
    157     my $dataStore = $dataStores->[0];
    158 
    159     $product = $dataStore->{outProduct};
    160 } else {
     111} elsif ($uri ne $new_uri) {
     112    # put a link to the file into the workdir
     113    if (-e $new_uri) {
     114        unlink $new_uri or die "failed to unlink $new_uri";
     115    }
     116    if (! symlink $uri, $new_uri) {
     117        die ("failed to link request file $uri to workdir $workdir");
     118    }
     119}
     120$uri = $new_uri;
     121
     122die "request file $uri not found" if ! -e $uri;
     123
     124#  if product was not defined (in database), use the default
     125if ($product eq "NULL") {
    161126    $product = $defaultDSProduct;
    162127}
    163 
    164 #
    165 # Here is one of the places that we assume that the output for a request goes directly into
    166 # the output fileset directory in the data store.
    167 #
    168 my $outProductDir = "$outputDataStoreRoot/$product";
    169 
    170 die "output product directory $outProductDir does not exist" if (! -e $outProductDir) ;
    171 die "file with name of output product directory $outProductDir exists but is not a directory"
    172     if (! -d $outProductDir);
    173128
    174129my $parse_cmd;
    … …  
    220175}
    221176
    222 $parse_cmd .= " --mode queue_job --req_id $req_id --product $product --out_dir $outProductDir --file $uri";
     177$parse_cmd .= " --mode queue_job --req_id $req_id --product $product --out_dir $workdir --file $uri";
    223178$parse_cmd .= " --dbname $dbname" if $dbname;
     179$parse_cmd .= " --verbose" if $verbose;
    224180
    225181my $newState = "run";

  • trunk/pstamp/scripts/pstamp_queue_requests.pl

    r19265 r21410  
    107107        my $command = "$dsproductls --uri $ds->{uri}/index.txt";
    108108        $command .= " --last_fileset $ds->{lastFileset}" if $ds->{lastFileset};
     109
    109110        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    110111            run(command => $command, verbose => $verbose);
    111112        unless ($success) {
     113            # dsproductls exit status is the http error code - 300
     114            my $exit_status = $error_code >> 8;
     115
     116            # don't die on "common faults"
     117            # perhaps after some number of these we should mark the data store as disabled
     118            next if ($exit_status == 200);  # 500 cannot connect (data store down)
     119            next if ($exit_status == 104);  # 404 not found
     120#            next if ($exit_status == 100);  # 400 bad request
     121
    112122            die("Unable to perform $command: $error_code");
    113123        }
    … …  
    143153                    run(command => $command, verbose => $verbose);
    144154            unless ($success) {
     155                # we don't want to die here. We need to set lastFileset
     156                # so that we don't re queue requsts that we have proccessed
    145157                die("Unable to perform $command: $error_code");
    146158            }
    … …  
    175187        }
    176188        $lastFileset = $fs_name;
    177         last if ($numFilesets >= $limit);
    178     }
    179 
    180     ## now update the last_fileset column in pstampDataStore
    181     ## XXX: we should perhaps do this while processing each fileset so that if a later
    182     ## one has an error we don't get repeats.
    183 
    184     if ($lastFileset) {
    185         # print "last fileset: $lastFileset\n";
     189
    186190        {
     191        ## now update the last_fileset column in pstampDataStore
    187192        my $command = "$pstamptool -ds_id $ds_id -moddatastore -last_fileset $lastFileset";
    188193        $command .= " -dbname $dbname" if $dbname;
    … …  
    193198            }
    194199        }
     200        last if ($numFilesets >= $limit);
    195201    }
    196202}

  • trunk/pstamp/scripts/pstampparse.pl

    r21242 r21410  
    8484die "REQ_NAME not found in $request_file_name"  if (!$req_name);
    8585die "wrong EXTVER $extver found in $request_file_name" if ($extver ne "1");
    86 
    87 if ($out_dir) {
    88     $out_dir .= "/$req_name";
    89     if (! -e $out_dir ) {
    90         mkdir $out_dir or die "cannot create output directory $out_dir";
    91     } elsif (! -d $out_dir ) {
    92         die "output fileset directory $out_dir exists but is not a directory";
    93     }
    94 }
    9586
    9687if ($req_id) {
    … …  
    187178    }
    188179
    189     # Call PStamp::Job's locate_images routinte to get the parameters for this request specification
     180    # Call PStamp::Job's locate_images routine to get the parameters for this request specification
    190181    my $images = locate_images($ipprc, $image_db, $req_type, $img_type, $id, $class_id,
    191             $x, $y, $mjd_min, $mjd_max, $filter);
     182            $x, $y, $mjd_min, $mjd_max, $filter, $verbose);
    192183
    193184    if (!$images) {

  • trunk/pstamp/scripts/request_finish.pl

    r19221 r21410  
    1313use File::Temp qw( tempfile );
    1414use File::Copy;
     15use File::Basename qw( dirname );
    1516
    1617use PS::IPP::Metadata::Config;
    … …  
    2021use PS::IPP::Config qw( :standard );
    2122
    22 my ( $req_id, $req_name, $req_file, $req_type, $product, $dbname, $verbose, $save_temps );
     23my ( $req_id, $req_name, $req_file, $req_type, $out_dir, $product, $dbname, $verbose, $save_temps, $redirect_output );
    2324
    2425GetOptions(
    … …  
    2728           'req_file=s' => \$req_file,
    2829           'req_type=s' => \$req_type,
     30           'out_dir=s'  => \$out_dir,
    2931           'product=s'  => \$product,
    3032           'dbname=s'   => \$dbname,
    3133           'verbose'    => \$verbose,
    3234           'save-temps' => \$save_temps,
     35           'redirect-output' => \$redirect_output,
    3336) or pod2usage( 2 );
    3437
    … …  
    4245$err .= "--req_name is required\n" if !$req_name;
    4346$err .= "--product is required\n" if !$product;
     47# $err .= "--out_dir is required\n" if !$out_dir;
    4448
    4549die "$err" if $err;
     50
     51
     52if (!$out_dir) {
     53    $out_dir = dirname($req_file);
     54}
     55
     56if ($redirect_output) {
     57    my $logDest = "$out_dir/reqfinish.$req_id.log";
     58    my $ipprc = PS::IPP::Config->new();
     59    $ipprc->redirect_output($logDest);
     60}
    4661
    4762my $missing_tools;
    … …  
    6378}
    6479if ($finish_cmd) {
    65     my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product";
     80    my $command = $finish_cmd . " --req_id $req_id --req_name $req_name --req_file $req_file --product $product --out_dir $out_dir";
    6681    $command   .= " --dbname $dbname" if $dbname;
    6782    $command   .= " --verbose" if $verbose;
Note: See TracChangeset for help on using the changeset viewer.