IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 33141


Ignore:
Timestamp:
Jan 24, 2012, 4:40:12 PM (14 years ago)
Author:
bills
Message:

more work on detectability queries

Location:
trunk/pstamp/scripts
Files:
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/pstamp/scripts/detectability_respond.pl

    r33126 r33141  
    4141my $EXTVER = 1.0;
    4242my $EXTNAME = 'MOPS_DETECTABILITY_RESPONSE';
    43 my ($req_id,$req_name,$product,$need_magic,$missing_tools,$project);
     43my ($req_id,$job_id,$req_name,$product,$need_magic,$missing_tools,$project);
    4444my ($request_file,$output,$workdir,$dbname,$dbserver,$verbose,$save_temps,$ignore_wisdom);
    4545GetOptions(
    … …  
    4747    'output=s'        =>      \$output,
    4848    'workdir=s'       =>      \$workdir,
     49    'job_id=s'        =>      \$job_id,
    4950    'dbname=s'        =>      \$dbname,
    5051    'dbserver=s'      =>      \$dbserver,
    … …  
    298299                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_GONE;
    299300                    }
    300                     elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] = 0)) {
     301                    elsif ($need_magic and ($query{$fpa_id}{MAGICKED}[$valid_index] eq 0)) {
    301302                        $query{$fpa_id}{FAULT}[$valid_index] = $PSTAMP_NOT_DESTREAKED;
    302303                    }
    … …  
    324325my %processing_request;
    325326
    326 open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
    327 foreach my $fpa_id (keys %query) {
    328     for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
    329         print WISDOM "$fpa_id\t";
    330         foreach my $key (keys %{ $query{$fpa_id} }) {
    331             print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
    332         }
    333         print WISDOM "\n";
    334         my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
    335         if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
    336             if ($data_state ne 'full') {
    337                
    338                 @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } =
    339                     ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
    340                      $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
     327if (!$job_id) {
     328    open(WISDOM,">$wisdom_file") or my_die("failed to open wisdom file $wisdom_file");
     329    foreach my $fpa_id (keys %query) {
     330        for (my $i = 0; $i <= $#{ $query{$fpa_id}{ROWNUM} }; $i++) {
     331            print WISDOM "$fpa_id\t";
     332            foreach my $key (keys %{ $query{$fpa_id} }) {
     333                print WISDOM "$key $query{$fpa_id}{$key}[$i]\t";
    341334            }
    342             push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
    343         }
    344     }
    345 }
    346 close(WISDOM);
    347 
    348 # If there is anything that needs to be updated, create the update request list, and then exit the program.
    349 my $exit_code = 0;
    350 my $update_request_file = "${workdir}/update_request.dat";
    351 open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
    352 foreach my $images (keys %update_request) {
    353     foreach my $fault (keys %{ $update_request{$images} }) {
    354         if ($fault == 25) {
    355             $exit_code = 25;
    356         }
    357         elsif ($fault != 0) {
    358             $exit_code = 21;
    359         }
    360         my $update_request = join ' ', @{ $update_request{$images}{$fault} };
    361         print UPDATE_REQUEST "$update_request\n";
    362     }
    363 }
    364 close(UPDATE_REQUEST);
    365 if ($exit_code != 0) {
    366     exit($exit_code);
     335            print WISDOM "\n";
     336            my $data_state = $query{$fpa_id}{DATA_STATE}[$i];
     337            if ($query{$fpa_id}{BAD_COMPONENT}[$i] == 0) {
     338                if ($data_state ne 'full') {
     339                   
     340                    @{ $update_request{$query{$fpa_id}{IMAGE}[$i]}{$query{$fpa_id}{FAULT}[$i]} } =
     341                        ($query{$fpa_id}{STATE}[$i],$query{$fpa_id}{STAGE}[$i],$query{$fpa_id}{STAGE_ID}[$i],
     342                         $query{$fpa_id}{COMPONENT_ID}[$i],$query{$fpa_id}{NEED_MAGIC}[$i],$query{$fpa_id}{IMAGE_DB}[$i]);
     343                }
     344                push @{ $processing_request{$fpa_id}{$query{$fpa_id}{IMAGE}[$i]} }, $i;
     345            }
     346        }
     347    }
     348    close(WISDOM);
     349
     350    # If there is anything that needs to be updated, create the update request list, and then exit the program.
     351    my $exit_code = 0;
     352    my $update_request_file = "${workdir}/update_request.dat";
     353    open(UPDATE_REQUEST,">$update_request_file") or my_die("failed to open update request_file $update_request_file");
     354    foreach my $images (keys %update_request) {
     355        foreach my $fault (keys %{ $update_request{$images} }) {
     356            if ($fault == 25) {
     357                $exit_code = 25;
     358            }
     359            elsif ($fault != 0) {
     360                $exit_code = 21;
     361            }
     362            my $update_request = join ' ', @{ $update_request{$images}{$fault} };
     363            print UPDATE_REQUEST "$update_request\n";
     364        }
     365    }
     366    close(UPDATE_REQUEST);
     367    if ($exit_code != 0) {
     368        exit($exit_code);
     369    }
    367370}
    368371
    … …  
    714717    $status = 0;
    715718#    print "$output\n";
     719    if (-e $output) {
     720        unlink $output or die "failed to unlink existing response file $output\n";
     721    }
    716722    my $outFits = Astro::FITS::CFITSIO::create_file( $output, $status );
    717723    check_fitsio( $status );

  • trunk/pstamp/scripts/dquery_finish.pl

    r27874 r33141  
    6666if ($product eq "NULL") {
    6767    # parsing failed just with fault = 0 (this leaves previously set fault in place
    68     stop_request($req_id, 0, $verbose);
     68    update_request($req_id, 0, $verbose);
    6969    exit 0;
    7070}
    … …  
    7373if (! -e $prod_dir ) {
    7474    # something must have gone wrong at the parse stage
    75     stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
     75    update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
    7676    die "product directory does not exist $prod_dir";
    7777}
    … …  
    8282    print STDERR "output fileset directory $outdir does not exist\n" if $verbose;
    8383    if (! mkdir $outdir ) {
    84         stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
     84        update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
    8585        die "cannot create output directory $outdir";
    8686    }
    8787} elsif (! -d $outdir) {
    88     stop_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
     88    update_request($req_id, $PS_EXIT_SYS_ERROR, $verbose);
    8989    die "output fileset directory $outdir exists but is not a directory";
    9090}
    … …  
    121121
    122122foreach my $job (@jobs) {
     123    next if $job->{parent_id};
    123124    my $job_id = $job->{job_id};
    124     my $response_file = "response${job_id}.fits";
     125    my $response_file = "response.${job_id}.fits";
    125126    my $response_path = "$outdir/$response_file";
    126127
    … …  
    157158}
    158159
    159 stop_request($req_id, $request_fault, $verbose);
     160update_request($req_id, $request_fault, $verbose);
    160161
    161162exit 0;
    162163
    163 sub stop_request {
     164sub update_request {
    164165    my $req_id = shift;
    165166    my $fault = shift;
    166167    my $verbose = shift;
    167168   
    168     my $command = "$pstamptool -updatereq -req_id $req_id -set_state stop";
    169     $command   .= " -set_fault $fault" if $fault;
     169    my $command = "$pstamptool -updatereq -req_id $req_id";
     170    if ($fault) {
     171        $command .= " -set_fault $fault";
     172    } else {
     173        $command .= " -set_state stop";
     174    }
    170175    $command   .= " -dbname $dbname" if $dbname;
    171176    $command   .= " -dbserver $dbserver" if $dbserver;

  • trunk/pstamp/scripts/dqueryparse.pl

    r33126 r33141  
    118118{
    119119    my $command = "$detectresponse --input $req_file --output $response_file --workdir $outdir";
     120    if ($job_id) {
     121        $command .= " --job_id $job_id";
     122    } else {
     123        $command .= " --ignore-wisdom";
     124    }
    120125    $command .= " --save-temps" if $save_temps;
    121126    $command .= " --verbose" if $verbose;
    122     $command .= " --ignore-wisdom" if !$job_id;
    123127
    124128    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    133137            my_die ("Update request indicated, but unable to find actual request!", $PS_EXIT_PROG_ERROR);
    134138        }
     139        warn("Some inputs are not available must be updated.";
    135140    }   
    136141   
    … …  
    139144my $result;
    140145unless ($job_id) {
     146    # We are running as a parse job
    141147    # If we returned correctly with a valid response file, get a job ID
    142148    # for the completed work, and move the response to a standardized name.
    … …  
    152158            chomp $job_id;
    153159            if ($job_id && -e $response_file) {
    154                 rename $response_file, "$outdir/response${job_id}.fits";
     160                rename $response_file, "$outdir/response.${job_id}.fits";
    155161            }
    156162            $result = 0;
    … …  
    222228        my_die ("Parse fault!!", $fault);
    223229    }
     230    # This does not set the request state to stop.  That will happen with the request_finish.pl script,
     231    # which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
     232    {
     233        my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
     234        $command .= " -set_fault $result" if $result;
     235
     236        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     237            run(command => $command, verbose => $verbose);
     238        unless ($success) {
     239            my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
     240        }
     241    }
    224242} else {
     243    # We are running as a job presumably because an input needed
     244    # to be regenerated
     245    if ($fault) {
    225246        # in some cases we will need to finish off the request
    226247        my_die ("Run fault!!", $fault);
    227 }
    228 
    229 # This does not set the request state to stop.  That will happen with the request_finish.pl script,
    230 # which will notice that we've inserted the stopped job and decide we're finished. Easy enough.
    231 {
    232     my $command = "$pstamptool -updatereq -req_id $req_id -set_name $req_name -set_outProduct $product";
    233     $command .= " -set_fault $result" if $result;
    234 
    235     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    236         run(command => $command, verbose => $verbose);
    237     unless ($success) {
    238         my_die("$command failed",$PS_EXIT_UNKNOWN_ERROR);
    239     }
    240 }
     248    }
     249    if (-e $response_file) {
     250        rename $response_file, "$outdir/response.${job_id}.fits";
     251    }
     252}
     253
    241254 
    242255exit 0;

  • trunk/pstamp/scripts/pstamp_job_run.pl

    r33126 r33141  
    306306    # Load the argument list that dqueryparse should have created the first time it ran, so we know how to
    307307    # run it again the same way.
    308     my $argslist = "$outputBase/parse.args";
     308    my $outdir = dirname($outputBase);
     309    my $argslist = "$outdir/parse.args";
    309310    open ARGSLIST, "<$argslist" or my_die("failed to open argslist file $argslist", $job_id, $PS_EXIT_UNKNOWN_ERROR);
    310311    my $argString = <ARGSLIST>;
    … …  
    319320    $command .= " --dbserver $dbserver" if $dbserver;
    320321    $command .= " --verbose" if $verbose;
     322    $command .= " --save-temps" if $save_temps;
    321323    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    322324        run(command => $command, verbose => $verbose);
    … …  
    616618
    617619    $exit_code = $PS_EXIT_PROG_ERROR unless $exit_code;
     620    if ($exit_code > 100) {
     621        carp ("invalid exit code: $exit_code changing to $PS_EXIT_UNKNOWN_ERROR");
     622        $exit_code = $PS_EXIT_UNKNOWN_ERROR;
     623    }
    618624    $job_state = 'run' unless $job_state;
    619625
Note: See TracChangeset for help on using the changeset viewer.