IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Aug 30, 2013, 4:48:58 PM (13 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/eam_branches/ipp-20130711/pstamp/scripts
Files:
7 edited
1 copied

Legend:

Unmodified
Added
Removed

  • branches/eam_branches/ipp-20130711/pstamp/scripts

  • branches/eam_branches/ipp-20130711/pstamp/scripts/Makefile.am

    r33324 r36071  
    2121        pstamp_get_image_job.pl \
    2222        psmkreq \
     23        psgetcalibinfo \
    2324        psstatus \
    2425        pstampstopfaulted \

  • branches/eam_branches/ipp-20130711/pstamp/scripts/pstamp_checkdependent.pl

    r35793 r36071  
    438438            $chips_ready = 0;
    439439            $chip->{fault} = $chip->{chip_fault};
     440            $chip->{data_group} = $chip->{chip_data_group};
    440441            push @chipsToUpdate, $chip;
    441442        } else {
    … …  
    575576    } elsif ($diff_mode == $IPP_DIFF_MODE_STACK_STACK ) {
    576577        # check the state of the input stack
    577         my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack2}";
     578        my $command = "$stacktool -sumskyfile -stack_id $skycell->{stack1}";
    578579        my $stack1 = runToolAndParseExpectOne($command, $verbose);
    579         my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack2}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
     580        my_die("failed to find stackSumSkyfile for stack_id $skycell->{stack1}", $PS_EXIT_UNKNOWN_ERROR) if !$stack1;
    580581
    581582        if ($stack1->{state} ne 'full') {
    582             print STDERR "input stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
     583            print STDERR "input stack $skycell->{stack1} for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
    583584            faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
    584585            return $PSTAMP_GONE;
    … …  
    590591
    591592        if ($stack2->{state} ne 'full') {
    592             print STDERR "template stack for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
     593            print STDERR "template stack $skycell->{stack2} for diffRun $diff_id $skycell_id is not in full state faulting jobs\n";
    593594            faultComponent('diff', $diff_id, $skycell_id, $PSTAMP_GONE);
    594595            return $PSTAMP_GONE;
    … …  
    850851    my ($stage, $stage_id, $component, $fault) = @_;
    851852
     853    # Stop faulting components. With the death of magic the likelyhood that something is truly gone
     854    # forever is much smaller. The false alarm rate is far too high
     855    return;
     856
    852857    my $command;
    853858    if ($stage eq 'chip') {

  • branches/eam_branches/ipp-20130711/pstamp/scripts/pstamp_cleanup.pl

    r35396 r36071  
    8080}
    8181
     82if (0) {
    8283my_die("Cleanup not yet supported for reqType: $reqType", $req_id, $PS_EXIT_UNKNOWN_ERROR)
    8384    if ($reqType ne "pstamp") and ($reqType ne "NULL") and ($reqType ne "dquery");
     85}
    8486
    8587my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files

  • branches/eam_branches/ipp-20130711/pstamp/scripts/pstamp_get_image_job.pl

    r33015 r36071  
    6565my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    6666
    67 my $data = $mdcParser->parse(join "", (<INPUT>)) or my_die("failed to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
     67my $data = $mdcParser->parse(join "", (<INPUT>))
     68    or my_die("failed to parse metadata config doc", $PS_EXIT_UNKNOWN_ERROR);
     69
    6870my $components = parse_md_list($data);
    6971my $n = scalar @$components;
    … …  
    8385    print STDERR "stage_id is $stage_id\n";
    8486    print STDERR "path_base is $path_base\n";
    85     print STDERR "path_base is $path_base\n";
    8687    print STDERR "CAMERA is $camera\n";
    8788    print STDERR "magicked is " . (defined $magicked ? $magicked : "undefined") . "\n";
    … …  
    8990
    9091if (!$camera or !$path_base or !$component or !$stage_id or !$stage) {
    91        my_die("failed to parse params from: $params_file", $PS_EXIT_UNKNOWN_ERROR);
     92       my_die("One or more parameters are missing in: $params_file", $PS_EXIT_UNKNOWN_ERROR);
    9293}
    9394

  • branches/eam_branches/ipp-20130711/pstamp/scripts/pstamp_job_run.pl

    r35609 r36071  
    5757my_die("output_base is required", $job_id, $PS_EXIT_PROG_ERROR) if !$outputBase;
    5858
    59 $options = 1 if !$options;
    60 
    61 # We don't need to muggle anymore. Ignore those options
    62 $options &=  ~($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED);
     59# ppstamp requires an input file
     60$options = $PSTAMP_SELECT_IMAGE if !$options;
    6361
    6462my $ipprc = PS::IPP::Config->new(); # IPP Configuration
    … …  
    7876my $ppstamp    = can_run('ppstamp') or (warn "Can't find ppstamp" and $missing_tools = 1);
    7977my $pstamp_get_image_job    = can_run('pstamp_get_image_job.pl') or (warn "Can't find pstamp_get_image_job.pl" and $missing_tools = 1);
     78my $psgetcalibinfo    = can_run('psgetcalibinfo') or (warn "Can't find psgetcalibinfo" and $missing_tools = 1);
    8079my $dquery_job_run = can_run('dquery_job_run.pl') or (warn "Can't find dquery_job_run.pl" and $missing_tools = 1);
    81 my $streaksreplace = can_run('streaksreplace')  or (warn "Can't find streaksreplace"  and $missing_tools = 1);
    82 my $magicdstool = can_run('magicdstool')  or (warn "Can't find magicdstool"  and $missing_tools = 1);
    8380my $whichnode = can_run('whichnode') or (warn "can't find whichnode" and $missing_tools = 1);
    8481
    … …  
    9693    my $argString;
    9794    $argString = $params->{job_args};
     95   
     96    my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
     97
    9898    my $stage = $params->{stage};
    99 
    100     # XXX: should we do any other sanity checking?
    101     my_die("argument list is empty", $job_id, $PS_EXIT_DATA_ERROR) if !$argString;
    102 
    103     my $nan_masked = 1;
    104     my $muggle = 0;
    105     if (!$params->{magicked}) {
    106         $nan_masked = 0;
    107     } elsif ($params->{magicked} and ($options & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED))) {
    108         # Attempt to find or create a muggle image
    109         $nan_masked = 0;
    110         $muggle = 1;
    111     }
     99    my_die("stage is not defined", $job_id, $PS_EXIT_DATA_ERROR) if !$stage;
     100
     101    if ($stage eq 'stack_summary') {
     102
     103        # remove options not supported by stack summary
     104        $options &= ~($PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_BACKMDL | $PSTAMP_SELECT_INVERSE
     105            | $PSTAMP_RESTORE_BACKGROUND);
     106
     107        # stackSummary outputs do not follow the usual IPP conventions for file names.
     108        # The parser (actually Job.pm) has deferred handling this until here
     109        update_stack_summary_filenames($params);
     110
     111    } elsif ($stage ne 'stack') {
     112        # ignore options only supported by stack and stack_summary
     113        $options &= ~($PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
     114    }
     115   
    112116
    113117    if ($stage eq "raw") {
    … …  
    122126    my @file_list = ($params->{image});
    123127   
    124     if ($nan_masked or ($options & $PSTAMP_SELECT_MASK)) {
     128    if ($options & $PSTAMP_SELECT_MASK) {
    125129        $mask = $params->{mask};
    126130        $fileArgs .= " -mask $mask";
    … …  
    162166    # find our output directory
    163167    my $outdir = dirname($outputBase);
    164     my ($tmpImage, $tmpMask, $tmpVariance, $tmproot);
    165    
    166     if ($muggle) {
    167         # first see if the original uncensored images are around
    168         if (check_for_backups($params, \$fileArgs)) {
    169             # We're good to go. fileArgs has been edited to contain the paths for the original uncensored images
    170             print "Making stamps from backup images\n";
    171         } elsif (($options & $PSTAMP_REQUIRE_UNCENSORED) and ($stage ne 'chip')) {
    172             # user required uncensored but since stage isn't chip we can't rebuild them
    173             my_die("uncensored inputs not available for job $job_id", $job_id, $PSTAMP_NOT_AVAILABLE, 'stop');
    174         } elsif (($options & $PSTAMP_REQUEST_UNCENSORED) and ($params->{state} ne 'full') and ($stage ne 'chip')) {
    175             # we can only restore pixels for chip stage images if the data has been updated.
    176             # the data will have been updated if the params->{state} the state when the job was queued is not 'full' (
    177             # XXX: we should probably be looking explicitly at the job and checking for a dep_id
    178             print "Run state was $params->{state}: will make stamps from destreaked $stage images.\n";
    179             # make stamps from uncensored images
    180             $muggle = 0;
     168
     169    my ($calib_fd, $calibfile);
     170    if ($stage eq 'chip' or $stage eq 'warp') {
     171        my $cam_id = $params->{cam_id};
     172        if (!$cam_id) {
     173            carp "no cam_id found in job params\n";
     174            exit $PS_EXIT_PROG_ERROR;
     175        }
     176        ($calib_fd, $calibfile) = tempfile ("$outdir/calib.XXXX", UNLINK => !$save_temps);
     177        close $calib_fd;
     178
     179        my $command = "$psgetcalibinfo --cam_id $cam_id --output $calibfile --dbname $params->{imagedb}";
     180        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     181            run(command => $command, verbose => $verbose);
     182
     183        my $exitStatus;
     184        if (WIFEXITED($error_code)) {
     185            $exitStatus = WEXITSTATUS($error_code);
    181186        } else {
    182             # Try and replace the streaks from the recovery images
    183 
    184             my $temp_dir = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR") or &my_die("Unable to find temporary directory in site configuration", $job_id, $PS_EXIT_CONFIG_ERROR);
    185             $tmproot = tempdir("$temp_dir/psjob.$job_id.XXXX", CLEANUP => !$save_temps);
    186             my $muggle_command = "$streaksreplace -stage $stage -tmproot $tmproot";
    187             # find the "directory" of the input path_base
    188             my $inputdir = dirname($image);
    189             my $base = basename($image);
    190             $tmpImage = "$tmproot/$base";
    191 
    192             @file_list = ();
    193 
    194             # XXX: We should get the recovery_path_base from the magicDSFile but that requires a bunch of file rule and
    195             # stage work. Import the rule here.
    196             my $recImage = "$inputdir/REC_$base";
    197             my $newFileArgs = " -file $tmpImage";
    198             $muggle_command .= " -image $image -recimage $recImage";
    199             push @file_list, $recImage;
    200 
    201             if ($mask) {
    202                 $base = basename($mask);
    203                 $tmpMask = "$tmproot/$base";
    204                 $newFileArgs .= " -mask $tmpMask";
    205                 my $recMask = "$inputdir/REC_$base";
    206                 $muggle_command .= " -mask $mask -recmask $recMask";
    207                 push @file_list, $recMask;
    208             }
    209 
    210             if ($variance) {
    211                 $base = basename($variance);
    212                 $tmpVariance = "$tmproot/$base";
    213                 $newFileArgs .= " -variance $tmpVariance";
    214                 my $recVariance = "$inputdir/REC_$base";
    215                 $muggle_command .= " -weight $variance -recweight $recVariance";
    216                 push @file_list, $recVariance;
    217             }
    218             if (check_files(0, @file_list)) {
    219                 # recovery files exist and are accessible restore the excised pixels
    220 
    221                 my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    222                     run(command => $muggle_command, verbose => $verbose);
    223                 unless ($success) {
    224                     my $exitStatus = WEXITSTATUS($error_code);
    225                     my_die( "streaksreplace failed with error code: $exitStatus", $job_id, $exitStatus);
    226                 }
    227 
    228                 # set the ppstamp file arguments
    229                 $fileArgs = $newFileArgs;
    230             } else {
    231                 if ($options & $PSTAMP_REQUIRE_UNCENSORED) {
    232                     my_die( "unable to restore uncensored images", $job_id, $PSTAMP_NOT_AVAILABLE);
    233                 }
    234                 # just make stamps from the censored images
    235                 print "Unable to restore uncensored images, will extract stamps from censored images\n";
    236                 # these files won't be used so zap them
    237                 ($tmpImage, $tmpMask, $tmpVariance, $tmproot) = (undef, undef, undef, undef);
    238             }
    239         }
    240     }
    241 
    242     # MAGIC IS DEAD
    243     $nan_masked = 0;
    244 
    245     my $command = "$ppstamp $outputBase $argString $fileArgs";
    246     $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
    247     $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
    248     $command .= " -stage $stage";
    249     $command .= " -censor_masked" if $nan_masked;
    250     $command .= " -dbname $dbname" if $dbname;
    251     $command .= " -dbserver $dbserver" if $dbserver;
    252     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    253         run(command => $command, verbose => $verbose);
     187            print STDERR "psgetcalibinfo failed error_code: $error_code\n";
     188            $exitStatus = $PS_EXIT_SYS_ERROR;
     189        }
     190        exit $exitStatus if $exitStatus;
     191
     192        if (-s $calibfile == 0) {
     193            print "no calibration information found for $cam_id\n";
     194            $calibfile = undef;
     195        }
     196    }
     197
     198    # unless the stage is stack_summary we use ppstamp to make postage stamps (including -wholefile)
     199    # otherwise we make copies of the input files to the outputs
     200    my $use_ppstamp = ($stage ne 'stack_summary');
    254201
    255202    my $exitStatus;
    256     if (WIFEXITED($error_code)) {
    257         $exitStatus = WEXITSTATUS($error_code);
    258     } else {
    259         print STDERR "ppstamp failed error_code: $error_code\n";
    260         $exitStatus = $PS_EXIT_SYS_ERROR;
     203    if ($use_ppstamp) {
     204        my $command = "$ppstamp $outputBase $argString $fileArgs";
     205        $command .= " -write_jpeg" if ($options & $PSTAMP_SELECT_JPEG);
     206        $command .= " -nocompress" if ($options & $PSTAMP_SELECT_UNCOMPRESSED);
     207        $command .= " -stage $stage";
     208        $command .= " -forheader $calibfile" if $calibfile;
     209        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     210            run(command => $command, verbose => $verbose);
     211
     212        if (WIFEXITED($error_code)) {
     213            $exitStatus = WEXITSTATUS($error_code);
     214        } else {
     215            print STDERR "ppstamp failed error_code: $error_code\n";
     216            $exitStatus = $PS_EXIT_SYS_ERROR;
     217        }
     218        # XXX: if stage is stack deal with EXP and NUM images if selected
     219    } else {
     220        $exitStatus = justCopyFiles($outputBase, \$options, $params);
    261221    }
    262222
    … …  
    270230
    271231        # Note: we are assuming the contents of the PSTAMP filerules here.
    272         my %extensions = ( $PSTAMP_SELECT_IMAGE    => "fits",
    273                            $PSTAMP_SELECT_MASK     => "mk.fits",
    274                            $PSTAMP_SELECT_VARIANCE => "wt.fits",
    275                            $PSTAMP_SELECT_SOURCES  => "cmf",
    276                            $PSTAMP_SELECT_JPEG     => "jpg");
    277 
    278         my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES);
     232        my %extensions = ( $PSTAMP_SELECT_IMAGE    => 'fits',
     233                           $PSTAMP_SELECT_MASK     => 'mk.fits',
     234                           $PSTAMP_SELECT_VARIANCE => 'wt.fits',
     235                           $PSTAMP_SELECT_SOURCES  => 'cmf',
     236                           $PSTAMP_SELECT_JPEG     => 'jpg',
     237                           $PSTAMP_SELECT_EXP      => 'exp.fits',
     238                           $PSTAMP_SELECT_NUM      => 'num.fits');
     239
     240        my $output_mask = $options & ($PSTAMP_SELECT_IMAGE | $PSTAMP_SELECT_MASK | $PSTAMP_SELECT_VARIANCE
     241            | $PSTAMP_SELECT_JPEG | $PSTAMP_SELECT_SOURCES | $PSTAMP_SELECT_EXP | $PSTAMP_SELECT_NUM);
    279242
    280243        foreach my $key (keys (%extensions)) {
    … …  
    523486}
    524487
    525 sub check_for_backups {
    526     my $params = shift;
    527     my $r_fileArgs = shift;
    528 
    529     my $command = "$magicdstool -destreakedfile -stage $params->{stage} -stage_id $params->{stage_id} -component $params->{component} -dbname $params->{imagedb}";
    530     my $results = runToolAndParse($command, $verbose);
    531     my $dsComponent  = $results->[0];
    532 
    533     if ($dsComponent and ($dsComponent->{data_state} eq 'full')) {
    534 
    535         print "magicDSFile state is full. Backup images should exist\n";
    536 
    537         # replace the file names with the backup paths
    538         # fileArgs has the form:  -file imagename [-mask maskname] [-weight weightname]
    539         my @args = split " ", $$r_fileArgs;
    540 
    541         my $newFileArgs;
    542         while (@args) {
    543             my $a = shift @args;
    544             my $f = shift @args;
    545             my_die( "unexpected fileArg list $$r_fileArgs", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$a or !$f;
    546 
    547             my $backup_image_name = $ipprc->destreaked_filename($f);
    548             my_die( "failed to extract backup image name from $f", $job_id, $PS_EXIT_PROG_ERROR, 'run') if !$backup_image_name;
    549             $newFileArgs .= " $a $backup_image_name";
    550         }
    551         $$r_fileArgs = $newFileArgs;
    552 
    553         return 1;
    554     }
    555 
    556     return 0;
    557 }
    558 
    559488my $neb;
    560489sub storage_object_exists
    … …  
    630559}
    631560
     561# stack_summary stage does not currently use proper file rules.
     562# The parser defers handlint this to us..
     563
     564sub update_stack_summary_filenames {
     565    my $params  = shift;
     566
     567    my $path_base = $params->{path_base};
     568
     569    $params->{image}  = $path_base . ".image.b1.fits";
     570    $params->{mask}   = $path_base . ".mask.b1.fits";
     571    $params->{weight} = $path_base . ".variance.b1.fits";
     572    $params->{jpeg}   = $path_base . ".image.0.b1.jpeg";
     573    $params->{exp}    = $path_base . ".exp.b1.fits";
     574    $params->{num}    = $path_base . ".num.b1.fits";
     575}
     576
     577sub myCopy {
     578    my ($dest, $src, $type, $dieOnFail) = @_;
     579
     580    my $result;
     581    my $resolved = $ipprc->file_resolve($src);
     582    if ($resolved and $ipprc->file_exists($resolved)) {
     583        print "Copying $src to $dest\n" if $verbose;;
     584        $result = copy($resolved, $dest);
     585    } else {
     586        my $msg = "Specified source $type image $src not found.";
     587        if ($dieOnFail) {
     588            $msg .= "\n";
     589        } else {
     590            $msg .= " ignoring\n";
     591        }
     592        carp $msg;
     593        $result = 0;
     594    }
     595   
     596    if (!$result and $dieOnFail) {
     597        &my_die("Unable to copy $type image", $job_id, $PS_EXIT_SYS_ERROR, 'run');
     598    }
     599    return $result;
     600}
     601
     602sub justCopyFiles {
     603    my ($outputBase, $r_options, $params) = @_;
     604
     605    print "Just copying files for $job_id.\n";
     606
     607    my $options = $$r_options;
     608    if ($options & $PSTAMP_SELECT_IMAGE) {
     609        myCopy("$outputBase.fits", $params->{image}, 'image', 1);
     610    }
     611    if ($options & $PSTAMP_SELECT_MASK) {
     612        if (!myCopy("$outputBase.mk.fits", $params->{mask}, 'mask', 0)) {
     613            $options &= ~$PSTAMP_SELECT_MASK;
     614        }
     615    }
     616    if ($options & $PSTAMP_SELECT_VARIANCE) {
     617        if (!myCopy("$outputBase.wt.fits", $params->{weight}, 'variance', 0)) {
     618            $options =  ~$PSTAMP_SELECT_VARIANCE;
     619        }
     620    }
     621    if ($options & $PSTAMP_SELECT_JPEG) {
     622        if (!myCopy("$outputBase.jpg", $params->{jpeg}, 'jpeg', 0)) {
     623            $options &= ~$PSTAMP_SELECT_JPEG;
     624        }
     625    }
     626    if ($options & $PSTAMP_SELECT_EXP) {
     627        if (!myCopy("$outputBase.exp.fits", $params->{exp}, 'exp', 0)) {
     628            $options &= ~$PSTAMP_SELECT_EXP;
     629        }
     630    }
     631    if ($options & $PSTAMP_SELECT_NUM) {
     632        if (!myCopy("$outputBase.num.fits", $params->{num}, 'num', 0)) {
     633            $options &= ~$PSTAMP_SELECT_NUM;
     634        }
     635    }
     636
     637    $$r_options = $options;
     638
     639    print "Done with copy.\n";
     640
     641    return 0;
     642}
     643
    632644sub my_die
    633645{

  • branches/eam_branches/ipp-20130711/pstamp/scripts/pstampparse.pl

    r35504 r36071  
    5151
    5252die "invalid mode '$mode'" unless ($mode eq "list_uri") or ($mode eq "queue_job");
    53 die "--file is required"     if !defined($request_file_name);
     53die "--file is required"   unless defined($request_file_name);
    5454
    5555if ($mode ne "list_uri") {
    56     die "req_id is required"   if !$req_id;
     56    die "req_id is required"  if !$req_id;
    5757    die "outdir is required"  if !$outdir;
    58     die "product is required"  if !$product;
     58    die "product is required" if !$product;
    5959} else {
    60     $req_id = 0;
    61     $outdir = "nowhere";
     60    # mode eq 'list_uri' (used for parser testing)
     61    # these values won't be used for anything but should be defined to avoid errors
     62    $req_id  = 0;
     63    $outdir  = "nowhere";
    6264    $product = "dummy";
    6365}
    … …  
    7678my $pstamptool  = can_run('pstamptool')  or (warn "Can't find pstamptool"  and $missing_tools = 1);
    7779my $pstampdump  = can_run('pstampdump') or (warn "Can't find pstampdump" and $missing_tools = 1);
    78 my $fields  = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
     80my $fields      = can_run('fields') or (warn "Can't find fields" and $missing_tools = 1);
    7981
    8082if ($missing_tools) {
    … …  
    116118if ($extver >= 2) {
    117119    # We have a version 2 file. Require that the new keywords be supplied.
    118     my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless defined $action;
    119 
    120     my_die("invalid action: $action supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
    121 
    122     my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST) unless $email;
     120    my_die("action not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST)
     121        unless defined $action;
     122
     123    my_die("invalid action: $action supplied in version $extver request file $request_file_name\n",
     124        $PSTAMP_INVALID_REQUEST)
     125        unless (uc($action) eq 'PROCESS' or uc($action) eq 'PREVIEW');
     126
     127    my_die("email not supplied in version $extver request file $request_file_name\n", $PSTAMP_INVALID_REQUEST)
     128        unless $email;
     129
    123130    # XXX check for "valid" $email
    124131} else {
    125132    # for version 1 file the action is process and email is not used
    126133    $action = 'PROCESS';
    127     $email = 'null';
     134    $email  = 'null';
    128135}
    129136
    … …  
    175182
    176183
    177 if ($req_id and !$no_update) {
     184{
    178185    # update the database with the request name. This will be used as the
    179     # the output data store's product name
     186    # the fileset name in the output data store
    180187    my $command = "$pstamptool -updatereq -req_id $req_id  -set_name $req_name";
    181188    $command .= " -set_username $email" if $email ne 'null';
    182189    $command .= " -set_outProduct $product";
    183190    $command .= " -set_label $label" if $label_changed;
    184     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    185         run(command => $command, verbose => $verbose);
    186     unless ($success) {
    187         my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
    188     }
    189 }
     191    unless ($no_update) {
     192        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
     193            run(command => $command, verbose => $verbose);
     194        unless ($success) {
     195            my_die("$command failed", $PS_EXIT_UNKNOWN_ERROR);
     196        }
     197    } else {
     198        print "skipping $command\n";
     199    }
     200}
     201
    190202if ($duplicate_req_name) {
    191203    exit 0;
    … …  
    212224}
    213225
    214 my $nRows = scalar @$rows;
     226my $nRows = $rows ? scalar @$rows : 0;
    215227print "\n$nRows rows read from request file\n";
     228
     229unless ($nRows) {
     230    # pstamp_job_run was invoked so the request file must have contained a valid header
     231    # a request file with no rows is invalid.
     232    # The print above will let the log file know a bit more.
     233    # Insert a faulted fake job and exit this program successfully.
     234    print STDERR "Invalid request file.\n";
     235    insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
     236    exit 0;
     237}
     238
    216239
    217240
    … …  
    219242my $imageList;
    220243my $stage;
     244my $big_limit = 100;    # XXX: this should be in a configuration file some where
    221245foreach my $row (@$rows) {
    222246
    223     if ($label eq 'WEB.UP' and ($nRows > 500 or $num_jobs > 500) and $req_id and !$no_update) {
    224         # this is a big request and it came from the upload page and doesn't have a specific label assigned
    225         # change it to the generic one that has lower with lower priority
    226         $label = 'WEB.BIG';
    227         print "\nChanging label for big WEB.UP request to $label\n";
     247
     248    if (!($label =~ /BIG/) and ($label =~ /PSI/ or $label =~ /WEB/) and ($nRows > $big_limit or $num_jobs > $big_limit)
     249        and $req_id and !$no_update) {
     250
     251        # This is a "big" request and it came from one of the "high priority" channels
     252        # and doesn't have a specific label assigned.
     253        # Change the label to a value that its jobs run with lower priority.
     254
     255        my $old_label = $label;
     256        $label = ($label =~ /WEB/) ? 'WEB.BIG' : 'PSI.BIG';
     257        print "\nChanging label for big $old_label request to $label\n";
    228258
    229259        my $command = "$pstamptool -updatereq -req_id $req_id  -set_label $label";
    … …  
    249279
    250280if (($action eq 'LIST' or $mode eq "queue_job") and ($num_jobs eq 0)) {
    251     print STDERR "no jobs created for $req_name\n" if $verbose;
    252     insertFakeJobForRow(undef, 0, $PSTAMP_INVALID_REQUEST);
     281    # this should not happen. The function above is required to insert a fake job for any
     282    # rows that did not yield any jobs.
     283    print STDERR "ERROR: zero jobs created for $req_name\n";
     284    insertFakeJobForRow(undef, 0, $PS_EXIT_PROG_ERROR);
    253285}
    254286
    … …  
    286318    }
    287319    if ($job_type eq 'get_image') {
    288         unless ($req_type eq 'byid' or $req_type eq 'byexp' or ($req_type eq 'byskycell' and $stage eq 'stack')) {
    289             print STDERR "REQ_TYPE must be 'byid' or 'byexp' or byskcyell for stacks for JOB_TYPE 'get_image'\n";
     320        # get_image jobs are quite expensive in terms of space so we are currently restricting them
     321        unless ($req_type eq 'byid' or $req_type eq 'byexp'
     322            or ($req_type eq 'byskycell' and $stage eq 'stack')
     323            or ($stage eq 'stack_summary')) {
     324            print STDERR "REQ_TYPE must be 'byid' or 'byexp' JOB_TYPE 'get_image' for IMG_TYPE $stage\n";
    290325            insertFakeJobForRow($row, 1, $PSTAMP_INVALID_REQUEST);
    291326            return 0;
    … …  
    295330    my $component = $row->{COMPONENT};
    296331    if (!defined $component or (lc($component) eq "null") or (lc($component) eq "all")) {
    297         if ($job_type eq 'get_image') {
     332        if ($job_type eq 'get_image' and ! ($stage eq 'stack' or $stage eq 'stack_summary')) {
    298333            $row->{COMPONENT} = 'all';
    299334        } else {
    … …  
    384419    }
    385420
    386     if (($req_type eq "byexp") and ($stage eq "stack")) {
    387         print STDERR "byexp not implemented for stack stage. row: $rownum\n";
     421    if (($req_type eq "byexp") and ($stage eq "stack" or $stage eq 'stack_summary')) {
     422        print STDERR "byexp not implemented for $stage stage. row: $rownum\n";
    388423        insertFakeJobForRow($row, 1, $PSTAMP_NOT_IMPLEMENTED);
    389424        return 0;
    … …  
    457492    my $start_locate = gettimeofday();
    458493
    459     print "\nCalling new_locate_images for row: $rownum\n";
     494    print "\nCalling locate_images_for_row for row: $rownum\n";
    460495
    461496    $imageList = locate_images_for_row($ipprc, $image_db, $camera, $row, $verbose);
    … …  
    521556    my $image_db   = $proj_hash->{dbname};
    522557    my $camera     = $proj_hash->{camera};
    523     my $need_magic    = $proj_hash->{need_magic};
     558    my $need_magic = $proj_hash->{need_magic};
    524559    # Since user can get unmagicked data "by coordinate" requests can go back in time
    525560    # to dredge unusable data from the "dark days"...
    … …  
    532567    $need_magic = 0 if $stage eq 'stack';
    533568
    534     if ($need_magic) {
    535 
    536         # this project requires that postage stamps be extracted from destreaked images
    537 
    538         if ($option_mask & ($PSTAMP_REQUEST_UNCENSORED | $PSTAMP_REQUIRE_UNCENSORED)) {
    539             # The user has requested uncensored stamps
    540 
    541             if (!$dest_requires_magic) {
    542                 # and this user's data store destination is allowed uncensored stamps, so accept the request
    543                 $need_magic = 0;
    544             } else {
    545                 print STDERR "Error row $rownum: User not authorized to to request uncensored stamps.\n";
    546                 if ($option_mask & $PSTAMP_REQUIRE_UNCENSORED) {
    547                     # user required uncensored stamps. Can't do it so fail.
    548                     foreach my $r (@$rowList) {
    549                         insertFakeJobForRow($r, 1, $PSTAMP_NOT_AUTHORIZED);
    550                         $num_jobs++;
    551                     }
    552                     return $num_jobs;
    553                 }
    554 
    555                 # user will accept censored stamps. alter OPTION_MASK and continue
    556 
    557                 print STDERR "    Will attempt to make destreaked stamps\n";
    558                 # zap the offending bit in the option mask
    559                 $option_mask = $option_mask ^ ($PSTAMP_REQUEST_UNCENSORED);
    560                 foreach my $r (@$rowList) {
    561                     $r->{OPTION_MASK} = $option_mask;
    562                 }
    563             }
    564         }
    565     }
    566    
     569    # XXX: magic is dead
     570    $need_magic = 0;
     571
    567572    my $numRows = scalar @$rowList;
    568573
    … …  
    701706    }
    702707    $base =~ s/.fits$//;
     708
     709    my $filter = $image->{filter};
     710    if (!$filter) {
     711        if ($stage eq 'diff') {
     712            $filter = $image->{filter_1};
     713        }
     714        if (!$filter) {
     715            # XXX: perhaps this should be a programming error...
     716            print STDERR "missing filter using 'X'\n";
     717            $filter = 'X';
     718        }
     719    }
     720    # use first character of filter
     721    $filter = substr($filter, 0, 1);
    703722           
    704     my $output_base = "$outdir/${rownum}_${job_num}_${base}";
     723    my $output_base = "$outdir/${rownum}_${job_num}_${filter}_${base}";
    705724    write_params($output_base, $image);
    706725
    … …  
    719738    if (!$no_update) {
    720739        # mode eq "queue_job"
     740        my $start_addjob = gettimeofday();
    721741        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    722742            run(command => $command, verbose => $verbose);
    … …  
    728748            my_die("failed to queue job for request $req_id", $PS_EXIT_UNKNOWN_ERROR);
    729749        }
     750        my $dtime_addjob = gettimeofday() - $start_addjob;
     751        print "Time to addjob for row $rownum $dtime_addjob\n";
    730752    } else {
    731753        print "skipping command: $command\n";
    … …  
    11601182    }
    11611183    if (($img_type eq "raw") or ($img_type eq "chip") or ($img_type eq "warp") or
    1162         ($img_type eq "stack") or ($img_type eq "diff")) {
     1184        ($img_type eq "stack") or ($img_type eq 'stack_summary') or ($img_type eq "diff")) {
    11631185        return 1;
    11641186    } else {
Note: See TracChangeset for help on using the changeset viewer.