IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
May 3, 2010, 8:50:52 AM (16 years ago)
Author:
eugene
Message:

updates from trunk

Location:
branches/simtest_nebulous_branches
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/simtest_nebulous_branches

  • branches/simtest_nebulous_branches/ippScripts/scripts/dist_component.pl

    r24872 r27840  
    88use Sys::Hostname;
    99my $host = hostname();
     10my $date = `date`;
    1011print "\n\n";
    11 print "Starting script $0 on $host\n\n";
     12print "Starting script $0 on $host at $date\n\n";
    1213
    1314use vars qw( $VERSION );
    … …  
    2829use Pod::Usage qw( pod2usage );
    2930
    30 # filerules that are not included in 'cleaned' distribution bundles
    31 # I could simplify the function below by using a two level hash but
    32 # that would be less clear I think
    33 my %chip_cleaned = ( 'PPIMAGE.CHIP' => 'image',
    34                      'PPIMAGE.CHIP.MASK' => 'mask',
    35                      'PPIMAGE.CHIP.VARIANCE' => 'variance' );
    36 my %camera_cleaned = ( 'PSASTRO.OUTPUT.MASK' => 'mask' );
    37 my %fake_cleaned;
    38 my %warp_cleaned = ( 'PSWARP.OUTPUT' => 'image',
    39                      'PSWARP.OUTPUT.MASK' => 'mask',
    40                      'PSWARP.OUTPUT.VARIANCE' => 'variance' );
    41 my %diff_cleaned = ( 'PPSUB.OUTPUT' => 'image',
    42                      'PPSUB.OUTPUT.MASK' => 'mask',
    43                      'PPSUB.OUTPUT.VARIANCE' => 'variance' );
    44 my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image',
    45                       'PPSTACK.OUTPUT.MASK' => 'mask',
    46                       'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
    47 
    48 
    4931# Look for programs we need
    5032my $missing_tools;
    5133my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
    52 my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
     34my $dist_make_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
    5335if ($missing_tools) {
    5436    warn("Can't find required tools.");
    … …  
    5739
    5840# Parse the command-line arguments
    59 my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean);
     41my ($dist_id, $camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $clean, $alt_path_base);
    6042my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality);
    6143my ($dbname, $save_temps, $verbose, $no_update, $logfile);
    … …  
    7456           'no_magic'       => \$no_magic,   # magic is not required for this distribution run
    7557           'magicked'       => \$magicked,   # magicked state for this component
     58           'alt_path_base=s'=> \$alt_path_base,  # path to alternate inputs
    7659           'outdir=s'       => \$outdir,     # "directory" for outputs
    7760           'clean'          => \$clean,      # create clean distribution
    … …  
    10083}
    10184
    102 $ipprc->define_camera($camera);
    103 
    10485my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    10586
    106 # making a clean bundle of raw images doesn't make sense
     87my ($rf, $rf_name) = tempfile("/tmp/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
     88close $rf;
    10789
    108 # create the output directories if it is not a nebulous path and it doesn't exist
    109 if (index($outdir, "neb://") != 0) {
    110     if (! -e $outdir ) {
    111         my $code = system "mkdir -p $outdir";
    112         &my_die("cannot create output directory $outdir", $dist_id, $component,
    113                 $code >> 8) if $code;
    114     }
    115 }
    116 
    117 # Get the list of data products for this component
    118 # note: We my_die in get_file_list if something goes wrong.
    119 
    120 my $file_list = get_file_list($stage, $component, $path_base, $clean);
    121 
    122 if (($stage ne 'raw') and ($stage ne 'fake')) {
    123     # If the file list is empty it is an error because we should at least get a config dump file
    124     # except for fake stage which doesn't do anything yet
    125     &my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list);
    126 }
    127 
    128 # set up directory for temporary files
    129 
    130 my $tmpdir  = "$outdir/tmpdir.$dist_id.$component";
    131 if (-e $tmpdir) {
    132     if (-d $tmpdir) {
    133         my $rc = system "rm -r $tmpdir";
    134         &my_die("cannot rm $tmpdir return code: $rc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if $rc;
    135     } else {
    136         unlink $tmpdir or &my_die("cannot delete $tmpdir", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
    137     }
    138 }
    139 mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $dist_id, $component,
    140                        $PS_EXIT_UNKNOWN_ERROR);
    141 
    142 #
    143 # we need to run set masked pixels to NAN in the image and variance images
    144 # unless
    145 #   1. we are building a clean distribution bundle
    146 #   2. magic is not required for this distRun
    147 #   3. the processing for the component produced no images (warp or diff with bad quality for example)
    148 my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack')) || $no_magic || $poor_quality);
    149 
    150 my ($image, $mask, $variance);
    151 
    152 # foreach my $file_rule (keys %$file_list) {
    153 my $num_files = 0;
    154 foreach my $file (@$file_list) {
    155     # check whether this file rule refers to an image, mask, or variance fits image
    156     my $file_rule = $file->{file_rule};
    157     my $image_type = get_image_type($stage, $file_rule);
    158 
    159     # if this is an image and we are building a clean bundle or if quality is bad skip this rule
    160     next if $image_type and ($clean or $poor_quality);
    161 
    162     # if magic is required, don't ship jpegs or binned fits images
    163     next if !$no_magic && (($file_rule =~ /.BIN1/) or ($file_rule =~ /.BIN2/) or ($file_rule =~ /.JPEG1/)
    164             or ($file_rule =~ /.JPEG2/));
    165 
    166     my $file_name = $file->{name};
    167     my $base = basename($file_name);
    168     my $path = $ipprc->file_resolve($file_name);
    169 
    170     if (!$path) {
    171         # skip this file if $poor_quality
    172         # this is for compatability with older runs which don't have the files list in the
    173         # config dump.
    174         # Once we give up on supporting that we can remove the next line. (If the file is in the list
    175         # it must exist)
    176         next if $poor_quality;
    177 
    178         &my_die("failed to resolve  $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR);
    179     }
    180 
    181     # open the file to make sure it exists (and to work around the failed mount phenomena)
    182     my $fh = open_with_retries($path);
    183     close $fh;
    184 
    185     # we need to pre-process the image before adding to the bundle. Save the path names.
    186     # the images will be created below
    187     $num_files++;
    188     if ($image_type && $nan_masked_pixels) {
    189         # save the
    190         if ($image_type eq 'image') {
    191             $image = $file_name;
    192         } elsif ($image_type eq 'mask') {
    193             $mask = $file_name;;
    194         } elsif ($image_type eq 'variance') {
    195             $variance = $file_name;;
    196         } else {
    197             &my_die("invalid image type found: $image_type", $dist_id, $component,
    198                        $PS_EXIT_PROG_ERROR);
    199         }
    200     } else {
    201         # create a symbolic link from the file in the nebulous repository
    202         # in the temporary directory
    203         symlink $path, "$tmpdir/$base";
    204     }
    205 }
    206 
    207 if ($nan_masked_pixels) {
    208     # One last check as to whether magic has been applied to the inputs
    209 
    210     # Note: the sql for disttool -pendingcomponent won't select a component that
    211     # requires magic and hasn't been magicked, but we check again here
    212 
    213     if (!($magicked or $no_magic)) {
    214         &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR);
    215     }
    216 
    217     &my_die("no image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image;
    218     &my_die("no mask image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask;
    219     &my_die("no variance image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
    220 
    221     my $class_id;
    222     # run streaksrelease to set masked pixels to NAN
    223     if ($stage eq "raw") {
    224         $class_id = $component;
    225         # we can use the chip mask because disttool demands that magic have been run
    226         # and so the camera mask and the chip mask are the same
    227         $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component);
    228         my $fh = open_with_retries($mask);
    229         close $fh;
    230     } elsif ($stage eq "chip") {
    231         $class_id = $component;
    232     }
    233 
    234     my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir";
    235     $command .= " -class_id $class_id" if $class_id;
    236     $command .= " -mask $mask" if $mask;
    237     $command .= " -chip_mask $mask" if ($stage eq 'chip' and $mask);
    238     $command .= " -weight $variance" if $variance;
    239     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    240         run(command => $command, verbose => $verbose);
    241     unless ($success) {
    242         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    243         &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
    244     }
    245 }
    246 
    247 
    248 my $file_name;
    249 my $bytes;
    250 my $md5sum;
    251 
    252 if ($num_files) {
    253     # create the tarfile
    254     # XXX TODO: create a file rule for the tar file name
    255     my $tbase = basename($path_base);
    256     $tbase .= ".$component" if $component;
    257     $file_name = "$tbase.tgz";
    258     my $tarfile = "$outdir/$file_name";
    259 
    260     my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";
     90my ($file_name, $bytes, $md5sum);
     91{
     92    my $command = "$dist_make_bundle --camera $camera --stage $stage --stage_id $stage_id";
     93    $command .= " --results_file $rf_name";
     94    $command .= " --component $component --path_base $path_base --outdir $outdir";
     95    $command .= " --chip_path_base $chip_path_base" if $chip_path_base;
     96    $command .= " --state $run_state" if defined $run_state;
     97    $command .= " --data_state $data_state" if defined $data_state;
     98    $command .= " --poor_quality" if defined $poor_quality;
     99    $command .= " --no_magic" if defined $no_magic;
     100    $command .= " --magicked" if defined $magicked;
     101    $command .= " --alt_path_base $alt_path_base" if defined $alt_path_base;
     102    $command .= " --clean" if defined $clean;
     103    $command .= " --save-temps" if defined $save_temps;
     104    $command .= " --dbname $dbname" if defined $dbname;
     105    $command .= " --verbose" if defined $verbose;
    261106
    262107    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    264109    unless ($success) {
    265110        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    266         &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
     111        &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code);
    267112    }
    268113
    269     # tell the module not to die on error
    270     $Digest::MD5::File::NOFATALS = 1;
    271     $md5sum = file_md5_hex($tarfile);
    272     &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum;
     114    open $rf, "<$rf_name" or &my_die("failed to open results file $rf_name", $dist_id, $component, $outdir, $PS_EXIT_UNKNOWN_ERROR);
    273115
    274     $bytes = -s $tarfile;
     116    my @lines = (<$rf>);
    275117
    276     delete_tmpdir($tmpdir);
    277 } else {
    278     # no files for this component
    279     $file_name = "none";
    280     $bytes = 0;
    281     $md5sum = "0";
     118    my $metadata = $mdcParser->parse (join "", @lines) or
     119        &my_die("Unable to parse metadata config doc", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR);
     120
     121    my $results = parse_md_list($metadata);
     122
     123    if ((scalar @$results) != 1) {
     124        my $n = scalar @$results;
     125        &my_die("Unexected number of results from dist_make_bundle.pl: $n", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR);
     126    }
     127    my $result = $results->[0];
     128
     129    $file_name = $result->{name};
     130    &my_die("undefined file name from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $file_name;
     131
     132    $bytes = $result->{bytes};
     133    &my_die("undefined file size from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $bytes;
     134
     135    $md5sum = $result->{md5sum};
     136    &my_die("undefined file md5sum from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $md5sum;
    282137}
     138
    283139{
    284     my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component";
     140    my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component -outdir $outdir";
    285141    $command .= " -name $file_name -bytes $bytes -md5sum $md5sum";
    286142    $command .= " -dbname $dbname" if defined $dbname;
    … …  
    291147        unless ($success) {
    292148            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    293             &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
     149            &my_die("Unable to perform $command: $error_code", $dist_id, $component, $outdir, $error_code);
    294150        }
    295151    } else {
    … …  
    301157
    302158### Pau.
    303 
    304 # return the image type (image, mask, or variance) if this file rule refers to
    305 # one of the big fits files that are not included in a clean distribution
    306 sub get_image_type {
    307     my $stage = shift;
    308     my $rule = shift;
    309     my $type;
    310 
    311     if ($stage eq "raw") {
    312         if ($rule eq "RAW.IMAGE") {
    313             $type = 'image';
    314         }
    315     } elsif ($stage eq "chip") {
    316         $type = $chip_cleaned{$rule};
    317     } elsif ($stage eq "camera") {
    318         $type = $camera_cleaned{$rule};
    319     } elsif ($stage eq "fake") {
    320         $type = $fake_cleaned{$rule};
    321     } elsif ($stage eq "warp") {
    322         $type = $warp_cleaned{$rule};
    323     } elsif ($stage eq "diff") {
    324         $type = $diff_cleaned{$rule};
    325     } elsif ($stage eq "stack") {
    326         $type = $stack_cleaned{$rule};
    327     } else {
    328         &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    329     }
    330     return $type;
    331 }
    332 
    333 sub open_with_retries {
    334     my $name = shift;
    335    
    336     my $tries = 1;
    337     my $max_tries = 5;
    338     my $opened = 0;
    339     while (!$opened && ($tries <= $max_tries)) {
    340         $opened = open(IN, "<$name");
    341         if (!$opened) {
    342             print STDERR "WARNING failed to open $name re-try $tries\n";
    343             $tries++;
    344             sleep 5;
    345         }
    346     }
    347    
    348     &my_die("failed to open $name after $max_tries tries\n", $dist_id, $component,
    349                     $PS_EXIT_DATA_ERROR) if (!$opened);
    350 
    351     return *IN;
    352 }
    353 
    354 sub get_file_list {
    355     my $stage = shift;
    356     my $component = shift;
    357     my $path_base = shift;
    358     my $clean = shift;
    359 
    360     my @file_list;
    361     if ($stage eq "raw") {
    362         # XXX: TODO for now disttool sets path_base is set to the uri of the file
    363         # eventually rawImfile will have a path_base that we'll need to add '.fits'
    364         # XXX do we want to distribute the registration log files?
    365         if (!$clean) {
    366             my %file;
    367             $file{file_rule} = "RAW.IMAGE";
    368             $file{name} = $path_base;
    369             push @file_list, \%file;
    370         }
    371         return \@file_list;
    372     }
    373 
    374     # we get the list of output data products for this run from the config dump file that
    375     # is created when the run is done
    376     my $config_file_rule;
    377     if ($stage eq "chip") {
    378         $config_file_rule = "PPIMAGE.CONFIG";
    379     } elsif ($stage eq "camera") {
    380         $config_file_rule = "PSASTRO.CONFIG";
    381     } elsif ($stage eq 'fake') {
    382         # XXX: fake is a no op now return an emtpy list
    383         return \@file_list;
    384     } elsif ($stage eq "warp") {
    385         $config_file_rule = "PSWARP.CONFIG";
    386     } elsif ($stage eq "diff") {
    387         $config_file_rule = "PPSUB.CONFIG";
    388     } elsif ($stage eq "stack") {
    389         $config_file_rule = "PPSTACK.CONFIG";
    390     } else {
    391         &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    392     }
    393     my $config_file = $ipprc->filename($config_file_rule, $path_base, $component) or
    394                 &my_die("can't get filename for config dump file: $config_file_rule", $dist_id, $component,
    395                     $PS_EXIT_CONFIG_ERROR);
    396 
    397     my %config_file_hash;
    398 
    399     # add the configuration file to the list
    400     $config_file_hash{file_rule} = $config_file_rule;
    401     $config_file_hash{name} = $config_file;
    402     push @file_list, \%config_file_hash;
    403 
    404     my $resolved = $ipprc->file_resolve($config_file);
    405 
    406     &my_die("failed to resolve name of config dump file: $config_file_rule", $dist_id, $component,
    407                     $PS_EXIT_CONFIG_ERROR) if (!$resolved);
    408 
    409     # we don't use the mdc parser because the perl parser is way is too slow for complicated config
    410     # files like this
    411     my $in = open_with_retries($resolved);
    412 
    413     my $line;
    414     while ($line = <$in>) {
    415         if ($line =~ /FILES.OUTPUT/) {
    416             # print "found FILES.OUTPUT\n";
    417             last;
    418         }
    419     }
    420     &my_die("config dump file does not contain FILES.OUTPUT: $config_file", $dist_id, $component,
    421                     $PS_EXIT_CONFIG_ERROR) if (!$line);
    422 
    423     while ($line = <$in>) {
    424         chomp $line;
    425         my ($key, $type, $val) = split " ", $line;
    426         # skip blank lines
    427         next if !$key;
    428         # we're done when we find END
    429         last if $key eq "END";
    430         # skip multi and other lines
    431         next if ($type ne "STR");
    432 
    433         # printf "%-32.32s   %s\n", $key, $val;
    434 
    435         &my_die("no value found for file rule $key in $resolved", $dist_id, $component,
    436                 $PS_EXIT_CONFIG_ERROR) if (!$val);
    437 
    438         my %file;
    439         $file{file_rule} = $key;
    440         $file{name} = $val;
    441         push @file_list, \%file;
    442     }
    443     close $in;
    444 
    445     return \@file_list;
    446 }
    447 
    448 
    449 
    450 
    451 sub file_check
    452 {
    453     my $file = shift;           # Name of file
    454     &my_die("Unable to find output file: $file", $dist_id, $component, $PS_EXIT_SYS_ERROR) unless
    455         $ipprc->file_exists($file);
    456 }
    457 
    458159sub my_die
    459160{
    460161    my $msg = shift;            # Warning message on die
    461     my $dist_id = shift;    # Magic DS identifier
    462     my $component = shift;      # class_id or skycell_id
     162    my $dist_id = shift;        # distRun.dist_id
     163    my $component = shift;      # class_id, skycell_id, or exposure
     164    my $outdir = shift;         # output directory
    463165    my $exit_code = shift;      # Exit code to add
    464166
    … …  
    468170    $command   .= " -dist_id $dist_id";
    469171    $command   .= " -component $component";
     172    $command   .= " -outdir $outdir";
    470173    $command   .= " -fault $exit_code";
    471174    $command   .= " -dbname $dbname" if defined $dbname;
    … …  
    482185    }
    483186
    484     delete_tmpdir();
    485 
    486187    carp($msg);
    487188    exit $exit_code;
    488189}
    489190
    490 sub delete_tmpdir
    491 {
    492     if (!$save_temps and $tmpdir and -e $tmpdir) {
    493         system "rm -r $tmpdir";
    494     }
    495 }
    496 
    497191__END__
Note: See TracChangeset for help on using the changeset viewer.