IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Nov 4, 2009, 11:33:11 AM (17 years ago)
Author:
bills
Message:

split dist_component.pl into two parts so we can build bundles in response to postage
stamp server requests.

  1. dist_component.pl calls dist_bundle.pl to create bundles for a distRun and updates the DB
  2. dist_bundle.pl actually creates the bundles doesn't touch the DB. Can be called by postage stamp server
File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/dist_component.pl

    r26015 r26029  
    5353my $missing_tools;
    5454my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
    55 my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
     55my $dist_make_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
    5656if ($missing_tools) {
    5757    warn("Can't find required tools.");
    … …  
    103103}
    104104
    105 $ipprc->define_camera($camera);
    106105
    107 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    108 
    109 # making a clean bundle of raw images doesn't make sense
    110 
    111 # create the output directories if it is not a nebulous path and it doesn't exist
    112 if (index($outdir, "neb://") != 0) {
    113     if (! -e $outdir ) {
    114         my $code = system "mkdir -p $outdir";
    115         &my_die("cannot create output directory $outdir", $dist_id, $component,
    116                 $code >> 8) if $code;
    117     }
    118 }
    119 
    120 # Get the list of data products for this component
    121 # note: We my_die in get_file_list if something goes wrong.
    122 
    123 my $file_list = get_file_list($stage, $component, $path_base, $clean);
    124 
    125 if (($stage ne 'raw') and ($stage ne 'fake')) {
    126     # If the file list is empty it is an error because we should at least get a config dump file
    127     # except for fake stage which doesn't do anything yet
    128     &my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list);
    129 }
    130 
    131 # set up directory for temporary files
    132 
    133 my $tmpdir  = "$outdir/tmpdir.$dist_id.$component";
    134 if (-e $tmpdir) {
    135     if (-d $tmpdir) {
    136         my $rc = system "rm -r $tmpdir";
    137         &my_die("cannot rm $tmpdir return code: $rc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if $rc;
    138     } else {
    139         unlink $tmpdir or &my_die("cannot delete $tmpdir", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
    140     }
    141 }
    142 mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $dist_id, $component,
    143                        $PS_EXIT_UNKNOWN_ERROR);
    144 
    145 #
    146 # we need to run set masked pixels to NAN in the image and variance images
    147 # unless
    148 #   1. we are building a clean distribution bundle
    149 #   2. magic is not required for this distRun
    150 #   3. the processing for the component produced no images (warp or diff with bad quality for example)
    151 my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack')) || $no_magic || $poor_quality);
    152 
    153 my ($image, $mask, $variance);
    154 my ($inv_image, $inv_mask, $inv_variance);
    155 
    156 # foreach my $file_rule (keys %$file_list) {
    157 my $num_files = 0;
    158 foreach my $file (@$file_list) {
    159     # check whether this file rule refers to an image, mask, or variance fits image
    160     my $file_rule = $file->{file_rule};
    161     my $image_type = get_image_type($stage, $file_rule);
    162 
    163     # if this is an image and we are building a clean bundle or if quality is bad skip this rule
    164     next if $image_type and ($clean or $poor_quality);
    165 
    166     # if magic is required, don't ship jpegs or binned fits images
    167     next if !$no_magic && (($file_rule =~ /.BIN1/) or ($file_rule =~ /.BIN2/) or ($file_rule =~ /.JPEG1/)
    168             or ($file_rule =~ /.JPEG2/));
    169 
    170     if ($stage eq "diff") {
    171         next if $file_rule =~ /CONV/;
    172     }
    173 
    174     my $file_name = $file->{name};
    175     my $base = basename($file_name);
    176     my $path = $ipprc->file_resolve($file_name);
    177 
    178     if (!$path) {
    179         # skip this file if $poor_quality
    180         # this is for compatability with older runs which don't have the files list in the
    181         # config dump.
    182         # Once we give up on supporting that we can remove the next line. (If the file is in the list
    183         # it must exist)
    184         next if $poor_quality;
    185 
    186         &my_die("failed to resolve  $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR);
    187     }
    188 
    189     # open the file to make sure it exists (and to work around the failed mount phenomena)
    190     my $fh = open_with_retries($path);
    191     close $fh;
    192 
    193     # we need to pre-process the image before adding to the bundle. Save the path names.
    194     # the images will be created below
    195     $num_files++;
    196     if ($image_type && $nan_masked_pixels) {
    197         # save the
    198         if ($image_type eq 'image') {
    199             $image = $file_name;
    200         } elsif ($image_type eq 'mask') {
    201             $mask = $file_name;
    202         } elsif ($image_type eq 'variance') {
    203             $variance = $file_name;;
    204         } elsif ($image_type eq 'inv_image') {
    205             $inv_image = $file_name;
    206         } elsif ($image_type eq 'inv_mask') {
    207             $inv_mask = $file_name;;
    208         } elsif ($image_type eq 'inv_variance') {
    209             $inv_variance = $file_name;;
    210         } else {
    211             &my_die("invalid image type found: $image_type", $dist_id, $component,
    212                        $PS_EXIT_PROG_ERROR);
    213         }
    214     } else {
    215         # create a symbolic link from the file in the nebulous repository
    216         # in the temporary directory
    217         symlink $path, "$tmpdir/$base";
    218     }
    219 }
    220 
    221 if ($nan_masked_pixels) {
    222     # One last check as to whether magic has been applied to the inputs
    223 
    224     # Note: the sql for disttool -pendingcomponent won't select a component that
    225     # requires magic and hasn't been magicked, but we check again here
    226 
    227     if (!($magicked or $no_magic)) {
    228         &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR);
    229     }
    230 
    231     &my_die("no image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image;
    232     &my_die("no mask image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask;
    233     &my_die("no variance image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
    234 
    235     my $class_id;
    236     # run streaksrelease to set masked pixels to NAN
    237     if ($stage eq "raw") {
    238         $class_id = $component;
    239         # we can use the chip mask because disttool demands that magic have been run
    240         # and so the camera mask and the chip mask are the same
    241         $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component);
    242         my $fh = open_with_retries($mask);
    243         close $fh;
    244     } elsif ($stage eq "chip") {
    245         $class_id = $component;
    246     }
    247 
    248     my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir";
    249     $command .= " -class_id $class_id" if $class_id;
    250     $command .= " -mask $mask" if $mask;
    251     $command .= " -chip_mask $mask" if ($stage eq 'chip' and $mask);
    252     $command .= " -weight $variance" if $variance;
    253     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    254         run(command => $command, verbose => $verbose);
    255     unless ($success) {
    256         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    257         &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
    258     }
    259     if ($inv_image) {
    260         $command = "$streaksrelease -stage $stage -image $inv_image -outroot $tmpdir";
    261         $command .= " -mask $inv_mask" if $inv_mask;
    262         $command .= " -weight $inv_variance" if $inv_variance;
    263         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    264             run(command => $command, verbose => $verbose);
    265         unless ($success) {
    266             $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
    267             &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
    268         }
    269     }
    270 }
    271 
    272 
    273 my $file_name;
    274 my $bytes;
    275 my $md5sum;
    276 
    277 if ($num_files) {
    278     # create the tarfile
    279     # XXX TODO: create a file rule for the tar file name
    280     my $tbase = basename($path_base);
    281     $tbase .= ".$component" if $component;
    282     $file_name = "$tbase.tgz";
    283     my $tarfile = "$outdir/$file_name";
    284 
    285     my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";
     106my ($file_name, $bytes, $md5sum);
     107{
     108    my $command = "$dist_make_bundle --dist_id $dist_id --camera $camera --stage $stage --stage_id $stage_id";
     109    $command .= " --component $component --path_base $path_base --outdir $outdir";
     110    $command .= " --state $run_state" if defined $run_state;
     111    $command .= " --data_state $data_state" if defined $data_state;
     112    $command .= " --poor_quality" if defined $poor_quality;
     113    $command .= " --no_magic" if defined $no_magic;
     114    $command .= " --magicked" if defined $magicked;
     115    $command .= " --clean" if defined $clean;
     116    $command .= " --save-temps" if defined $save_temps;
     117    $command .= " --dbname $dbname" if defined $dbname;
     118    $command .= " --verbose" if defined $verbose;
    286119
    287120    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
    … …  
    291124        &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
    292125    }
     126   # parse stdout_buf to get ($file_name, $bytes, $md5sum);
     127}
    293128
    294     # tell the module not to die on error
    295     $Digest::MD5::File::NOFATALS = 1;
    296     $md5sum = file_md5_hex($tarfile);
    297     &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum;
     129my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    298130
    299     $bytes = -s $tarfile;
    300 
    301     delete_tmpdir($tmpdir);
    302 } else {
    303     # no files for this component
    304     $file_name = "none";
    305     $bytes = 0;
    306     $md5sum = "0";
    307 }
    308131{
    309132    my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component -outdir $outdir";
    … …  
    326149
    327150### Pau.
    328 
    329 # return the image type (image, mask, or variance) if this file rule refers to
    330 # one of the big fits files that are not included in a clean distribution
    331 sub get_image_type {
    332     my $stage = shift;
    333     my $rule = shift;
    334     my $type;
    335 
    336     if ($stage eq "raw") {
    337         if ($rule eq "RAW.IMAGE") {
    338             $type = 'image';
    339         }
    340     } elsif ($stage eq "chip") {
    341         $type = $chip_cleaned{$rule};
    342     } elsif ($stage eq "camera") {
    343         $type = $camera_cleaned{$rule};
    344     } elsif ($stage eq "fake") {
    345         $type = $fake_cleaned{$rule};
    346     } elsif ($stage eq "warp") {
    347         $type = $warp_cleaned{$rule};
    348     } elsif ($stage eq "diff") {
    349         $type = $diff_cleaned{$rule};
    350     } elsif ($stage eq "stack") {
    351         $type = $stack_cleaned{$rule};
    352     } else {
    353         &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    354     }
    355     return $type;
    356 }
    357 
    358 sub open_with_retries {
    359     my $name = shift;
    360    
    361     my $tries = 1;
    362     my $max_tries = 5;
    363     my $opened = 0;
    364     while (!$opened && ($tries <= $max_tries)) {
    365         $opened = open(IN, "<$name");
    366         if (!$opened) {
    367             print STDERR "WARNING failed to open $name re-try $tries\n";
    368             $tries++;
    369             sleep 5;
    370         }
    371     }
    372    
    373     &my_die("failed to open $name after $max_tries tries\n", $dist_id, $component,
    374                     $PS_EXIT_DATA_ERROR) if (!$opened);
    375 
    376     return *IN;
    377 }
    378 
    379 sub get_file_list {
    380     my $stage = shift;
    381     my $component = shift;
    382     my $path_base = shift;
    383     my $clean = shift;
    384 
    385     my @file_list;
    386     if ($stage eq "raw") {
    387         # XXX: TODO for now disttool sets path_base is set to the uri of the file
    388         # eventually rawImfile will have a path_base that we'll need to add '.fits'
    389         # XXX do we want to distribute the registration log files?
    390         if (!$clean) {
    391             my %file;
    392             $file{file_rule} = "RAW.IMAGE";
    393             $file{name} = $path_base;
    394             push @file_list, \%file;
    395         }
    396         return \@file_list;
    397     }
    398 
    399     # we get the list of output data products for this run from the config dump file that
    400     # is created when the run is done
    401     my $config_file_rule;
    402     if ($stage eq "chip") {
    403         $config_file_rule = "PPIMAGE.CONFIG";
    404     } elsif ($stage eq "camera") {
    405         $config_file_rule = "PSASTRO.CONFIG";
    406     } elsif ($stage eq 'fake') {
    407         # XXX: fake is a no op now return an emtpy list
    408         return \@file_list;
    409     } elsif ($stage eq "warp") {
    410         $config_file_rule = "PSWARP.CONFIG";
    411     } elsif ($stage eq "diff") {
    412         $config_file_rule = "PPSUB.CONFIG";
    413     } elsif ($stage eq "stack") {
    414         $config_file_rule = "PPSTACK.CONFIG";
    415     } else {
    416         &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    417     }
    418     my $config_file = $ipprc->filename($config_file_rule, $path_base, $component) or
    419                 &my_die("can't get filename for config dump file: $config_file_rule", $dist_id, $component,
    420                     $PS_EXIT_CONFIG_ERROR);
    421 
    422     my %config_file_hash;
    423 
    424     # add the configuration file to the list
    425     $config_file_hash{file_rule} = $config_file_rule;
    426     $config_file_hash{name} = $config_file;
    427     push @file_list, \%config_file_hash;
    428 
    429     my $resolved = $ipprc->file_resolve($config_file);
    430 
    431     &my_die("failed to resolve name of config dump file: $config_file_rule", $dist_id, $component,
    432                     $PS_EXIT_CONFIG_ERROR) if (!$resolved);
    433 
    434     # we don't use the mdc parser because the perl parser is way is too slow for complicated config
    435     # files like this
    436     my $in = open_with_retries($resolved);
    437 
    438     my $line;
    439     while ($line = <$in>) {
    440         if ($line =~ /FILES.OUTPUT/) {
    441             # print "found FILES.OUTPUT\n";
    442             last;
    443         }
    444     }
    445     &my_die("config dump file does not contain FILES.OUTPUT: $config_file", $dist_id, $component,
    446                     $PS_EXIT_CONFIG_ERROR) if (!$line);
    447 
    448     while ($line = <$in>) {
    449         chomp $line;
    450         my ($key, $type, $val) = split " ", $line;
    451         # skip blank lines
    452         next if !$key;
    453         # we're done when we find END
    454         last if $key eq "END";
    455         # skip multi and other lines
    456         next if ($type ne "STR");
    457 
    458         # printf "%-32.32s   %s\n", $key, $val;
    459 
    460         &my_die("no value found for file rule $key in $resolved", $dist_id, $component,
    461                 $PS_EXIT_CONFIG_ERROR) if (!$val);
    462 
    463         my %file;
    464         $file{file_rule} = $key;
    465         $file{name} = $val;
    466         push @file_list, \%file;
    467     }
    468     close $in;
    469 
    470     return \@file_list;
    471 }
    472 
    473 
    474 
    475 
    476 sub file_check
    477 {
    478     my $file = shift;           # Name of file
    479     &my_die("Unable to find output file: $file", $dist_id, $component, $PS_EXIT_SYS_ERROR) unless
    480         $ipprc->file_exists($file);
    481 }
    482 
    483151sub my_die
    484152{
    … …  
    507175    }
    508176
    509     delete_tmpdir();
    510 
    511177    carp($msg);
    512178    exit $exit_code;
    513179}
    514180
    515 sub delete_tmpdir
    516 {
    517     if (!$save_temps and $tmpdir and -e $tmpdir) {
    518         system "rm -r $tmpdir";
    519     }
    520 }
    521 
    522181__END__
Note: See TracChangeset for help on using the changeset viewer.