IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 23693


Ignore:
Timestamp:
Apr 2, 2009, 5:06:33 PM (17 years ago)
Author:
bills
Message:

get the list of files to be added to a distribution bundle for a component from the
config dump that is made when the component is processed. For backward compatability
for now I fall back to the static list if FILES.OUTPUT mdc is not found in the dump

File:
1 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippScripts/scripts/dist_component.pl

    r23688 r23693  
    8383my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
    8484
    85 my $basename = basename($path_base);
    86 
    8785# making a clean bundle of raw images doesn't make sense
    8886if (($stage eq "raw") and $clean) {
    89     # well I suppose we could ship the log file....
     87    # well I suppose we could ship the registration log file. naw.
    9088    &my_die("cannot create clean run at raw stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    9189}
    … …  
    10098}
    10199
    102 my $file_list = get_file_list($clean, $stage, $path_base);
    103 
    104 &my_die("failed to compute product list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$file_list;
    105 
    106 &my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !scalar @$file_list;
    107 
     100my $file_list = get_file_list($stage, $component, $path_base);
     101
     102# we die in get_file_list if something goes wrong.
     103# If the file list is empty  it is an error because we should at least get a config dump file
     104
     105&my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar keys %$file_list);
    108106
    109107# set up directory for temporary files
    … …  
    121119                       $PS_EXIT_UNKNOWN_ERROR);
    122120
     121# XXX get this from camera and a recipe file
     122# or this with the quality flag
     123my $need_to_nan_masked = !$clean && ($camera eq "GPC1");
     124
     125my ($image, $mask, $variance);
     126
    123127my @base_list;
    124 foreach my $file (@$file_list) {
    125     my $base = basename($file);
    126     my $path = $ipprc->file_resolve($file);
    127     # if file is not found, just skip it.
    128     # This is not always the right thing to do.
    129     # For example if warpSkyfile is ignored, there won't be many of the data products
    130     # the new config dump will only list the files actually produced.
    131     next if !$path;
    132 
     128foreach my $file_rule (keys %$file_list) {
     129    # if this is an image, mask, or variance image and we are building a clean distribution bundle,
     130    # skip it
     131    my $image_type = get_image_type($stage, $file_rule);
     132    next if $clean && $image_type;
     133
     134    my $file_name = $file_list->{$file_rule};
     135    my $base = basename($file_name);
     136    my $path = $ipprc->file_resolve($file_name);
     137
     138    &my_die("failed to resolve  $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR) if !$path;
     139
     140    # open the file to make sure it exists (and to work around the failed mount phenomena)
     141    my $fh = open_with_retries($file_name);
     142    close $fh;
     143
     144    if ($image_type && $need_to_nan_masked) {
     145        # save the
     146        if ($image_type eq 'image') {
     147            $image = $path
     148        } elsif ($image_type eq 'mask') {
     149            $mask = $path;
     150        } elsif ($image_type eq 'variance') {
     151            $variance = $path;
     152        } else {
     153            &my_die("invalid image type found: $image_type", $dist_id, $component,
     154                       $PS_EXIT_PROG_ERROR);
     155        }
     156    } else {
     157        # just symlink the existing file into the temporary directory
     158        symlink $path, "$tmpdir/$base";
     159    }
    133160    push @base_list, $base;
    134     symlink $path, "$tmpdir/$base";
    135 }
    136 
    137 if (!$clean) {
    138 
     161}
     162
     163my $run_streaksrelease;
     164if ($need_to_nan_masked) {
     165    # make sure that the needed images exist.
     166    # XXX: they won't be found for a warp or diff with bad quality
     167    # we probably should be passing in the component's quality flag
     168    if (0) {
     169        if (($stage ne "warp") and ($stage ne "diff")) {
     170            &my_die("no image found", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image;
     171            &my_die("no mask image found", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask;
     172            &my_die("no variance image found", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
     173        }
     174    }
     175    $run_streaksrelease = 1 if $image;
     176}
     177
     178if ($run_streaksrelease) {
    139179    # Here is where we determine whether or not a GPC1 image can be released.
     180    # XXX: actually the sql for disttool -pendingcomponent won't pass a component that
     181    # requires magic and hasn't been magicked, but we check again here
     182    # XXX: get this from a recipe
    140183
    141184    if ($camera eq "GPC1" and !$magicked) {
    … …  
    143186    }
    144187
     188    my $class_id;
    145189    # run streaksrelease to set masked pixels to NAN
    146     my ($image, $mask, $variance, $class_id);
    147190    if ($stage eq "raw") {
    148191        $class_id = $component;
    149         $image = $path_base;
    150         $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $class_id);
     192        # we can use the chip mask because disttool demands that magic have been run
     193        # and so the camera mask and the chip mask are the same
     194        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component);
     195        my $fh = open_with_retries($mask);
     196        close $fh;
    151197    } elsif ($stage eq "chip") {
    152198        $class_id = $component;
    153         $image = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
    154         $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
    155         $variance = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
    156     } elsif ($stage eq "warp") {
    157         $mask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $path_base);
    158         $variance = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $path_base);
    159     } elsif ($stage eq "diff") {
    160         $mask   = $ipprc->filename("PPSUB.OUTPUT.MASK", $path_base);
    161         $variance = $ipprc->filename("PPSUB.OUTPUT.VARIANCE", $path_base);
    162     } else {
    163         &my_die("not ready for stage: $stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
    164     }
    165 
    166     push @base_list, $image;
    167     push @base_list, $mask if $mask;
    168     push @base_list, $variance if $variance;
     199    }
    169200
    170201    my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir";
    … …  
    234265### Pau.
    235266
     267# filerules that are not included in 'cleaned' distribution bundles
     268# I could simplify the function below by using a two level hash but
     269# that would be less clear I think
     270my %chip_cleaned = ( 'PPIMAGE.CHIP' => 'image',
     271                     'PPIMAGE.CHIP.MASK' => 'mask',
     272                     'PPIMAGE.CHIP.VARIANCE' => 'variance' );
     273my %camera_cleaned = ( 'PSASTRO.OUTPUT.MASK' => 'mask' );
     274my %fake_cleaned;
     275my %warp_cleaned = ( 'PSWARP.OUTPUT' => 'image',
     276                     'PSWARP.OUTPUT.MASK' => 'mask',
     277                     'PSWARP.OUTPUT.VARIANCE' => 'variance' );
     278my %diff_cleaned = ( 'PPSUB.OUTPUT' => 'image',
     279                     'PPSUB.OUTPUT.MASK' => 'mask',
     280                     'PPSUB.OUTPUT.VARIANCE' => 'variance' );
     281my %stack_cleaned = ( 'PPSTACK.OUTPUT' => 'image',
     282                      'PPSTACK.OUTPUT.MASK' => 'mask',
     283                      'PPSTACK.OUTPUT.VARIANCE' => 'variance' );
     284
     285# return the image type (image, mask, or variance) if this file rule refers to
     286# one of the big fits files that are not included in a clean distribution
     287sub get_image_type {
     288    my $stage = shift;
     289    my $rule = shift;
     290    my $type;
     291
     292    if ($stage eq "raw") {
     293        if ($rule eq "RAW.IMAGE") {
     294            $type = 'image';
     295        }
     296    } elsif ($stage eq "chip") {
     297        $type = $chip_cleaned{$rule};
     298    } elsif ($stage eq "camera") {
     299        $type = $camera_cleaned{$rule};
     300    } elsif ($stage eq "fake") {
     301        $type = $fake_cleaned{$rule};
     302    } elsif ($stage eq "warp") {
     303        $type = $warp_cleaned{$rule};
     304    } elsif ($stage eq "diff") {
     305        $type = $diff_cleaned{$rule};
     306    } elsif ($stage eq "stack") {
     307        $type = $stack_cleaned{$rule};
     308    } else {
     309        &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
     310    }
     311    return $type;
     312}
     313
     314sub open_with_retries {
     315    my $name = shift;
     316   
     317    my $tries = 1;
     318    my $max_tries = 5;
     319    my $opened = 0;
     320    while (!$opened && ($tries <= $max_tries)) {
     321        $opened = open(IN, "<$name");
     322        if (!$opened) {
     323            print STDERR "WARNING failed to open $name re-try $tries\n";
     324            $tries++;
     325            sleep 5;
     326        }
     327    }
     328   
     329    &my_die("failed to open $name after $max_tries tries\n", $dist_id, $component,
     330                    $PS_EXIT_DATA_ERROR) if (!$opened);
     331
     332    return *IN;
     333}
    236334
    237335sub get_file_list {
    238     my $clean = shift;
    239336    my $stage = shift;
     337    my $component = shift;
    240338    my $path_base = shift;
    241339
    242     my @file_list;
     340    my %file_list;
    243341    if ($stage eq "raw") {
    244         push @file_list, $path_base;
     342        # XXX: TODO for  now path_base is set to the uri of the file
     343        # eventually we'll have a path_base that we'll need to resolve
     344        $file_list{"RAW.IMAGE"} = $path_base;
     345        return \%file_list;
     346    }
     347
     348    # we get the list of output data products for this run from the config dump file that
     349    # is created when the run is done
     350    my $config_file_rule;
     351    if ($stage eq "chip") {
     352        $config_file_rule = "PPIMAGE.CONFIG";
     353    } elsif ($stage eq "camera") {
     354        $config_file_rule = "PSASTRO.CONFIG";
     355    } elsif ($stage eq "warp") {
     356        $config_file_rule = "PSWARP.CONFIG";
     357    } elsif ($stage eq "diff") {
     358        $config_file_rule = "PPSUB.CONFIG";
     359    } elsif ($stage eq "stack") {
     360        $config_file_rule = "PPSTACK.CONFIG";
    245361    } else {
    246         # TODO: these data will eventually come from the CONFIG dump
    247 
     362        &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
     363    }
     364    my $config_file = $ipprc->filename($config_file_rule, $path_base, $component) or
     365                &my_die("can't get filename for config dump file: $config_file_rule", $dist_id, $component,
     366                    $PS_EXIT_CONFIG_ERROR);
     367
     368    my $resolved = $ipprc->file_resolve($config_file);
     369
     370    &my_die("failed to resolve name of config dump file: $config_file_rule", $dist_id, $component,
     371                    $PS_EXIT_CONFIG_ERROR) if (!$resolved);
     372
     373    # we don't use the mdc parser because the perl parser is way is too slow for complicated config
     374    # files like this
     375    my $in = open_with_retries($resolved);
     376
     377    my $line;
     378    while ($line = <$in>) {
     379        if ($line =~ /FILES.OUTPUT/) {
     380            # print "found FILES.OUTPUT\n";
     381            last;
     382        }
     383    }
     384
     385    if ($line) {
     386        # this is a new style config dump that contains the list of files
     387
     388        my %files;
     389        while ($line = <$in>) {
     390            chomp $line;
     391            my ($key, $type, $val) = split " ", $line;
     392            # skip blank lines
     393            next if !$key;
     394            # we're done when we find END
     395            last if $key eq "END";
     396            # skip multi and other lines
     397            next if ($type ne "STR");
     398
     399            # printf "%-32.32s   %s\n", $key, $val;
     400
     401            &my_die("no value found for file rule $key in $resolved", $dist_id, $component,
     402                    $PS_EXIT_CONFIG_ERROR) if (!$val);
     403
     404            $file_list{key} = $val;
     405        }
     406        close $in;
     407    } else {
     408        # did not find FILES.OUTPUT in the config file, assume we have an old style file
     409        # and build the distro from the static list which is at the end of this script
    248410        my $clean_mdc = get_legacy_file_mdc();
    249411
    250412        my $mdlist = $mdcParser->parse($clean_mdc) or
    251                 &my_die("failed to parse clean.mdc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
     413                &my_die("failed to parse legacy file mdc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
    252414
    253415        my $product_lists = parse_md_list($mdlist) or
    … …  
    268430        foreach my $rule (keys %$product_list) {
    269431            next if $rule eq "STAGE";
    270             my $keep_on_clean = $product_list->{$rule};
    271432            my $fn = $ipprc->filename($rule, $path_base, $component) or
    272433                &my_die("Missing entry from camera config: $rule", $dist_id, $component,
    273434                    $PS_EXIT_CONFIG_ERROR);
    274     #        printf "%-16.16s\t%s\t%s\n",  $rule, $clean, $fn;
    275             if (!$clean or $keep_on_clean) {
    276                 push @file_list, $fn;
    277             }
    278         }
    279     }
    280 
    281     return \@file_list;
     435            #        printf "%-16.16s\t%s\t%s\n",  $rule, $clean, $fn;
     436
     437            $file_list{$rule} = $fn;
     438        }
     439    }
     440
     441    return \%file_list;
    282442}
    283443
Note: See TracChangeset for help on using the changeset viewer.