Changeset 26032

Timestamp:

Nov 4, 2009, 3:44:47 PM (17 years ago)

Author:

bills

Message:

restore revision 26015. Last one isn't ready yet

File:

• trunk/ippScripts/scripts/dist_component.pl (modified) (5 diffs)

trunk/ippScripts/scripts/dist_component.pl

@@ -53 +53 @@
 my $missing_tools;
 my $disttool   = can_run('disttool') or (warn "Can't find disttool" and $missing_tools = 1);
-my $dist_make_bundle   = can_run('dist_bundle.pl') or (warn "Can't find dist_bundle.pl" and $missing_tools = 1);
+my $streaksrelease   = can_run('streaksrelease') or (warn "Can't find streaksrelease" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -103 +103 @@
 }
 
-
-my ($file_name, $bytes, $md5sum);
-{
-    my $command = "$dist_make_bundle --dist_id $dist_id --camera $camera --stage $stage --stage_id $stage_id";
-    $command .= " --component $component --path_base $path_base --outdir $outdir";
-    $command .= " --state $run_state" if defined $run_state;
-    $command .= " --data_state $data_state" if defined $data_state;
-    $command .= " --poor_quality" if defined $poor_quality;
-    $command .= " --no_magic" if defined $no_magic;
-    $command .= " --magicked" if defined $magicked;
-    $command .= " --clean" if defined $clean;
-    $command .= " --save-temps" if defined $save_temps;
-    $command .= " --dbname $dbname" if defined $dbname;
-    $command .= " --verbose" if defined $verbose;
-
+$ipprc->define_camera($camera);
+
+my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# making a clean bundle of raw images doesn't make sense
+
+# create the output directories if it is not a nebulous path and it doesn't exist
+if (index($outdir, "neb://") != 0) {
+    if (! -e $outdir ) {
+        my $code = system "mkdir -p $outdir";
+        &my_die("cannot create output directory $outdir", $dist_id, $component,
+                $code >> 8) if $code;
+    }
+}
+
+# Get the list of data products for this component
+# note: We my_die in get_file_list if something goes wrong. 
+
+my $file_list = get_file_list($stage, $component, $path_base, $clean);
+
+if (($stage ne 'raw') and ($stage ne 'fake')) {
+    # If the file list is empty it is an error because we should at least get a config dump file
+    # except for fake stage which doesn't do anything yet
+    &my_die("empty file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if (!scalar @$file_list);
+}
+
+# set up directory for temporary files
+
+my $tmpdir  = "$outdir/tmpdir.$dist_id.$component";
+if (-e $tmpdir) {
+    if (-d $tmpdir) {
+        my $rc = system "rm -r $tmpdir";
+        &my_die("cannot rm $tmpdir return code: $rc", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if $rc;
+    } else {
+        unlink $tmpdir or &my_die("cannot delete $tmpdir", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $dist_id, $component,
+                       $PS_EXIT_UNKNOWN_ERROR);
+
+#
+# we need to run set masked pixels to NAN in the image and variance images
+# unless
+#   1. we are building a clean distribution bundle
+#   2. magic is not required for this distRun
+#   3. the processing for the component produced no images (warp or diff with bad quality for example)
+my $nan_masked_pixels = ! ($clean || (($stage eq "camera") || ($stage eq 'fake') || ($stage eq 'stack')) || $no_magic || $poor_quality);
+
+my ($image, $mask, $variance);
+my ($inv_image, $inv_mask, $inv_variance);
+
+# foreach my $file_rule (keys %$file_list) {
+my $num_files = 0;
+foreach my $file (@$file_list) {
+    # check whether this file rule refers to an image, mask, or variance fits image
+    my $file_rule = $file->{file_rule};
+    my $image_type = get_image_type($stage, $file_rule);
+
+    # if this is an image and we are building a clean bundle or if quality is bad skip this rule
+    next if $image_type and ($clean or $poor_quality);
+
+    # if magic is required, don't ship jpegs or binned fits images
+    next if !$no_magic && (($file_rule =~ /.BIN1/) or ($file_rule =~ /.BIN2/) or ($file_rule =~ /.JPEG1/)
+            or ($file_rule =~ /.JPEG2/));
+
+    if ($stage eq "diff") {
+        next if $file_rule =~ /CONV/;
+    }
+
+    my $file_name = $file->{name};
+    my $base = basename($file_name);
+    my $path = $ipprc->file_resolve($file_name);
+
+    if (!$path) {
+        # skip this file if $poor_quality
+        # this is for compatability with older runs which don't have the files list in the
+        # config dump.
+        # Once we give up on supporting that we can remove the next line. (If the file is in the list
+        # it must exist)
+        next if $poor_quality;
+
+        &my_die("failed to resolve  $file_name", $dist_id, $component, $PS_EXIT_DATA_ERROR);
+    }
+
+    # open the file to make sure it exists (and to work around the failed mount phenomena)
+    my $fh = open_with_retries($path);
+    close $fh;
+
+    # we need to pre-process the image before adding to the bundle. Save the path names.
+    # the images will be created below
+    $num_files++;
+    if ($image_type && $nan_masked_pixels) {
+        # save the 
+        if ($image_type eq 'image') {
+            $image = $file_name;
+        } elsif ($image_type eq 'mask') {
+            $mask = $file_name;
+        } elsif ($image_type eq 'variance') {
+            $variance = $file_name;;
+        } elsif ($image_type eq 'inv_image') {
+            $inv_image = $file_name;
+        } elsif ($image_type eq 'inv_mask') {
+            $inv_mask = $file_name;;
+        } elsif ($image_type eq 'inv_variance') {
+            $inv_variance = $file_name;;
+        } else {
+            &my_die("invalid image type found: $image_type", $dist_id, $component,
+                       $PS_EXIT_PROG_ERROR);
+        }
+    } else {
+        # create a symbolic link from the file in the nebulous repository
+        # in the temporary directory
+        symlink $path, "$tmpdir/$base";
+    }
+}
+
+if ($nan_masked_pixels) {
+    # One last check as to whether magic has been applied to the inputs
+
+    # Note: the sql for disttool -pendingcomponent won't select a component that 
+    # requires magic and hasn't been magicked, but we check again here
+
+    if (!($magicked or $no_magic)) {
+        &my_die("cannot create distribution bundle ${stage}_id $stage_id because the data has not been magic desreaked", $dist_id, $component, $PS_EXIT_DATA_ERROR);
+    }
+
+    &my_die("no image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$image;
+    &my_die("no mask image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$mask;
+    &my_die("no variance image found in file list", $dist_id, $component, $PS_EXIT_CONFIG_ERROR) if !$variance;
+
+    my $class_id;
+    # run streaksrelease to set masked pixels to NAN
+    if ($stage eq "raw") {
+        $class_id = $component;
+        # we can use the chip mask because disttool demands that magic have been run
+        # and so the camera mask and the chip mask are the same
+        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component);
+        my $fh = open_with_retries($mask);
+        close $fh;
+    } elsif ($stage eq "chip") {
+        $class_id = $component;
+    }
+
+    my $command = "$streaksrelease -stage $stage -image $image -outroot $tmpdir";
+    $command .= " -class_id $class_id" if $class_id;
+    $command .= " -mask $mask" if $mask;
+    $command .= " -chip_mask $mask" if ($stage eq 'chip' and $mask);
+    $command .= " -weight $variance" if $variance;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
         run(command => $command, verbose => $verbose);
@@ -124 +257 @@
         &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
     }
-   # parse stdout_buf to get ($file_name, $bytes, $md5sum);
-}
-
-my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
-
+    if ($inv_image) {
+        $command = "$streaksrelease -stage $stage -image $inv_image -outroot $tmpdir";
+        $command .= " -mask $inv_mask" if $inv_mask;
+        $command .= " -weight $inv_variance" if $inv_variance;
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
+        }
+    }
+}
+
+
+my $file_name;
+my $bytes;
+my $md5sum;
+
+if ($num_files) {
+    # create the tarfile
+    # XXX TODO: create a file rule for the tar file name
+    my $tbase = basename($path_base);
+    $tbase .= ".$component" if $component;
+    $file_name = "$tbase.tgz";
+    my $tarfile = "$outdir/$file_name";
+
+    my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $dist_id, $component, $error_code);
+    }
+
+    # tell the module not to die on error
+    $Digest::MD5::File::NOFATALS = 1;
+    $md5sum = file_md5_hex($tarfile);
+    &my_die("unable to compute md5sum for $tarfile", $dist_id, $component, $PS_EXIT_UNKNOWN_ERROR) if !$md5sum;
+
+    $bytes = -s $tarfile;
+
+    delete_tmpdir($tmpdir);
+} else {
+    # no files for this component
+    $file_name = "none";
+    $bytes = 0;
+    $md5sum = "0";
+}
 {
     my $command = "$disttool -addprocessedcomponent -dist_id $dist_id -component $component -outdir $outdir";
@@ -149 +326 @@
 
 ### Pau.
+
+# return the image type (image, mask, or variance) if this file rule refers to 
+# one of the big fits files that are not included in a clean distribution
+sub get_image_type {
+    my $stage = shift;
+    my $rule = shift;
+    my $type;
+
+    if ($stage eq "raw") {
+        if ($rule eq "RAW.IMAGE") {
+            $type = 'image';
+        }
+    } elsif ($stage eq "chip") {
+        $type = $chip_cleaned{$rule};
+    } elsif ($stage eq "camera") {
+        $type = $camera_cleaned{$rule};
+    } elsif ($stage eq "fake") {
+        $type = $fake_cleaned{$rule};
+    } elsif ($stage eq "warp") {
+        $type = $warp_cleaned{$rule};
+    } elsif ($stage eq "diff") {
+        $type = $diff_cleaned{$rule};
+    } elsif ($stage eq "stack") {
+        $type = $stack_cleaned{$rule};
+    } else {
+        &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
+    }
+    return $type;
+}
+
+sub open_with_retries {
+    my $name = shift;
+    
+    my $tries = 1;
+    my $max_tries = 5;
+    my $opened = 0;
+    while (!$opened && ($tries <= $max_tries)) {
+        $opened = open(IN, "<$name");
+        if (!$opened) {
+            print STDERR "WARNING failed to open $name re-try $tries\n";
+            $tries++;
+            sleep 5;
+        }
+    }
+    
+    &my_die("failed to open $name after $max_tries tries\n", $dist_id, $component,
+                    $PS_EXIT_DATA_ERROR) if (!$opened);
+
+    return *IN;
+}
+
+sub get_file_list {
+    my $stage = shift;
+    my $component = shift;
+    my $path_base = shift;
+    my $clean = shift;
+
+    my @file_list;
+    if ($stage eq "raw") {
+        # XXX: TODO for now disttool sets path_base is set to the uri of the file
+        # eventually rawImfile will have a path_base that we'll need to add '.fits'
+        # XXX do we want to distribute the registration log files?
+        if (!$clean) {
+            my %file;
+            $file{file_rule} = "RAW.IMAGE";
+            $file{name} = $path_base;
+            push @file_list, \%file;
+        }
+        return \@file_list;
+    }
+
+    # we get the list of output data products for this run from the config dump file that
+    # is created when the run is done
+    my $config_file_rule;
+    if ($stage eq "chip") {
+        $config_file_rule = "PPIMAGE.CONFIG";
+    } elsif ($stage eq "camera") {
+        $config_file_rule = "PSASTRO.CONFIG";
+    } elsif ($stage eq 'fake') {
+        # XXX: fake is a no op now return an emtpy list
+        return \@file_list;
+    } elsif ($stage eq "warp") {
+        $config_file_rule = "PSWARP.CONFIG";
+    } elsif ($stage eq "diff") {
+        $config_file_rule = "PPSUB.CONFIG";
+    } elsif ($stage eq "stack") {
+        $config_file_rule = "PPSTACK.CONFIG";
+    } else {
+        &my_die("$stage is not a valid stage", $dist_id, $component, $PS_EXIT_CONFIG_ERROR);
+    }
+    my $config_file = $ipprc->filename($config_file_rule, $path_base, $component) or
+                &my_die("can't get filename for config dump file: $config_file_rule", $dist_id, $component,
+                    $PS_EXIT_CONFIG_ERROR);
+
+    my %config_file_hash;
+
+    # add the configuration file to the list
+    $config_file_hash{file_rule} = $config_file_rule;
+    $config_file_hash{name} = $config_file;
+    push @file_list, \%config_file_hash;
+
+    my $resolved = $ipprc->file_resolve($config_file);
+
+    &my_die("failed to resolve name of config dump file: $config_file_rule", $dist_id, $component,
+                    $PS_EXIT_CONFIG_ERROR) if (!$resolved);
+
+    # we don't use the mdc parser because the perl parser is way is too slow for complicated config
+    # files like this
+    my $in = open_with_retries($resolved);
+
+    my $line;
+    while ($line = <$in>) {
+        if ($line =~ /FILES.OUTPUT/) {
+            # print "found FILES.OUTPUT\n";
+            last;
+        }
+    }
+    &my_die("config dump file does not contain FILES.OUTPUT: $config_file", $dist_id, $component,
+                    $PS_EXIT_CONFIG_ERROR) if (!$line);
+
+    while ($line = <$in>) {
+        chomp $line;
+        my ($key, $type, $val) = split " ", $line;
+        # skip blank lines
+        next if !$key;
+        # we're done when we find END
+        last if $key eq "END";
+        # skip multi and other lines
+        next if ($type ne "STR");
+
+        # printf "%-32.32s   %s\n", $key, $val;
+
+        &my_die("no value found for file rule $key in $resolved", $dist_id, $component,
+                $PS_EXIT_CONFIG_ERROR) if (!$val);
+
+        my %file;
+        $file{file_rule} = $key;
+        $file{name} = $val;
+        push @file_list, \%file;
+    }
+    close $in;
+
+    return \@file_list;
+}
+
+
+
+
+sub file_check
+{
+    my $file = shift;           # Name of file
+    &my_die("Unable to find output file: $file", $dist_id, $component, $PS_EXIT_SYS_ERROR) unless
+        $ipprc->file_exists($file);
+}
+
 sub my_die
 {
@@ -175 +507 @@
     }
 
+    delete_tmpdir();
+
     carp($msg);
     exit $exit_code;
 }
 
+sub delete_tmpdir
+{
+    if (!$save_temps and $tmpdir and -e $tmpdir) {
+        system "rm -r $tmpdir";
+    }
+}
+
 __END__