Changeset 30676 for branches/eam_branches/ipp-20110213/ippScripts

Timestamp:

Feb 17, 2011, 3:20:07 PM (15 years ago)

Author:

eugene

Message:

merge changes from the trunk

Location:

branches/eam_branches/ipp-20110213/ippScripts/scripts

Files:

• dist_bundle.pl (modified) (1 diff)
• ipp_apply_burntool_single.pl (modified) (1 prop)
• ipp_cleanup.pl (modified) (1 diff)
• magic_destreak_cleanup.pl (modified) (4 diffs)
• magic_destreak_revert.pl (modified) (3 diffs)
• receive_file.pl (modified) (6 diffs)
• register_imfile.pl (modified) (2 diffs)
• stack_skycell.pl (modified) (2 diffs)

branches/eam_branches/ipp-20110213/ippScripts/scripts/dist_bundle.pl

@@ -287 +287 @@
         # we can use the chip mask because disttool demands that magic have been run
         # and so the camera mask and the chip mask are the same
-        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component);
+        #$mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $chip_path_base, $component);
+        # XXX: hack the pending query passes in the camera path base
+        $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $chip_path_base, $component);
         my $mask_resolved = $ipprc->file_resolve($mask);
         my $fh = open_with_retries($mask_resolved);

branches/eam_branches/ipp-20110213/ippScripts/scripts/ipp_cleanup.pl

@@ -1900 +1900 @@
 {
     my $files = shift; # reference to a list of files to unlink
-#     my $test_verbose = 1;
-
-#     if ($test_verbose == 1) {
-#       open(TMPLOG,">>/tmp/czw.cleanup.log");
-#       flock(TMPLOG,2);
-#     }
-
-    # this script is, of course, very dangerous.
+
     foreach my $file (@$files) {
         print STDERR "unlinking $stage $stage_id $file";
-        unless ($ipprc->file_exists($file)) {
-            print STDERR "\t File not found\n";
-        }
-        else {
-            print STDERR "\n";
-        }
-
-#       if ($test_verbose == 1) {
-#           print TMPLOG "$stage $stage_id $file";
-
-#           else {
-#               print TMPLOG "\n";
-#           }
-#       }
-
-        $ipprc->file_delete($file);
-    }
-
-#     if ($test_verbose == 1) {
-#       flock(TMPLOG,8);
-#       close(TMPLOG);
-#     }
+
+        my $error_code = $ipprc->kill_file($file);
+
+        &my_die("failed to kill $file", $stage, $stage_id, $PS_EXIT_CONFIG_ERROR) if $error_code;
+    }
 
     return 1;

branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_cleanup.pl

@@ -44 +44 @@
            'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
            'camera=s'       => \$camera,     # camera for evaluating file rules
-           'stage=s'        => \$stage,     # camera for evaluating file rules
+           'stage=s'        => \$stage,      # ipp stage for this magicDSRun
            'save-temps'     => \$save_temps, # Save temporary files?
            'dbname=s'       => \$dbname,     # Database name
@@ -160 +160 @@
         my $backup_path_base = $comp->{backup_path_base};
         my ($bimage, $bmask, $bch_mask, $bweight, $bsources, $bastrom);
-        my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
+#        my ($rimage, $rmask, $rch_mask, $rweight, $rsources, $rastrom);
 
         if ($stage eq "chip") {
@@ -219 +219 @@
         }
 
-        delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask, $rch_mask);
+        delete_files($bimage, $bmask, $bweight, $bsources, $bastrom, $bch_mask);
 
         if ($stage eq "diff" and $warp_warp) {
@@ -252 +252 @@
     foreach my $file (@_) {
         if ($file) {
-            if ($ipprc->file_exists($file)) {
-                if (!$no_update) {
-                    print STDERR "deleting $file\n" if $verbose;
-                    $ipprc->file_delete($file, 1) or my_die("Failed to delete $file", $magic_ds_id, $PS_EXIT_UNKNOWN_ERROR);
-                    } else {
-                    print STDERR "skipping delete $file\n";
-                }
-            } else {
-                print STDERR "$file not found\n" if $verbose;
-            }
+            my $error_code = $ipprc->kill_file($file);
+            my_die("Failed to delete $file", $magic_ds_id, $error_code) if $error_code;
         }
     }

branches/eam_branches/ipp-20110213/ippScripts/scripts/magic_destreak_revert.pl

@@ -191 +191 @@
     # Note that when destreaking as part of an update $magicked is non-zero.
     # In this case we do not touch the camera stage mask so there is no need to revert it
-    if (!$magicked and $dynamicMasks) {
+    if ($dynamicMasks and (!$magicked or ($run_state eq 'goto_restored'))) {
         $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
         $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
@@ -210 +210 @@
     $bweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $backup_path_base, $class_id);
 
-    $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $class_id);
-    # This is somewhat kludgey but it works whether the mask is camera mask or chip mask
-    $rmask   = dirname($recovery_path_base) . "/REC_" . basename($mask);
-    $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $class_id);
-    $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $class_id);
+    if ($recovery_path_base) {
+        $rimage  = $ipprc->filename("PPIMAGE.CHIP", $recovery_path_base, $class_id);
+        # This is somewhat kludgey but it works whether the mask is camera mask or chip mask
+        $rmask   = dirname($recovery_path_base) . "/REC_" . basename($mask);
+        $rch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $recovery_path_base, $class_id);
+        $rweight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $recovery_path_base, $class_id);
+    }
 } elsif ($stage eq "camera") {
     $astrom =  $ipprc->filename("PSASTRO.OUTPUT", $path_base);
@@ -227 +229 @@
     $bweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $backup_path_base);
     $bsources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $backup_path_base);
-    $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
-    $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
-    $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
+    if ($recovery_path_base) {
+        $rimage  = $ipprc->filename("PSWARP.OUTPUT", $recovery_path_base);
+        $rmask   = $ipprc->filename("PSWARP.OUTPUT.MASK", $recovery_path_base);
+        $rweight = $ipprc->filename("PSWARP.OUTPUT.VARIANCE", $recovery_path_base);
+    }
 } elsif ($stage eq "diff") {
     my $name = "PPSUB.OUTPUT";

branches/eam_branches/ipp-20110213/ippScripts/scripts/receive_file.pl

@@ -41 +41 @@
 
 # Parse the command-line arguments
-my ( $file_id, $source, $product, $fileset, $fileset_id, $file, $component, $bytes, $md5sum, $workdir, $dirinfo_uri, $dbname, $verbose, $no_update, $save_temps );
+my ( $file_id, $source, $product, $fileset, $fileset_id, $file, $component, $bytes, $md5sum, $workdir, $dirinfo_uri, $no_extract, $dbname, $verbose, $no_update, $save_temps );
 
 GetOptions(
@@ -54 +54 @@
            'md5sum=s'          => \$md5sum, # md5sum for file from data store
            'workdir=s'         => \$workdir, # Working directory for output
+           'no-extract'        => \$no_extract, # Do not extract the tarfiles
            'dirinfo=s'    => \$dirinfo_uri, # file containing the destination directories for this component
            'dbname=s'          => \$dbname,    # Database name
@@ -85 +86 @@
 my $mdcParser = PS::IPP::Metadata::Config->new;
 
-
+# select a directory for the dirinfo and dbinfo files
+# XXX: perhaps this directory should be set by the script and passed in
+# rather than computed here.
+
+my ($day, $month, $year) = (localtime)[3,4,5];
+my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
+my $dir_for_info_files = caturi($workdir, $datestr, $fileset);
+
+
+my $is_tarfile = 0;
+if ($file =~ m|.*\.tgz$|) {        # XXX: perhaps get this off of file type ?
+    $is_tarfile = 1;
+}
+
+my $filename;
+if ($is_tarfile and $no_extract) {
+    $filename = "$dir_for_info_files/$file"; 
+} else {
+    $filename = "$tempdir/$file";
+}
 
 # Retrieve file
-my $filename = "$tempdir/$file";  # Target filename
 {
     my $uri = "$source/$product/$fileset/$file"; # URI for datastore file
@@ -107 +126 @@
 my ($destdir, $components, $dirinfo_lines) = read_dirinfo_file($dirinfo_file_to_read, $file_id);
 
-# select a directory for the dirinfo and dbinfo files
-# XXX: perhaps this directory should be set by the script and passed in
-# rather than computed here.
-
-my ($day, $month, $year) = (localtime)[3,4,5];
-my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
-my $dir_for_info_files = caturi($workdir, $datestr, $fileset);
 
 # Deal with file
@@ -279 +291 @@
 
 
-} elsif ($file =~ m|.*\.tgz$|) {        # XXX: perhaps get this off of file type ?
+} elsif ($is_tarfile and !$no_extract) {
     # Get contents of tarball
     my @files = ();
@@ -339 +351 @@
         }
     }
+} elsif ($is_tarfile and $no_extract) {
+    print "skipping extraction of tarfile $filename\n";
 } else {
     &my_die( "Unrecognised file: $file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);

branches/eam_branches/ipp-20110213/ippScripts/scripts/register_imfile.pl

@@ -199 +199 @@
     $command .= " -data_state full";
 }
+elsif (is_ccim($tmp_exp_name,$exp_type)) {
+    printf STDERR "This is a camera commanded detrend exposure that should not cause a burn.\n";
+    $command .= " -data_state full";
+}    
 else {
     printf STDERR "Need to check burntool.\n";
@@ -380 +384 @@
 }
 
+sub is_ccim 
+{
+    my $exp_name = shift;
+    my $exp_type = shift;
+ 
+    # needs to match regtool.c checks for "is_ccim".
+    if ($exp_name =~ /c/) {
+        if (($exp_type eq 'DOMEFLAT')||
+            ($exp_type eq 'DARK')||
+            ($exp_type eq 'BIAS')) {
+            return(1);
+        }
+    }
+    return(0);
+}
+
 sub is_daytime
 {

branches/eam_branches/ipp-20110213/ippScripts/scripts/stack_skycell.pl

@@ -172 +172 @@
 my $photometry = metadataLookupBool($recipe, 'PHOTOMETRY'); # perform photometry?
 my $output_nocomp = metadataLookupBool($recipe, 'OUTPUT.NOCOMP'); # change filerules to produced uncompressed output images
-
+my $output_logflux = metadataLookupBool($recipe, 'OUTPUT.LOGFLUX'); # change filerules to produce logflux compressed output images.
+
+if ($output_nocomp and $output_logflux) {
+    &my_die("Unable to not compress and logflux compress simultaneously. Check config.",$stack_id, $PS_EXIT_CONFIG_ERROR);
+}
 
 # Generate MDC file with the inputs
@@ -252 +256 @@
         $command .= " -F PPSTACK.UNCONV.EXPWT PPSTACK.UNCONV.EXPWT.NOCOMP";
     }
+    if ($output_logflux) {
+        $command .= " -R PPSTACK.OUTPUT FITS.TYPE COMP_STACK "; # Just this one output component?
+        $command .= " -R PPSTACK.OUTPUT.VARIANCE FITS.TYPE COMP_STACK "; 
+        $command .= " -R PPSTACK.OUTPUT.EXPWT FITS.TYPE COMP_STACK ";
+        $command .= " -R PPSTACK.UNCONV FITS.TYPE COMP_STACK ";
+        $command .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE COMP_STACK ";
+        $command .= " -R PPSTACK.UNCONV.EXPWT FITS.TYPE COMP_STACK ";
+    }
     $command .= " -threads $threads" if defined $threads;
     $command .= " -debug-stack" if defined $debug;