Changeset 42099

Timestamp:

Feb 28, 2022, 4:35:35 PM (4 years ago)

Author:

eugene

Message:

move JPEGS to list of files produced with or without convolution; add recipe option to select a DEEP exposure output format; do not fail on invalid warp inputs if SKIP.MISSING.INPUTS is true

Location:

trunk/ippScripts/scripts

Files:

• stack_skycell.pl (modified) (7 diffs)
• warp_overlap.pl (modified) (1 prop)

trunk/ippScripts/scripts/stack_skycell.pl

@@ -66 +66 @@
     and defined $run_state;
 
-# lists of file rules which we expect to produce output output
+# lists of file rules which we expect to produce output when convolving
 my @outputList = qw(
 PPSTACK.OUTPUT
@@ -74 +74 @@
 PPSTACK.OUTPUT.EXPNUM       
 PPSTACK.OUTPUT.EXPWT
-PPSTACK.OUTPUT.JPEG1        
-PPSTACK.OUTPUT.JPEG2        
-PPSTACK.CONFIG              
+PPSTACK.TARGET.PSF          
 );
+
 # produced if we run photometry
 my @outputListPhotom = qw(
 PSPHOT.OUT.CMF.MEF
 );
-# extra outputs when convolving
+
+# list of file rules produced if convolution is false
 my @outputListUnconv = qw(
-PPSTACK.TARGET.PSF          
 PPSTACK.UNCONV
 PPSTACK.UNCONV.MASK
@@ -91 +90 @@
 PPSTACK.UNCONV.EXPNUM       
 PPSTACK.UNCONV.EXPWT
+PPSTACK.OUTPUT.JPEG1        
+PPSTACK.OUTPUT.JPEG2        
+PPSTACK.CONFIG              
 );
 
@@ -207 +209 @@
 my $output_nocomp = metadataLookupBool($recipe, 'OUTPUT.NOCOMP'); # change filerules to produced uncompressed output images
 my $output_logflux = metadataLookupBool($recipe, 'OUTPUT.LOGFLUX'); # change filerules to produce logflux compressed output images.
+my $output_deepexp = metadataLookupBool($recipe, 'OUTPUT.DEEPEXP'); # change filerules to produce exptime map with more depth
 my $replicate_outputs = (defined($neb) and metadataLookupBool($recipe, 'OUTPUT.REPLICATE')); # replicate output images
+my $skip_missing_inputs = metadataLookupBool($recipe, 'SKIP.MISSING.INPUTS'); # ignore missing inputs 
 if ($output_nocomp and $output_logflux) {
     &my_die("Unable to not compress and logflux compress simultaneously. Check config.",$stack_id, $PS_EXIT_CONFIG_ERROR);
+}
+if ($output_nocomp and $output_deepexp) {
+    &my_die("Unable to not compress and use deepexp simultaneously. Check config.",$stack_id, $PS_EXIT_CONFIG_ERROR);
 }
 my $stack_type = metadataLookupStr($recipe, 'STACK.TYPE');
@@ -222 +229 @@
     if ($file->{ignored}) { next; }
 
+    # check for the input files
+    my $image    = $ipprc->filename("PSWARP.OUTPUT", $file->{path_base} ); # Image name
+    my $mask     = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name
+    my $weight   = $ipprc->filename( "PSWARP.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name
+    my $psf      = $convolve ? $ipprc->filename( "PSPHOT.PSF.SKY.SAVE", $file->{path_base} ) : undef; # PSF name
+    my $sources  = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name
+    my $bkgmodel = $ipprc->filename("PSWARP.OUTPUT.BKGMODEL", $file->{path_base});
+
+    my $have_image   = $ipprc->file_exists( $image );
+    my $have_mask    = $ipprc->file_exists( $mask );
+    my $have_weight  = $ipprc->file_exists( $weight );
+    my $have_sources = $ipprc->file_exists( $sources );
+    my $have_psf;
+    if ($convolve) { $have_psf = $ipprc->file_exists( $psf ); }
+
+    if ($skip_missing_inputs) {
+        my $missing_inputs = 0;
+        unless ($have_image)  { print "WARNING: MISSING INPUT $image  \n"; $missing_inputs = 1; }
+        unless ($have_mask)   { print "WARNING: MISSING INPUT $mask   \n"; $missing_inputs = 1; }
+        unless ($have_weight) { print "WARNING: MISSING INPUT $weight \n"; $missing_inputs = 1; }
+        unless ($have_sources){ print "WARNING: MISSING INPUT $sources\n"; $missing_inputs = 1; }
+        if ($convolve) { unless ($have_psf) { print "WARNING: MISSING INPUT $psf\n"; $missing_inputs = 1;}}
+        if ($missing_inputs) { next; }
+    } else {
+        &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_image;
+        &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_mask;
+        &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_weight;
+        &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $have_sources;
+        &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) if ($convolve and not $have_psf);
+    }
+
     print $listFile "INPUT$num\tMETADATA\n";
+
+    print $listFile "\tIMAGE\tSTR\t"    . $image    . "\n";
+    print $listFile "\tMASK\tSTR\t"     . $mask     . "\n";
+    print $listFile "\tVARIANCE\tSTR\t" . $weight   . "\n";
+    print $listFile "\tPSF\tSTR\t"      . $psf      . "\n" if $convolve;
+    print $listFile "\tSOURCES\tSTR\t"  . $sources  . "\n";
+    print $listFile "\tBKGMODEL\tSTR\t" . $bkgmodel . "\n" if $ipprc->file_exists( $bkgmodel );
+
+    print $listFile "END\n\n";
     $num++;
-
-#    my $image = $file->{uri};   # Image name
-    my $image = $ipprc->filename("PSWARP.OUTPUT", $file->{path_base} ); # Image name
-    my $mask = $ipprc->filename( "PSWARP.OUTPUT.MASK", $file->{path_base} ); # Mask name
-    my $weight = $ipprc->filename( "PSWARP.OUTPUT.VARIANCE", $file->{path_base} ); # Weight name
-    my $psf = $convolve ? $ipprc->filename( "PSPHOT.PSF.SKY.SAVE", $file->{path_base} ) : undef; # PSF name
-    my $sources = $ipprc->filename("PSWARP.OUTPUT.SOURCES", $file->{path_base}); # Sources name
-
-    my $bkgmodel = $ipprc->filename("PSWARP.OUTPUT.BKGMODEL", $file->{path_base});
-
-    &my_die("Image $image does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $image );
-    &my_die("Mask $mask does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $mask );
-    &my_die("Weight $weight does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $weight );
-    &my_die("PSF $psf does not exist", $stack_id, $PS_EXIT_SYS_ERROR) if ($convolve and not $ipprc->file_exists( $psf ));
-    &my_die("Sources $sources does not exist", $stack_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists( $sources );
-
-    print $listFile "\tIMAGE\tSTR\t" . $image . "\n";
-    print $listFile "\tMASK\tSTR\t" . $mask . "\n";
-    print $listFile "\tVARIANCE\tSTR\t" . $weight . "\n";
-    print $listFile "\tPSF\tSTR\t" . $psf . "\n" if $convolve;
-    print $listFile "\tSOURCES\tSTR\t" . $sources . "\n";
-    print $listFile "\tBKGMODEL\tSTR\t" . $bkgmodel . "\n" if $ipprc->file_exists( $bkgmodel );
-
-    print $listFile "END\n\n";
 }
 close($listFile);
@@ -255 +278 @@
 # Prepare the output files
 my @outputFiles;
-prepare_outputs(\@outputFiles, \@outputList, $outroot);
+prepare_outputs(\@outputFiles, \@outputList,       $outroot) if $convolve;
 prepare_outputs(\@outputFiles, \@outputListPhotom, $outroot) if $photometry;
-prepare_outputs(\@outputFiles, \@outputListUnconv, $outroot) if $convolve;
+prepare_outputs(\@outputFiles, \@outputListUnconv, $outroot); # unconvolved output always go here
 
 # we need the output image name for the database
@@ -309 +332 @@
     }
     if ($output_logflux) {
-        $command .= " -R PPSTACK.OUTPUT FITS.TYPE COMP_STACK "; # Just this one output component?
-        $command .= " -R PPSTACK.OUTPUT.VARIANCE FITS.TYPE COMP_STACK "; 
-#       $command .= " -R PPSTACK.OUTPUT.EXPWT FITS.TYPE COMP_STACK ";
-        $command .= " -R PPSTACK.UNCONV FITS.TYPE COMP_STACK ";
-        $command .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE COMP_STACK ";
-#       $command .= " -R PPSTACK.UNCONV.EXPWT FITS.TYPE COMP_STACK ";
+        $command .= " -R PPSTACK.OUTPUT          FITS.TYPE STK_UNIONS_COMP";
+        $command .= " -R PPSTACK.OUTPUT.VARIANCE FITS.TYPE STK_UNIONS_COMP"; 
+        $command .= " -R PPSTACK.UNCONV          FITS.TYPE STK_UNIONS_COMP";
+        $command .= " -R PPSTACK.UNCONV.VARIANCE FITS.TYPE STK_UNIONS_COMP";
+    }
+    # if the total exposure time is long (> 6500 sec), need to use
+    # a different compression options.
+    if ($output_deepexp) {
+        $command .= " -R PPSTACK.OUTPUT.EXP FITS.TYPE EXP_UNIONS ";
+        $command .= " -R PPSTACK.UNCONV.EXP FITS.TYPE EXP_UNIONS ";
     }
     $command .= " -threads $threads" if defined $threads;