Changeset 37068 for branches/eam_branches/ipp-ops-20130712/ippScripts/scripts

Timestamp:

Jul 17, 2014, 12:36:19 PM (12 years ago)

Author:

eugene

Message:

merge changes (from past YEAR) from trunk

Location:

branches/eam_branches/ipp-ops-20130712

Files:

• . (modified) (1 prop)
• ippScripts/scripts/background_chip.pl (modified) (3 diffs)
• ippScripts/scripts/chip_imfile.pl (modified) (1 prop)
• ippScripts/scripts/destreak_restore_camera.pl (modified) (1 prop)
• ippScripts/scripts/dist_advancerun.pl (modified) (5 diffs, 1 prop)
• ippScripts/scripts/dist_bundle.pl (modified) (4 diffs, 1 prop)
• ippScripts/scripts/dist_defineruns.pl (modified) (5 diffs)
• ippScripts/scripts/dist_make_fileset.pl (modified) (1 diff)
• ippScripts/scripts/ipp_apply_burntool_single.pl (modified) (1 diff, 1 prop)
• ippScripts/scripts/lap_science.pl (modified) (1 prop)
• ippScripts/scripts/magic_destreak.pl (modified) (1 prop)
• ippScripts/scripts/nightly_science.pl (modified) (1 prop)
• ippScripts/scripts/psphot_fullforce_summary.pl (copied) (copied from trunk/ippScripts/scripts/psphot_fullforce_summary.pl )
• ippScripts/scripts/psphot_fullforce_warp.pl (copied) (copied from trunk/ippScripts/scripts/psphot_fullforce_warp.pl )
• ippScripts/scripts/publish_file.pl (modified) (1 prop)
• ippScripts/scripts/queuestaticsky.pl (modified) (2 diffs, 1 prop)
• ippScripts/scripts/rawcheck.pl (modified) (1 prop)
• ippScripts/scripts/receive_advance.pl (modified) (3 diffs)
• ippScripts/scripts/receive_file.pl (modified) (2 diffs)
• ippScripts/scripts/register_imfile.pl (modified) (1 prop)
• ippScripts/scripts/sc_mk_stack_mdc.pl (copied) (copied from trunk/ippScripts/scripts/sc_mk_stack_mdc.pl )
• ippScripts/scripts/sc_prepare_camera.pl (copied) (copied from trunk/ippScripts/scripts/sc_prepare_camera.pl )
• ippScripts/scripts/sc_prepare_chip.pl (copied) (copied from trunk/ippScripts/scripts/sc_prepare_chip.pl )
• ippScripts/scripts/sc_prepare_stack.pl (copied) (copied from trunk/ippScripts/scripts/sc_prepare_stack.pl )
• ippScripts/scripts/sc_prepare_warp.pl (copied) (copied from trunk/ippScripts/scripts/sc_prepare_warp.pl )
• ippScripts/scripts/sc_remote_exec.pl (copied) (copied from trunk/ippScripts/scripts/sc_remote_exec.pl )
• ippScripts/scripts/sc_transfer_tool.pl (copied) (copied from trunk/ippScripts/scripts/sc_transfer_tool.pl )
• ippScripts/scripts/sc_validate_files.pl (copied) (copied from trunk/ippScripts/scripts/sc_validate_files.pl )
• ippScripts/scripts/sc_validate_processing.pl (copied) (copied from trunk/ippScripts/scripts/sc_validate_processing.pl )
• ippScripts/scripts/skycell_jpeg.pl (modified) (1 prop)
• ippScripts/scripts/stack_bkg_mk_mdc.pl (copied) (copied from trunk/ippScripts/scripts/stack_bkg_mk_mdc.pl )
• ippScripts/scripts/stack_skycell.pl (modified) (4 diffs, 1 prop)
• ippScripts/scripts/staticsky.pl (modified) (4 diffs)
• ippScripts/scripts/warp_skycell.pl (modified) (1 prop)

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/background_chip.pl

@@ -34 +34 @@
 my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
+my $ppImage = can_run('ppImage') or (warn "Can't find ppImage" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -134 +135 @@
 }
 my $in_wt = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $in_path, $class_id);
-my $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id);
+my $in_bg;
+if (1) {
+    $in_bg = $ipprc->filename("PPIMAGE.BACKMDL", $cam_path_base, $class_id);
+}
+else {
+    $in_bg = $ipprc->filename("PSPHOT.BACKMDL", $in_path, $class_id);
+}
 my $in_pattern = $ipprc->filename("PPIMAGE.PATTERN", $in_path, $class_id);
 my $in_config = $ipprc->filename("PPIMAGE.CONFIG", $in_path, $class_id);
@@ -220 +227 @@
 }
 
+my $do_binned_images = 1;   # Generate the binned images that are useful for making JPEGs
+if ($do_binned_images) {
+    my $command = "$ppImage -file $out_image -mask $out_mask $outroot -recipe PPIMAGE PPIMAGE_PA -Db PHOTOM F";
+    my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform ppImage: $error_code", $chip_bg_id, $class_id, $error_code);
+    }
+}
+
+
 if (!$quality and !$no_op) {
     &my_die("Couldn't find expected output file: $out_image", $chip_bg_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($out_image);

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_advancerun.pl

@@ -69 +69 @@
 my $stacktool   = can_run('stacktool') or (warn "Can't find stacktool" and $missing_tools = 1);
 my $bgtool = can_run('bgtool') or (warn "Can't find bgtool" and $missing_tools = 1);
+my $fftool = can_run('fftool') or (warn "Can't find fftool" and $missing_tools = 1);
 my $staticskytool = can_run('staticskytool') or (warn "Can't find staticskytool" and $missing_tools = 1);
 if ($missing_tools) {
@@ -77 +78 @@
 
 my $tool_cmd;
+my $tool_cmd2;
 my $list_mode;
 my $component_key;
@@ -127 +129 @@
     $list_mode = "-diffskyfile";
     $component_key = "skycell_id";
+} elsif ($stage eq "ff") {
+    $tool_cmd = "$fftool -ff_id";
+    $list_mode = "-result";
+    $component_key = "dist_component";
+    $tool_cmd2 = "$fftool -summary -ff_id"
 } else {
     &my_die("Unexpected stage: $stage", $dist_id, $PS_EXIT_CONFIG_ERROR);
@@ -132 +139 @@
 
 $tool_cmd .= " $stage_id";
+$tool_cmd2 .= " $stage_id" if $tool_cmd2;
 
 my $exportarg = '-exportrun';
@@ -215 +223 @@
     my $components = parse_md_list($metadata) or
         &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    if ($tool_cmd2) {
+        my $command = "$tool_cmd2";
+        $command .= " -dbname $dbname" if defined $dbname;
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+            unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+        }
+        if (@$stdout_buf == 0) {
+            &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+        }
+        my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+        my $more_components = parse_md_list($metadata) or
+            &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+        if (scalar @$more_components) {
+            push @$components, @$more_components;
+        }
+    }
 
     open MANIFEST, ">$resolved" or

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_bundle.pl

@@ -67 +67 @@
                       );
 my %empty_cleaned = ();
-
-
+my %empty_cleaned = ();
 
 # Look for programs we need
@@ -496 +495 @@
     } elsif ($stage eq "stack") {
         $type = $stack_cleaned{$rule};
-    } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary') {
+    } elsif ($stage eq "sky" or $stage eq 'skycal' or $stage eq 'stack_summary' or $stage eq 'ff') {
         $type = $empty_cleaned{$rule};
+    } elsif ($stage eq "skycal") {
+        $type = $skycal_cleaned{$rule};
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -588 +589 @@
         # stack_summary stage does not use a config dump file.
         return build_stack_summary_file_list($path_base, \@file_list);
+    } elsif ($stage eq "ff") {
+        if ($component eq 'summary') {
+            # full force summary does not use a config dump file. Cobble together a file list
+            return build_ff_summary_file_list($path_base, \@file_list);
+        } else {
+            $config_file_rule = "PSPHOT.SKY.CONFIG";
+        }
     } else {
         &my_die("$stage is not a valid stage", $component, $PS_EXIT_CONFIG_ERROR);
@@ -742 +750 @@
 }
 
+# psphotFullForceSummary does not produce a config dump file cobble together a file list
+sub build_ff_summary_file_list {
+    my ($path_base, $file_list) = @_;
+
+    my %cmf ;
+    $cmf{file_rule} = "SOURCES";
+    $cmf{name} = "$path_base.cmf";
+    push @$file_list, \%cmf;
+
+    my %log;
+    $log{file_rule} = "LOG";
+    $log{name} = "$path_base.log";
+    push @$file_list, \%log;
+
+    return $file_list;
+}
+
 sub my_die
 {

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_defineruns.pl

@@ -42 +42 @@
 
 # Parse the command-line arguments
-my ($stage, $stage_limit, $dist_root, $workdir, $no_magic);
+my ($stage, $stage_limit, $dist_root, $workdir, $no_magic, $rerun);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 my @labels;
@@ -51 +51 @@
            'workdir=s'      => \$workdir,     # workdir
            'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
+           'rerun'          => \$rerun,      # queue new runs even if one exists
 #           'dist_root=s'    => \$dist_root,  # root of distribution work area
            'no_magic'       => \$no_magic,   # queue runs without requiring magic (for testing only)
@@ -56 +57 @@
            'verbose'        => \$verbose,    # Print stuff?
            'no-update'      => \$no_update,  # Don't update the database
+           'pretend'        => \$no_update,  # Don't update the database
            'save-temps'     => \$save_temps, # Save temporary files?
            'logfile=s'      => \$logfile,
@@ -88 +90 @@
     push @stages, $stage;
 } else {
-    @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle);
+    @stages = qw(chip chip_bg camera fake warp warp_bg diff stack SSdiff sky skysingle ff);
 }
 
@@ -104 +106 @@
         $command .= " -singlefilter" if $single;
         $command .= " -no_magic" if $no_magic;
+        $command .= " -rerun" if $rerun;
         $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/dist_make_fileset.pl

@@ -223 +223 @@
     my $dbname = shift;
 
-    if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky' or $stage eq 'skycal') {
+    if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff') or $stage eq 'sky' or $stage eq 'skycal' or $stage eq "ff") {
         return "";
     }

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/ipp_apply_burntool_single.pl

@@ -45 +45 @@
     defined $dbname;
 
+# XXX watch out for TC3 and GPC2 here:
 unless (defined $camera) {
     if ($this_uri =~ /ota/) {

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/queuestaticsky.pl

@@ -35 +35 @@
 };
 
+# don't update the database if we are pretending
+$no_update = 1 if $pretend;
 
 my $missing_tools;
@@ -116 +118 @@
         }
     }
+    last if $pretend;   # only do the first (full) set if we are in pretend mode
 }
 {

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/receive_advance.pl

@@ -65 +65 @@
     my ($stage) = $filename =~ m|^dbinfo\.(\S+)\.\d+\.mdc$|; # Stage of interest
     my $tool_name;
+    my $tool_mode = '-importrun';
     if ($stage eq "raw") {
         $tool_name = "regtool";
@@ -73 +74 @@
     } elsif ($stage eq "sky") {
         $tool_name = "staticskytool";
+    } elsif ($stage eq "skycal") {
+        $tool_name = "staticskytool";
+        $tool_mode = '-importskycal';
     } else {
         $tool_name = "${stage}tool";
@@ -81 +85 @@
     &my_die("Unable to resolve $dbinfo_uri\n", $PS_EXIT_UNKNOWN_ERROR) unless $file;
 
-    my $command = "$tool -importrun -infile $file"; # Command to execute
+    my $command = "$tool $tool_mode -infile $file"; # Command to execute
     $command .= " -dbname $dbname" if defined $dbname;
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/receive_file.pl

@@ -223 +223 @@
     } elsif ($runType eq 'staticskyRun') {
         $stage = 'sky';
-        # XXX: This should be skycell, but the distribution code uses exposure
         $comp_name = 'skycell_id';
-#        $current_component = $comp_name;
+    } elsif ($runType eq 'skycalRun') {
+        $stage = 'skycal';
+        $comp_name = 'skycell_id';
     } else {
         &my_die( "unexpected run type line found in $filename: $runType\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
@@ -238 +239 @@
     }
 
-    if ($stage eq 'sky') {
-        # the dbinfo file for a skyRun only has one component and it doesn't contain
+    if ($stage eq 'sky' or $stage eq 'skycal') {
+        # the dbinfo file for staticskyRun and skycalRun only have one component and they don't contain
         # skycell_id which is the way components are listed in the dirinfo file.
         my @ids = keys %$components;
-        &my_die( "unexpected number of components scalar @ids found in staticsky dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if scalar @ids != 1;
+        my $nComponents = scalar @ids;
+        &my_die( "unexpected number of components $nComponents found in $stage dirinfo file\n", $file_id, $PS_EXIT_UNKNOWN_ERROR) if $nComponents  != 1;
         $current_component = $ids[0];
     }

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/stack_skycell.pl

@@ -40 +40 @@
 }
 
-my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps);
+my ($stack_id, $dbname, $outroot, $debug, $run_state, $threads, $reduction, $verbose, $no_update, $no_op, $redirect, $save_temps, $delete_convolved_images);
 GetOptions(
     'stack_id|d=s'      => \$stack_id, # Stack identifier
@@ -55 +55 @@
     'redirect-output'   => \$redirect,
     'save-temps'        => \$save_temps, # Save temporary files?
+    'delete-convolved'  => \$delete_convolved_images,
 ) or pod2usage( 2 );
 
@@ -92 +93 @@
 );
 
+
 my $ipprc = PS::IPP::Config->new() or my_die( "Unable to set up", $stack_id, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 $| = 1;
@@ -360 +362 @@
 
     if (!$quality) {
+        
+
         check_outputs(\@outputFiles, $replicate_outputs);
+
+        if (($convolve)&&($delete_convolved_images)) { # We made convolved products, but do not wish to keep them anymore
+            my @products_to_clear = ('PPSTACK.OUTPUT',
+                                     'PPSTACK.OUTPUT.MASK',
+                                     'PPSTACK.OUTPUT.VARIANCE',
+                                     'PPSTACK.OUTPUT.EXP',
+                                     'PPSTACK.OUTPUT.EXPNUM',
+                                     'PPSTACK.OUTPUT.EXPWT');
+            foreach my $product (@products_to_clear) {
+                my $file = $ipprc->filename($product,$outroot,$skycell_id);
+                print "Deleting unwanted convolved product: $file\n";
+                my $error= $ipprc->kill_file($file);
+                if ($error_code) {
+                    print "Failed to delete unwanted convolved product: $error_code\n";
+                }
+            }
+        }
+
     }
 

branches/eam_branches/ipp-ops-20130712/ippScripts/scripts/staticsky.pl

@@ -138 +138 @@
         my $configuration = $ipprc->filename("PSPHOT.STACK.CONFIG", $outroot);
 
+        my $needConvolvedImages = 0;
+
         foreach my $file (@$files) {
             print $listFile "INPUT   METADATA\n";
@@ -146 +148 @@
             my $stack_id = $file->{stack_id};
 
-            my $imageCnv  = $ipprc->filename("PPSTACK.OUTPUT",          $path_base ); # Image name
-            my $maskCnv   = $ipprc->filename("PPSTACK.OUTPUT.MASK",     $path_base ); # Mask name
-            my $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name
-            my $expnumCnv = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM",   $path_base ); # Expnum name
+            my ($imageCnv, $maskCnv, $weightCnv, $expnumCnv);
+            if ($needConvolvedImages) {
+                $imageCnv  = $ipprc->filename("PPSTACK.OUTPUT",          $path_base ); # Image name
+                $maskCnv   = $ipprc->filename("PPSTACK.OUTPUT.MASK",     $path_base ); # Mask name
+                $weightCnv = $ipprc->filename("PPSTACK.OUTPUT.VARIANCE", $path_base ); # Weight name
+                $expnumCnv = $ipprc->filename("PPSTACK.OUTPUT.EXPNUM",   $path_base ); # Expnum name
+            }
 
             my $imageRaw  = $ipprc->filename("PPSTACK.UNCONV",          $path_base ); # Image name
@@ -166 +171 @@
             &my_die("Couldn't find input: $weightRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightRaw");
             &my_die("Couldn't find input: $expnumRaw", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumRaw");
-            &my_die("Couldn't find input: $imageCnv",  $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv");
-            &my_die("Couldn't find input: $maskCnv",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv");
-            &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv");
-            &my_die("Couldn't find input: $expnumCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumCnv");
-            &my_die("Couldn't find input: $psfCnv",    $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv");
+            if ($needConvolvedImages) {
+                &my_die("Couldn't find input: $imageCnv",  $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$imageCnv");
+                &my_die("Couldn't find input: $maskCnv",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$maskCnv");
+                &my_die("Couldn't find input: $weightCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$weightCnv");
+                &my_die("Couldn't find input: $expnumCnv", $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$expnumCnv");
+                &my_die("Couldn't find input: $psfCnv",    $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$psfCnv");
+            }
             &my_die("Couldn't find input: $sources",   $sky_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists("$sources");
 
@@ -178 +185 @@
             print $listFile "  RAW:VARIANCE  STR  " . $weightRaw . "\n";
             print $listFile "  RAW:EXPNUM    STR  " . $expnumRaw . "\n";
-
-            print $listFile "  CNV:IMAGE     STR  " . $imageCnv  . "\n";
-            print $listFile "  CNV:MASK      STR  " . $maskCnv   . "\n";
-            print $listFile "  CNV:VARIANCE  STR  " . $weightCnv . "\n";
-            print $listFile "  CNV:EXPNUM    STR  " . $expnumCnv . "\n";
-            print $listFile "  CNV:PSF       STR  " . $psfCnv    . "\n";
-
             print $listFile "  SOURCES       STR  " . $sources   . "\n";
+            if ($needConvolvedImages) {
+                print $listFile "  CNV:IMAGE     STR  " . $imageCnv  . "\n";
+                print $listFile "  CNV:MASK      STR  " . $maskCnv   . "\n";
+                print $listFile "  CNV:VARIANCE  STR  " . $weightCnv . "\n";
+                print $listFile "  CNV:EXPNUM    STR  " . $expnumCnv . "\n";
+                print $listFile "  CNV:PSF       STR  " . $psfCnv    . "\n";
+            }
 
             print $listFile "END\n\n";