Changeset 26721 for branches/eam_branches/20091201/ippScripts/scripts/magic_destreak.pl

Timestamp:

Jan 29, 2010, 10:51:08 AM (16 years ago)

Author:

eugene

Message:

various cleanup and automation operations

File:

• branches/eam_branches/20091201/ippScripts/scripts/magic_destreak.pl (modified) (13 diffs)

branches/eam_branches/20091201/ippScripts/scripts/magic_destreak.pl

@@ -15 +15 @@
 
 use IPC::Cmd 0.36 qw( can_run run );
-use File::Temp qw( tempfile );
+use File::Temp qw( tempfile tempdir );
 use File::Basename qw( basename dirname );
 use PS::IPP::Metadata::Config;
@@ -32 +32 @@
 my $camtool = can_run('camtool') or (warn "Can't find camtool" and $missing_tools = 1);
 my $censorObjects = can_run('censorObjects') or (warn "Can't find censorObjects" and $missing_tools = 1);
+my $ppStatsFromMetadata = can_run('ppStatsFromMetadata') or (warn "Can't find ppStatsFromMetadata" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -38 +39 @@
 
 # Parse the command-line arguments
-my ($magic_ds_id, $camera, $streaks, $inv_streaks, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
+my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base);
 my ($outroot, $recoveryroot);
 my ($replace, $release);
@@ -48 +49 @@
            'streaks=s'      => \$streaks,    # file containing the list of streaks
            'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
+           'exp_id=s'       => \$exp_id,     # exp_id, chip_id, warp_id, or diff_id
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
@@ -74 +76 @@
     defined $stage and
     defined $stage_id and
+    defined $exp_id and
     defined $component and
     defined $uri and
@@ -145 +148 @@
     }
 }
+
+my $statsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.stats";
 
 my $backup_path_base;
@@ -166 +171 @@
         if (! -e $recoveryroot ) {
             my $code = system "mkdir -p $recoveryroot";
-            &my_die("cannot create output directory $outroot", $magic_ds_id, $component,
+            &my_die("cannot create output directory $recoveryroot", $magic_ds_id, $component,
                     $code >> 8) if $code;
         }
@@ -173 +178 @@
     $recovery_path_base = "$recoveryroot/$basename/";
 }
+
+my $temp_dir;
 
 if ($stage ne "camera") {
@@ -195 +202 @@
                     &my_die("Unable to parse metadata list", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
 
+            $temp_dir = tempdir( CLEANUP => !$save_temps);
             ($sfh, $skycell_list) = tempfile( "/tmp/skycell_list.XXXX", UNLINK => !$save_temps);
 
             foreach my $skycell (@$skycells) {
-                my $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                my $skycell_uri;
+                if ($skycell->{data_state} eq "full") {
+                    $skycell_uri = $ipprc->filename("PPSUB.OUTPUT", $skycell->{path_base});
+                } else {
+                    # diff run must have been cleaned up, need to create this skycell file on the fly
+                    my $skycell_id = $skycell->{skycell_id};
+                    $skycell_uri = "$temp_dir/$skycell_id";
+                    $ipprc->skycell_file($skycell->{tess_id}, $skycell_id, $skycell_uri, $verbose) or
+                        &my_die("failed to create skycell file for $skycell_id", $magic_ds_id, $component, $PS_EXIT_PROG_ERROR);
+                }
                 print $sfh "$skycell_uri\n";
             }
-            close $sfh;
+            close $sfh
         }
     }
@@ -260 +277 @@
         my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
 
+        $command .= " -stats $statsFile";
         $command .= " -class_id $class_id" if defined $class_id;
         $command .= " -recovery $recoveryroot" if defined $recoveryroot;
@@ -288 +306 @@
         $sources = $ipprc->filename("PPSUB.INVERSE.SOURCES", $path_base);
 
+        # Note: we create a stats file for the inverse procesing but we don't look at the results
+        my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
+
         my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+        $command .= " -stats $invStatsFile";
 
         $command .= " -recovery $recoveryroot" if defined $recoveryroot;
@@ -371 +393 @@
 }
 
-# if recovery and/or backup files were expected make sure they exist
+my $statsFlags;
+if ($stage ne "camera") {
+    file_check($statsFile);
+    {
+        my $resolvedStatsFile = $ipprc->file_resolve($statsFile);
+        my $command = "$ppStatsFromMetadata $resolvedStatsFile - STREAKSREMOVE";
+
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform ppStatsFromMetadata: $error_code", $magic_ds_id, $component, $error_code);
+        }
+        foreach my $line (@$stdout_buf) {
+            $statsFlags .= " $line";
+        }
+        chomp $statsFlags;
+    }
+}
+
+# XXX: if recovery and/or backup files were expected make sure they exist
 
 # Input result into database
@@ -381 +423 @@
     $command   .= " -backup_path_base $backup_path_base" if $backup_path_base;
     $command   .= " -recovery_path_base $recovery_path_base" if $recovery_path_base;
+    $command   .= " $statsFlags" if $statsFlags;
     $command   .= " -dbname $dbname" if defined $dbname;