Changeset 29501

Timestamp:

Oct 20, 2010, 3:38:35 PM (16 years ago)

Author:

bills

Message:

merge in new magic scripts

Location:

tags/ipp-20100823/ippScripts/scripts

Files:

• magic_destreak.pl (modified) (9 diffs, 1 prop)
• magic_process.pl (modified) (15 diffs, 1 prop)

tags/ipp-20100823/ippScripts/scripts/magic_destreak.pl

@@ -42 +42 @@
 # Parse the command-line arguments
 my ($magic_ds_id, $camera, $streaks, $inv_streaks, $exp_id, $stage, $stage_id, $component, $uri, $path_base, $cam_path_base, $cam_reduction);
+my ($streaks_path_base, $inv_streaks_path_base);
 my ($outroot, $recoveryroot, $magicked);
 my ($replace, $release);
@@ -49 +50 @@
            'magic_ds_id=s'  => \$magic_ds_id,# Magic destreak run identifier
            'camera=s'       => \$camera,     # camera for evaluating file rules
+           'streaks_path_base=s'      => \$streaks_path_base,    # path_base for streaks data
+           'inv_streaks_path_base=s'  => \$inv_streaks_path_base, #path_base for streaks from inverse diff
+           'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
            'streaks=s'      => \$streaks,    # file containing the list of streaks
            'inv_streaks=s'  => \$inv_streaks,# file containing the list of streaks from the inverse diff
@@ -78 +82 @@
     defined $camera and
     defined $streaks and
+    defined $streaks_path_base and
     defined $stage and
     defined $stage_id and
@@ -108 +113 @@
     &my_die("Invalid value for stage: $stage", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
 }
+$inv_streaks_path_base = undef if defined($inv_streaks_path_base) and ($inv_streaks_path_base eq "NULL");
 $inv_streaks = undef if defined($inv_streaks) and ($inv_streaks eq "NULL");
 
@@ -240 +246 @@
     my ($allstreaks_fh, $allstreaks_name);
 
+    # Set name of streaks files from their path_base. Note. ne 'NULL' test is for backward compatability
+    # with runs that don't have path_base set yet
+    if ($streaks_path_base ne 'NULL') {
+        $streaks = "$streaks_path_base.streaks";
+    }
+    if ($inv_streaks_path_base and ($inv_streaks_path_base ne 'NULL')) {
+        $inv_streaks = "$inv_streaks_path_base.streaks";
+    }
+
+    my $streaks_resolved = $ipprc->file_resolve($streaks) or &my_die("failed to resolve streaks file $streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
+    my $inv_streaks_resolved;
+    if ($inv_streaks) {
+        $inv_streaks_resolved = $ipprc->file_resolve($inv_streaks) or &my_die("failed to resolve inverse streaks file $inv_streaks", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR);
+    }
+        
+
     if ($stage eq "raw") {
         $image = $uri;
@@ -301 +323 @@
         $sources    = $ipprc->filename("PPSUB.OUTPUT.SOURCES", $path_base);
 
-        if ($inv_streaks) {
+        if ($inv_streaks_resolved) {
             # create a temporary file containing the contents of the
             # two streaks files
@@ -307 +329 @@
                     UNLINK => !$save_temps);
 
-            combine_streaks($allstreaks_fh, $streaks, $inv_streaks);
+            combine_streaks($allstreaks_fh, $streaks_resolved, $inv_streaks_resolved);
 
             # apply the combined streaks to both the forward and inverse diffs
-            $streaks = $allstreaks_name;
+            $streaks_resolved = $allstreaks_name;
         }
     }
 
     {
-        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
 
         $command .= " -stats $statsFile";
@@ -340 +362 @@
         }
     }
-    if (($stage eq "diff") and $inv_streaks) {
+    if (($stage eq "diff") and $inv_streaks_resolved) {
         $image   = $ipprc->filename("PPSUB.INVERSE", $path_base);
         $mask    = $ipprc->filename("PPSUB.INVERSE.MASK", $path_base);
@@ -349 +371 @@
         my $invStatsFile = "$outroot/$exp_id.mds.$magic_ds_id.$stage_id.$component.inv.stats";
 
-        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks -image $image";
+        my $command = "$streaksremove -stage $stage -tmproot $tmproot -streaks $streaks_resolved -image $image";
         $command .= " -stats $invStatsFile";
 

tags/ipp-20100823/ippScripts/scripts/magic_process.pl

@@ -17 +17 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
+use File::Copy;
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -41 +42 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile, $final_outroot);
 
 GetOptions(
@@ -49 +50 @@
            'dbname=s'        => \$dbname,     # Database name
            'baseroot=s'      => \$baseroot,   # Output root name
+           'final-outroot=s' => \$final_outroot,   # location for final outputs
            'save-temps'      => \$save_temps, # Save temporary files?
            'verbose'         => \$verbose,    # Print stuff?
@@ -86 +88 @@
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# list of VerifyStreaks input and output files to copy to nebulous 
+my @verify_outputs = qw(
+clusterPos.txt
+duplicate.png
+mask.png
+original.png
+original.fits
+residual.png
+residual.fits
+clusters.list
+);
 
 ### Get a list of inputs
@@ -311 +325 @@
 }
 
-
 ### Input result into database
 {
@@ -334 +347 @@
 
 if ($node eq "root") {
+    # XXXX: Since we just added the result above, all of these my_dies are going to fail
+    # with a duplicate row error.
+    # see more comments below
     my $streaks_file = "$outroot.streaks";
     my $resolved = $ipprc->file_resolve($streaks_file);
@@ -349 +365 @@
     }
 
-    &run_verifystreaks($baseroot);
 
     my $exp_id;                 # Exposure identifier
@@ -366 +381 @@
     }
 
+    &run_verifystreaks($baseroot, $exp_id);
+
     {
         my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file
@@ -383 +400 @@
     }
 
+    my $output_streaks = $final_outroot . ".streaks";
+    if ($output_streaks and ($output_streaks ne $streaks_file)) {
+        copy_to_nebulous($ipprc, $streaks_file, $output_streaks, 1);
+        foreach my $f (@verify_outputs) {
+            my $src = "$baseroot.verify/${exp_id}_$f";
+            my $dest = "$final_outroot.${f}";
+            copy_to_nebulous($ipprc, $src, $dest, 1);
+        }
+    } else {
+        $output_streaks = $streaks_file;
+    }
 
     {
         my $command = "$magictool -addmask";
         $command   .= " -magic_id $magic_id";
-        $command   .= " -uri $streaks_file";
+        $command   .= " -uri $final_outroot";
         $command   .= " -streaks $num_streaks";
         $command   .= " -dbname $dbname" if defined $dbname;
@@ -397 +425 @@
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                # This isn't going to work because our result was already entered.
+                # XXX: This my_die isn't going to work because the result for the root node was already
+                # added to the database up above.
+                # There is a magicMask has a fault column. Maybe we should 
+                # use that and add a new revert mode that deletes the magicMask and the magicNodeResult
+                # for the root node.
                 &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code);
             }
@@ -406 +438 @@
 }
 
+
 ### Pau.
 
@@ -411 +444 @@
 
     my $baseroot = shift;
+    my $exp_id = shift;
 
     unless ($VerifyStreaks) {
@@ -431 +465 @@
     my @files = <$baseroot.*.clusters>;
 
-    unless (open ($FILE, ">$outdir/clusters.list")) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+    my $clusters_list = "$outdir/${exp_id}_clusters.list";
+    unless (open ($FILE, ">$clusters_list")) {
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
@@ -441 +476 @@
     close ($FILE);
     if ($status) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
 
-    my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap";
+    my $command = "$VerifyStreaks --out $outdir --clusters $clusters_list $baseroot.root.streakMap";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -513 +548 @@
 }
 
+# Copy a file to nebulous and optionally replicate it
+# We should consider making this an ipprc function. For now try it here so we can print
+# the right error messages
+sub copy_to_nebulous {
+    my $ipprc = shift;
+    my $src = shift;
+    my $dest = shift;
+    my $replicate = shift;
+
+    print "copying $src to $dest\n";
+
+    $ipprc->file_exists($src) or
+        &my_die("expected output file does not exist: $src", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    # copy the file to it's final destination - which is presumably in nebulous
+    # we delete it so that all instances are deleted in case it has been replicated
+    if ($ipprc->file_exists($dest)) {
+        $ipprc->file_delete($dest) or 
+                &my_die("failed to delete existing file: $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+    my $dest_resolved = $ipprc->file_resolve($dest, 'create');
+    &my_die("failed to resolve $dest", $PS_EXIT_UNKNOWN_ERROR) if !$dest_resolved;
+
+    copy ($src, $dest_resolved) or 
+        &my_die("failed to copy $src to $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    if ($replicate and (file_scheme($dest) eq 'neb')) {
+        my $neb = $ipprc->nebulous();
+        $neb->setxattr($dest, 'user.copies', 2, 'create') or 
+            &my_die("failed to set user.copies for $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+        $neb->replicate($dest) or
+            &my_die("failed to replicate $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+
 sub my_die
 {