Changeset 29515 for branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_process.pl

Timestamp:

Oct 21, 2010, 2:45:13 PM (16 years ago)

Author:

eugene

Message:

merge changes from trunk

File:

• branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_process.pl (modified) (15 diffs)

branches/eam_branches/ipp-20100823/ippScripts/scripts/magic_process.pl

@@ -17 +17 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
+use File::Copy;
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -41 +42 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile, $final_outroot);
 
 GetOptions(
@@ -49 +50 @@
            'dbname=s'        => \$dbname,     # Database name
            'baseroot=s'      => \$baseroot,   # Output root name
+           'final-outroot=s' => \$final_outroot,   # location for final outputs
            'save-temps'      => \$save_temps, # Save temporary files?
            'verbose'         => \$verbose,    # Print stuff?
@@ -86 +88 @@
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# list of VerifyStreaks input and output files to copy to nebulous 
+my @verify_outputs = qw(
+clusterPos.txt
+duplicate.png
+mask.png
+original.png
+original.fits
+residual.png
+residual.fits
+clusters.list
+);
 
 ### Get a list of inputs
@@ -311 +325 @@
 }
 
-
 ### Input result into database
 {
@@ -334 +347 @@
 
 if ($node eq "root") {
+    # XXXX: Since we just added the result above, all of these my_dies are going to fail
+    # with a duplicate row error.
+    # see more comments below
     my $streaks_file = "$outroot.streaks";
     my $resolved = $ipprc->file_resolve($streaks_file);
@@ -349 +365 @@
     }
 
-    &run_verifystreaks($baseroot);
 
     my $exp_id;                 # Exposure identifier
@@ -366 +381 @@
     }
 
+    &run_verifystreaks($baseroot, $exp_id);
+
     {
         my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file
@@ -383 +400 @@
     }
 
+    my $output_streaks = $final_outroot . ".streaks";
+    if ($output_streaks and ($output_streaks ne $streaks_file)) {
+        copy_to_nebulous($ipprc, $streaks_file, $output_streaks, 1);
+        foreach my $f (@verify_outputs) {
+            my $src = "$baseroot.verify/${exp_id}_$f";
+            my $dest = "$final_outroot.${f}";
+            copy_to_nebulous($ipprc, $src, $dest, 1);
+        }
+    } else {
+        $output_streaks = $streaks_file;
+    }
 
     {
         my $command = "$magictool -addmask";
         $command   .= " -magic_id $magic_id";
-        $command   .= " -uri $streaks_file";
+        $command   .= " -path_base $final_outroot";
         $command   .= " -streaks $num_streaks";
         $command   .= " -dbname $dbname" if defined $dbname;
@@ -397 +425 @@
             unless ($success) {
                 $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                # This isn't going to work because our result was already entered.
+                # XXX: This my_die isn't going to work because the result for the root node was already
+                # added to the database up above.
+                # There is a magicMask has a fault column. Maybe we should 
+                # use that and add a new revert mode that deletes the magicMask and the magicNodeResult
+                # for the root node.
                 &my_die("Unable to perform magictool -addmask: $error_code", $magic_id, $node, $error_code);
             }
@@ -406 +438 @@
 }
 
+
 ### Pau.
 
@@ -411 +444 @@
 
     my $baseroot = shift;
+    my $exp_id = shift;
 
     unless ($VerifyStreaks) {
@@ -431 +465 @@
     my @files = <$baseroot.*.clusters>;
 
-    unless (open ($FILE, ">$outdir/clusters.list")) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+    my $clusters_list = "$outdir/${exp_id}_clusters.list";
+    unless (open ($FILE, ">$clusters_list")) {
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
@@ -441 +476 @@
     close ($FILE);
     if ($status) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
 
-    my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap";
+    my $command = "$VerifyStreaks --out $outdir --clusters $clusters_list $baseroot.root.streakMap";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -513 +548 @@
 }
 
+# Copy a file to nebulous and optionally replicate it
+# We should consider making this an ipprc function. For now try it here so we can print
+# the right error messages
+sub copy_to_nebulous {
+    my $ipprc = shift;
+    my $src = shift;
+    my $dest = shift;
+    my $replicate = shift;
+
+    print "copying $src to $dest\n";
+
+    $ipprc->file_exists($src) or
+        &my_die("expected output file does not exist: $src", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    # copy the file to it's final destination - which is presumably in nebulous
+    # we delete it so that all instances are deleted in case it has been replicated
+    if ($ipprc->file_exists($dest)) {
+        $ipprc->file_delete($dest) or 
+                &my_die("failed to delete existing file: $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+    my $dest_resolved = $ipprc->file_resolve($dest, 'create');
+    &my_die("failed to resolve $dest", $PS_EXIT_UNKNOWN_ERROR) if !$dest_resolved;
+
+    copy ($src, $dest_resolved) or 
+        &my_die("failed to copy $src to $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    if ($replicate and (file_scheme($dest) eq 'neb')) {
+        my $neb = $ipprc->nebulous();
+        $neb->setxattr($dest, 'user.copies', 2, 'create') or 
+            &my_die("failed to set user.copies for $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+        $neb->replicate($dest) or
+            &my_die("failed to replicate $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+
 sub my_die
 {