Changeset 29493 for trunk/ippScripts/scripts/magic_process.pl

Timestamp:

Oct 20, 2010, 9:39:26 AM (16 years ago)

Author:

bills

Message:

changes to implment saving magic streaks file and verify outputs in nebulous and to add data_state to Background tables to support cleanup

File:

• trunk/ippScripts/scripts/magic_process.pl (modified) (12 diffs)

trunk/ippScripts/scripts/magic_process.pl

@@ -17 +17 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
+use File::Copy;
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -41 +42 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile, $final_outroot);
 
 GetOptions(
@@ -49 +50 @@
            'dbname=s'        => \$dbname,     # Database name
            'baseroot=s'      => \$baseroot,   # Output root name
+           'final-outroot=s' => \$final_outroot,   # location for final outputs
            'save-temps'      => \$save_temps, # Save temporary files?
            'verbose'         => \$verbose,    # Print stuff?
@@ -86 +88 @@
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
+
+# list of VerifyStreaks input and output files to copy to nebulous 
+my @verify_outputs = qw(
+clusterPos.txt
+duplicate.png
+mask.png
+original.png
+original.fits
+residual.png
+residual.fits
+clusters.list
+);
 
 ### Get a list of inputs
@@ -312 +326 @@
 
 
-### Input result into database
-{
-    my $command = "$magictool -addresult";
-    $command   .= " -magic_id $magic_id";
-    $command   .= " -node $node";
-    $command   .= " -path_base $outroot";
-    $command   .= " -dbname $dbname" if defined $dbname;
-
-    # Add the processed file to the database
-    unless ($no_update) {
-        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-            run(command => $command, verbose => $verbose);
-        unless ($success) {
-            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-            &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code);
-        }
-    } else {
-        print "Skipping command: $command\n";
-    }
-}
-
 if ($node eq "root") {
     my $streaks_file = "$outroot.streaks";
@@ -349 +342 @@
     }
 
-    &run_verifystreaks($baseroot);
 
     my $exp_id;                 # Exposure identifier
@@ -366 +358 @@
     }
 
+    &run_verifystreaks($baseroot, $exp_id);
+
     {
         my $astrom = $ipprc->filename("PSASTRO.OUTPUT", $cam_path); # Astrometry file
@@ -383 +377 @@
     }
 
+    my $output_streaks = $final_outroot . ".streaks";
+    if ($output_streaks and ($output_streaks ne $streaks_file)) {
+        copy_to_nebulous($ipprc, $streaks_file, $output_streaks, 1);
+        foreach my $f (@verify_outputs) {
+            my $src = "$baseroot.verify/${exp_id}_$f";
+            my $dest = "$final_outroot.${f}";
+            copy_to_nebulous($ipprc, $src, $dest, 1);
+        }
+    } else {
+        $output_streaks = $streaks_file;
+    }
+
+### Input result into database
+# XXXX: if this succeeds but addmask fails the magicRun is in a goofy state XXXX
+{
+    my $command = "$magictool -addresult";
+    $command   .= " -magic_id $magic_id";
+    $command   .= " -node $node";
+    $command   .= " -path_base $outroot";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform magictool -addresult: $error_code", $magic_id, $node, $error_code);
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+}
+
 
     {
         my $command = "$magictool -addmask";
         $command   .= " -magic_id $magic_id";
-        $command   .= " -uri $streaks_file";
+        $command   .= " -uri $final_outroot";
         $command   .= " -streaks $num_streaks";
         $command   .= " -dbname $dbname" if defined $dbname;
@@ -411 +439 @@
 
     my $baseroot = shift;
+    my $exp_id = shift;
 
     unless ($VerifyStreaks) {
@@ -431 +460 @@
     my @files = <$baseroot.*.clusters>;
 
-    unless (open ($FILE, ">$outdir/clusters.list")) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+    my $clusters_list = "$outdir/${exp_id}_clusters.list";
+    unless (open ($FILE, ">$clusters_list")) {
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
@@ -441 +471 @@
     close ($FILE);
     if ($status) {
-        print "failed to create cluster file $outdir/clusters.list\n";
+        print "failed to create cluster file $clusters_list\n";
         return 1;
     }
 
-    my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap";
+    my $command = "$VerifyStreaks --out $outdir --clusters $clusters_list $baseroot.root.streakMap";
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -513 +543 @@
 }
 
+# Copy a file to nebulous and optionally replicate it
+# We should consider making this an ipprc function. For now try it here so we can print
+# the right error messages
+sub copy_to_nebulous {
+    my $ipprc = shift;
+    my $src = shift;
+    my $dest = shift;
+    my $replicate = shift;
+
+    print "copying $src to $dest\n";
+
+    $ipprc->file_exists($src) or
+        &my_die("expected output file does not exist: $src", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    # copy the file to it's final destination - which is presumably in nebulous
+    # we delete it so that all instances are deleted in case it has been replicated
+    if ($ipprc->file_exists($dest)) {
+        $ipprc->file_delete($dest) or 
+                &my_die("failed to delete existing file: $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+    my $dest_resolved = $ipprc->file_resolve($dest, 'create');
+    &my_die("failed to resolve $dest", $PS_EXIT_UNKNOWN_ERROR) if !$dest_resolved;
+
+    copy ($src, $dest_resolved) or 
+        &my_die("failed to copy $src to $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+    if ($replicate and (file_scheme($dest) eq 'neb')) {
+        my $neb = $ipprc->nebulous();
+        $neb->setxattr($dest, 'user.copies', 2, 'create') or 
+            &my_die("failed to set user.copies for $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+
+        $neb->replicate($dest) or
+            &my_die("failed to replicate $dest", $magic_id, $node, $PS_EXIT_UNKNOWN_ERROR);
+    }
+}
+
 sub my_die
 {