Changeset 27840 for branches/simtest_nebulous_branches/ippScripts/scripts/magic_destreak_revert.pl

Timestamp:

May 3, 2010, 8:50:52 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simtest_nebulous_branches

Files:

• . (modified) (1 prop)
• ippScripts/scripts/magic_destreak_revert.pl (modified) (17 diffs)

branches/simtest_nebulous_branches/ippScripts/scripts/magic_destreak_revert.pl

@@ -8 +8 @@
 use Sys::Hostname;
 my $host = hostname();
+my $date = `date`;
 print "\n\n";
-print "Starting script $0 on $host\n\n";
+print "Starting script $0 on $host at $date\n\n";
 
 use vars qw( $VERSION );
@@ -30 +31 @@
 my $magicdstool   = can_run('magicdstool') or (warn "Can't find magicdstool" and $missing_tools = 1);
 my $isdestreaked = can_run('isdestreaked') or (warn "Can't find isdestreaked" and $missing_tools = 1);
+my $ppConfigDump = can_run('ppConfigDump') or (warn "Can't find ppConfigDump" and $missing_tools = 1);
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -36 +38 @@
 
 # Parse the command-line arguments
-my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $inverse, $cam_path_base);
+my ($magic_ds_id, $camera, $streaks, $stage, $stage_id, $component, $uri, $path_base, $bothways, $cam_path_base, $cam_reduction);
 my ($outroot, $recoveryroot, $replace, $release, $bytes, $md5sum);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
@@ -48 +50 @@
            'path_base=s'    => \$path_base,  # path_base of the input
            'cam_path_base=s'=> \$cam_path_base,  # path_base of the associated camera run
+           'cam_reduction=s'=> \$cam_reduction,  # reduction class of the associated camera run
            'outroot=s'      => \$outroot,     # "directory" for temporary images (may be nebulous)
            'recoveryroot=s' => \$recoveryroot,# "directory" for saving the images of excised pixels
            'replace=s'      => \$replace,    # replace the input images with the results.
+           'bothways=s'     => \$bothways,   # run has inverse files (bothways diff)
            'save-temps'     => \$save_temps, # Save temporary files?
            'dbname=s'       => \$dbname,     # Database name
@@ -78 +82 @@
 $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_ds_id, $component, $PS_EXIT_SYS_ERROR ) if $logfile;
 
+$cam_reduction = 'DEFAULT' if !defined $cam_reduction or ($cam_reduction eq "NULL");
+my $recipe_psastro = $ipprc->reduction($cam_reduction, 'PSASTRO'); # Recipe to use
+&my_die("Unrecognised PSASTRO recipe", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) unless defined $recipe_psastro;
+
 my $nebulousServer = metadataLookupStr( $ipprc->{_siteConfig}, 'NEB_SERVER' );
 &my_die("cannot find NEB_SERVER in site configuration", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) if !$nebulousServer;
@@ -111 +119 @@
 }
 
-&my_die("replace not allowed for non-nebulous files", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR) 
+&my_die("replace not allowed for non-nebulous files", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR)
     if ($replace eq "T") and !$nebulousInput;
 
@@ -155 +163 @@
     $bimage = $backup_path_base . ".fits";
 } elsif ($stage eq "chip") {
+    # Check to see if we're using dynamic masks
+    my $dynamicMasks;               # Use dynamic masks?
+    {
+        # Get the PSASTRO recipe
+        my $command = "$ppConfigDump -camera $camera -recipe PSASTRO $recipe_psastro -dump-recipe PSASTRO -";
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform ppConfigDump: $error_code", $magic_ds_id, $component,
+                    $PS_EXIT_CONFIG_ERROR);
+        }
+        my $recipeData = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse metadata config doc", $magic_ds_id, $component,
+                    $PS_EXIT_CONFIG_ERROR);
+
+        $dynamicMasks = metadataLookupBool($recipeData, 'REFSTAR_MASK');
+    }
+
     # we use the mask output from the camera stage for input and replace
     # the output of the chip stage with that mask as well.
     $image  = $ipprc->filename("PPIMAGE.CHIP", $path_base, $class_id);
-    $mask   = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
-    $ch_mask= $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
     $weight = $ipprc->filename("PPIMAGE.CHIP.VARIANCE", $path_base, $class_id);
+
+    if ($dynamicMasks) {
+        $mask = $ipprc->filename("PSASTRO.OUTPUT.MASK", $cam_path_base, $class_id);
+        $ch_mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
+    } else {
+        $mask = $ipprc->filename("PPIMAGE.CHIP.MASK", $path_base, $class_id);
+    }
 
     $bimage  = $ipprc->filename("PPIMAGE.CHIP", $backup_path_base, $class_id);
@@ -191 +223 @@
 }
 
-revert_files($image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
-
-if ($stage eq "diff") {
+revert_files($replace, $image, $mask, $weight, $sources, $astrom, $bimage, $bmask, $bweight, $bsources, $bastrom);
+
+if ($stage eq "diff" and $bothways) {
     my $name = "PPSUB.INVERSE";
     $image  = $ipprc->filename($name, $path_base);
@@ -203 +235 @@
     $bweight = $ipprc->filename("$name.VARIANCE", $backup_path_base);
     $bsources = $ipprc->filename("$name.SOURCES", $backup_path_base);
-    revert_files($image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef);
+    revert_files($replace, $image, $mask, $weight, $sources, undef, $bimage, $bmask, $bweight, $bsources, undef);
 }
 
 # now revert the row in the database
 {
-    my $command = "$magicdstool -revertdestreakedfile";
+    my $command = "$magicdstool -revertdestreakedfile -i_am_sure";
     $command   .= " -magic_ds_id $magic_ds_id";
     $command   .= " -component $component";
@@ -231 +263 @@
 
 sub revert_files {
+    my $replace = shift;
+    return if !$replace;
+
     my $image = shift;
     my $mask = shift;
@@ -249 +284 @@
     if ($mask) {
         if (!revert_file($mask, $bmask)) {
-            if ($stage eq 'chip') {
-                # don't fail if the mask file (from the camera stage) fails to revert.
-                # It probably doesn't exist
-                # XXX: Handle this properly
-                print STDERR "failure to revert mask file, ignored\n";
-            } else {
-                &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
-            }
+            &my_die("failed to restore mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
         }
     }
@@ -264 +292 @@
             &my_die("failed to restore chip mask file", $magic_ds_id, $component, $PS_EXIT_CONFIG_ERROR);
         }
-    } 
+    }
 
 
@@ -291 +319 @@
         run(command => $command, verbose => $verbose);
 
+    if (!defined $error_code) {
+        print STDERR "run( $command ) returned undef error_code!!!!!\n";
+        return $PS_EXIT_UNKNOWN_ERROR;
+    }
     return $error_code >> 8;
 }
@@ -298 +330 @@
     my $original = shift;
     my $backup   = shift;
+    my $force = 1; # force deletion of backup files in nebulous
 
     my $o_path = $ipprc->file_resolve($original);
@@ -323 +356 @@
 
     if ($backup_is_not_destreaked) {
+        if ($original_is_not_destreaked) {
+            # this is unexpected result. throw an error so this can be checked manually
+            print STDERR "both files appear to not be destreaked\n";
+            print STDERR "original: $original\n";
+            print STDERR "backup:   $backup\n";
+            my $o_basename = basename($o_path);
+            my $b_basename = basename($b_path);
+            if (($o_basename =~ /SR_/) and !($b_basename =~ /SR_/)) {
+                print " basenames are as expected it is safe to revert\n";
+            } else {
+                return 0;
+            }
+        }
         # XXX TODO if stage is raw, check that backup has the correct size and md5sum
 
+        if ($verbose) {
+            print "ready to swap $backup\n";
+            print "           to $original\n";
+        }
         # Do we need to make this test? After the swap we're going to delete the file anyways
-        print "ready to swap $backup to $original\n" if $verbose;
 
         if (! $nebulous->swap($backup, $original)) {
-            print "failed to swap $backup to $original\n";
+            print "failed to swap $backup\n";
+            print "            to $original\n";
             return 0;
         }
 
-        if ($b_path and -e $b_path) {
+        if ($b_path) {
             print "ready to delete backup\n" if $verbose;
-            if (! $ipprc->file_delete($backup)) {
+            if (! $ipprc->file_delete($backup, $force)) {
                 print "failed to delete $backup\n";
                 return 0;
@@ -344 +394 @@
         print "original uri: $original is not a destreaked file no need to swap backup_result: $backup_result\n";
         # delete the 'backup' (destreaked target) file if it exists
-        if ($b_path and -e $b_path) {
-            if (! $ipprc->file_delete($backup)) {
+        if ($b_path) {
+            if (! $ipprc->file_delete($backup, $force)) {
                 print "failed to delete $backup\n";
                 return 0;
@@ -386 +436 @@
     $exit_code = $PS_EXIT_PROG_ERROR unless defined $exit_code;
 
-    # we wouldn't be here unless an entry already exists in the database
-#    my $command = "$magicdstool -adddestreakedfile";
-#    $command   .= " -magic_ds_id $magic_ds_id";
-#    $command   .= " -component $component";
-#    $command   .= " -fault $exit_code";
-#    $command   .= " -dbname $dbname" if defined $dbname;
-
-#    # Add the processed file to the database
-#    unless ($no_update) {
-#        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-#            run(command => $command, verbose => $verbose);
-#        unless ($success) {
-#            carp("failed to update database for $magic_ds_id $component");
-#        }
-#    } else {
-#        print "Skipping command: $command\n";
-#    }
-
-    carp($msg);
+    # fault the whole run if one of the components fails to revert
+    my $command = "$magicdstool -updaterun";
+    $command   .= " -magic_ds_id $magic_ds_id";
+    $command   .= " -set_state failed_revert";
+    $command   .= " -dbname $dbname" if defined $dbname;
+
+    # Add the processed file to the database
+    unless ($no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            carp("failed to update database for $magic_ds_id $component");
+        }
+    } else {
+        print "Skipping command: $command\n";
+    }
+
     exit $exit_code;
 }