Changeset 27537 for branches/pap_delete/ippScripts/scripts/magic_process.pl

Timestamp:

Mar 30, 2010, 3:45:08 PM (16 years ago)

Author:

Paul Price

Message:

Branching to branches/pap_delete for development of deletion of convolved diffs.

• ppSub will now only write the convolved images if explicitly requested on the command-line.
• diff_skyfile.pl only requests convolved images for warps (because we don't need to run magic on stacks).
• magic_process.pl deletes the convolved images when done.
• If magic_process.pl can't find the convolved image to use (e.g., it's already been deleted because we're re-running magic), then it is created on the fly using ppSubConvolve and the previously calculated convolution kernel.

Location:

branches/pap_delete

Files:

• . (copied) (copied from trunk )
• ippScripts/scripts/magic_process.pl (modified) (5 diffs)

branches/pap_delete/ippScripts/scripts/magic_process.pl

@@ -30 +30 @@
 my $missing_tools;
 my $magictool      = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $difftool       = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $ppSubConvolve = can_run('ppSubConvolve') or (warn "Can't find ppSubConvolve" and $missing_tools = 1);
 my $detectstreaks = can_run('DetectStreaks') or (warn "Can't find DetectStreaks" and $missing_tools = 1);
 my $VerifyStreaks = can_run('VerifyStreaks') or (warn "Can't find VerifyStreaks, will not produce png images");
@@ -116 +118 @@
     $command .= " --threshold 2.35";
 
+    my @deletions;          # Files to delete
     if (scalar @$inputs == 1 and $node ne "root") {
         #
@@ -135 +138 @@
         }
 
-        my $template = resolve_template($innode);
-        &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR)
-            unless defined $template;
+        my $diff_base = $innode->{diff_path_base}; # Base name for diff
+        my $tempName = $innode->{inverse} ? "PPSUB.INPUT.CONV" : "PPSUB.REF.CONV"; # File rule of interest
+        my $template = $ipprc->file_resolve($ipprc->filename($tempName, $diff_base));
+        &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless defined $template;
+
+        # Delete the convolved products when done (we can recreate them as we need)
+        push @deletions, $template;
+        push @deletions, $ipprc->filename($tempName . ".MASK", $diff_base);
+        push @deletions, $ipprc->filename($tempName . ".VARIANCE", $diff_base);
+
+        unless ($ipprc->file_exists($template)) {
+            # Template doesn't exist (or can't be found); try to recreate it
+            my $tempPath = "/tmp/magic.$magic_id.$node.template";
+
+            my $kernel = $ipprc->filename("PPSUB.OUTPUT.KERNELS", $diff_base); # Name of kernel file
+            &my_die("Unable to find kernel file", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless $ipprc->file_exists($kernel);
+
+            my ($image, $mask);   # Image and mask
+            {
+                my $command = "$difftool -inputskyfile -diff_id $diff_id -skycell_id $node"; # Command to run
+                $command .= " -dbname $dbname" if defined $dbname;
+                if ($node->{inverse}) {
+                    # Want the input because we're magicking the reference
+                    $command .= " -input";
+                } else {
+                    # Want the reference because we're magicking the input
+                    $command .= " -template";
+                }
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to determine convolution inputs: $error_code", $magic_id, $node, $error_code);
+                }
+
+                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata config doc", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+
+                my $inputs = parse_md_list($metadata) or
+                    &my_die("Unable to parse metadata list", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+                &my_die("Unexpected number of outputs", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless scalar @$inputs == 1;
+                my $input = $$inputs[0];
+                my $path = $input->{path_base}; # Path of interest
+                if (defined $input->{warp_id} and $input->{warp_id} > 0) {
+                    $image = $ipprc->filename("PSWARP.OUTPUT", $path);
+                    $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path);
+                } elsif (defined $input->{stack_id} and $input->{stack_id} > 0) {
+                    $image = $ipprc->filename("PPSTACK.UNCONV", $path);
+                    $mask = $ipprc->filename("PPSTACK.UNCONV.MASK", $path);
+                }
+                &my_die("Unable to determine image and mask name", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless defined $image and defined $mask;
+            }
+
+            {
+                &my_die("Unable to find image and mask", $magic_id, $node, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image) and $ipprc->file_exists($mask);
+
+                my $command = "$ppSubConvolve $tempPath -image $image -mask $mask -kernel $kernel";
+                $command .= " -reference" unless $node->{inverse};
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to create template image: $error_code", $magic_id, $node, $error_code);
+                }
+
+                $template = $ipprc->filename("PPSUB.INPUT.CONV", $tempPath) or &my_die("Unable to determine filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+                $template = $ipprc->file_resolve($template) or &my_die("Unable to resolve filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to find created template", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($template);
+                push @deletions, $template;
+            }
+        }
 
         $command .= " --detect --image $image --mask $mask --weight $weight -k $template";
@@ -226 +298 @@
         print "Skipping command: $command\n";
     }
+
+    foreach my $file (@deletions) {
+        $ipprc->file_delete($file);
+    }
 }
 
@@ -387 +463 @@
 }
 
-sub resolve_template
-{
-    my $node = shift;
-
-    my $path_base = $node->{diff_path_base}; # Base name for name
-    my $image = $node->{inverse} ? "PPSUB.INPUT.CONV" : "PPSUB.REF.CONV"; # File rule of interest
-
-    $image = $ipprc->file_resolve($ipprc->filename($image, $path_base));
-
-    return $image
-}
-
 sub file_check
 {