Changeset 27596 for trunk

Timestamp:

Apr 5, 2010, 10:16:10 AM (16 years ago)

Author:

Paul Price

Message:

Merging branches/pap_delete/ into trunk after successful testing. This provides automatic deletion of the convolved images from the diff stage.

Location:

trunk

Files:

• . (modified) (1 prop)
• Ohana/src/relastro (modified) (1 prop)
• ippScripts/scripts/diff_skycell.pl (modified) (4 diffs)
• ippScripts/scripts/magic_process.pl (modified) (6 diffs)
• ippScripts/scripts/magic_tree.pl (modified) (1 diff)
• ippconfig/recipes/ppSub.config (modified) (1 diff)
• ppSub/src/ppSub.h (modified) (1 diff)
• ppSub/src/ppSubArguments.c (modified) (2 diffs)
• ppSub/src/ppSubCamera.c (modified) (5 diffs)
• ppSub/src/ppSubData.c (modified) (1 diff)

trunk/ippScripts/scripts/diff_skycell.pl

@@ -102 +102 @@
 my $camera;                     # Camera
 my ($inputMagic, $templateMagic); # Are the inputs been magicked?
+my ($saveInConv, $saveRefConv);   # Save the input or reference convolved images?
 # Prescan to decide if this is or is not a stack stack diff. The check above confirms we only have two entries.
 if ((${ $files }[0]->{warp_id} == 0)&&
@@ -140 +141 @@
             $templateSources = "PSWARP.OUTPUT.SOURCES";
             $templateMagic = $file->{magicked};
+            $saveRefConv = 1;
         }
     } else {
@@ -162 +164 @@
             $inputVariance = "PSWARP.OUTPUT.VARIANCE";
             $inputSources = "PSWARP.OUTPUT.SOURCES";
+            $saveInConv = 1;
         }
     }
@@ -284 +287 @@
         $command .= " -ipprc $configurationReal";
     }
+    $command .= " -save-inconv" if defined $saveInConv;
+    $command .= " -save-refconv" if defined $saveRefConv;
     $command .= " -recipe PPSUB $recipe_ppSub";
     $command .= " -recipe PSPHOT $recipe_psphot";

trunk/ippScripts/scripts/magic_process.pl

@@ -30 +30 @@
 my $missing_tools;
 my $magictool      = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $difftool       = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $ppSubConvolve = can_run('ppSubConvolve') or (warn "Can't find ppSubConvolve" and $missing_tools = 1);
 my $detectstreaks = can_run('DetectStreaks') or (warn "Can't find DetectStreaks" and $missing_tools = 1);
 my $VerifyStreaks = can_run('VerifyStreaks') or (warn "Can't find VerifyStreaks, will not produce png images");
@@ -80 +82 @@
 $baseroot = $ipprc->file_resolve($baseroot);
 my $outroot = "$baseroot.$node";
+$ipprc->outroot_prepare($outroot);
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -116 +119 @@
     $command .= " --threshold 2.35";
 
+    my @deletions;          # Files to delete
     if (scalar @$inputs == 1 and $node ne "root") {
         #
@@ -135 +139 @@
         }
 
-        my $template = resolve_template($innode);
-        &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR)
-            unless defined $template;
+        my $diff_base = $innode->{diff_path_base}; # Base name for diff
+        my $tempName = $innode->{inverse} ? "PPSUB.INPUT.CONV" : "PPSUB.REF.CONV"; # File rule of interest
+        my $template = $ipprc->file_resolve($ipprc->filename($tempName, $diff_base));
+        &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless defined $template;
+
+        # Delete the convolved products when done (we can recreate them as we need)
+        push @deletions, $template;
+        push @deletions, $ipprc->filename($tempName . ".MASK", $diff_base);
+        push @deletions, $ipprc->filename($tempName . ".VARIANCE", $diff_base);
+
+        unless ($ipprc->file_exists($template)) {
+            # Template doesn't exist (or can't be found); try to recreate it
+            my $tempPath = "/tmp/magic.$magic_id.$node.template";
+
+            my $kernel = $ipprc->filename("PPSUB.OUTPUT.KERNELS", $diff_base); # Name of kernel file
+            &my_die("Unable to find kernel file", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless $ipprc->file_exists($kernel);
+
+            my ($image, $mask);   # Image and mask
+            {
+                my $command = "$difftool -inputskyfile -diff_id $diff_id -skycell_id $node"; # Command to run
+                $command .= " -dbname $dbname" if defined $dbname;
+                if ($innode->{inverse}) {
+                    # Want the input because we're magicking the reference
+                    $command .= " -input";
+                } else {
+                    # Want the reference because we're magicking the input
+                    $command .= " -template";
+                }
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to determine convolution inputs: $error_code", $magic_id, $node, $error_code);
+                }
+
+                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata config doc", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+
+                my $inputs = parse_md_list($metadata) or
+                    &my_die("Unable to parse metadata list", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+                &my_die("Unexpected number of outputs", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless scalar @$inputs == 1;
+                my $input = $$inputs[0];
+                my $path = $input->{path_base}; # Path of interest
+                if (defined $input->{warp_id} and $input->{warp_id} > 0) {
+                    $image = $ipprc->filename("PSWARP.OUTPUT", $path);
+                    $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path);
+                } elsif (defined $input->{stack_id} and $input->{stack_id} > 0) {
+                    $image = $ipprc->filename("PPSTACK.UNCONV", $path);
+                    $mask = $ipprc->filename("PPSTACK.UNCONV.MASK", $path);
+                }
+                &my_die("Unable to determine image and mask name", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless defined $image and defined $mask;
+            }
+
+            {
+                &my_die("Unable to find image and mask", $magic_id, $node, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image) and $ipprc->file_exists($mask);
+
+                my $command = "$ppSubConvolve $tempPath -image $image -mask $mask -kernel $kernel";
+                $command .= " -reference" unless $innode->{inverse};
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to create template image: $error_code", $magic_id, $node, $error_code);
+                }
+
+                $template = $ipprc->filename("PPSUB.INPUT.CONV", $tempPath) or &my_die("Unable to determine filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+                $template = $ipprc->file_resolve($template) or &my_die("Unable to resolve filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to find created template", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($template);
+                push @deletions, $template;
+            }
+        }
 
         $command .= " --detect --image $image --mask $mask --weight $weight -k $template";
@@ -223 +296 @@
             file_check( $output );
         }
+
+        foreach my $file (@deletions) {
+            print "Deleting $file...\n";
+            $ipprc->file_delete($file);
+        }
+
     } else {
         print "Skipping command: $command\n";
     }
+
 }
 
@@ -387 +467 @@
 }
 
-sub resolve_template
-{
-    my $node = shift;
-
-    my $path_base = $node->{diff_path_base}; # Base name for name
-    my $image = $node->{inverse} ? "PPSUB.INPUT.CONV" : "PPSUB.REF.CONV"; # File rule of interest
-
-    $image = $ipprc->file_resolve($ipprc->filename($image, $path_base));
-
-    return $image
-}
-
 sub file_check
 {

trunk/ippScripts/scripts/magic_tree.pl

@@ -171 +171 @@
     if ($xi == 0 and $eta == 0) {
         $phi = 0;
-        $eta = 0;
+        $theta = 0;
     } else {
         $phi = atan2($eta,$xi) + pi/2;

trunk/ippconfig/recipes/ppSub.config

@@ -76 +76 @@
 INVERSE         BOOL    FALSE           # Generate inverse subtraction?
 PHOTOMETRY      BOOL    FALSE           # Perform photometry?
-SAVE.CONVOLVED  BOOL    TRUE            # Save convolved images?
 
 

trunk/ppSub/src/ppSub.h

@@ -45 +45 @@
     bool photometry;                    // Perform photometry?
     bool inverse;                       // Output inverse subtraction as well?
+    bool saveInConv;                    // Save convolved input?
+    bool saveRefConv;                   // Save convolved reference?
     psString stamps;                    // Stamps file
     pmPSF *psf;                         // Point Spread Function

trunk/ppSub/src/ppSubArguments.c

@@ -87 +87 @@
     psMetadataAddBool(arguments, PS_LIST_TAIL, "-photometry", 0, "Perform photometry?", NULL);
     psMetadataAddF32(arguments, PS_LIST_TAIL, "-zp", 0, "Zero point for photometry", NAN);
-    psMetadataAddBool(arguments, PS_LIST_TAIL, "-inverse", 0, "Generate inverse subtractions?", NULL);
+    psMetadataAddBool(arguments, PS_LIST_TAIL, "-inverse", 0, "Generate inverse subtractions?", false);
+    psMetadataAddBool(arguments, PS_LIST_TAIL, "-save-inconv", 0, "Save input convolved images?", false);
+    psMetadataAddBool(arguments, PS_LIST_TAIL, "-save-refconv", 0, "Save reference convolved images?", false);
     psMetadataAddBool(arguments, PS_LIST_TAIL, "-visual", 0, "Show diagnostic plots", NULL);
 
@@ -154 +156 @@
     }
 
+    data->saveInConv = psMetadataLookupBool(NULL, arguments, "-save-inconv");
+    data->saveRefConv = psMetadataLookupBool(NULL, arguments, "-save-refconv");
+
     if (psMetadataLookupBool(NULL, arguments, "-visual")) {
         pmVisualSetVisual(true);

trunk/ppSub/src/ppSubCamera.c

@@ -219 +219 @@
     data->photometry = psMetadataLookupBool(NULL, recipe, "PHOTOMETRY");
 
-    bool mdok;                          // Status of MD lookup
-    bool saveConv = psMetadataLookupBool(&mdok, recipe, "SAVE.CONVOLVED"); // Save convolved images?
-
     // Convolved input image
     pmFPAfile *inConvImage = defineOutputFile(config, input, true, "PPSUB.INPUT.CONV", PM_FPA_FILE_IMAGE);
@@ -230 +227 @@
         return false;
     }
-    if (saveConv) {
-        inConvImage->save = true;
-        inConvMask->save = true;
-    }
+    inConvImage->save = data->saveInConv;
+    inConvMask->save = data->saveInConv;
     if (inVar) {
         pmFPAfile *inConvVar = defineOutputFile(config, inConvImage, false, "PPSUB.INPUT.CONV.VARIANCE",
@@ -241 +236 @@
             return false;
         }
-        if (saveConv) {
-            inConvVar->save = true;
-        }
+        inConvVar->save = data->saveInConv;
     }
 
@@ -254 +247 @@
         return false;
     }
-    if (saveConv) {
-        refConvImage->save = true;
-        refConvMask->save = true;
-    }
+    refConvImage->save = data->saveRefConv;
+    refConvMask->save = data->saveRefConv;
     if (refVar) {
         pmFPAfile *refConvVar = defineOutputFile(config, refConvImage, false, "PPSUB.REF.CONV.VARIANCE",
@@ -265 +256 @@
             return false;
         }
-        if (saveConv) {
-            refConvVar->save = true;
-        }
+        refConvVar->save = data->saveRefConv;
     }
 

trunk/ppSub/src/ppSubData.c

@@ -33 +33 @@
     data->photometry = false;
     data->inverse = false;
+    data->saveInConv = false;
+    data->saveRefConv = false;
     data->stamps = NULL;
     data->psf = NULL;