Changeset 27840 for branches/simtest_nebulous_branches/ippScripts/scripts/magic_process.pl

Timestamp:

May 3, 2010, 8:50:52 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simtest_nebulous_branches

Files:

• . (modified) (1 prop)
• ippScripts/scripts/magic_process.pl (modified) (12 diffs)

branches/simtest_nebulous_branches/ippScripts/scripts/magic_process.pl

@@ -8 +8 @@
 use Sys::Hostname;
 my $host = hostname();
+my $date = `date`;
 print "\n\n";
-print "Starting script $0 on $host\n\n";
+print "Starting script $0 on $host at $date\n\n";
 
 use vars qw( $VERSION );
@@ -30 +31 @@
 my $missing_tools;
 my $magictool      = can_run('magictool') or (warn "Can't find magictool" and $missing_tools = 1);
+my $difftool       = can_run('difftool') or (warn "Can't find difftool" and $missing_tools = 1);
+my $ppSubConvolve = can_run('ppSubConvolve') or (warn "Can't find ppSubConvolve" and $missing_tools = 1);
 my $detectstreaks = can_run('DetectStreaks') or (warn "Can't find DetectStreaks" and $missing_tools = 1);
+my $VerifyStreaks = can_run('VerifyStreaks') or (warn "Can't find VerifyStreaks, will not produce png images");
 if ($missing_tools) {
     warn("Can't find required tools.");
@@ -37 +41 @@
 
 # Parse the command-line arguments
-my ($magic_id, $node, $camera, $dbname, $outroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
+my ($magic_id, $node, $camera, $dbname, $baseroot, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
 GetOptions(
@@ -44 +48 @@
            'camera=s'        => \$camera,     # Camera name
            'dbname=s'        => \$dbname,     # Database name
-           'outroot=s'       => \$outroot,    # Output root name
+           'baseroot=s'      => \$baseroot,   # Output root name
            'save-temps'      => \$save_temps, # Save temporary files?
            'verbose'         => \$verbose,    # Print stuff?
@@ -53 +57 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --magic_id --camera --node --outroot",
+pod2usage( -msg => "Required options: --magic_id --camera --node --baseroot",
            -exitval => 3) unless
     defined $magic_id and
     defined $node and
     defined $camera and
-    defined $outroot;
+    defined $baseroot;
 
 my $ipprc = PS::IPP::Config->new( $camera ) or my_die( "Unable to set up", $magic_id, $node, $PS_EXIT_CONFIG_ERROR ); # IPP configuration
 $ipprc->redirect_output($logfile) or my_die( "Unable to redirect output", $magic_id, $node, $PS_EXIT_SYS_ERROR ) if $logfile;
 
-# DetectStreaks doesn't know about nebulous. It expects to be able to append strings to outroot
-# to form valid file names.
-# So forbid nebulous path in outroot. We could relax this by change DetectStreaks to take all
-# of the file names as arguments or by teaching it about Nebulous
-if ($outroot =~ 'neb:/') {
+# DetectStreaks doesn't know about nebulous. It expects to be able to
+# append strings to baseroot to form valid file names.  So forbid
+# nebulous path in baseroot. We could relax this by change
+# DetectStreaks to take all of the file names as arguments or by
+# teaching it about Nebulous
+if ($baseroot =~ 'neb:/') {
     &my_die("DetectStreaks does not support nebulous paths in outroot", $magic_id, $node, $PS_EXIT_CONFIG_ERROR);
 }
 
+# most filenames are of the form $baseroot.$node.*, but VerifyStreaks
+# needs access to $baseroot.*, so we construct $outroot =
+# $baseroot.$node in here
+
 # resolve any path:// or file:// in outroot
-$outroot = $ipprc->file_resolve($outroot);
+$baseroot = $ipprc->file_resolve($baseroot);
+my $outroot = "$baseroot.$node";
+$ipprc->outroot_prepare($outroot);
 
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
@@ -103 +114 @@
     $command .= " --verbose" if $verbose;
 
-    # added per Paul Sydney's request Jan 6, 2009
-    # removed per Paul Sydney March 30, 2009
-    # $command .= " -D 3";
-
-
-    # added per Paul Sydney's Jan 8, 2009 to help solve the STS problem
-    # removed because it didn't help the STS problem and it quadrupled the number of false streaks for m31
-    # Paul asked me to remove this parameter Jan 20,2009
-#    $command .= " -C 0";
-
-
-#    $command .= " -u 6";
-#    $command .= " --test";
-
+    ### To enable debugging output:
+    #    $command .= " --test";
+
+    ### per email from Paul Sydney 2010.02.11, we should use threshold of 2.35 to catch the faint streak(s)
+    $command .= " --threshold 2.35";
+
+    my @deletions;          # Files to delete
     if (scalar @$inputs == 1 and $node ne "root") {
         #
@@ -136 +140 @@
         }
 
-        my $template = resolve_template($innode);
-        &my_die("failed to resolve template", $magic_id, $node, $PS_EXIT_DATA_ERROR)
-            unless defined $template;
+        my $diff_base = $innode->{diff_path_base}; # Base name for diff
+        my $tempName = $innode->{inverse} ? "PPSUB.INPUT.CONV" : "PPSUB.REF.CONV"; # File rule of interest
+        my $template = $ipprc->file_resolve($ipprc->filename($tempName, $diff_base));
+
+        # Delete the convolved products when done (we can recreate them as we need)
+        push @deletions, $ipprc->filename($tempName . ".MASK", $diff_base);
+        push @deletions, $ipprc->filename($tempName . ".VARIANCE", $diff_base);
+
+        unless (defined $template and $ipprc->file_exists($template)) {
+            # Template doesn't exist (or can't be found); try to recreate it
+            my $tempPath = "/tmp/magic.$magic_id.$node.template";
+
+            my $kernel = $ipprc->filename("PPSUB.OUTPUT.KERNELS", $diff_base); # Name of kernel file
+            &my_die("Unable to find kernel file", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless $ipprc->file_exists($kernel);
+
+            my ($image, $mask);   # Image and mask
+            {
+                my $command = "$difftool -inputskyfile -diff_id $diff_id -skycell_id $node"; # Command to run
+                $command .= " -dbname $dbname" if defined $dbname;
+                if ($innode->{inverse}) {
+                    # Want the input because we're magicking the reference
+                    $command .= " -input";
+                } else {
+                    # Want the reference because we're magicking the input
+                    $command .= " -template";
+                }
+
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to determine convolution inputs: $error_code", $magic_id, $node, $error_code);
+                }
+
+                my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+                    &my_die("Unable to parse metadata config doc", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+
+                my $inputs = parse_md_list($metadata) or
+                    &my_die("Unable to parse metadata list", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+                &my_die("Unexpected number of outputs", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless scalar @$inputs == 1;
+                my $input = $$inputs[0];
+                my $path = $input->{path_base}; # Path of interest
+                if (defined $input->{warp_id} and $input->{warp_id} > 0) {
+                    $image = $ipprc->filename("PSWARP.OUTPUT", $path);
+                    $mask = $ipprc->filename("PSWARP.OUTPUT.MASK", $path);
+                } elsif (defined $input->{stack_id} and $input->{stack_id} > 0) {
+                    $image = $ipprc->filename("PPSTACK.UNCONV", $path);
+                    $mask = $ipprc->filename("PPSTACK.UNCONV.MASK", $path);
+                }
+                &my_die("Unable to determine image and mask name", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless defined $image and defined $mask;
+            }
+
+            {
+                &my_die("Unable to find image and mask", $magic_id, $node, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($image) and $ipprc->file_exists($mask);
+
+                my $command = "$ppSubConvolve $tempPath -image $image -mask $mask -kernel $kernel";
+                $command .= " -reference" unless $innode->{inverse};
+                my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                    run(command => $command, verbose => $verbose);
+                unless ($success) {
+                    $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                    &my_die("Unable to create template image: $error_code", $magic_id, $node, $error_code);
+                }
+
+                $template = $ipprc->filename("PPSUB.INPUT.CONV", $tempPath) or &my_die("Unable to determine filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+                $template = $ipprc->file_resolve($template) or &my_die("Unable to resolve filename for created template", $magic_id, $node, $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to find created template", $magic_id, $node, $PS_EXIT_PROG_ERROR) unless $ipprc->file_exists($template);
+            }
+        }
+
+        &my_die("Cannot find template", $magic_id, $node, $PS_EXIT_DATA_ERROR) unless defined $template and $ipprc->file_exists($template);
+        push @deletions, $template;
 
         $command .= " --detect --image $image --mask $mask --weight $weight -k $template";
-
-        # XXX: get all of these parameters from a recipe
-
-        # set threshold to 2.3 sigma (Default specified by Paul Sydney)
-        $command .= ' -t 2.3';
-
-        # set threshold to 2.75 sigma (Test by EAM : uncorrelated noise is typically ~20% higher than expected)
-        # XXX we have fixed the output variance to address this
-        # $command .= ' -t 2.75';
-
-        # set threshold to 4 sigma (for testing)
-        # $command .= ' -t 4';
-
-        # set Minimum percent visible required between pixels clusters in a valid streak [0.0 - 1.0]
-        # default is 0.2
-        $command .= ' -P 0.2';
-
-        #$command .= ' -t 4' . ' -P 0.25';
-
-        # add -S "Number of pixels to shift for determining maximum differences in mean images default:8"
-        $command .= ' -S';
-
-        # add 3 x 3 binning
-        $command .= ' -b 3';
 
         # create the list of inputs used at this stage. At higher levels the
@@ -248 +297 @@
             file_check( $output );
         }
+
+        foreach my $file (@deletions) {
+            print "Deleting $file...\n";
+            $ipprc->file_delete($file);
+        }
+
     } else {
         print "Skipping command: $command\n";
     }
+
 }
 
@@ -290 +346 @@
     }
 
+    &run_verifystreaks($baseroot);
+
     my $command = "$magictool -addmask";
     $command   .= " -magic_id $magic_id";
@@ -310 +368 @@
 }
 
-
-
 ### Pau.
+
+sub run_verifystreaks {
+
+    my $baseroot = shift;
+
+    unless ($VerifyStreaks) {
+        print STDERR "skipping VerifyStreaks\n";
+        return 1;
+    }
+
+    # VerifyStreaks --out $outdir --clusters $outdir/clusters.list $rootname.root.streakMap
+
+    my $outdir = "$baseroot.verify";
+
+    my($status) = system ("mkdir -p $outdir");
+    if ($status) {
+        print STDERR "failed to create output directory $outdir\n";
+        return 1;
+    }
+
+    my $FILE;
+
+    my @files = <$baseroot.*.clusters>;
+
+    unless (open ($FILE, ">$outdir/clusters.list")) {
+        print "failed to create cluster file $outdir/clusters.list\n";
+        return 1;
+    }
+    foreach my $file (@files) {
+        $file =~ s|.clusters$||;
+        print $FILE "$file\n";
+    }
+    close ($FILE);
+    if ($status) {
+        print "failed to create cluster file $outdir/clusters.list\n";
+        return 1;
+    }
+
+    my $command = "$VerifyStreaks --out $outdir --clusters $outdir/clusters.list $baseroot.root.streakMap";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        print "failed to run VerifyStreaks:\n";
+        return 1;
+    }
+
+    return 0;
+}
 
 sub open_list_file {
@@ -363 +468 @@
 }
 
-sub resolve_template
-{
-    my $node = shift;
-
-    my $path_base = $node->{diff_path_base}; # Base name for name
-    my $image = $node->{inverse} ? "PPSUB.INPUT.CONV" : "PPSUB.REF.CONV"; # File rule of interest
-
-    $image = $ipprc->file_resolve($ipprc->filename($image, $path_base));
-
-    return $image
-}
-
 sub file_check
 {
@@ -397 +490 @@
         $command .= " -node $node";
         $command .= " -fault $exit_code";
+        $command .= " -path_base $outroot" if defined $outroot;
         $command .= " -dbname $dbname" if defined $dbname;
         system($command);