Changeset 27840 for branches/simtest_nebulous_branches/ippScripts/scripts/chip_imfile.pl

Timestamp:

May 3, 2010, 8:50:52 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simtest_nebulous_branches

Files:

• . (modified) (1 prop)
• ippScripts/scripts/chip_imfile.pl (modified) (10 diffs)

branches/simtest_nebulous_branches/ippScripts/scripts/chip_imfile.pl

@@ -8 +8 @@
 use Sys::Hostname;
 my $host = hostname();
+my $date = `date`;
 print "\n\n";
-print "Starting script $0 on $host\n\n";
+print "Starting script $0 on $host at $date\n\n";
 
 use DateTime;
@@ -19 +20 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use PS::IPP::Metadata::Config;
+use PS::IPP::Metadata::List qw( parse_md_list );
 use PS::IPP::Config 1.01 qw( :standard );
 use File::Temp qw( tempfile );
@@ -52 +54 @@
     'reduction=s'       => \$reduction, # Reduction class
     'run-state=s'       => \$run_state, # current state of the run (new, update)
-    'magicked'          => \$magicked,  # magicked state of input file
+    'magicked=s'        => \$magicked,  # magicked state of input file
     'deburned=s'        => \$deburned,  # does deburned image exist?
     'threads=s'         => \$threads,   # Number of threads to use for ppImage
@@ -133 +135 @@
     $command = "$ppConfigDump -camera $camera -dump-recipe PPIMAGE -recipe PPIMAGE $recipe_ppImage -";
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
+        run(command => $command, verbose => 0);
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
@@ -143 +145 @@
     ## XXX make the feature more general?
     my $useDeburnedImage = metadataLookupBool($recipeData, 'USE.DEBURNED.IMAGE');
-    if ($useDeburnedImage && $deburned) {
-        $uri =~ s/fits$/burn.fits/;
-        &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($uri);
+
+    if ($useDeburnedImage) {
+        my $useBestBurntool  = metadataLookupBool($recipeData, 'USE.BEST.BURNTOOL');
+
+        ## Check that we have required programs:
+#       print STDERR "Inside burntool loop!\n";
+        my $regtool  = can_run('regtool') or &my_die ("Can't find regtool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        my $funpack  = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        my $burntool = can_run('burntool') or &my_die ("Can't find burntool", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+
+        ## Check the current burntool processing version:
+        my $regtool_state_cmd = "$regtool -processedimfile -exp_id $exp_id -class_id $class_id -limit 1";
+        if (defined($dbname)) {
+            $regtool_state_cmd .= " -dbname $dbname";
+        }
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $regtool_state_cmd, verbose => $verbose);
+
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform regtool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+
+        my $regData = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse regtool metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        my $regData2 = parse_md_list($regData);
+
+        my $burntoolState = 999;
+        foreach my $regEntry (@$regData2) {
+           print "$regEntry->{exp_id} $regEntry->{burntool_state}\n";
+
+            if ($regEntry->{exp_id} == $exp_id) {
+                $burntoolState = $regEntry->{burntool_state};
+            }
+        }
+        if ($burntoolState == 999) {
+            &my_die("Unable to find burntool_state in metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+        if ($burntoolState == 32767) {
+            $burntoolState = 0;
+        }
+
+        ## Read camera config to get the current good burntool state :
+        ## XXX This is extremely slow. Any better way to do this?
+        my $camera_config_cmd = "$ppConfigDump -camera $camera -dump-camera -";
+        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $camera_config_cmd, verbose => 0);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform ppConfigDump: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+
+        my $camData = $mdcParser->parse(join "", @$stdout_buf) or
+            &my_die("Unable to parse ppConfigDump metadata", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+
+        my $burntoolStateGood;
+        foreach my $camEntry (@$camData) {
+            if ($camEntry->{name} eq "BURNTOOL.STATE.GOOD") {
+                $burntoolStateGood = $camEntry->{value};
+            }
+        }
+
+        print "burntool state vs burntoolStateGood : $burntoolState vs $burntoolStateGood\n";
+        if (abs($burntoolState) != $burntoolStateGood) {
+            if ($useBestBurntool) {
+                &my_die("Image burntool version does not match current accepted version.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+            else {
+                warn ("Image burntool version does not match current accepted version. Continuing as requested.");
+                # Internally pretend that we're at a good version.
+                if ($burntoolState < -10) {
+                    $burntoolState = -1 * $burntoolStateGood;
+                }
+                elsif ($burntoolState > 10) {
+                    $burntoolState = $burntoolStateGood;
+                }
+                elsif ($burntoolState == 0) {
+                    # You've told me to use a deburned image, and that you don't care if it's the most recent. The database has told me
+                    # that burntool has never been run on this image. We'll hope the database is wrong, but I don't really trust that.
+                    warn ("burntool_state suggests no table will be found. This will likely crash.");
+                    $burntoolState = -1 * $burntoolStateGood;
+                }
+                else {
+                    &my_die("No valid burntool table will be found.", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+                }
+            }
+        }
+
+        ## We now know that we have an image that has been burntooled.
+        my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.deburned.XXXX",
+                                              UNLINK => !$save_temps, SUFFIX => '.fits' );
+
+        # get the UNIX version of the (possible) neb: or path: filename
+        my $uriReal = $ipprc->file_resolve( $uri );
+
+        # funpack into the temp file.
+        my $funpack_cmd = "$funpack -S $uriReal > $tempName";
+        ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $funpack_cmd, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+
+        ## Construct commands to apply the pre-calculated burntool trailfits to the pixel data.
+        my ($burntoolTable_uri, $burntoolTable_uriReal);
+        my $burntool_cmd = "$burntool ";
+
+        if ($burntoolState == -1 * $burntoolStateGood) {   # Burntool information stored in an external table.
+            $burntoolTable_uri = $uri;
+            $burntoolTable_uri =~ s/fits$/burn.tbl/;
+            $burntoolTable_uriReal = $ipprc->file_resolve( $burntoolTable_uri );
+            unless ($ipprc->file_exists($burntoolTable_uri)) {
+                &my_die("Couldn't find burntool table: $burntoolTable_uri",$exp_id,$chip_id,$class_id, $PS_EXIT_SYS_ERROR);
+            }
+
+            $burntool_cmd .= "$tempName in=${burntoolTable_uriReal} persist=t apply=t";
+        }
+        elsif ($burntoolState == $burntoolStateGood) { # Burntool information stored in a header table.
+            $burntool_cmd .= "$tempName persist=t apply=t";
+        }
+        else {
+            &my_die("Image data not properly burntooled, impossible state", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+
+        ## Run burntool to change the pixels of tempfile, and repoint $uri to that file.
+        ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf) =
+            run(command => $burntool_cmd, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            &my_die("Unable to perform burntool: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+
+        $uri = $tempName;
+        unless ($ipprc->file_exists($uri)) {
+            &my_die("Couldn't find deburned input file: $uri\n", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+        }
+
+#       print STDERR "$uri $uriReal $tempName $burntoolTable_uri $burntoolTable_uriReal $burntool_cmd $save_temps\n";
+#       exit(100);
     }
 
@@ -152 +290 @@
     if ($tiltystreakApply) {
 
-        my $tiltystreakByClass = metadataLookupMD($recipeData, 'TILTYSTREAK.BY.CLASS');
-        my $tiltystreakClassApply = metadataLookupBool($tiltystreakByClass, "APPLY.$class_id");
-        if ($tiltystreakClassApply) {
-
-            my $tiltystreakClassOptions = metadataLookupStr($tiltystreakByClass, "OPTIONS.$class_id");
-            if ($tiltystreakClassOptions eq "none") {
-                $tiltystreakClassOptions = "";
-            }
-
-            my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
-
-            # XXX make these optional (must be built by psbuild as well...)
-            my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-            my $tiltystreak = can_run('tiltystreak') or &my_die ("Can't find tiltystreak", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-
-            # create an temporary output file:
-            my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.tmp.XXXX", UNLINK => !$save_temps );
-
-            print "uri: $uri\n";
-
-            # get the UNIX version of the (possible) neb: or path: filename
-            my $uriReal = $ipprc->file_resolve( $uri );
-
-            print "uriReal: $uriReal\n";
-
-            # unpack the data (is a NOP if not compressed)
-            $command = "$funpack -S $uriReal > $tempName";
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-            }
-
-            # run tiltystreak
-            $command = "$tiltystreak $tempName $tiltystreakClassOptions";
-            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                &my_die("Unable to perform tiltystreak: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
-            }
-
-            # supply the output file as the new input file
-            $uri = $tempName;
-        }
+        my $tiltystreakByClass = metadataLookupMD($recipeData, 'TILTYSTREAK.BY.CLASS');
+        my $tiltystreakClassApply = metadataLookupBool($tiltystreakByClass, "APPLY.$class_id");
+        if ($tiltystreakClassApply) {
+
+            my $tiltystreakClassOptions = metadataLookupStr($tiltystreakByClass, "OPTIONS.$class_id");
+            if ($tiltystreakClassOptions eq "none") {
+                $tiltystreakClassOptions = "";
+            }
+
+            my ($success, $error_code, $full_buf, $stdout_buf, $stderr_buf);
+
+            # XXX make these optional (must be built by psbuild as well...)
+            my $funpack = can_run('funpack') or &my_die ("Can't find funpack", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            my $tiltystreak = can_run('tiltystreak') or &my_die ("Can't find tiltystreak", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+
+            # create an temporary output file:
+            my ($tempFile, $tempName) = tempfile( "/tmp/chip.$exp_id.$class_id.tmp.XXXX", UNLINK => !$save_temps );
+
+            print "uri: $uri\n";
+
+            # get the UNIX version of the (possible) neb: or path: filename
+            my $uriReal = $ipprc->file_resolve( $uri );
+
+            print "uriReal: $uriReal\n";
+
+            # unpack the data (is a NOP if not compressed)
+            $command = "$funpack -S $uriReal > $tempName";
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform funpack: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+
+            # run tiltystreak
+            $command = "$tiltystreak $tempName $tiltystreakClassOptions";
+            ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) = run(command => $command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                &my_die("Unable to perform tiltystreak: $error_code", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR);
+            }
+
+            # supply the output file as the new input file
+            $uri = $tempName;
+        }
     }
 
@@ -239 +377 @@
     if ($do_stats) {
         my $outputStatsReal = $ipprc->file_resolve($outputStats);
-        &my_die("Couldn't find expected output file: $outputStats", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless -f $outputStatsReal;
+        &my_die("Couldn't find expected output file: $outputStats", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputStatsReal);
 
         # measure chip stats
@@ -253 +391 @@
         }
         chomp $cmdflags;
-        if (($camera ne "GPC1") or ($class_id ne "XY27") or ($exp_id < 53171)) {
-            ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
-        } else {
-            # hack to kick ota 27 out of subsequent analysis by setting quality flag to a non-zero value
-            my ($before, $after) = split " -quality ", $cmdflags;
-
-            # get the current value
-            ($quality) = $after =~ /^(\d+)/;
-            $after = substr($after, length($quality));
-
-            # replace it if it is zero
-            $quality = 42 if !$quality;
-
-            # rebuild the cmdflags
-            $cmdflags = $before . " -quality $quality";
-            $cmdflags .= " $after" if $after;
-        }
+        ($quality) = $cmdflags =~ /-quality (\d+)/; # Quality flag
     }
 
@@ -278 +400 @@
         &my_die("Couldn't find expected output file: $outputBin1\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin1);
         &my_die("Couldn't find expected output file: $outputBin2\n",   $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($outputBin2);
+        if ($run_state eq 'new') {
+            &my_die("Couldn't find expected output file: $configuration", $exp_id, $chip_id, $class_id, $PS_EXIT_SYS_ERROR) unless $ipprc->file_exists($configuration);
+        }
     }
 
@@ -291 +416 @@
     $command .= " -uri $outputImage";
     $command .= " -path_base $outroot";
-    $command .= " -magicked" if $magicked;
+    $command .= " -magicked $magicked" if $magicked;
     $command .= " -hostname $host" if defined $host;
     $command .= " -dbname $dbname" if defined $dbname;