Changeset 42848 for trunk/tools

Timestamp:

May 9, 2025, 2:04:40 PM (12 months ago)

Author:

eugene

Message:

rawfix / burntool updates

Location:

trunk/tools/eam/rawfix.20230221/src

Files:

• check.rawfix.sh (modified) (1 diff)
• ipp_apply_burntool_psvideo.pl (modified) (9 diffs)
• rawfix.burntool.pt (modified) (4 diffs)
• rawfix.fixchip.pt (modified) (1 diff)

trunk/tools/eam/rawfix.20230221/src/check.rawfix.sh

@@ -75 +75 @@
   if ($mystate == "burntool.new")    goto good_state
   if ($mystate == "burntool.done")   goto good_state
+  if ($mystate == "burntool.redo")   goto good_state
   if ($mystate == "burntool.fail")   goto good_state
 

trunk/tools/eam/rawfix.20230221/src/ipp_apply_burntool_psvideo.pl

@@ -4 +4 @@
 # for all exposures for a given time period.
 # USAGE: ipp_apply_burntool.pl --dbname gpc1 --class_id XY00 --dateobs_begin YYYY/MM/DD --dateobs_end YYYY/MM/DD
+# OPTION: --check-status : scan the exposures/chips and report if the desired burntool file exists
 
 use warnings;
@@ -22 +23 @@
 print "Starting script $0 on $host at $date\n\n";
 
-my ( $class_id, $dateobs_begin, $dateobs_end, $convert, $dbname, $logfile, $verbose, $save_temps, $rerun, $psvidOutput);
+my ( $class_id, $dateobs_begin, $dateobs_end, $convert, $dbname, $logfile, $check_status, $verbose, $save_temps, $rerun, $psvidOutput);
 GetOptions(
     'class_id=s'        => \$class_id, # chip identifier
@@ -28 +29 @@
     'dateobs_end=s'     => \$dateobs_end, # exposure date/time range stop
     'dbname|d=s'        => \$dbname, # Database name
-    'psvidOutput|d=s'   => \$psvidOutput, # output base for psvideo data
+    'psvidOutput=s'     => \$psvidOutput, # output base for psvideo data
     'logfile=s'         => \$logfile,
     'rerun'             => \$rerun,
+    'check-status'      => \$check_status,
     'verbose'           => \$verbose,   # Print to stdout
     'save-temps'        => \$save_temps, # Save temporary files?
-
     ) or pod2usage( 2 );
 
@@ -74 +75 @@
 my $burntoolStateGood = 15;
 if (0) { 
+    # XXX note: I am disabling this automatic check of the burntool value
+    # and hard-wiring 15
     # Determine the value of a "good" burntool run.
     # XXX this program should always generate the latest version
@@ -154 +157 @@
     my $state = $file->{burntool_state};
 
+    my $exp_name = $file->{exp_name};
+    if ($exp_name =~ /a$/) {
+        print "STRK: skipping $exp_name, streak exposure\n";
+        next;
+    }
+
     my $rawImfile = $file->{uri};
     my $rawImfileReal = $ipprc->file_resolve($rawImfile, 0);
@@ -161 +170 @@
     }
 
+    if ($rawImfileReal =~ /fits.fz$/) {
+        print "COMP: skipping $exp_name, lossy-compressed data\n";
+        next;
+    }
+
     my $outTable  = $file->{uri};
     
     my $data_state = $file->{data_state};
     if ($data_state eq "lossy") {
-        print "## skipping $rawImfile : data_state $data_state\n";
-        next;
-    }
-
-    if($burntoolStateGood == 15) {
+        print "SKIP: skipping $rawImfile : data_state $data_state\n";
+        next;
+    }
+
+    if ($burntoolStateGood == 15) {
         $outTable =~ s/fits$/burn.v15.tbl/;         
     } else {
-        $outTable =~ s/fits$/burn.tbl/;             
-    }
+        # $outTable =~ s/fits$/burn.tbl/;           
+        die "invalid burntoolStateGood $burntoolStateGood\n";
+    }
+
+    if ($check_status) {
+        my $outTableReal = $ipprc->file_resolve($outTable, 0);
+        if (defined $outTableReal and ($state == $outState)) {
+            print "GOOD: $outTableReal for $exp_id, $class_id, $state\n";
+        } else {
+            if (defined $outTableReal) {
+                print "FAIL: $outTableReal for $exp_id, $class_id, $state\n";
+            } else {
+                print "MISS: $rawImfile for $exp_id, $class_id, $state\n";
+            }
+        }
+        next;
+    }
+
     my $outTableReal = $ipprc->file_resolve($outTable, 1);
 
@@ -222 +252 @@
         $status = vsystem ("$funpack -S $rawImfileReal > $tempPixels", $REALRUN);
         if ($status) {
-            print "FAIL: funpack fails for $rawImfile\n";
-            &my_die("failed on funpack",$exp_id,$class_id);
+
+            ## funpack can fail in two specific ways that might be recoverable:
+            ## 1) the file is corrupt (bad MD5SUM)
+            ## 2) the file is missing
+
+            ## in either case, IFF a file exists in the other location with the right MD5SUM, we can use it instead
+
+            my $nebInstances = $ipprc->find_instances($rawImfile) or my_die("error with find_instance");
+            my $md5tru = $file->{md5sum};
+            my $isBad = 1;
+            my $goodInstance = "NONE";
+
+            foreach my $uri (@$nebInstances) {
+                # strip the leading file://
+                $uri =~ s|file://||;
+                
+                # get the md5sum
+                if (-e $uri) {
+                    my $fh;
+                    open($fh,$uri);
+                    my $md5sum = Digest::MD5->new->addfile($fh)->hexdigest;
+                    close ($fh);
+                    if ($md5sum eq $md5tru) { 
+                        print "uri: $uri $md5sum eq $md5tru\n";
+                        $status = vsystem ("$funpack -S $uri > $tempPixels", $REALRUN);
+                        if ($status) {
+                            print "FALT: funpack failed for $rawImfile and $uri\n";
+                            &my_die("failed on funpack",$exp_id,$class_id);
+                        } else {
+                            $isBad = 0;
+                            $goodInstance = $uri;
+                        }
+                    } else {
+                        print "MDBD: $uri $md5sum vs $md5tru\n";
+                    }
+                } else {
+                    print "GONE: $uri NOTFOUND\n";
+                }
+            }
+
+            if ($isBad) {
+                print "FAIL: funpack failed for $rawImfile, not recovered\n";
+                &my_die("failed on funpack",$exp_id,$class_id);
+            } else {
+                print "FIXT: funpack failed for $rawImfile on $rawImfileReal but recovered $goodInstance\n";
+            }
         }
 
@@ -261 +335 @@
 
         print "\n";
+    } else {
+        print "DONE: skipping $rawImfile : btState $state vs $burntoolStateGood : data_state $data_state\n";
     }
 
@@ -275 +351 @@
 }
 
+print STDERR "exit SUCCESS\n";
 exit 0;
 

trunk/tools/eam/rawfix.20230221/src/rawfix.burntool.pt

@@ -82 +82 @@
     # save the page so we can use it below
 
+    substr $DATEOBS 0 4 YEAR
+
     # write an entry point in the log for this night
-    sprintf tmpline "%s/%s/log.burntool.runhosts.v%d" $mypath $DATEOBS $ITER
+    sprintf tmpline "%s/%s/%s/log.burntool.runhosts.v%d" $mypath $YEAR $DATEOBS $ITER
     date -var mydate
-    exec echo "# start runhosts $mydate" >> $tmpline
+    exec echo "\\# start runhosts $mydate" >> $tmpline
 
     # get_hosts_fixchip.sh selects the most used machine at ITC
@@ -180 +182 @@
     # if we are able to run the 'exec', use a short retry time to launch all outstanding jobs
     periods -exec 0.05
-    command ipp_apply_burntool_psvideo.pl --dateobs_begin $STARTDATE --dateobs_end $ENDDATE --class_id $CHIP_ID --dbname gpc1 --psvidOutput $psvidOutput
+    command ipp_apply_burntool_psvideo.pl --rerun --dateobs_begin $STARTDATE --dateobs_end $ENDDATE --class_id $CHIP_ID --dbname gpc1 --psvidOutput $psvidOutput
+    ## NOTE the rerun option is set here
   end
 
@@ -270 +273 @@
 ######################### UTILITIES #################################
 
+macro burntool.loadhosts.state
+  if ($0 != 2)
+    echo "USAGE: burntool.loadhosts.state (state)"
+    break
+  end
+
+  task nights.burntool.loadhosts
+    active $1
+  end
+end
+
+macro burntool.clean.state
+  if ($0 != 2)
+    echo "USAGE: burntool.clean.state (state)"
+    break
+  end
+
+  task nights.burntool.clean
+    active $1
+  end
+end
+
+macro burntool.run.state
+  if ($0 != 2)
+    echo "USAGE: burntool.run.state (state)"
+    break
+  end
+
+  task nights.burntool.run
+    active $1
+  end
+end
+
 macro nights.burntool.exit
   if ($0 != 2)
@@ -279 +315 @@
   book getword burntool.book $options:1 NHOST -var Nhosts
   book getword burntool.book $options:1 NDONE -var Ndone
-  book getword burntool.book $options:1 stage -var ITER
+  book getword burntool.book $options:1 iter  -var ITER
   $Ndone ++
   book setword burntool.book $options:1 NDONE $Ndone

trunk/tools/eam/rawfix.20230221/src/rawfix.fixchip.pt

@@ -91 +91 @@
     sprintf tmpline "%s/%s/%s/log.fixchip.runhosts.v%d" $mypath $YEAR $DATEOBS $ITER
     date -var mydate
-    exec echo "# start runhosts $mydate" >> $tmpline
+    exec echo "\\# start runhosts $mydate" >> $tmpline
 
     # the command returns these fields: dateobs stage fixhost chipID