Changeset 30183

Timestamp:

Dec 26, 2010, 7:02:49 AM (15 years ago)

Author:

eugene

Message:

merge fixes from trunk

Location:

tags/ipp-20101215

Files:

• ippScripts/scripts/register_imfile.pl (modified) (10 diffs, 1 prop)
• ippTasks/nightly_science.pro (modified) (1 diff, 1 prop)
• ippTasks/register.pro (modified) (2 diffs, 1 prop)
• ippTools/share/regtool_pendingburntoolimfile.sql (modified) (1 diff, 1 prop)
• ippTools/src/regtool.c (modified) (23 diffs, 1 prop)

tags/ipp-20101215/ippScripts/scripts/register_imfile.pl

@@ -84 +84 @@
 
 my $cmdflags;
-my $burntoolStateGood;
-my $burntoolState;
+my $burntoolStateTarget;
+my $burntoolStateCurrent;
 # Run ppStats on the input file
 {
@@ -116 +116 @@
 
     # Manually parse the burntool_state entry.
-    $burntoolState = 0;
+    $burntoolStateCurrent = 0;
     my $isGPC1 = 0;
-    $burntoolStateGood = 0;
+    $burntoolStateTarget = 0;
     foreach my $line (split /\n/, $out1) {
         if ($line =~ /FPA.BURNTOOL.APPLIED/) {
             $line =~ s/^\s+//;
-            $burntoolState = (split /\s+/, $line)[2];
+            $burntoolStateCurrent = (split /\s+/, $line)[2];
         }
         if ($line =~ /FPA.CAMERA/) {
@@ -132 +132 @@
     }
     if ($isGPC1 != 1) {
-        $burntoolState = 0; # If it's not GPC1, you shouldn't have run burntool.
-    }
-    elsif (($isGPC1 == 1) && ($burntoolState == 1)) {
-#       print STDERR "In the good region: >>$burntoolState<<\n";
+        $burntoolStateCurrent = 0; # If it's not GPC1, you shouldn't have run burntool.
+    }
+    elsif (($isGPC1 == 1) && ($burntoolStateCurrent == 1)) {
+#       print STDERR "In the good region: >>$burntoolStateCurrent<<\n";
         my $ppConfigDump_cmd = "$ppConfigDump -camera GPC1 -dump-camera -";
         my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
@@ -149 +149 @@
             if ($line =~ /BURNTOOL.STATE.GOOD/) {
                 $line =~ s/^\s+//;
-                $burntoolStateGood = (split /\s+/, $line)[2];
+                $burntoolStateTarget = (split /\s+/, $line)[2];
                 last;
             }
         }
-        $burntoolState = $burntoolStateGood; # Positive because this has the header table.
-
-    }
-    $cmdflags .= " -burntool_state $burntoolState ";
+        # XXX why was this being equated??
+        # $burntoolState = $burntoolStateGood; # Positive because this has the header table.
+
+    }
+    $cmdflags .= " -burntool_state $burntoolStateTarget ";
 }
 
 $now_time = localtime();
 printf STDERR "\ndone with ppStats: %s\n", $now_time if $verbose;
+printf STDERR "\nburntool state current: %d target: %d\n", $burntoolStateCurrent, $burntoolStateTarget;
 
 # we require at a minimum: -telescope, -inst, -filelevel, -class_id, -exp_type
@@ -178 +180 @@
 $command .= " -hostname $host" if defined $host;
 $command .= " -dbname $dbname" if defined $dbname;
-if (abs($burntoolState) == $burntoolStateGood) {
+if (abs($burntoolStateCurrent) == $burntoolStateTarget) {
     $command .= " -data_state full";
 }
@@ -222 +224 @@
 # lock file?
 # if exp_type = DARK and date > MIDNIGHT HST { wait }
-# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log"); 
+# system("ipp_apply_burntool.pl --class_id $class_id --dateobs_begin $date_start --dateobs_end $date_end --dbname gpc1 --logfile /data/$host.0/burntool_logs/$class_id.$start_date.log");
 
 $now_time = localtime();
@@ -242 +244 @@
 # We now have an imfile in the database, check if we can burntool it.  If not, continue on.
 
-if (abs($burntoolState) != $burntoolStateGood) { 
+if (abs($burntoolStateCurrent) != $burntoolStateTarget) {
     my $mdcParser  = PS::IPP::Metadata::Config->new;
 
@@ -249 +251 @@
     $check_date =~ s/T.*$//;
     my $exp_name   = $tmp_exp_name;
-    
+
 
     my $bt_check_command = "$regtool -checkburntoolimfile ";
     $bt_check_command .= " -class_id $class_id ";
     $bt_check_command .= " -date $check_date ";
-    $bt_check_command .= " -valid_burntool $burntoolStateGood ";
+    $bt_check_command .= " -valid_burntool $burntoolStateTarget ";
     $bt_check_command .= " -exp_name $exp_name ";
     $bt_check_command .= " -dbname $dbname" if defined $dbname;
 
     my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run ( command => $bt_check_command, verbose => $verbose);
+        run ( command => $bt_check_command, verbose => $verbose);
     unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die_for_update("Unable to perform regtool: $error_code", 
-                           $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR);
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die_for_update("Unable to perform regtool: $error_code",
+                           $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR);
     }
 
@@ -270 +272 @@
     my $exposures;
     while ( scalar @whole > 0 ) {
-        my $value = shift @whole;
-        push @single, $value;
-        if ($value =~ /^\s*END\s*$/) {
-            push @single, "\n";
-            
-            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
-            &my_die_for_update("Unable to parse output from regtool", 
-                               $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR) unless
-                                   defined $list;
-            push @{ $exposures }, @$list;
-            @single = ();
-        }
-    }
-    
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) );
+            &my_die_for_update("Unable to parse output from regtool",
+                               $exp_id, $exp_name, $class_id, $PS_EXIT_SYS_ERROR) unless
+                                   defined $list;
+            push @{ $exposures }, @$list;
+            @single = ();
+        }
+    }
+
     # We only care about the final entry, as that contains the exposure we are.
-    
+
     my $regtool_update = "$regtool -updateprocessedimfile ";
     $regtool_update .= "-dbname $dbname " if defined $dbname;
@@ -292 +294 @@
     my $burntool_data = pop(@{ $exposures });
     if ($burntool_data->{burnable} == 0) {
-        $regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
-        unless ($no_update) {
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                warn ("Unable to perform regtool -addprocessedimfile: $error_code");
-                exit($error_code);
-            }
-        } else {
-            print "skipping command: $command\n";
-        }
+        $regtool_update .= " -burntool_state 0 -set_state pending_burntool ";
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                IPC::Cmd::run(command => $regtool_update, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                warn ("Unable to perform regtool -addprocessedimfile: $error_code");
+                exit($error_code);
+            }
+        } else {
+            print "skipping command: $command\n";
+        }
     }
     else {
-        my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname ";
-        $apply_command .= " --class_id $class_id --exp_id $exp_id ";
-        $apply_command .= " --this_uri $burntool_data->{uri} ";
-        $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
-        $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
-        $apply_command .= " --verbose " if $verbose;
-        print "$apply_command\n";
-        unless ($no_update) {
-            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-                IPC::Cmd::run(command => $apply_command, verbose => $verbose);
-            unless ($success) {
-                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-                warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
-                exit($error_code);
-            }
-        }
-    }    
+        my $apply_command = "$ippApplyBurntoolSingle --dbname $dbname ";
+        $apply_command .= " --class_id $class_id --exp_id $exp_id ";
+        $apply_command .= " --this_uri $burntool_data->{uri} ";
+        $apply_command .= " --previous_uri $burntool_data->{previous_uri} " if defined $burntool_data->{previous_uri};
+        $apply_command .= " --imfile_state $burntool_data->{imfile_state} ";
+        $apply_command .= " --verbose " if $verbose;
+        print "$apply_command\n";
+        unless ($no_update) {
+            my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+                IPC::Cmd::run(command => $apply_command, verbose => $verbose);
+            unless ($success) {
+                $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+                warn ("Unable to perform ipp_apply_burntool_single.pl: $error_code");
+                exit($error_code);
+            }
+        }
+    }
 
 

tags/ipp-20101215/ippTasks/nightly_science.pro

@@ -132 +132 @@
 #
 macro ns.add.date
+  if ($0 != 2) 
+    echo "USAGE: ns.add.date (YYYY-MM-DD)"
+    break
+  end
+
    book newpage nsData $1
    book setword nsData $1 nsState NEW

tags/ipp-20101215/ippTasks/register.pro

@@ -79 +79 @@
 macro register.revert.off
   task register.imfile.revert
+    active false
+  end
+end
+
+macro burntool.on
+  task register.burntool.load
+    active true
+  end
+  task register.burntool.run
+    active true
+  end
+end
+
+macro burntool.off
+  task register.burntool.load
+    active false
+  end
+  task register.burntool.run
     active false
   end
@@ -529 +547 @@
     $today = `date -u +%Y-%m-%d`
 # debugging purposes
-#    $today = "2010-12-06"
+#   $today = "2010-12-25"
     $run = $run -date $today -valid_burntool $valid_burntool_value
     if ($DB:n == 0)

tags/ipp-20101215/ippTools/share/regtool_pendingburntoolimfile.sql

@@ -23 +23 @@
             LEFT JOIN pzDownloadExp USING(exp_name)
             LEFT JOIN newExp ON exp_name = tmp_exp_name
-            LEFT JOIN newImfile USING(exp_id)
+            LEFT JOIN newImfile USING(exp_id)
             LEFT JOIN rawExp USING(exp_id, exp_name)
             LEFT JOIN rawImfile ON (rawImfile.exp_name = newExp.tmp_exp_name AND rawImfile.tmp_class_id = newImfile.tmp_class_id)
        WHERE
              summitExp.dateobs >= '@DATE@T00:00:00'
-             AND summitExp.dateobs <= '@DATE@T23:59:59'
+             AND summitExp.dateobs <= '@DATE@T23:59:59'
 ORDER BY newImfile.tmp_class_id,summitExp.dateobs

tags/ipp-20101215/ippTools/src/regtool.c

@@ -74 +74 @@
         // imfile
         MODECASE(REGTOOL_MODE_PENDINGIMFILE,         pendingimfileMode);
-        MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE,   checkburntoolimfileMode);
-        MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode);
+        MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE,   checkburntoolimfileMode);
+        MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode);
         MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE,    addprocessedimfileMode);
         MODECASE(REGTOOL_MODE_PROCESSEDIMFILE,       processedimfileMode);
@@ -201 +201 @@
   psFree(query);
   query = rep;
-  
+
   psStringSubstitute(&query,exp_name,"@EXP_NAME@");
   psStringSubstitute(&query,class_id,"@CLASS_ID@");
@@ -215 +215 @@
   }
   psFree(query);
-  
+
   psArray *output = p_psDBFetchResult(config->dbh);
   if (!output) {
@@ -227 +227 @@
       psError(PXTOOLS_ERR_PROG, false, "unknown error");
     }
-    
+
     return false;
   }
@@ -243 +243 @@
 
     if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
-        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
+        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
       ok_to_burn = false;
     }
@@ -255 +255 @@
       already_burned = false;
     }
-    
+
     if (previous_uri) {
       psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
@@ -264 +264 @@
     previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri"));
   }
-  
+
   // negate simple so the default is true
   if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) {
@@ -271 +271 @@
     return false;
   }
-  
+
   psFree(output);
-  
+
 
   return(true);
-  
+
 }
 
@@ -297 +297 @@
   psFree(query);
   query = rep;
-  
+
   psStringSubstitute(&query,date,"@DATE@");
 
-  //  fprintf(stderr,"%s",query);
+  // fprintf(stderr,"%s",query);
 
   if (!p_psDBRunQuery(config->dbh, query)) {
@@ -309 +309 @@
   }
   psFree(query);
-  
+
   psArray *output = p_psDBFetchResult(config->dbh);
   if (!output) {
@@ -321 +321 @@
       psError(PXTOOLS_ERR_PROG, false, "unknown error");
     }
-    
+
     return false;
   }
+
+  // fprintf (stderr, "found %ld rows\n", output->n);
   if (!psArrayLength(output)) {
     psTrace("regtool", PS_LOG_INFO, "no rows found");
@@ -337 +339 @@
   psString this_class_id = NULL;
   psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool.
-  
+
   for (long i = 0; i < output->n; i++) {
     psMetadata *row = output->data[i];
@@ -356 +358 @@
     if (previous_class_id) {
       if (strcmp(this_class_id,previous_class_id) != 0) {
-        ok_to_burn = true;
-        already_burned = true;
-        previous_uri = NULL;
+        ok_to_burn = true;
+        already_burned = true;
+        previous_uri = NULL;
       }
     }
@@ -364 +366 @@
     // Write the URIs as well.
     this_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Duplicate, but helpful for my debugging.
-    psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); 
+    psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri);
     if (previous_uri) {
       psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
@@ -372 +374 @@
     // Convert bits of the SQL query into booleans describing the data state
     if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
-        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
-      //      printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id);
-      ok_to_burn = false;
+        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
+        // printf("I claim this isn't downloaded or registered? %s %s\n",this_uri,this_class_id);
+        ok_to_burn = false;
     }
     if (already_burned == false) {
-      //      printf("already_burned looks false %s %s\n",this_uri,this_class_id);
-      ok_to_burn = false;
+        // printf("already_burned looks false %s %s\n",this_uri,this_class_id);
+        ok_to_burn = false;
     }
     if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) {
@@ -391 +393 @@
     // Check the uri for this exposure
     if (!this_uri) {
-        ok_to_burn = false;
-        already_burned = false;
-
-        // Save this round for next round.
-        psFree(previous_class_id);
-        psFree(previous_uri);
-        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
-        psStringSubstitute(&previous_class_id,"XY","ota");
-        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
-        continue;
-    }      
-
-    //    printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded"));
-
-    // If the state of this imfile is not "pending_burntool" then we can't burn it. 
+        ok_to_burn = false;
+        already_burned = false;
+
+        // Save this round for next round.
+        psFree(previous_class_id);
+        psFree(previous_uri);
+        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
+        psStringSubstitute(&previous_class_id,"XY","ota");
+        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
+        continue;
+    }
+
+    // printf("STATUS: %s %s %s %s (%d %d) %d %d %d\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned,psMetadataLookupS32(NULL,row,"burntool_state"),psMetadataLookupS32(NULL,row,"is_registered"),psMetadataLookupS32(NULL,row,"is_downloaded"));
+
+    // If the state of this imfile is not "pending_burntool" then we can't burn it.
     if (!ignore_state) {
       psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state");
 
       if (!imfile_state) { // imfile state is NULL, so we probably aren't registered.
-        ok_to_burn = false;
-
-        // Save this round for next round.
-        psFree(previous_class_id);
-        psFree(previous_uri);
-        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
-        psStringSubstitute(&previous_class_id,"XY","ota");
-        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
-        continue;
+        ok_to_burn = false;
+
+        // Save this round for next round.
+        psFree(previous_class_id);
+        psFree(previous_uri);
+        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
+        psStringSubstitute(&previous_class_id,"XY","ota");
+        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
+        continue;
       }
-      if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states  
-        // Save this round for next round.
-        psFree(previous_class_id);
-        psFree(previous_uri);
-        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
-        psStringSubstitute(&previous_class_id,"XY","ota");
-        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
-        continue;
+      if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states
+        // Save this round for next round.
+        psFree(previous_class_id);
+        psFree(previous_uri);
+        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
+        psStringSubstitute(&previous_class_id,"XY","ota");
+        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
+        continue;
       }
     }
-    
+
     // Determine if we've already suggested an entry for this ota, and if not, copy this
     // suggestion into our output result list.
@@ -439 +441 @@
       previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
       psStringSubstitute(&previous_class_id,"XY","ota");
-      previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
+      previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
       continue;
     }
@@ -450 +452 @@
     previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     psStringSubstitute(&previous_class_id,"XY","ota");
-    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.         
-  }
-  
+    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.
+  }
+
   // negate simple so the default is true
   if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) {
@@ -460 +462 @@
     return false;
   }
-  
+
   psFree(output);
   psFree(results);
-  
+
 
   return(true);
-  
-}
-
-      
+
+}
+
+
 
 static bool addprocessedimfileMode(pxConfig *config)
@@ -545 +547 @@
     PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false);
     PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false);
-    
+
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
@@ -618 +620 @@
         bytes,
         md5sum,
-        video_cells,
+        video_cells,
         0   // burntool_state
     )) {
@@ -806 +808 @@
     PXOPT_LOOKUP_STR(set_state, config->args, "-set_state", false, false);
 
-    if ((fault == INT16_MAX) && !isfinite(burntool_state) && !(set_state)) {
+    if ((fault == INT16_MAX) && (burntool_state == INT16_MAX) && !set_state) {
         psError(PS_ERR_UNKNOWN, false, "one of -fault or -burntool_state or -set_state must be selected");
         return false;
     }
-    if ((fault != INT16_MAX) && isfinite(burntool_state)) {
+    if ((fault != INT16_MAX) && (burntool_state != INT16_MAX)) {
         psError(PS_ERR_UNKNOWN, false, "only one of -fault or -burntool_state must be selected");
         return false;
@@ -1312 +1314 @@
     if (!pxchipQueueByExpTag(config,
                 exp_id,
-                             chip_workdir,
+                             chip_workdir,
                 label,
-                             data_group,
-                             dist_group,
+                             data_group,
+                             dist_group,
                 reduction,
                 NULL,       // expgroup