Changeset 30049 for trunk/ippTools/src/regtool.c

Timestamp:

Dec 15, 2010, 1:29:28 PM (15 years ago)

Author:

watersc1

Message:

Merge of realtime burntool and new nightly science code into trunk.

Location:

trunk

Files:

• . (modified) (1 prop)
• ippTools/src (modified) (1 prop)
• ippTools/src/regtool.c (modified) (8 diffs)

trunk/ippTools/src/regtool.c

@@ -34 +34 @@
 static bool pendingimfileMode(pxConfig *config);
 static bool addprocessedimfileMode(pxConfig *config);
+static bool checkburntoolimfileMode(pxConfig *config);
+static bool pendingburntoolimfileMode(pxConfig *config);
 static bool processedimfileMode(pxConfig *config);
 static bool revertprocessedimfileMode(pxConfig *config);
@@ -72 +74 @@
         // imfile
         MODECASE(REGTOOL_MODE_PENDINGIMFILE,         pendingimfileMode);
+        MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE,   checkburntoolimfileMode);
+        MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode);
         MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE,    addprocessedimfileMode);
         MODECASE(REGTOOL_MODE_PROCESSEDIMFILE,       processedimfileMode);
@@ -176 +180 @@
 }
 
+static bool checkburntoolimfileMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config,false);
+
+  // required
+  PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false);
+  PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
+  PXOPT_LOOKUP_STR(date,     config->args, "-date", true, false);
+  PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false);
+  // optional
+  PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false);
+  PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false);
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  psString query = pxDataGet("regtool_checkburntoolimfile.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+    return false;
+  }
+  psString rep = psStringCopy(query);
+  psFree(query);
+  query = rep;
+  
+  psStringSubstitute(&query,exp_name,"@EXP_NAME@");
+  psStringSubstitute(&query,class_id,"@CLASS_ID@");
+  psStringSubstitute(&query,date,"@DATE@");
+
+  fprintf(stderr,"%s",query);
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
+    psError(PXTOOLS_ERR_PROG, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+  
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+    
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("regtool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  psString previous_uri = NULL;
+  bool ok_to_burn = true;
+  bool already_burned = true;
+  for (long i = 0; i < output->n; i++) {
+    psMetadata *row = output->data[i];
+
+    if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
+        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
+      ok_to_burn = false;
+    }
+    if (already_burned == false) {
+      ok_to_burn = false;
+    }
+    if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) {
+      already_burned = true;
+    }
+    else {
+      already_burned = false;
+    }
+    
+    if (previous_uri) {
+      psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
+      psFree(previous_uri);
+    }
+    psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn);
+    psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned);
+    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri"));
+  }
+  
+  // negate simple so the default is true
+  if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) {
+    psError(PXTOOLS_ERR_PROG, false, "failed to print array");
+    psFree(output);
+    return false;
+  }
+  
+  psFree(output);
+  
+
+  return(true);
+  
+}
+
+static bool pendingburntoolimfileMode(pxConfig *config)
+{
+  PS_ASSERT_PTR_NON_NULL(config,false);
+
+  // required
+  PXOPT_LOOKUP_STR(date,     config->args, "-date", true, false);
+  PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false);
+  // optional
+  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+  PXOPT_LOOKUP_BOOL(ignore_state, config->args, "-ignore_state", false);
+  psString query = pxDataGet("regtool_pendingburntoolimfile.sql");
+  if (!query) {
+    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
+    return false;
+  }
+  psString rep = psStringCopy(query);
+  psFree(query);
+  query = rep;
+  
+  psStringSubstitute(&query,date,"@DATE@");
+
+  fprintf(stderr,"%s",query);
+
+  if (!p_psDBRunQuery(config->dbh, query)) {
+    // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
+    psError(PXTOOLS_ERR_PROG, false, "database error");
+    psFree(query);
+    return false;
+  }
+  psFree(query);
+  
+  psArray *output = p_psDBFetchResult(config->dbh);
+  if (!output) {
+    psErrorCode err = psErrorCodeLast();
+    switch (err) {
+    case PS_ERR_DB_CLIENT:
+      psError(PXTOOLS_ERR_SYS, false, "database error");
+    case PS_ERR_DB_SERVER:
+      psError(PXTOOLS_ERR_PROG, false, "database error");
+    default:
+      psError(PXTOOLS_ERR_PROG, false, "unknown error");
+    }
+    
+    return false;
+  }
+  if (!psArrayLength(output)) {
+    psTrace("regtool", PS_LOG_INFO, "no rows found");
+    psFree(output);
+    return true;
+  }
+
+  psString previous_uri = NULL;
+  psString this_uri = NULL;
+  bool ok_to_burn = true;
+  bool already_burned = true;
+  psString previous_class_id = NULL;
+  psString this_class_id = NULL;
+  psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool.
+  
+  for (long i = 0; i < output->n; i++) {
+    psMetadata *row = output->data[i];
+
+    // Add the information about this row and the previous, if it exists.
+    // Write the class_id stuff for debugging.
+    this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
+    psStringSubstitute(&this_class_id,"XY","ota");
+    psMetadataAddStr(row,PS_LIST_TAIL,"this_class_id",PS_META_REPLACE,"",this_class_id);
+    if (previous_class_id) {
+      psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id);
+    }
+    
+    // Determine if we've crossed a class_id boundary, as this resets the bits.
+    if (previous_class_id) {
+      if (strcmp(this_class_id,previous_class_id) != 0) {
+        ok_to_burn = true;
+        already_burned = true;
+        previous_uri = NULL;
+      }
+    }
+
+    // Write the URIs as well.
+    this_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Duplicate, but helpful for my debugging.
+    psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri); 
+    if (previous_uri) {
+      psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
+    }
+
+    // Check the two status variables
+    // Convert bits of the SQL query into booleans describing the data state
+    if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
+        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
+      ok_to_burn = false;
+    }
+    if (already_burned == false) {
+      ok_to_burn = false;
+    }
+    if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) {
+      already_burned = true;
+    }
+    else {
+      already_burned = false;
+    }
+    psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn);
+    psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned);
+
+    // Check the uri for this exposure
+    if (!this_uri) {
+        ok_to_burn = false;
+        already_burned = false;
+
+        // Save this round for next round.
+        psFree(previous_class_id);
+        psFree(previous_uri);
+        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
+        psStringSubstitute(&previous_class_id,"XY","ota");
+        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
+        continue;
+    }      
+
+    //    printf("STATUS: %s %s %s %s (%d %d)\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned);
+
+    // If the state of this imfile is not "pending_burntool" then we can't burn it. 
+    if (!ignore_state) {
+      psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state");
+
+      if (!imfile_state) { // imfile state is NULL, so we probably aren't registered.
+        ok_to_burn = false;
+
+        // Save this round for next round.
+        psFree(previous_class_id);
+        psFree(previous_uri);
+        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
+        psStringSubstitute(&previous_class_id,"XY","ota");
+        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
+        continue;
+      }
+      if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states  
+        // Save this round for next round.
+        psFree(previous_class_id);
+        psFree(previous_uri);
+        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
+        psStringSubstitute(&previous_class_id,"XY","ota");
+        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
+        continue;
+      }
+    }
+    
+    // Determine if we've already suggested an entry for this ota, and if not, copy this
+    // suggestion into our output result list.
+    if (!ok_to_burn || already_burned) {
+      // Save this round for next round.
+      psFree(previous_class_id);
+      psFree(previous_uri);
+      previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
+      psStringSubstitute(&previous_class_id,"XY","ota");
+      previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
+      continue;
+    }
+    // If we're here, then we think we could potentially burntool this file.
+    psArrayAdd(results,results->n,row);
+
+    // Save this round for next round.
+    psFree(previous_class_id);
+    psFree(previous_uri);
+    previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
+    psStringSubstitute(&previous_class_id,"XY","ota");
+    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.         
+  }
+  
+  // negate simple so the default is true
+  if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) {
+    psError(PXTOOLS_ERR_PROG, false, "failed to print array");
+    psFree(output);
+    psFree(results);
+    return false;
+  }
+  
+  psFree(output);
+  psFree(results);
+  
+
+  return(true);
+  
+}
+
+      
 
 static bool addprocessedimfileMode(pxConfig *config)
@@ -250 +539 @@
     PXOPT_LOOKUP_S32(bytes, config->args,  "-bytes", false, false);
     PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false);
-
+    PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false);
+    
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
@@ -323 +613 @@
         bytes,
         md5sum,
+        video_cells,
         0   // burntool_state
     )) {
@@ -809 +1100 @@
     PXOPT_LOOKUP_F32(moon_phase, config->args, "-moon_phase", false, false);
     PXOPT_LOOKUP_STR(label,  config->args, "-label", false, false);
+    PXOPT_LOOKUP_STR(data_group,  config->args, "-data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group,  config->args, "-dist_group", false, false);
+    PXOPT_LOOKUP_STR(chip_workdir,config->args, "-chip_workdir", false, false);
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
 
@@ -1008 +1302 @@
     // else continue on...
 
-
+    chip_workdir = chip_workdir ? chip_workdir : workdir;
     // insert an entry into the chipPendingExp table
     // this can only be run as the newExp's state has been set to stop
     if (!pxchipQueueByExpTag(config,
                 exp_id,
-                workdir,
+                             chip_workdir,
                 label,
-                NULL,       // data_group
-                NULL,       // dist_group
+                             data_group,
+                             dist_group,
                 reduction,
                 NULL,       // expgroup
@@ -1092 +1386 @@
     PXOPT_COPY_F64(config->args,   where,  "-posang_min", "posang", ">=");
     PXOPT_COPY_F64(config->args,   where,  "-posang_max", "posang", "<");
-    PXOPT_COPY_STR(config->args,   where,  "-object", "object", "==");
+    PXOPT_COPY_STR(config->args,   where,  "-object", "object", "LIKE");
+    PXOPT_COPY_STR(config->args,   where,  "-obs_mode", "obs_mode", "LIKE");
+    PXOPT_COPY_STR(config->args,   where,  "-comment", "comment", "LIKE");
     PXOPT_COPY_F32(config->args,   where,  "-sun_angle_min", "sun_angle", ">=");
     PXOPT_COPY_F32(config->args,   where,  "-sun_angle_max", "sun_angle", "<");