IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 30049 for trunk/ippTools/src


Ignore:
Timestamp:
Dec 15, 2010, 1:29:28 PM (15 years ago)
Author:
watersc1
Message:

Merge of realtime burntool and new nightly science code into trunk.

Location:
trunk
Files:
8 edited

Legend:

Unmodified
Added
Removed

  • trunk

  • trunk/ippTools/src

  • trunk/ippTools/src/chiptool.c

    r29902 r30049  
    271271
    272272    // default
     273    PXOPT_LOOKUP_BOOL(unique, config->args, "-unique", false);
    273274    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
    274275    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    275 
     276   
    276277    // find the exp_id of all the exposures that we want to queue up.
    277278    psString query = pxDataGet("chiptool_find_rawexp.sql");
    … …  
    288289    psFree(where);
    289290
     291    if (unique) {
     292      psStringAppend(&query, "AND chip_id IS NULL");
     293    }
     294   
    290295    if (!p_psDBRunQuery(config->dbh, query)) {
    291296        psError(PS_ERR_UNKNOWN, false, "database error");

  • trunk/ippTools/src/chiptoolConfig.c

    r29902 r30049  
    6161    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_dist_group",  0,      "define distribution group", NULL);
    6262    psMetadataAddStr(definebyqueryArgs, PS_LIST_TAIL, "-set_note",  0,           "define note", NULL);
     63    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-unique",   0,           "only queue exposures that have no previous chipRun", false);
    6364    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-pretend",  0,            "do not actually modify the database", false);
    6465    psMetadataAddBool(definebyqueryArgs, PS_LIST_TAIL, "-simple",  0,            "use the simple output format", false);

  • trunk/ippTools/src/pztool.c

    r27053 r30049  
    112112    PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", true, false);
    113113    PXOPT_LOOKUP_STR(uri, config->args, "-uri", true, false);
    114 
     114   
    115115    if (!pzDataStoreInsert(config->dbh,
    116116            camera,

  • trunk/ippTools/src/regtool.c

    r27748 r30049  
    3434static bool pendingimfileMode(pxConfig *config);
    3535static bool addprocessedimfileMode(pxConfig *config);
     36static bool checkburntoolimfileMode(pxConfig *config);
     37static bool pendingburntoolimfileMode(pxConfig *config);
    3638static bool processedimfileMode(pxConfig *config);
    3739static bool revertprocessedimfileMode(pxConfig *config);
    … …  
    7274        // imfile
    7375        MODECASE(REGTOOL_MODE_PENDINGIMFILE,         pendingimfileMode);
     76        MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE,   checkburntoolimfileMode);
     77        MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode);
    7478        MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE,    addprocessedimfileMode);
    7579        MODECASE(REGTOOL_MODE_PROCESSEDIMFILE,       processedimfileMode);
    … …  
    176180}
    177181
     182static bool checkburntoolimfileMode(pxConfig *config)
     183{
     184  PS_ASSERT_PTR_NON_NULL(config,false);
     185
     186  // required
     187  PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false);
     188  PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
     189  PXOPT_LOOKUP_STR(date,     config->args, "-date", true, false);
     190  PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false);
     191  // optional
     192  PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false);
     193  PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false);
     194  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     195  psString query = pxDataGet("regtool_checkburntoolimfile.sql");
     196  if (!query) {
     197    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     198    return false;
     199  }
     200  psString rep = psStringCopy(query);
     201  psFree(query);
     202  query = rep;
     203 
     204  psStringSubstitute(&query,exp_name,"@EXP_NAME@");
     205  psStringSubstitute(&query,class_id,"@CLASS_ID@");
     206  psStringSubstitute(&query,date,"@DATE@");
     207
     208  fprintf(stderr,"%s",query);
     209
     210  if (!p_psDBRunQuery(config->dbh, query)) {
     211    // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
     212    psError(PXTOOLS_ERR_PROG, false, "database error");
     213    psFree(query);
     214    return false;
     215  }
     216  psFree(query);
     217 
     218  psArray *output = p_psDBFetchResult(config->dbh);
     219  if (!output) {
     220    psErrorCode err = psErrorCodeLast();
     221    switch (err) {
     222    case PS_ERR_DB_CLIENT:
     223      psError(PXTOOLS_ERR_SYS, false, "database error");
     224    case PS_ERR_DB_SERVER:
     225      psError(PXTOOLS_ERR_PROG, false, "database error");
     226    default:
     227      psError(PXTOOLS_ERR_PROG, false, "unknown error");
     228    }
     229   
     230    return false;
     231  }
     232  if (!psArrayLength(output)) {
     233    psTrace("regtool", PS_LOG_INFO, "no rows found");
     234    psFree(output);
     235    return true;
     236  }
     237
     238  psString previous_uri = NULL;
     239  bool ok_to_burn = true;
     240  bool already_burned = true;
     241  for (long i = 0; i < output->n; i++) {
     242    psMetadata *row = output->data[i];
     243
     244    if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
     245        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
     246      ok_to_burn = false;
     247    }
     248    if (already_burned == false) {
     249      ok_to_burn = false;
     250    }
     251    if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) {
     252      already_burned = true;
     253    }
     254    else {
     255      already_burned = false;
     256    }
     257   
     258    if (previous_uri) {
     259      psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
     260      psFree(previous_uri);
     261    }
     262    psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn);
     263    psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned);
     264    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri"));
     265  }
     266 
     267  // negate simple so the default is true
     268  if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) {
     269    psError(PXTOOLS_ERR_PROG, false, "failed to print array");
     270    psFree(output);
     271    return false;
     272  }
     273 
     274  psFree(output);
     275 
     276
     277  return(true);
     278 
     279}
     280
     281static bool pendingburntoolimfileMode(pxConfig *config)
     282{
     283  PS_ASSERT_PTR_NON_NULL(config,false);
     284
     285  // required
     286  PXOPT_LOOKUP_STR(date,     config->args, "-date", true, false);
     287  PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false);
     288  // optional
     289  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     290  PXOPT_LOOKUP_BOOL(ignore_state, config->args, "-ignore_state", false);
     291  psString query = pxDataGet("regtool_pendingburntoolimfile.sql");
     292  if (!query) {
     293    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     294    return false;
     295  }
     296  psString rep = psStringCopy(query);
     297  psFree(query);
     298  query = rep;
     299 
     300  psStringSubstitute(&query,date,"@DATE@");
     301
     302  fprintf(stderr,"%s",query);
     303
     304  if (!p_psDBRunQuery(config->dbh, query)) {
     305    // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
     306    psError(PXTOOLS_ERR_PROG, false, "database error");
     307    psFree(query);
     308    return false;
     309  }
     310  psFree(query);
     311 
     312  psArray *output = p_psDBFetchResult(config->dbh);
     313  if (!output) {
     314    psErrorCode err = psErrorCodeLast();
     315    switch (err) {
     316    case PS_ERR_DB_CLIENT:
     317      psError(PXTOOLS_ERR_SYS, false, "database error");
     318    case PS_ERR_DB_SERVER:
     319      psError(PXTOOLS_ERR_PROG, false, "database error");
     320    default:
     321      psError(PXTOOLS_ERR_PROG, false, "unknown error");
     322    }
     323   
     324    return false;
     325  }
     326  if (!psArrayLength(output)) {
     327    psTrace("regtool", PS_LOG_INFO, "no rows found");
     328    psFree(output);
     329    return true;
     330  }
     331
     332  psString previous_uri = NULL;
     333  psString this_uri = NULL;
     334  bool ok_to_burn = true;
     335  bool already_burned = true;
     336  psString previous_class_id = NULL;
     337  psString this_class_id = NULL;
     338  psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool.
     339 
     340  for (long i = 0; i < output->n; i++) {
     341    psMetadata *row = output->data[i];
     342
     343    // Add the information about this row and the previous, if it exists.
     344    // Write the class_id stuff for debugging.
     345    this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     346    psStringSubstitute(&this_class_id,"XY","ota");
     347    psMetadataAddStr(row,PS_LIST_TAIL,"this_class_id",PS_META_REPLACE,"",this_class_id);
     348    if (previous_class_id) {
     349      psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id);
     350    }
     351   
     352    // Determine if we've crossed a class_id boundary, as this resets the bits.
     353    if (previous_class_id) {
     354      if (strcmp(this_class_id,previous_class_id) != 0) {
     355        ok_to_burn = true;
     356        already_burned = true;
     357        previous_uri = NULL;
     358      }
     359    }
     360
     361    // Write the URIs as well.
     362    this_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Duplicate, but helpful for my debugging.
     363    psMetadataAddStr(row,PS_LIST_TAIL,"this_uri",PS_META_REPLACE,"",this_uri);
     364    if (previous_uri) {
     365      psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
     366    }
     367
     368    // Check the two status variables
     369    // Convert bits of the SQL query into booleans describing the data state
     370    if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
     371        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
     372      ok_to_burn = false;
     373    }
     374    if (already_burned == false) {
     375      ok_to_burn = false;
     376    }
     377    if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) {
     378      already_burned = true;
     379    }
     380    else {
     381      already_burned = false;
     382    }
     383    psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn);
     384    psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned);
     385
     386    // Check the uri for this exposure
     387    if (!this_uri) {
     388        ok_to_burn = false;
     389        already_burned = false;
     390
     391        // Save this round for next round.
     392        psFree(previous_class_id);
     393        psFree(previous_uri);
     394        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     395        psStringSubstitute(&previous_class_id,"XY","ota");
     396        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
     397        continue;
     398    }     
     399
     400    //    printf("STATUS: %s %s %s %s (%d %d)\n",this_uri,previous_uri,this_class_id,previous_class_id,ok_to_burn,already_burned);
     401
     402    // If the state of this imfile is not "pending_burntool" then we can't burn it.
     403    if (!ignore_state) {
     404      psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state");
     405
     406      if (!imfile_state) { // imfile state is NULL, so we probably aren't registered.
     407        ok_to_burn = false;
     408
     409        // Save this round for next round.
     410        psFree(previous_class_id);
     411        psFree(previous_uri);
     412        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     413        psStringSubstitute(&previous_class_id,"XY","ota");
     414        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
     415        continue;
     416      }
     417      if (strcmp("pending_burntool",imfile_state) != 0) { // Probably the state is full, do not twiddle states 
     418        // Save this round for next round.
     419        psFree(previous_class_id);
     420        psFree(previous_uri);
     421        previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     422        psStringSubstitute(&previous_class_id,"XY","ota");
     423        previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
     424        continue;
     425      }
     426    }
     427   
     428    // Determine if we've already suggested an entry for this ota, and if not, copy this
     429    // suggestion into our output result list.
     430    if (!ok_to_burn || already_burned) {
     431      // Save this round for next round.
     432      psFree(previous_class_id);
     433      psFree(previous_uri);
     434      previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     435      psStringSubstitute(&previous_class_id,"XY","ota");
     436      previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.       
     437      continue;
     438    }
     439    // If we're here, then we think we could potentially burntool this file.
     440    psArrayAdd(results,results->n,row);
     441
     442    // Save this round for next round.
     443    psFree(previous_class_id);
     444    psFree(previous_uri);
     445    previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     446    psStringSubstitute(&previous_class_id,"XY","ota");
     447    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri")); // Save for next round.         
     448  }
     449 
     450  // negate simple so the default is true
     451  if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) {
     452    psError(PXTOOLS_ERR_PROG, false, "failed to print array");
     453    psFree(output);
     454    psFree(results);
     455    return false;
     456  }
     457 
     458  psFree(output);
     459  psFree(results);
     460 
     461
     462  return(true);
     463 
     464}
     465
     466     
    178467
    179468static bool addprocessedimfileMode(pxConfig *config)
    … …  
    250539    PXOPT_LOOKUP_S32(bytes, config->args,  "-bytes", false, false);
    251540    PXOPT_LOOKUP_STR(md5sum, config->args, "-md5sum", false, false);
    252 
     541    PXOPT_LOOKUP_BOOL(video_cells, config->args, "-video_cells", false);
     542   
    253543    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
    254544    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
    … …  
    323613        bytes,
    324614        md5sum,
     615        video_cells,
    325616        0   // burntool_state
    326617    )) {
    … …  
    8091100    PXOPT_LOOKUP_F32(moon_phase, config->args, "-moon_phase", false, false);
    8101101    PXOPT_LOOKUP_STR(label,  config->args, "-label", false, false);
     1102    PXOPT_LOOKUP_STR(data_group,  config->args, "-data_group", false, false);
     1103    PXOPT_LOOKUP_STR(dist_group,  config->args, "-dist_group", false, false);
     1104    PXOPT_LOOKUP_STR(chip_workdir,config->args, "-chip_workdir", false, false);
    8111105    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
    8121106
    … …  
    10081302    // else continue on...
    10091303
    1010 
     1304    chip_workdir = chip_workdir ? chip_workdir : workdir;
    10111305    // insert an entry into the chipPendingExp table
    10121306    // this can only be run as the newExp's state has been set to stop
    10131307    if (!pxchipQueueByExpTag(config,
    10141308                exp_id,
    1015                 workdir,
     1309                             chip_workdir,
    10161310                label,
    1017                 NULL,       // data_group
    1018                 NULL,       // dist_group
     1311                             data_group,
     1312                             dist_group,
    10191313                reduction,
    10201314                NULL,       // expgroup
    … …  
    10921386    PXOPT_COPY_F64(config->args,   where,  "-posang_min", "posang", ">=");
    10931387    PXOPT_COPY_F64(config->args,   where,  "-posang_max", "posang", "<");
    1094     PXOPT_COPY_STR(config->args,   where,  "-object", "object", "==");
     1388    PXOPT_COPY_STR(config->args,   where,  "-object", "object", "LIKE");
     1389    PXOPT_COPY_STR(config->args,   where,  "-obs_mode", "obs_mode", "LIKE");
     1390    PXOPT_COPY_STR(config->args,   where,  "-comment", "comment", "LIKE");
    10951391    PXOPT_COPY_F32(config->args,   where,  "-sun_angle_min", "sun_angle", ">=");
    10961392    PXOPT_COPY_F32(config->args,   where,  "-sun_angle_max", "sun_angle", "<");

  • trunk/ippTools/src/regtool.h

    r27149 r30049  
    2626    REGTOOL_MODE_NONE      = 0x0,
    2727    REGTOOL_MODE_PENDINGIMFILE,
     28    REGTOOL_MODE_CHECKBURNTOOLIMFILE,
     29    REGTOOL_MODE_PENDINGBURNTOOLIMFILE,
    2830    REGTOOL_MODE_ADDPROCESSEDIMFILE,
    2931    REGTOOL_MODE_PROCESSEDIMFILE,

  • trunk/ippTools/src/regtoolConfig.c

    r27748 r30049  
    5353    ADD_OPT(Bool, pendingimfileArgs, "-simple",    "use the simple output format", false);
    5454
     55    // -checkburntoolimfile
     56    psMetadata *checkburntoolimfileArgs = psMetadataAlloc();
     57    ADD_OPT(Str,  checkburntoolimfileArgs, "-exp_name",       "define the exp_name (required)",         NULL);
     58    ADD_OPT(Str,  checkburntoolimfileArgs, "-class_id",       "define class ID (required)",         NULL);
     59    ADD_OPT(Str,  checkburntoolimfileArgs, "-date",           "select the date to process (required)", NULL);
     60    ADD_OPT(S32,  checkburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0);
     61    ADD_OPT(Str,  checkburntoolimfileArgs, "-inst",           "define the camera name",     NULL);
     62    ADD_OPT(Str,  checkburntoolimfileArgs, "-telescope",      "define the telescope name",     NULL);
     63    ADD_OPT(Bool, checkburntoolimfileArgs, "-simple",    "use the simple output format",          false);
     64
     65    // -pendingburntoolimfile
     66    psMetadata *pendingburntoolimfileArgs = psMetadataAlloc();
     67    ADD_OPT(Str,  pendingburntoolimfileArgs, "-date",           "select the date to process (required)", NULL);
     68    ADD_OPT(S32,  pendingburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0);
     69    ADD_OPT(Bool, pendingburntoolimfileArgs, "-simple",    "use the simple output format",          false);
     70    ADD_OPT(Bool, pendingburntoolimfileArgs, "-ignore_state",   "ignore the data_state when deciding what to work on",  false);
     71    ADD_OPT(U64,  pendingburntoolimfileArgs, "-limit",     "limit result set to N items",  0);
     72   
    5573    // -addprocessedimfile
    5674    psMetadata *addprocessedimfileArgs = psMetadataAlloc();
    … …  
    128146    ADD_OPT(S32,  addprocessedimfileArgs, "-bytes",          "define bytes",                0);
    129147    ADD_OPT(S16,  addprocessedimfileArgs, "-burntool_state",        "set burntool state", 0);
     148    ADD_OPT(Bool, addprocessedimfileArgs, "-video_cells",    "define if chip has video cells", false);
    130149    ADD_OPT(S16,  addprocessedimfileArgs, "-fault",           "set fault code",                  0);
    131150    ADD_OPT(S16,  addprocessedimfileArgs, "-quality",        "set quality flag", 0);
    … …  
    249268    ADD_OPT(F32,  addprocessedexpArgs, "-moon_phase",       "define moon phase (0.0 = new)",   NAN);
    250269    ADD_OPT(Str,  addprocessedexpArgs, "-label",            "define label for chip stage (non-detrend data only)", NULL);
     270    ADD_OPT(Str,  addprocessedexpArgs, "-data_group",       "define data_group for chip stage (non-detrend data only)", NULL);
     271    ADD_OPT(Str,  addprocessedexpArgs, "-dist_group",       "define dist_group for chip stage (non-detrend data only)", NULL);
     272    ADD_OPT(Str,  addprocessedexpArgs, "-chip_workdir",     "define workdir for chip stage (non-detrend data only)", NULL);
    251273    ADD_OPT(Str,  addprocessedexpArgs, "-hostname",         "define host name", NULL);
    252274    ADD_OPT(S16,  addprocessedexpArgs, "-fault",             "set fault code", 0);
    … …  
    291313    psMetadataAddF64(processedexpArgs,  PS_LIST_TAIL, "-posang_max",    0,            "search by max rotator position angle", NAN);
    292314    psMetadataAddStr(processedexpArgs,  PS_LIST_TAIL, "-object",        0,            "search by exposure object", NULL);
     315    psMetadataAddStr(processedexpArgs,  PS_LIST_TAIL, "-obs_mode",      0,            "search by exposure obs_mode", NULL);
     316    psMetadataAddStr(processedexpArgs,  PS_LIST_TAIL, "-comment",       0,            "search by exposure comment", NULL);
     317   
    293318    psMetadataAddF32(processedexpArgs,  PS_LIST_TAIL, "-sun_angle_min",    0,         "define min solar angle", NAN);
    294319    psMetadataAddF32(processedexpArgs,  PS_LIST_TAIL, "-sun_angle_max",    0,         "define max solar angle", NAN);
    … …  
    395420
    396421    PXOPT_ADD_MODE("-pendingimfile",           "", REGTOOL_MODE_PENDINGIMFILE, pendingimfileArgs);
     422    PXOPT_ADD_MODE("-checkburntoolimfile",     "", REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileArgs);
     423    PXOPT_ADD_MODE("-pendingburntoolimfile",   "", REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileArgs);
    397424    PXOPT_ADD_MODE("-addprocessedimfile",      "", REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs);
    398425    PXOPT_ADD_MODE("-processedimfile",         "", REGTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs);
Note: See TracChangeset for help on using the changeset viewer.