IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29908


Ignore:
Timestamp:
Dec 3, 2010, 5:46:26 PM (15 years ago)
Author:
watersc1
Message:

Realtime burntool code. Initial tests indicate it should work fine

Location:
branches/czw_branch/20101203
Files:
2 added
5 edited

Legend:

Unmodified
Added
Removed

  • branches/czw_branch/20101203/ippTasks/register.pro

    r25418 r29908  
    88book init regPendingImfile
    99book init regPendingExp
     10book init regPendingBurntoolImfile
    1011
    1112macro register.reset
    1213  book init regPendingImfile
    1314  book init regPendingExp
     15  book init regPendingBurntoolImfile
    1416end
    1517
    … …  
    1820  book listbook regPendingImfile
    1921  book listbook regPendingExp
     22  book listbook regPendingBurntoolImfile
    2023end
    2124
    … …  
    3336    active true
    3437  end
     38  task register.burntool.load
     39    active true
     40  end
     41  task register.burntool.run
     42    active true
     43  end
    3544end
    3645
    … …  
    4655  end
    4756  task register.exp.run
     57    active false
     58  end
     59  task register.burntool.load
     60    active false
     61  end
     62  task register.burntool.run
    4863    active false
    4964  end
    … …  
    429444end
    430445
     446
     447# select imfiles that can now be burntooled.
     448task       register.burntool.load
     449  host     local
     450
     451  periods  -poll $LOADPOLL
     452  periods  -exec $LOADEXEC
     453  periods  -timeout 30
     454  npending 1
     455
     456  # silently drop stdout
     457  stdout NULL
     458  stderr $LOGDIR/register.burntool.load.log
     459 
     460  # select entried from the current DB; cycle to the next DB, if it exists
     461  task.exec
     462    $run = regtool -pendingburntoolimfile
     463    if ($DB:n == 0)
     464      option DEFAULT
     465    else
     466      # save the DB name for the exit tasks
     467      option $DB:$regPendingBurntoolImfile_DB
     468      $run = $run -dbname $DB:$regPendingBurntoolImfile_DB
     469      $regPendingBurntoolImfile_DB ++
     470      if ($regPendingBurntoolImfile_DB >= $DB:n) set regPendingBurntoolImfile_DB = 0
     471    end
     472
     473    $today = `date +%Y-%m-%d`
     474    $run = $run -date $today -valid_burntool $valid_burntool_value
     475    add_poll_args run
     476    command $run
     477  end
     478
     479  # success
     480  task.exit $EXIT_SUCCESS
     481    # convert 'stdout' to book format
     482    ipptool2book stdout regPendingBurntoolImfile -key exp_id:class_id -uniq -setword dbname $options:0 -setword pantaskState INIT
     483    book shuffle regPendingBurntoolImfile
     484    if ($VERBOSE > 2)
     485      book listbook regPendingBurntoolImfile
     486    end
     487
     488    # delete existing entries in the appropriate pantasksStates
     489    process_cleanup regPendingBurntoolImfile
     490  end
     491
     492  # locked list
     493  task.exit   default
     494    showcommand failure
     495  end
     496  task.exit   crash
     497    showcommand crash
     498  end
     499  task.exit   timeout
     500    showcommand timeout
     501  end
     502end
     503
     504# run the ipp_apply_burntool_single.pl script on the pending images
     505task          register.burntool.run
     506  periods     -poll $RUNPOLL
     507  periods     -exec $RUNEXEC
     508  periods     -timeout 30
     509
     510  task.exec
     511    if ($NETWORK == 0) break
     512    book npages regPendingBurntoolImfile -var N
     513    if ($N == 0)
     514      periods -exec $RUNEXEC
     515      break
     516    end
     517    periods -exec 0.05
     518
     519    # look for new images to burn in regPendingBurntoolImfile
     520    book getpage regPendingBurntoolImfile 0 -var pageName -key pantaskState INIT
     521    if ("$pageName" == "NULL") break
     522
     523    book setword regPendingBurntoolImfile $pageName pantaskState RUN
     524
     525    book getword regPendingBurntoolImfile $pageName exp_id      -var EXP_ID
     526    book getword regPendingBurntoolImfile $pageName class_id    -var CLASS_ID
     527    book getword regPendingBurntoolImfile $pageName uri         -var THIS_URI
     528    book getword regPendingBurntoolImfile $pageName previous_uri -var PREVIOUS_URI
     529    book getword regPendingBurntoolImfile $pageName camera      -var CAMERA
     530    book getword regPendingBurntoolImfile $pageName dbname      -var DBNAME
     531    set.host.for.camera $CAMERA $CLASS_ID
     532
     533    stdout $LOGDIR/register.burntool.run.log
     534    stderr $LOGDIR/register.burntool.run.log
     535
     536    $run = ipp_apply_burntool_single.pl --exp_id $EXP_ID --class_id $CLASS_ID --this_uri $THIS_URI
     537    if ("$PREVIOUS_URI" != "NULL")
     538      $run = $run --previous_uri $PREVIOUS_URI
     539    end
     540
     541    add_standard_args run
     542
     543    options $pageName
     544
     545    if ($VERBOSE > 1)
     546      echo command $run
     547    end
     548    command $run
     549  end
     550
     551  # default exit status
     552  task.exit default
     553    process_exit regPendingBurntoolImfile $options:0 $JOB_STATUS
     554  end
     555
     556  # locked list
     557  task.exit crash
     558    showcommand crash
     559    echo "hostname: $JOB_HOSTNAME"
     560    book setword regPendingBurntoolImfile $options:0 pantaskState CRASH
     561  end
     562
     563  # operation timed out
     564  task.exit  timeout
     565    showcommand timeout
     566    book setword regPendingBurntoolImfile $options:0 pantaskState TIMEOUT
     567  end
     568end
     569
     570   
     571   
     572 
    431573## XXX add a global path to output files 

  • branches/czw_branch/20101203/ippTools/share/Makefile.am

    r29717 r29908  
    316316        receivetool_revert.sql \
    317317        receivetool_toadvance.sql \
     318        regtool_checkburntoolimfile.sql \
    318319        regtool_create_dup_table.sql \
    319320        regtool_export_exp.sql \
    … …  
    331332        regtool_updatebyquery.sql \
    332333        regtool_updatebyqueryimfile.sql \
     334        regtool_pendingburntoolimfile.sql \
    333335        stacktool_associationdefine_select.sql \
    334336        stacktool_definebyquery_insert.sql \

  • branches/czw_branch/20101203/ippTools/src/regtool.c

    r27748 r29908  
    3434static bool pendingimfileMode(pxConfig *config);
    3535static bool addprocessedimfileMode(pxConfig *config);
     36static bool checkburntoolimfileMode(pxConfig *config);
     37static bool pendingburntoolimfileMode(pxConfig *config);
    3638static bool processedimfileMode(pxConfig *config);
    3739static bool revertprocessedimfileMode(pxConfig *config);
    … …  
    7274        // imfile
    7375        MODECASE(REGTOOL_MODE_PENDINGIMFILE,         pendingimfileMode);
     76        MODECASE(REGTOOL_MODE_CHECKBURNTOOLIMFILE,   checkburntoolimfileMode);
     77        MODECASE(REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileMode);
    7478        MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE,    addprocessedimfileMode);
    7579        MODECASE(REGTOOL_MODE_PROCESSEDIMFILE,       processedimfileMode);
    … …  
    176180}
    177181
     182static bool checkburntoolimfileMode(pxConfig *config)
     183{
     184  PS_ASSERT_PTR_NON_NULL(config,false);
     185
     186  // required
     187  PXOPT_LOOKUP_STR(exp_name, config->args, "-exp_name", true, false);
     188  PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
     189  PXOPT_LOOKUP_STR(date,     config->args, "-date", true, false);
     190  PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false);
     191  // optional
     192  PXOPT_LOOKUP_STR(camera, config->args, "-inst", false, false);
     193  PXOPT_LOOKUP_STR(telescope, config->args, "-telescope", false, false);
     194  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     195  psString query = pxDataGet("regtool_checkburntoolimfile.sql");
     196  if (!query) {
     197    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     198    return false;
     199  }
     200  psString rep = psStringCopy(query);
     201  psFree(query);
     202  query = rep;
     203 
     204  psStringSubstitute(&query,exp_name,"@EXP_NAME@");
     205  psStringSubstitute(&query,class_id,"@CLASS_ID@");
     206  psStringSubstitute(&query,date,"@DATE@");
     207
     208  fprintf(stderr,"%s",query);
     209
     210  if (!p_psDBRunQuery(config->dbh, query)) {
     211    // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
     212    psError(PXTOOLS_ERR_PROG, false, "database error");
     213    psFree(query);
     214    return false;
     215  }
     216  psFree(query);
     217 
     218  psArray *output = p_psDBFetchResult(config->dbh);
     219  if (!output) {
     220    psErrorCode err = psErrorCodeLast();
     221    switch (err) {
     222    case PS_ERR_DB_CLIENT:
     223      psError(PXTOOLS_ERR_SYS, false, "database error");
     224    case PS_ERR_DB_SERVER:
     225      psError(PXTOOLS_ERR_PROG, false, "database error");
     226    default:
     227      psError(PXTOOLS_ERR_PROG, false, "unknown error");
     228    }
     229   
     230    return false;
     231  }
     232  if (!psArrayLength(output)) {
     233    psTrace("regtool", PS_LOG_INFO, "no rows found");
     234    psFree(output);
     235    return true;
     236  }
     237
     238  psString previous_uri = NULL;
     239  bool ok_to_burn = true;
     240  bool already_burned = true;
     241  for (long i = 0; i < output->n; i++) {
     242    psMetadata *row = output->data[i];
     243
     244    if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
     245        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
     246      ok_to_burn = false;
     247    }
     248    if (already_burned == false) {
     249      ok_to_burn = false;
     250    }
     251    if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) {
     252      already_burned = true;
     253    }
     254    else {
     255      already_burned = false;
     256    }
     257   
     258    if (previous_uri) {
     259      psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
     260      psFree(previous_uri);
     261    }
     262    psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn);
     263    psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned);
     264    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri"));
     265  }
     266 
     267  // negate simple so the default is true
     268  if (!ippdbPrintMetadatas(stdout, output, "regBurntoolImfile", !simple)) {
     269    psError(PXTOOLS_ERR_PROG, false, "failed to print array");
     270    psFree(output);
     271    return false;
     272  }
     273 
     274  psFree(output);
     275 
     276
     277  return(true);
     278 
     279}
     280
     281static bool pendingburntoolimfileMode(pxConfig *config)
     282{
     283  PS_ASSERT_PTR_NON_NULL(config,false);
     284
     285  // required
     286  PXOPT_LOOKUP_STR(date,     config->args, "-date", true, false);
     287  PXOPT_LOOKUP_S32(valid_burntool, config->args, "-valid_burntool", true, false);
     288  // optional
     289  PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     290  PXOPT_LOOKUP_BOOL(ignore_state, config->args, "-ignore_state", false);
     291  psString query = pxDataGet("regtool_pendingburntoolimfile.sql");
     292  if (!query) {
     293    psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
     294    return false;
     295  }
     296  psString rep = psStringCopy(query);
     297  psFree(query);
     298  query = rep;
     299 
     300  psStringSubstitute(&query,date,"@DATE@");
     301
     302  fprintf(stderr,"%s",query);
     303
     304  if (!p_psDBRunQuery(config->dbh, query)) {
     305    // XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
     306    psError(PXTOOLS_ERR_PROG, false, "database error");
     307    psFree(query);
     308    return false;
     309  }
     310  psFree(query);
     311 
     312  psArray *output = p_psDBFetchResult(config->dbh);
     313  if (!output) {
     314    psErrorCode err = psErrorCodeLast();
     315    switch (err) {
     316    case PS_ERR_DB_CLIENT:
     317      psError(PXTOOLS_ERR_SYS, false, "database error");
     318    case PS_ERR_DB_SERVER:
     319      psError(PXTOOLS_ERR_PROG, false, "database error");
     320    default:
     321      psError(PXTOOLS_ERR_PROG, false, "unknown error");
     322    }
     323   
     324    return false;
     325  }
     326  if (!psArrayLength(output)) {
     327    psTrace("regtool", PS_LOG_INFO, "no rows found");
     328    psFree(output);
     329    return true;
     330  }
     331
     332  psString previous_uri = NULL;
     333  bool ok_to_burn = true;
     334  bool already_burned = true;
     335  psString previous_class_id = NULL;
     336  psString this_class_id = NULL;
     337  psArray *results = psArrayAllocEmpty(60); // List of suggested imfiles to burntool.
     338 
     339  for (long i = 0; i < output->n; i++) {
     340    psMetadata *row = output->data[i];
     341
     342    // Use tmp_class_id because class_id may not be populated for all imfiles
     343    this_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     344    psStringSubstitute(&this_class_id,"XY","ota");
     345    // Determine if we've crossed a class_id boundary.
     346    if (previous_class_id) {
     347      if (strcmp(this_class_id,previous_class_id) != 0) {
     348        ok_to_burn = true;
     349        already_burned = true;
     350        previous_uri = NULL;
     351      }
     352    }
     353
     354    // If the state of this imfile is not "pending_burntool" then skip it, because
     355    if (!ignore_state) {
     356      psString imfile_state = psMetadataLookupStr(NULL,row,"imfile_state");
     357
     358      if (!imfile_state) {
     359        continue;
     360      }
     361      if (strcmp("pending_burntool",imfile_state) != 0) {
     362        continue;
     363      }
     364    }
     365   
     366    // Convert bits of the SQL query into booleans describing the data state
     367    if ((psMetadataLookupS32(NULL,row,"is_downloaded") != 1)||
     368        (psMetadataLookupS32(NULL,row,"is_registered") != 1)) {
     369      ok_to_burn = false;
     370    }
     371    if (already_burned == false) {
     372      ok_to_burn = false;
     373    }
     374    if (abs(psMetadataLookupS32(NULL,row,"burntool_state")) == valid_burntool) {
     375      already_burned = true;
     376    }
     377    else {
     378      already_burned = false;
     379    }
     380    psMetadataAddBool(row,PS_LIST_TAIL,"burnable",PS_META_REPLACE,"",ok_to_burn);
     381    psMetadataAddBool(row,PS_LIST_TAIL,"already_burned",PS_META_REPLACE,"",already_burned);
     382
     383    // We'll need to return the previous uri so we know where the input table is:
     384    if (previous_uri) {
     385      psMetadataAddStr(row,PS_LIST_TAIL,"previous_uri",PS_META_REPLACE,"",previous_uri);
     386      psFree(previous_uri);
     387    }
     388    previous_uri = psStringCopy(psMetadataLookupStr(NULL,row,"uri"));
     389
     390    // Write the class_id stuff at the end for debugging.
     391    psMetadataAddStr(row,PS_LIST_TAIL,"this_class_id",PS_META_REPLACE,"",this_class_id);
     392    if (previous_class_id) {
     393      psMetadataAddStr(row,PS_LIST_TAIL,"previous_class_id",PS_META_REPLACE,"",previous_class_id);
     394      psFree(previous_class_id);
     395    }
     396    // Use tmp_class_id because class_id may not be populated for all imfiles
     397    previous_class_id = psStringCopy(psMetadataLookupStr(NULL,row,"tmp_class_id"));
     398    psStringSubstitute(&previous_class_id,"XY","ota");
     399    // Determine if we've already suggested an entry for this ota, and if not, copy this
     400    // suggestion into our output result list.
     401    if (!ok_to_burn || already_burned) {
     402      continue;
     403    }
     404   
     405    psArrayAdd(results,results->n,row);
     406    // If we're here, then we think we could potentially burntool this file.
     407
     408  }
     409 
     410  // negate simple so the default is true
     411  if (!ippdbPrintMetadatas(stdout, results, "regPendingBurntoolImfile", !simple)) {
     412    psError(PXTOOLS_ERR_PROG, false, "failed to print array");
     413    psFree(output);
     414    psFree(results);
     415    return false;
     416  }
     417 
     418  psFree(output);
     419  psFree(results);
     420 
     421
     422  return(true);
     423 
     424}
     425
     426     
    178427
    179428static bool addprocessedimfileMode(pxConfig *config)

  • branches/czw_branch/20101203/ippTools/src/regtool.h

    r27149 r29908  
    2626    REGTOOL_MODE_NONE      = 0x0,
    2727    REGTOOL_MODE_PENDINGIMFILE,
     28    REGTOOL_MODE_CHECKBURNTOOLIMFILE,
     29    REGTOOL_MODE_PENDINGBURNTOOLIMFILE,
    2830    REGTOOL_MODE_ADDPROCESSEDIMFILE,
    2931    REGTOOL_MODE_PROCESSEDIMFILE,

  • branches/czw_branch/20101203/ippTools/src/regtoolConfig.c

    r27748 r29908  
    5353    ADD_OPT(Bool, pendingimfileArgs, "-simple",    "use the simple output format", false);
    5454
     55    // -checkburntoolimfile
     56    psMetadata *checkburntoolimfileArgs = psMetadataAlloc();
     57    ADD_OPT(Str,  checkburntoolimfileArgs, "-exp_name",       "define the exp_name (required)",         NULL);
     58    ADD_OPT(Str,  checkburntoolimfileArgs, "-class_id",       "define class ID (required)",         NULL);
     59    ADD_OPT(Str,  checkburntoolimfileArgs, "-date",           "select the date to process (required)", NULL);
     60    ADD_OPT(S32,  checkburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0);
     61    ADD_OPT(Str,  checkburntoolimfileArgs, "-inst",           "define the camera name",     NULL);
     62    ADD_OPT(Str,  checkburntoolimfileArgs, "-telescope",      "define the telescope name",     NULL);
     63    ADD_OPT(Bool, checkburntoolimfileArgs, "-simple",    "use the simple output format",          false);
     64
     65    // -pendingburntoolimfile
     66    psMetadata *pendingburntoolimfileArgs = psMetadataAlloc();
     67    ADD_OPT(Str,  pendingburntoolimfileArgs, "-date",           "select the date to process (required)", NULL);
     68    ADD_OPT(S32,  pendingburntoolimfileArgs, "-valid_burntool", "define the good burntool value (required)", 0);
     69    ADD_OPT(Bool, pendingburntoolimfileArgs, "-simple",    "use the simple output format",          false);
     70    ADD_OPT(Bool, pendingburntoolimfileArgs, "-ignore_state",   "ignore the data_state when deciding what to work on",  false);
     71    ADD_OPT(U64,  pendingburntoolimfileArgs, "-limit",     "limit result set to N items",  0);
     72   
    5573    // -addprocessedimfile
    5674    psMetadata *addprocessedimfileArgs = psMetadataAlloc();
    … …  
    395413
    396414    PXOPT_ADD_MODE("-pendingimfile",           "", REGTOOL_MODE_PENDINGIMFILE, pendingimfileArgs);
     415    PXOPT_ADD_MODE("-checkburntoolimfile",     "", REGTOOL_MODE_CHECKBURNTOOLIMFILE, checkburntoolimfileArgs);
     416    PXOPT_ADD_MODE("-pendingburntoolimfile",   "", REGTOOL_MODE_PENDINGBURNTOOLIMFILE, pendingburntoolimfileArgs);
    397417    PXOPT_ADD_MODE("-addprocessedimfile",      "", REGTOOL_MODE_ADDPROCESSEDIMFILE, addprocessedimfileArgs);
    398418    PXOPT_ADD_MODE("-processedimfile",         "", REGTOOL_MODE_PROCESSEDIMFILE, processedimfileArgs);
Note: See TracChangeset for help on using the changeset viewer.