Changeset 17712

Timestamp:

May 16, 2008, 9:51:26 AM (18 years ago)

Author:

Paul Price

Message:

Adding difftool -queue

Location:

trunk/ippTools

Files:

• share/Makefile.am (modified) (1 diff)
• share/difftool_queue.sql (added)
• src/difftool.c (modified) (8 diffs)
• src/difftool.h (modified) (1 diff)
• src/difftoolConfig.c (modified) (2 diffs)

trunk/ippTools/share/Makefile.am

@@ -45 +45 @@
         difftool_todiffskyfile.sql \
         difftool_revertdiffskyfile.sql \
+        difftool_queue.sql \
         flatcorr_create_tmp_chiprundone.sql \
         flatcorr_completely_processed_chiprun.sql \

trunk/ippTools/src/difftool.c

@@ -40 +40 @@
 static bool revertdiffskyfileMode(pxConfig *config);
 static bool definepoprunMode(pxConfig *config);
+static bool queueMode(pxConfig *config);
 
 static bool setdiffRunState(pxConfig *config, const char *diff_id, const char *state);
@@ -70 +71 @@
         MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,      definepoprunMode);
+        MODECASE(DIFFTOOL_MODE_QUEUE,             queueMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -460 +462 @@
 
     if (!diffSkyfileInsert(config->dbh,
-               (psS64)atoll(diff_id),
-               uri,
-               path_base,
-               bg,
-               bg_stdev,
-               dtime_diff,
-               hostname,
-               good_frac,
-               code
-      )) {
+                           (psS64)atoll(diff_id),
+                           uri,
+                           path_base,
+                           bg,
+                           bg_stdev,
+                           dtime_diff,
+                           hostname,
+                           good_frac,
+                           code
+          )) {
         if (!psDBRollback(config->dbh)) {
             psError(PS_ERR_UNKNOWN, false, "database error");
@@ -628 +630 @@
 }
 
-
-static bool definepoprunMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    // required options
-    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
-    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
-    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
-
-    // optional
-    PXOPT_LOOKUP_STR(template_stack_id, config->args, "-template_stack_id", false, false);
-    PXOPT_LOOKUP_STR(template_warp_id, config->args, "-template_warp_id", false, false);
-
-    if (template_stack_id && template_warp_id) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one template can be defined.");
-        return false;
-    }
-    if (!template_stack_id && !template_warp_id) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                "No template has been defined (-template_stack_id or -template_warp_id)");
-        return false;
-    }
-
-    PXOPT_LOOKUP_STR(input_stack_id, config->args, "-input_stack_id", false, false);
-    PXOPT_LOOKUP_STR(input_warp_id, config->args, "-input_warp_id", false, false);
-    if (input_stack_id && input_warp_id) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one input can be defined.");
-        return false;
-    }
-    if (!input_stack_id && !input_warp_id) {
-        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
-                "No input has been defined (-input_stack_id or -input_warp_id)");
+// Generate a single populated run
+static bool populatedrun(const char *workdir, // Working directory
+                         const char *skycell_id, // Skycell identifier
+                         const char *tess_id, // Tessellation identifier
+                         psS64 input_warp_id, // Warp identifier for input image, PS_MAX_S64 for none
+                         psS64 input_stack_id, // Stack identifier for input image, PS_MAX_S64 for none
+                         psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
+                         psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
+                         pxConfig *config // Configuration
+                         )
+{
+    PS_ASSERT_STRING_NON_EMPTY(workdir, false);
+    PS_ASSERT_STRING_NON_EMPTY(skycell_id, false);
+    PS_ASSERT_STRING_NON_EMPTY(tess_id, false);
+    if ((input_warp_id == PS_MAX_S64 && input_stack_id == PS_MAX_S64) ||
+        (input_warp_id != PS_MAX_S64 && input_stack_id != PS_MAX_S64)) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, input must be defined.");
+        return false;
+    }
+    if ((template_warp_id == PS_MAX_S64 && template_stack_id == PS_MAX_S64) ||
+        (template_warp_id != PS_MAX_S64 && template_stack_id != PS_MAX_S64)) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, template must be defined.");
         return false;
     }
@@ -683 +674 @@
     }
 
-
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -702 +692 @@
             run->diff_id,
             true,
-            template_stack_id ? (psS64)atoll(template_stack_id) : PS_MAX_S64, // defined or NULL
-            template_warp_id ? (psS64)atoll(template_warp_id) : PS_MAX_S64, // defined or NULL
+            template_stack_id,
+            template_warp_id,
             skycell_id,
             tess_id,
@@ -719 +709 @@
             run->diff_id,
             false,
-            input_stack_id ? (psS64)atoll(input_stack_id) : PS_MAX_S64, // defined or NULL
-            input_warp_id ? (psS64)atoll(input_warp_id) : PS_MAX_S64, // defined or NULL
+            input_stack_id,
+            input_warp_id,
             skycell_id,
             tess_id,
@@ -758 +748 @@
     return true;
 }
+
+
+static bool definepoprunMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required options
+    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
+    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
+    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
+
+    // optional
+    PXOPT_LOOKUP_STR(template_stack_id, config->args, "-template_stack_id", false, false);
+    PXOPT_LOOKUP_STR(template_warp_id, config->args, "-template_warp_id", false, false);
+
+    if (template_stack_id && template_warp_id) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one template can be defined.");
+        return false;
+    }
+    if (!template_stack_id && !template_warp_id) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                "No template has been defined (-template_stack_id or -template_warp_id)");
+        return false;
+    }
+
+    PXOPT_LOOKUP_STR(input_stack_id, config->args, "-input_stack_id", false, false);
+    PXOPT_LOOKUP_STR(input_warp_id, config->args, "-input_warp_id", false, false);
+    if (input_stack_id && input_warp_id) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one input can be defined.");
+        return false;
+    }
+    if (!input_stack_id && !input_warp_id) {
+        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
+                "No input has been defined (-input_stack_id or -input_warp_id)");
+        return false;
+    }
+
+    return populatedrun(workdir, skycell_id, tess_id,
+                        input_warp_id ? (psS64)atoll(input_warp_id) : PS_MAX_S64,
+                        input_stack_id ? (psS64)atoll(input_stack_id) : PS_MAX_S64,
+                        template_warp_id ? (psS64)atoll(template_warp_id) : PS_MAX_S64,
+                        template_stack_id ? (psS64)atoll(template_stack_id) : PS_MAX_S64,
+                        config);
+
+    return true;
+}
+
+
+static bool queueMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    // required options
+    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
+
+    // default
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
+
+    // find all things to queue
+    psString query = pxDataGet("difftool_queue.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (config->where) {
+        psString whereClause = psDBGenerateWhereSQL(config->where, NULL);
+        psStringAppend(&query, " %s", whereClause);
+        psFree(whereClause);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+            case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+            case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+            default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+        }
+
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("difftool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    long numGood = 0;                   // Number of good rows added
+    for (long i = 0; i < output->n; i++) {
+        psMetadata *row = output->data[i]; // Output row from query
+
+        // Selected parameters
+        bool mdok;                      // Status of MD lookup
+        const char *skycell_id = psMetadataLookupStr(&mdok, row, "skycell_id");
+        if (!mdok || !skycell_id) {
+            psWarning("skycell_id not found --- ignoring row %ld", i);
+            continue;
+        }
+        const char *tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
+        if (!mdok || !tess_id) {
+            psWarning("tess_id not found --- ignoring row %ld", i);
+            continue;
+        }
+        psS64 stack_id = psMetadataLookupS64(&mdok, row, "stack_id");
+        if (!mdok) {
+            psWarning("stack_id not found --- ignoring row %ld", i);
+            continue;
+        }
+        psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
+        if (!mdok) {
+            psWarning("warp_id not found --- ignoring row %ld", i);
+            continue;
+        }
+
+        if (!populatedrun(workdir, skycell_id, tess_id, warp_id, PS_MAX_S64, PS_MAX_S64, stack_id, config)) {
+            psWarning("Unable to add run for %s,%s,%" PRId64 ",%" PRId64, skycell_id, tess_id,
+                      warp_id, stack_id);
+            psErrorClear();
+            continue;
+        }
+
+        numGood++;
+    }
+
+    psLogMsg("difftool", PS_LOG_INFO, "%ld runs added.", numGood);
+
+    psFree(output);
+
+    return true;
+}

trunk/ippTools/src/difftool.h

@@ -34 +34 @@
     DIFFTOOL_MODE_REVERTDIFFSKYFILE,
     DIFFTOOL_MODE_DEFINEPOPRUN,
+    DIFFTOOL_MODE_QUEUE,
 } difftoolMode;
 

trunk/ippTools/src/difftoolConfig.c

@@ -174 +174 @@
             "define stack ID for input", NULL);
 
+    // -queue
+    psMetadata *queueArgs = psMetadataAlloc();
+    psMetadataAddStr(queueArgs, PS_LIST_TAIL, "-workdir", 0,
+            "define workdir (required)", NULL);
+    psMetadataAddTime(queueArgs, PS_LIST_TAIL, "-registered",  0,
+            "time detrend run was registered", now);
+    psMetadataAddBool(queueArgs, PS_LIST_TAIL, "-simple",  0,
+            "use the simple output format", false);
+
     psFree(now);
 
@@ -188 +197 @@
     PXOPT_ADD_MODE("-revertdiffskyfile","", DIFFTOOL_MODE_REVERTDIFFSKYFILE, revertdiffskyfileArgs);
     PXOPT_ADD_MODE("-definepoprun",     "", DIFFTOOL_MODE_DEFINEPOPRUN,      definepoprunArgs);
+    PXOPT_ADD_MODE("-queue",            "", DIFFTOOL_MODE_QUEUE,             queueArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {