Changeset 35960

Timestamp:

Aug 15, 2013, 4:05:13 PM (13 years ago)

Author:

bills

Message:

various changes for support of distribution of skycal results

Location:

trunk

Files:

• ippScripts/scripts/dist_advancerun.pl (modified) (2 diffs)
• ippTools/share/Makefile.am (modified) (1 diff)
• ippTools/share/disttool_pending_skycal.sql (modified) (1 diff)
• ippTools/share/disttool_toadvance.sql (modified) (1 diff)
• ippTools/share/staticskytool_export_skycalresult.sql (added)
• ippTools/share/staticskytool_export_skycalrun.sql (added)
• ippTools/share/staticskytool_skycalresult.sql (modified) (1 diff)
• ippTools/src/staticskytool.c (modified) (4 diffs)
• ippTools/src/staticskytool.h (modified) (1 diff)
• ippTools/src/staticskytoolConfig.c (modified) (2 diffs)

trunk/ippScripts/scripts/dist_advancerun.pl

@@ -115 +115 @@
     $list_mode = "-result";
     $component_key = "skycell_id";
+} elsif ($stage eq "skycal") {
+    $tool_cmd = "$staticskytool -skycal_id";
+    $list_mode = "-skycalresult";
+    $component_key = "skycell_id";
 } elsif ($stage eq "diff") {
     $tool_cmd = "$difftool -diff_id";
@@ -134 +138 @@
 } elsif ($stage eq 'warp_bg') {
     $exportarg = '-exportwarp';
+} elsif ($stage eq 'skycal') {
+    $exportarg = '-exportskycalrun';
 }
 

trunk/ippTools/share/Makefile.am

@@ -410 +410 @@
         staticskytool_export_input.sql \
         staticskytool_export_result.sql \
+        staticskytool_export_skycalrun.sql \
+        staticskytool_export_skycalresult.sql \
         staticskytool_inputs.sql \
         staticskytool_todo.sql \

trunk/ippTools/share/disttool_pending_skycal.sql

@@ -38 +38 @@
 JOIN chipRun
     USING(chip_id)
-    -- ON camRun.chip_id = chipRun.chip_id
 JOIN rawExp 
      USING (exp_id)

trunk/ippTools/share/disttool_toadvance.sql

@@ -262 +262 @@
         AND distComponent.fault = 0
 UNION
+-- skycal stage
+-- NOTE this assumes that there is only one component per skycalRun
+-- (one skycell)
+SELECT
+    distRun.dist_id,
+    stage,
+    stage_id,
+    outroot,
+    label,
+    clean
+    FROM distRun
+    JOIN skycalResult on stage_id = skycal_id
+    LEFT JOIN distComponent
+        ON distRun.dist_id = distComponent.dist_id
+    WHERE
+        distRun.state = 'new'
+        AND distRun.fault = 0
+        AND distRun.stage = 'skycal'
+        AND distComponent.component IS NOT NULL
+        AND distComponent.fault = 0
+UNION
 -- SSdiff stage
 SELECT

trunk/ippTools/share/staticskytool_skycalresult.sql

@@ -2 +2 @@
     skycalResult.*,
     skycalRun.stack_id,
+    skycalRun.workdir,
     stackRun.filter,
     skycalRun.state,

trunk/ippTools/src/staticskytool.c

@@ -50 +50 @@
 static bool revertskycalresultMode(pxConfig *config);
 static bool updateskycalresultMode(pxConfig *config);
+static bool exportskycalrunMode(pxConfig *config);
+static bool importskycalrunMode(pxConfig *config);
 
 static bool setstaticskyRunState(pxConfig *config, psS64 sky_id, const char *state);
@@ -88 +90 @@
         MODECASE(STATICSKYTOOL_MODE_REVERTSKYCALRESULT,revertskycalresultMode);
         MODECASE(STATICSKYTOOL_MODE_SKYCALRESULT,      skycalresultMode);
+        MODECASE(STATICSKYTOOL_MODE_EXPORTSKYCALRUN,   exportskycalrunMode);
+        MODECASE(STATICSKYTOOL_MODE_IMPORTSKYCALRUN,   importskycalrunMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -895 +899 @@
 bool importrunMode(pxConfig *config)
 {
-# if (0)
-  unsigned int nFail;
-
-  int numImportTables = 2;
-
-  char tables[2] [80] = {"stackInputSkyfile", "stackSumSkyfile"};
+  return false;
+}
+
+bool exportskycalrunMode(pxConfig *config)
+{
+  typedef struct ExportTable {
+    char tableName[80];
+    char sqlFilename[80];
+  } ExportTable;
+
+  int numExportTables = 2;
 
   PS_ASSERT_PTR_NON_NULL(config, NULL);
 
-  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
-
-  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
-
-#ifdef notdef
-  fprintf (stderr, "---- input ----\n");
-  psMetadataPrint (stderr, input, 1);
-#endif
-
-  if (!pxCheckImportVersion(config, input)) {
-      psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed");
+  // XXX unused PXOPT_LOOKUP_S64(det_id, config->args, "-sky_id", true,  false);
+  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+  PXOPT_LOOKUP_BOOL(clean, config->args, "-clean", false);
+  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
+
+  FILE *f = fopen (outfile, "w");
+  if (f == NULL) {
+      psError(PS_ERR_UNKNOWN, false, "failed to open output file");
       return false;
   }
-  psMetadataItem *item = psMetadataLookup (input, "stackRun");
-  psAssert (item, "entry not in input?");
-  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-
-  psMetadataItem *entry = psListGet (item->data.list, 0);
-  assert (entry);
-  assert (entry->type == PS_DATA_METADATA);
-  stackRunRow *stackRun = stackRunObjectFromMetadata (entry->data.md);
-  stackRunInsertObject (config->dbh, stackRun);
-
-  // fprintf (stdout, "---- stack run ----\n");
-  // psMetadataPrint (stderr, entry->data.md, 1);
-
-  for (int i = 0; i < numImportTables; i++) {
-    psMetadataItem *item = psMetadataLookup (input, tables[i]);
-    psAssert (item, "entry not in input?");
-    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-
-    switch (i) {
-      case 0:
-        for (int i = 0; i < item->data.list->n; i++) {
-          entry = psListGet (item->data.list, i);
-          assert (entry);
-          assert (entry->type == PS_DATA_METADATA);
-          stackInputSkyfileRow *stackInputSkyfile = stackInputSkyfileObjectFromMetadata (entry->data.md);
-          stackInputSkyfileInsertObject (config->dbh, stackInputSkyfile);
-
-          // fprintf (stdout, "---- row %d ----\n", i);
-          // psMetadataPrint (stderr, entry->data.md, 1);
-        }
-        break;
-
-      case 1:
-        for (int i = 0; i < item->data.list->n; i++) {
-          entry = psListGet (item->data.list, i);
-          assert (entry);
-          assert (entry->type == PS_DATA_METADATA);
-          stackSumSkyfileRow *stackSumSkyfile = stackSumSkyfileObjectFromMetadata (entry->data.md);
-          stackSumSkyfileInsertObject (config->dbh, stackSumSkyfile);
-
-          // fprintf (stdout, "---- row %d ----\n", i);
-          // psMetadataPrint (stderr, entry->data.md, 1);
-        }
-        break;
-    }
+
+  if (!pxExportVersion(config, f)) {
+    psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file");
+    return false;
   }
-
-# endif
-  return true;
+  psMetadata *where = psMetadataAlloc();
+  PXOPT_COPY_S64(config->args, where, "-skycal_id", "skycal_id", "==");
+
+  ExportTable tables [] = {
+    {"skycalRun", "staticskytool_export_skycalrun.sql"},
+    {"skycalResult", "staticskytool_export_skycalresult.sql"},
+  };
+
+  for (int i=0; i < numExportTables; i++) {
+    psString query = pxDataGet(tables[i].sqlFilename);
+    if (!query) {
+      psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+      return false;
+    }
+
+    if (where && psListLength(where->list)) {
+      psString whereClause = psDBGenerateWhereSQL(where, NULL);
+      psStringAppend(&query, " %s", whereClause);
+      psFree(whereClause);
+    }
+
+    // treat limit == 0 as "no limit"
+    if (limit) {
+      psString limitString = psDBGenerateLimitSQL(limit);
+      psStringAppend(&query, " %s", limitString);
+      psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      psFree(query);
+      return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+      psError(PS_ERR_UNKNOWN, false, "database error");
+      return false;
+    }
+    if (!psArrayLength(output)) {
+      psError(PS_ERR_UNKNOWN, true, "no rows found");
+      psFree(output);
+      return false;
+    }
+
+    if (clean) {
+        if (!strcmp(tables[i].tableName, "skycalRun")) {
+            if (!pxSetStateCleaned("skycalRun", "state", output)) {
+                psFree(output);
+                psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
+                return false;
+            }
+        }
+    }
+
+      // we must write the export table in non-simple (true) format
+    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+  }
+
+    fclose (f);
+
+    return true;
+}
+
+bool importskycalrunMode(pxConfig *config)
+{
+  return false;
 }
 
@@ -986 +1022 @@
 
 }
-
-# if (0)
-        // now we need to loop over all requested filters and check that each is found
-        // XXX is this needed?  haven't we required that we only match the requested filters
-        // and that N unique filters are found?
-
-        psVector *found = psVectorAlloc(inputs->n, PS_TYPE_U8);
-        psVectorInit (found, 0);
-
-        psMetadataItem *filter = NULL;
-        psListIterator *iter = psListIteratorAlloc (filters->data.list, PS_LIST_HEAD, false);
-        while ((filter = psListGetAndIncrement(iter))) {
-
-            bool foundOne = false;
-
-            psAssert (filter->type == PS_DATA_STR, "filter is not a string?");
-            for (int j = 0; !foundOne && (j < inputs->n); j++) {
-                if (found->data.U8[j]) continue;
-                psMetadata *inputRow = inputs->data[j]; // Row from select
-
-                // pull out the skycell_id, tess_id, filter
-                psString inFilter = psMetadataLookupStr(&status, row, "filter");
-                psAssert (status);
-
-                found->data.U8[j] = true;
-                foundOne = true;
-            }
-            if (!foundOne) {
-                // this required filter was not found in the inputs, skip the entry
-                skip();
-            }
-        }           
-# endif
 
 static bool defineskycalrunMode(pxConfig *config)

trunk/ippTools/src/staticskytool.h

@@ -43 +43 @@
     STATICSKYTOOL_MODE_SKYCALRESULT,
     STATICSKYTOOL_MODE_REVERTSKYCALRESULT,
+    STATICSKYTOOL_MODE_EXPORTSKYCALRUN,
+    STATICSKYTOOL_MODE_IMPORTSKYCALRUN,
 } staticskytoolMode;
 

trunk/ippTools/src/staticskytoolConfig.c

@@ -269 +269 @@
     psMetadataAddBool(skycalresultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
 
+    // -exportskycalrun
+    psMetadata *exportskycalrunArgs = psMetadataAlloc();
+    psMetadataAddS64(exportskycalrunArgs, PS_LIST_TAIL, "-skycal_id", 0, "export this skycal ID (required)", 0);
+    psMetadataAddStr(exportskycalrunArgs, PS_LIST_TAIL, "-outfile", 0, "export to this file (required)", NULL);
+    psMetadataAddU64(exportskycalrunArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
+    psMetadataAddBool(exportskycalrunArgs, PS_LIST_TAIL, "-clean", 0, "mark tables as cleaned", false);
+
+    // -importrun
+    psMetadata *importskycalrunArgs = psMetadataAlloc();
+    psMetadataAddStr(importskycalrunArgs, PS_LIST_TAIL, "-infile", 0, "import from this file (required)", NULL);
+  
+
     psFree(now);
 
@@ -291 +303 @@
     PXOPT_ADD_MODE("-updateskycal",  "revert faulted skycal run", STATICSKYTOOL_MODE_UPDATESKYCALRESULT, updateskycalresultArgs);
     PXOPT_ADD_MODE("-skycalresult",  "Get result of skycal run",  STATICSKYTOOL_MODE_SKYCALRESULT, skycalresultArgs);
+    PXOPT_ADD_MODE("-exportskycalrun",     "Export skycal run",   STATICSKYTOOL_MODE_EXPORTSKYCALRUN, exportskycalrunArgs);
+    PXOPT_ADD_MODE("-importskycalrun",     "Import skycal run",   STATICSKYTOOL_MODE_IMPORTSKYCALRUN, importskycalrunArgs);
 
     if (!pxGetOptions(stderr, argc, argv, config, modes, argSets)) {