Changeset 28091

Timestamp:

May 25, 2010, 1:38:50 PM (16 years ago)

Author:

eugene

Message:

updates to staticskytool (now mostly working)

Location:

trunk/ippTools

Files:

• notes.txt (modified) (1 diff)
• share/Makefile.am (modified) (1 diff)
• share/pxadmin_create_tables.sql (modified) (5 diffs)
• share/staticskytool_definebyquery.sql (modified) (1 diff)
• share/staticskytool_definebyquery_inputs.sql (added)
• share/staticskytool_definebyquery_select.sql (added)
• share/staticskytool_inputs.sql (added)
• share/staticskytool_result.sql (added)
• share/staticskytool_todo.sql (added)
• src/Makefile.am (modified) (3 diffs)
• src/chiptool.c (modified) (1 diff)
• src/pxtools.c (modified) (5 diffs)
• src/staticskytool.c (modified) (43 diffs)
• src/staticskytoolConfig.c (modified) (6 diffs)

trunk/ippTools/notes.txt

@@ +1 @@
+
+2010.05.24 EAM
+
+  staticskytool:
+
+    definebyquery : choose input stacks that meet a list of
+                    restrictions.  Require a specific list of filters? 
+                    Require a minimum number of inputs?
+
+      eg: sst definebyquery -filterlist g,r,i,z,y [-label md04%] [-ra_min x] [-dec_min x] ...
+
+      can we do the filter selects in the database?
 
 2008.07.19 EAM

trunk/ippTools/share/Makefile.am

@@ -289 +289 @@
      stacktool_sumskyfile.sql \
      stacktool_tosum.sql \
+     staticskytool_definebyquery_select.sql \
+     staticskytool_definebyquery_inputs.sql \
+     staticskytool_inputs.sql \
+     staticskytool_todo.sql \
+     staticskytool_result.sql \
      warptool_change_skyfile_data_state.sql \
      warptool_change_run_state.sql \

trunk/ippTools/share/pxadmin_create_tables.sql

@@ -1617 +1617 @@
 -- A static sky analysis set
 CREATE TABLE staticskyRun (
-      ss_id BIGINT AUTO_INCREMENT, -- unique identifier
+      sky_id BIGINT AUTO_INCREMENT, -- unique identifier
       state VARCHAR(64) NOT NULL,  -- state of run (new, full, etc.)
       workdir VARCHAR(255) NOT NULL, -- working directory
@@ -1626 +1626 @@
       note VARCHAR(255),             -- note
       registered TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- time run was registered
-      PRIMARY KEY(ss_id),
+      PRIMARY KEY(sky_id),
       KEY(state),
       KEY(label),
@@ -1635 +1635 @@
 -- Inputs for static sky analysis
 CREATE TABLE staticskyInput (
-      ss_id BIGINT,             -- static sky identifier
+      sky_id BIGINT,             -- static sky identifier
       stack_id BIGINT,          -- stack identifier
-      PRIMARY KEY(ss_id,stack_id),
+      PRIMARY KEY(sky_id,stack_id),
       KEY(stack_id),
-      FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id),
+      FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id),
       FOREIGN KEY(stack_id) REFERENCES stackSumSkyfile(stack_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
@@ -1645 +1645 @@
 -- Result of static sky analysis
 CREATE TABLE staticskyResult (
-      ss_id BIGINT,             -- static sky identifier
+      sky_id BIGINT,             -- static sky identifier
       path_base VARCHAR(255) NOT NULL, -- root name for outputs
       dtime_phot FLOAT,                -- elapsed time for photometry
@@ -1654 +1654 @@
       quality SMALLINT NOT NULL,       -- bad quality flag
       fault SMALLINT NOT NULL,         -- fault code
-      PRIMARY KEY(ss_id),
+      PRIMARY KEY(sky_id),
       KEY(good_frac),
       KEY(fault),
       KEY(quality),
-      FOREIGN KEY(ss_id) REFERENCES staticskyRun(ss_id)
+      FOREIGN KEY(sky_id) REFERENCES staticskyRun(sky_id)
 ) ENGINE=innodb DEFAULT CHARSET=latin1;
 

trunk/ippTools/share/staticskytool_definebyquery.sql

@@ -1 @@
-SELECT
-    tess_id,
-    skycell_id,
-    COUNT(*) AS num,
-    COUNT(DISTINCT filter) AS num_filter
-FROM stackRun
-JOIN stackSumSkyfile USING(stack_id)
-WHERE stackRun.state = 'full'
-    AND stackSumSkyfile.fault = 0
-    AND stackSumSkyfile.quality = 0
--- WHERE hook %s
-GROUP BY
-    tess_id,
-    skycell_id
+SELECT 
+       tess_id, 
+       skycell_id, 
+       num_filter 
+FROM (
+  SELECT
+      tess_id,
+      skycell_id,
+      COUNT(DISTINCT filter) AS num_filter
+  FROM stackRun
+  JOIN stackSumSkyfile USING(stack_id)
+  WHERE stackRun.state = 'full'
+      AND stackSumSkyfile.fault = 0
+      AND stackSumSkyfile.quality = 0
+  -- WHERE hook 1 %s 
+  -- restrict by stackSumSkyfile.good_frac, stackRun.skycell_id, stackRun.label
+  GROUP BY
+      tess_id,
+      skycell_id
+  ) AS TMP
+WHERE num_filter == %d
+

trunk/ippTools/src/Makefile.am

@@ -22 +22 @@
         regtool \
         stacktool \
+        staticskytool \
         warptool \
         receivetool \
@@ -66 +67 @@
         regtool.h \
         stacktool.h \
+        staticskytool.h \
         warptool.h \
         pubtool.h
@@ -197 +199 @@
     stacktoolConfig.c
 
+staticskytool_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
+staticskytool_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la
+staticskytool_SOURCES = \
+    staticskytool.c \
+    staticskytoolConfig.c
+
 pxadmin_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULES_CFLAGS) $(IPPDB_CFLAGS)
 pxadmin_LDADD = $(PSLIB_LIBS) $(PSMODULES_LIBS) $(IPPDB_LIBS) libpxtools.la

trunk/ippTools/src/chiptool.c

@@ -141 +141 @@
     )
 {
-    // start a transaction so we don't end up with an exp without any associted
+    // start a transaction so we don't end up with an exp without any associated
     // imfiles
     if (!psDBTransaction(config->dbh)) {

trunk/ippTools/src/pxtools.c

@@ -49 +49 @@
 
 psString pxMergeCodeVersions(psString version1, psString version2) {
-  psString out = NULL;
-
-  bool mod1 = false;
-  bool mod2 = false;
-
-  psS32 num1;
-  psS32 num2;
-  psS32 numO;
-
-  if (!version1) {
-    psStringAppend(&out,version2);
+    psString out = NULL;
+
+    bool mod1 = false;
+    bool mod2 = false;
+
+    psS32 num1;
+    psS32 num2;
+    psS32 numO;
+
+    if (!version1) {
+        psStringAppend(&out, "%s", version2);
+        return(out);
+    }
+    if (!version2) {
+        psStringAppend(&out, "%s", version1);
+        return(out);
+    }
+  
+    if (strchr(version1,'M')) {
+        psStringSubstitute(&version1,"M","");
+        mod1 = true;
+    }
+    if (strchr(version2,'M')) {
+        psStringSubstitute(&version2,"M","");
+        mod2 = true;
+    }
+
+    num1 = strtol(version1,NULL,10);
+    num2 = strtol(version2,NULL,10);
+
+    if (num1 >= num2) {
+        numO = num1;
+    }
+    else {
+        numO = num2;
+    }
+  
+    psStringAppend(&out,"%" PRId32,numO);
+    if (mod1 || mod2) {
+        psStringAppend(&out,"M");
+    }
     return(out);
-  }
-  if (!version2) {
-    psStringAppend(&out,version1);
-    return(out);
-  }
-  
-  if (strchr(version1,'M')) {
-    psStringSubstitute(&version1,"M","");
-    mod1 = true;
-  }
-  if (strchr(version2,'M')) {
-    psStringSubstitute(&version2,"M","");
-    mod2 = true;
-  }
-
-  num1 = strtol(version1,NULL,10);
-  num2 = strtol(version2,NULL,10);
-
-  if (num1 >= num2) {
-    numO = num1;
-  }
-  else {
-    numO = num2;
-  }
-  
-  psStringAppend(&out,"%" PRId32,numO);
-  if (mod1 || mod2) {
-    psStringAppend(&out,"M");
-  }
-  return(out);
 }
 
@@ -96 +96 @@
                          psS64 *maskfrac_npix, psF32 *maskfrac_static, psF32 *maskfrac_dynamic,
                          psF32 *maskfrac_magic, psF32 *maskfrac_advisory) {
-  psString query = pxDataGet(dbQFile);
+    psString query = pxDataGet(dbQFile);
 /*   psString text_id = NULL; */
 /*   psStringAppend(&text_id," %" PRId64,stage_id); */
 /*   psStringSubstitute(&query,text_id,"@STAGE_ID@"); */
 /*   psFree(text_id); */
-  if (!p_psDBRunQueryF(config->dbh, query, stage_id)) {
-    psError(PS_ERR_UNKNOWN, false, "database error");
+    if (!p_psDBRunQueryF(config->dbh, query, stage_id)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        return(false);
+    }
     psFree(query);
-    return(false);
-  }
-  psFree(query);
-  psArray *output = p_psDBFetchResult(config->dbh);
-  if (!output) {
-    psError(PS_ERR_UNKNOWN, false, "database error");
-    return(false);
-  }
-  
-  for (long i = 0; i < psArrayLength(output); i++) {
-    psMetadata *row = output->data[i];
-
-    psS32 this_npix = psMetadataLookupS32(NULL, row, "maskfrac_npix");
-    psF32 this_static = psMetadataLookupF32(NULL, row, "maskfrac_static");
-    psF32 this_dynamic = psMetadataLookupF32(NULL, row, "maskfrac_dynamic");
-    psF32 this_magic = psMetadataLookupF32(NULL, row, "maskfrac_magic");
-    psF32 this_advisory = psMetadataLookupF32(NULL, row, "maskfrac_advisory");
-    psString this_version = psMetadataLookupStr(NULL, row, "software_ver");
-
-    *software_ver = pxMergeCodeVersions(*software_ver,this_version);
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return(false);
+    }
+  
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+
+        psS32 this_npix = psMetadataLookupS32(NULL, row, "maskfrac_npix");
+        psF32 this_static = psMetadataLookupF32(NULL, row, "maskfrac_static");
+        psF32 this_dynamic = psMetadataLookupF32(NULL, row, "maskfrac_dynamic");
+        psF32 this_magic = psMetadataLookupF32(NULL, row, "maskfrac_magic");
+        psF32 this_advisory = psMetadataLookupF32(NULL, row, "maskfrac_advisory");
+        psString this_version = psMetadataLookupStr(NULL, row, "software_ver");
+
+        *software_ver = pxMergeCodeVersions(*software_ver,this_version);
 /*     printf("%ld : %d %f %f %f %f <-> %ld %f %f %f %f\n",i,this_npix,this_static,this_dynamic,this_magic,this_advisory, */
 /*         *maskfrac_npix,*maskfrac_static,*maskfrac_dynamic,*maskfrac_magic,*maskfrac_advisory); */
-    if (this_npix > 0) {
-      *maskfrac_static = ((*maskfrac_static * *maskfrac_npix) + (this_npix * this_static)) / (this_npix + *maskfrac_npix);
-      *maskfrac_dynamic = ((*maskfrac_dynamic * *maskfrac_npix) + (this_npix * this_dynamic)) / (this_npix + *maskfrac_npix);
-      *maskfrac_magic = ((*maskfrac_magic * *maskfrac_npix) + (this_npix * this_magic)) / (this_npix + *maskfrac_npix);
-      *maskfrac_advisory = ((*maskfrac_advisory * *maskfrac_npix) + (this_npix * this_advisory)) / (this_npix + *maskfrac_npix);
-      *maskfrac_npix += this_npix;
-    }
-  }
-  psFree(output);
-  return(true);
+        if (this_npix > 0) {
+            *maskfrac_static = ((*maskfrac_static * *maskfrac_npix) + (this_npix * this_static)) / (this_npix + *maskfrac_npix);
+            *maskfrac_dynamic = ((*maskfrac_dynamic * *maskfrac_npix) + (this_npix * this_dynamic)) / (this_npix + *maskfrac_npix);
+            *maskfrac_magic = ((*maskfrac_magic * *maskfrac_npix) + (this_npix * this_magic)) / (this_npix + *maskfrac_npix);
+            *maskfrac_advisory = ((*maskfrac_advisory * *maskfrac_npix) + (this_npix * this_advisory)) / (this_npix + *maskfrac_npix);
+            *maskfrac_npix += this_npix;
+        }
+    }
+    psFree(output);
+    return(true);
 }
 
 bool pxSetRunSoftware(pxConfig *config, const psString tableName, const psString stage_id_name, const psS64 stage_id,
                       psString software_ver) {
-  char *query = "UPDATE %s SET software_ver = '%s' WHERE %s = %" PRId64;
+    char *query = "UPDATE %s SET software_ver = '%s' WHERE %s = %" PRId64;
 /*   printf(query,tableName,software_ver,stage_id_name,stage_id); */
-  if (!p_psDBRunQueryF(config->dbh,query,tableName,software_ver,stage_id_name,stage_id)) {
-    psError(PS_ERR_UNKNOWN, false,
-            "failed to set software version for %s %" PRId64,stage_id_name,stage_id);
-    return(false);
-  }
-  
-  return(true);
+    if (!p_psDBRunQueryF(config->dbh,query,tableName,software_ver,stage_id_name,stage_id)) {
+        psError(PS_ERR_UNKNOWN, false,
+                "failed to set software version for %s %" PRId64,stage_id_name,stage_id);
+        return(false);
+    }
+  
+    return(true);
 }
 
@@ -154 +154 @@
                       psS64 maskfrac_npix, psF32 maskfrac_static, psF32 maskfrac_dynamic,
                       psF32 maskfrac_magic, psF32 maskfrac_advisory) {
-  char *query = "UPDATE %s SET maskfrac_npix = %f, maskfrac_static = %f, maskfrac_dynamic = %f, maskfrac_magic = %f, maskfrac_advisory = %f WHERE %s = %" PRId64;
-  if (!p_psDBRunQueryF(config->dbh,query,tableName,(float) maskfrac_npix,maskfrac_static,
-                       maskfrac_dynamic, maskfrac_magic,maskfrac_advisory,stage_id_name,stage_id)) {
-    psError(PS_ERR_UNKNOWN, false,
-            "failed to set maskfrac stats for %s %" PRId64,stage_id_name,stage_id);
-    return(false);
-  }
-  
-
-
-  return(true);
+    char *query = "UPDATE %s SET maskfrac_npix = %f, maskfrac_static = %f, maskfrac_dynamic = %f, maskfrac_magic = %f, maskfrac_advisory = %f WHERE %s = %" PRId64;
+    if (!p_psDBRunQueryF(config->dbh,query,tableName,(float) maskfrac_npix,maskfrac_static,
+                         maskfrac_dynamic, maskfrac_magic,maskfrac_advisory,stage_id_name,stage_id)) {
+        psError(PS_ERR_UNKNOWN, false,
+                "failed to set maskfrac stats for %s %" PRId64,stage_id_name,stage_id);
+        return(false);
+    }
+  
+
+
+    return(true);
 }
 
@@ -293 +293 @@
     char *comma = ",";
 
-#   define addColumn(_tab, _val) \
-        do { \
-            if (_val) { \
-                psStringAppend(pQuery, "%s %s.%s = '%s'", separator, _tab, #_val, _val); \
-                separator = comma; \
-            } \
-        } while (0)
+#   define addColumn(_tab, _val)                                        \
+    do {                                                                \
+        if (_val) {                                                     \
+            psStringAppend(pQuery, "%s %s.%s = '%s'", separator, _tab, #_val, _val); \
+            separator = comma;                                          \
+        }                                                               \
+    } while (0)
 
     addColumn(runTable, state);
@@ -436 +436 @@
         if (import_version && strcmp(import_version, schema_version)) {
             psError(PS_ERR_UNKNOWN, true, "input file schema_version: %s does not match data base: %s",
-                import_version, schema_version);
+                    import_version, schema_version);
             return false;
         } else if (!import_version) {

trunk/ippTools/src/staticskytool.c

@@ -1 +1 @@
 /*
- * stacktool.c
+ * staticskytool.c
  *
  * Copyright (C) 2007-2010  Joshua Hoblitt, Paul Price
@@ -29 +29 @@
 
 #include "pxtools.h"
-#include "stacktool.h"
+#include "staticskytool.h"
 
 static bool definebyqueryMode(pxConfig *config);
-static bool definerunMode(pxConfig *config);
 static bool updaterunMode(pxConfig *config);
-static bool addinputskyfileMode(pxConfig *config);
-static bool inputskyfileMode(pxConfig *config);
-static bool tosumMode(pxConfig *config);
-static bool addsumskyfileMode(pxConfig *config);
-static bool sumskyfileMode(pxConfig *config);
-static bool revertsumskyfileMode(pxConfig *config);
-static bool pendingcleanuprunMode(pxConfig *config);
-static bool pendingcleanupskyfileMode(pxConfig *config);
-static bool donecleanupMode(pxConfig *config);
-static bool updatesumskyfileMode(pxConfig *config);
+static bool inputsMode(pxConfig *config);
+static bool todoMode(pxConfig *config);
+static bool addresultMode(pxConfig *config);
+static bool resultMode(pxConfig *config);
+static bool revertMode(pxConfig *config);
+static bool updateresult(pxConfig *config);
 static bool exportrunMode(pxConfig *config);
 static bool importrunMode(pxConfig *config);
-
-static bool setstackRunState(pxConfig *config, psS64 stack_id, const char *state);
 
 # define MODECASE(caseName, func) \
@@ -60 +53 @@
     psLibInit(NULL);
 
-    pxConfig *config = stacktoolConfig(NULL, argc, argv);
+    pxConfig *config = staticskytoolConfig(NULL, argc, argv);
     if (!config) {
         psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
@@ -67 +60 @@
 
     switch (config->mode) {
-        MODECASE(STACKTOOL_MODE_DEFINEBYQUERY, definebyqueryMode);
-        MODECASE(STACKTOOL_MODE_UPDATERUN,     updaterunMode);
-        MODECASE(STACKTOOL_MODE_INPUTS,        inputsMode);
-        MODECASE(STACKTOOL_MODE_TODO,          todoMode);
-        MODECASE(STACKTOOL_MODE_ADDRESULT,     addresultMode);
-        MODECASE(STACKTOOL_MODE_RESULT,        resultMode);
-        MODECASE(STACKTOOL_MODE_REVERT,        revertMode);
-        MODECASE(STACKTOOL_MODE_UPDATERESULT,  updateresultMode);
-        MODECASE(STACKTOOL_MODE_EXPORTRUN,     exportrunMode);
-        MODECASE(STACKTOOL_MODE_IMPORTRUN,     importrunMode);
+        MODECASE(STATICSKYTOOL_MODE_DEFINEBYQUERY,     definebyqueryMode);
+        MODECASE(STATICSKYTOOL_MODE_UPDATERUN,         updaterunMode);
+        MODECASE(STATICSKYTOOL_MODE_INPUTS,            inputsMode);
+        MODECASE(STATICSKYTOOL_MODE_TODO,              todoMode);
+        MODECASE(STATICSKYTOOL_MODE_ADDRESULT,         addresultMode);
+        MODECASE(STATICSKYTOOL_MODE_RESULT,            resultMode);
+        MODECASE(STATICSKYTOOL_MODE_REVERT,            revertMode);
+        MODECASE(STATICSKYTOOL_MODE_UPDATERESULT,      updateresult);
+        MODECASE(STATICSKYTOOL_MODE_EXPORTRUN,         exportrunMode);
+        MODECASE(STATICSKYTOOL_MODE_IMPORTRUN,         importrunMode);
         default:
             psAbort("invalid option (this should not happen)");
@@ -114 +107 @@
     PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false);
 
-    psMetadata *where = psMetadataAlloc();
-    psMetadata *having = psMetadataAlloc(); // HAVING clause
-
-    PXOPT_COPY_F32(config->args, where, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">=");
-    PXOPT_COPY_STR(config->args, where, "-select_skycell_id",    "stackRun.skycell_id",       "==");
-    pxAddLabelSearchArgs(config, where, "-select_label",         "stackRun.label",            "LIKE");
-    pxAddLabelSearchArgs(config, where, "-select_filter",        "stackRun.filter",           "LIKE");
-
-    // these are used to build the HAVING restriction
-    PXOPT_COPY_S32(config->args, having, "-min_num",    "num",        ">=");
-    PXOPT_COPY_S32(config->args, having, "-min_filter", "num_filter", ">=");
+    psMetadata *whereMD = psMetadataAlloc();
+
+    PXOPT_COPY_STR(config->args, whereMD, "-select_skycell_id",    "stackRun.skycell_id",       "==");
+    PXOPT_COPY_STR(config->args, whereMD, "-select_tess_id",       "stackRun.tess_id",          "==");
+    PXOPT_COPY_F32(config->args, whereMD, "-select_good_frac_min", "stackSumSkyfile.good_frac", ">=");
+    pxAddLabelSearchArgs(config, whereMD, "-select_label",         "stackRun.label",            "LIKE");
+    pxAddLabelSearchArgs(config, whereMD, "-select_filter",        "stackRun.filter",           "LIKE");
+
+    // find the number of requested filters:
+    psMetadataItem *filters = psMetadataLookup(config->args, "-select_filter");
+    psAssert (filters, "-select_filter must exist");
+    psAssert (filters->type == PS_DATA_METADATA_MULTI, "-select_filter should be a multi container");
+    psAssert (filters->data.list->n, "-select_filter should at least have a place-holder");
+    int num_filter = filters->data.list->n;
+    if (num_filter < 2) {
+        psError(PXTOOLS_ERR_CONFIG, false, "invalid request: only 1 filter selected");
+        psFree(whereMD);
+        return false;
+    }
 
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
-
-    if (!psListLength(where->list)) {
-        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
-        psFree(where);
-        psFree(having);
-        return false;
-    }
 
     psString select = pxDataGet("staticskytool_definebyquery_select.sql");
     if (!select) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        psFree(where);
-        psFree(having);
-        return false;
-    }
-
-    psString where1 = psStringCopy("");
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&where1, "\nAND %s", whereClause);
-        psFree(whereClause);
-    }
-
-    psString where2 = psStringCopy("");
-    if (label) {
-        psStringAppend(&where2, "\nWHERE stackRun.label = '%s'", label);
-    }
-
-    // Restriction on aggregated quantities using HAVING
-    {
-        psString havingClause = NULL;   // HAVING string
-        if (psListLength(having->list)) {
-            havingClause = psDBGenerateWhereConditionSQL(having, NULL);
-        }
-
-        if (min_new > 0) {
-            if (havingClause) {
-                psStringAppend(&havingClause, " AND");
-            }
-            psStringAppend(&havingClause,
-                           " (num_warp - num_stack >= %d OR (num_warp >= %d AND num_stack IS NULL))",
-                           min_new, min_new);
-        }
-        if (isfinite(min_frac)) {
-            if (havingClause) {
-                psStringAppend(&havingClause, " AND");
-            }
-            // Avoiding division by zero
-            psStringAppend(&havingClause, " (num_warp >= %f * num_stack OR num_stack IS NULL)",
-                           (double)min_frac);
-        }
-        if (havingClause) {
-            psStringAppend(&select, " HAVING %s", havingClause);
-            psFree(havingClause);
-        }
-    }
-    psFree(having);
-
-    if (!p_psDBRunQueryF(config->dbh, select, where1, where2)) {
+        psFree(whereMD);
+        return false;
+    }
+
+    if (!psListLength(whereMD->list)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        psFree(whereMD);
+        return false;
+    }
+
+    // this 'where' is used for both staticskytool_definebyquery_select.sql and staticskytool_definebyquery_inputs.sql 
+    psString where = NULL;
+    psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL);
+    psStringAppend(&where, "\nAND %s", whereClause);
+    psFree(whereClause);
+    psFree(whereMD);
+
+    if (!p_psDBRunQueryF(config->dbh, select, where, num_filter)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(select);
-        psFree(where1);
-        psFree(where2);
         return false;
     }
     psFree(select);
-    psFree(where1);
-    psFree(where2);
+
+    // we now have a list of (tess_id, skycell_id) that (potentially) meet out needs
+    // we now need to loop over all of these and for each pair, select the best set of
+    // inputs
 
     psArray *output = p_psDBFetchResult(config->dbh);
@@ -212 +176 @@
     }
     if (!psArrayLength(output)) {
-        psWarning("stacktool: no rows found");
+        psWarning("staticskytool: no rows found");
         psFree(output);
         psFree(where);
@@ -219 +183 @@
     if (pretend) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "staticskyInput", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -230 +194 @@
     }
 
-    psString insert = NULL;             // Insertion query
-    if (randomLimit > 0) {
-        insert = pxDataGet("stacktool_definebyquery_insert_random_part1.sql");
-    } else {
-        insert = pxDataGet("stacktool_definebyquery_insert.sql");
-    }
-    if (!insert) {
+    psString inputsSQL = pxDataGet("staticskytool_definebyquery_inputs.sql");
+    if (!inputsSQL) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
         psFree(where);
@@ -242 +201 @@
     }
 
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&insert, " AND %s", whereClause);
-        psFree(whereClause);
-    }
-    psFree(where);
-
-    if (randomLimit > 0) {
-        psString part2 = pxDataGet("stacktool_definebyquery_insert_random_part2.sql");
-        if (!part2) {
-            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-            psFree(insert);
-            return false;
-        }
-        psStringAppend(&insert, "%s", part2);
-        psFree(part2);
-    }
-
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(where);
-        return false;
-    }
-
-    psArray *list = psArrayAllocEmpty(16); // List of runs, to print
     for (long i = 0; i < output->n; i++) {
         psMetadata *row = output->data[i]; // Row from select
@@ -276 +210 @@
             psError(PS_ERR_UNKNOWN, false, "failed to lookup skycell_id");
             psFree(output);
-            psFree(insert);
-            psFree(list);
+            psFree(inputsSQL);
             if (!psDBRollback(config->dbh)) {
                 psError(PS_ERR_UNKNOWN, false, "database error");
@@ -288 +221 @@
             psError(PS_ERR_UNKNOWN, false, "failed to lookup tess_id");
             psFree(output);
-            psFree(insert);
-            psFree(list);
+            psFree(inputsSQL);
             if (!psDBRollback(config->dbh)) {
                 psError(PS_ERR_UNKNOWN, false, "database error");
@@ -296 +228 @@
         }
 
-        psString filter = psMetadataLookupStr(&status, row, "filter");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to lookup filter");
+        // query for the inputs
+        if (!p_psDBRunQueryF(config->dbh, inputsSQL, tess_id, skycell_id, where)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
             psFree(output);
-            psFree(insert);
-            psFree(list);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        // create a new stackRun for this stack
-        stackRunRow *run = stackRunRowAlloc(
-            0,                          // ID
-            "new",                      // state
-            workdir,
-            label,
-            data_group ? data_group : label,
-            dist_group,
-            reduction,
-            dvodb,
-            registered,
-            skycell_id,
-            tess_id,
-            filter,
-            note);
-
-        if (!stackRunInsertObject(config->dbh, run)) {
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(inputsSQL);
+            return false;
+        }
+        
+        psArray *inputs = p_psDBFetchResult(config->dbh);
+        if (!inputs) {
+            psErrorCode err = psErrorCodeLast();
+            switch (err) {
+              case PS_ERR_DB_CLIENT:
+                psError(PXTOOLS_ERR_SYS, false, "database error");
+              case PS_ERR_DB_SERVER:
+                psError(PXTOOLS_ERR_PROG, false, "database error");
+              default:
+                psError(PXTOOLS_ERR_PROG, false, "unknown error");
+            }
+            psFree(inputs);
             psFree(output);
-            psFree(run);
-            psFree(insert);
-            psFree(list);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        // figure out the new stack_id
-        psS64 stack_id = psDBLastInsertID(config->dbh);
-        run->stack_id = stack_id;
-
-        psArrayAdd(list, list->n, run);
-        psFree(run);
-
-        // Create a suitable insertion query for this run
-        psString thisInsert = psStringCopy(insert);
-        psString idString = NULL;
-        psStringAppend(&idString, "%" PRId64, stack_id);
-        psStringSubstitute(&thisInsert, idString, "@STACK_ID@");
-        psFree(idString);
-
-        // replace @FILTER@, @SKYCELL_ID@, @RANDOM_LIMIT@
-        psStringSubstitute(&thisInsert, filter, "@FILTER@");
-        psStringSubstitute(&thisInsert, skycell_id, "@SKYCELL_ID@");
-
-        if (randomLimit > 0) {
-          psString limString = NULL;
-          psStringAppend(&limString, "%d", randomLimit);
-          psStringSubstitute(&thisInsert, limString, "@RANDOM_LIMIT@");
-          psFree(limString);
-        }
-
-        // XXX this insert uses a select to generate the list of warp_ids for the stack,
-        // we have applied a set of criteria above (WHERE) to select the relevant warps
-        // this insert needs to use exactly the same restrictions (race condition is probably not critical)
-        // the insert below seems to only restrict matches to the skycell, tess, and filter
-        if (!p_psDBRunQuery(config->dbh, thisInsert)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(thisInsert);
-            psFree(insert);
+            psFree(inputsSQL);
+            return false;
+        }
+        if (!psArrayLength(inputs)) {
+            psWarning("staticskytool ERROR: no rows found for known tess_id, skycell_id?");
+            continue;
+        }
+
+        // XXX if we are unable to guarantee that all selected inputs have all and only the
+        // requested filters, see the code at the bottom of this file (ifdef'ed out for now)
+
+        // insert a new staticsky entry and find its new id
+        if (!psDBTransaction(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(inputs);
             psFree(output);
-            psFree(list);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psFree(thisInsert);
-
-# if (0)
-        {
-          psArray *output = p_psDBFetchResult(config->dbh);
-          if (!output) {
-            psErrorCode err = psErrorCodeLast();
-            switch (err) {
-            case PS_ERR_DB_CLIENT:
-              psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-              psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-              psError(PXTOOLS_ERR_PROG, false, "unknown error");
-            }
-            psFree(where);
-            return false;
-          }
-          if (!psArrayLength(output)) {
-            psWarning("stacktool (definebyquery, insert): no rows found");
+            psFree(inputsSQL);
+            return false;
+        }
+
+        // create a chipRun
+        if (!staticskyRunInsert(config->dbh,
+                                0x0,         // sky_id
+                                "new",       // state
+                                workdir,
+                                label,
+                                data_group,
+                                dist_group,
+                                reduction,
+                                registered,
+                                note
+                )
+            ) {
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction");
+            }
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(inputs);
             psFree(output);
-            psFree(where);
-            return true;
-          }
-          // negative simple so the default is true
-          if (!ippdbPrintMetadatas(stdout, output, "stackSkycells", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            psFree(where);
-            return false;
-          }
-          psFree(output);
-          psFree(where);
-          return true;
-        }
-# endif
-
-    }
+            psFree(inputsSQL);
+            return false;
+        }
+
+        psS64 sky_id =  psDBLastInsertID(config->dbh);
+
+        // loop over the possible inputs and record only the valid ones
+        for (int j = 0; j < inputs->n; j++) {
+            psMetadata *inputRow = inputs->data[j]; // Row from select
+            
+            // pull out the skycell_id, tess_id, filter
+            psS64 stack_id = psMetadataLookupS64(&status, inputRow, "stack_id");
+            psAssert(status, "failed to find stack_id?");
+            
+            // add a staticskyInput entry
+            if (!staticskyInputInsert(config->dbh, sky_id, stack_id)) {
+                if (!psDBRollback(config->dbh)) {
+                    psError(PS_ERR_UNKNOWN, false, "database error failed to rollback transaction");
+                }
+                psError(PS_ERR_UNKNOWN, false, "database error");
+                psFree(inputs);
+                psFree(output);
+                psFree(inputsSQL);
+                return false;
+            }
+        }
+        psFree (inputs);
+        
+        if (!psDBCommit(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(output);
+            psFree(inputsSQL);
+            return false;
+        }
+    }
+    psFree(inputsSQL);
     psFree(output);
-
-    if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(list);
-        return false;
-    }
-
-    if (!stackRunPrintObjects(stdout, list, !simple)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print object");
-        psFree(list);
-        return false;
-    }
-    psFree(list);
     return true;
 }
@@ -438 +332 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-#ifdef notdef
-    PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", false, false);
-    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
-    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
-#endif
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-stack_id",  "stack_id",   "==");
-    PXOPT_COPY_STR(config->args, where, "-label",     "label",     "==");
-    PXOPT_COPY_STR(config->args, where, "-state",     "state",     "==");
+    PXOPT_COPY_S64(config->args, where, "-sky_id",  "sky_id",   "==");
+    PXOPT_COPY_STR(config->args, where, "-label",   "label",    "==");
+    PXOPT_COPY_STR(config->args, where, "-state",   "state",    "==");
     if (!psListLength(where->list)) {
         psFree(where);
@@ -453 +342 @@
     }
 
-    psString query = psStringCopy("UPDATE stackRun");
+    psString query = psStringCopy("UPDATE staticskyRun");
 
     // pxUpdateRun gets parameters from config->args and updates
-    bool result = pxUpdateRun(config, where, &query, "stackRun", "stack_id", "stackSumSkyfile", true);
-
+    bool result = pxUpdateRun(config, where, &query, "staticskyRun", "sky_id", "staticskyResult", true);
     psFree(query);
     psFree(where);
 
     return result;
-
-#ifdef notdef
-    // Hack-y work around to make stacktool more like the other tools, without breaking other stuff (hopefully).
-
-    if ((state)&&(stack_id)) {
-        // set detRun.state to state
-        return setstackRunState(config, stack_id, state);
-    }
-
-    if ((state)&&(label)) {
-      return setstackRunStateByLabel(config, label, state);
-    }
-
-    psError(PS_ERR_UNKNOWN, false, "Required options not found.");
-    return false;
-#endif
 }
 
@@ -485 +357 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    // XXX require at least a stack id (add better search options)
-    PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false);
+    // XXX require at least a sky id (add better search options)
+    PXOPT_LOOKUP_S64(sky_id, config->args, "-sky_id", true, false);
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
+    PXOPT_COPY_S64(config->args, where, "-sky_id", "sky_id", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = pxDataGet("stacktool_inputskyfile.sql");
+    psString query = pxDataGet("staticskytool_inputs.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -502 +373 @@
 
     if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, "stackInputSkyfile");
-        psStringAppend(&query, " AND %s", whereClause);
+        psString whereClause = psDBGenerateWhereConditionSQL(where, "staticskyInput");
+        psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
     }
@@ -537 +408 @@
     }
     if (!psArrayLength(output)) {
-        psTrace("stacktool", PS_LOG_INFO, "no rows found");
+        psTrace("staticskytool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
@@ -544 +415 @@
     if (psArrayLength(output)) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "stackInputSkyfile", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "staticskyInput", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -561 +432 @@
     PS_ASSERT_PTR_NON_NULL(config, false);
 
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-stack_id", "stack_id", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "stackRun.label", "==");
+    psMetadata *whereMD = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, whereMD, "-sky_id", "sky_id", "==");
+    pxAddLabelSearchArgs (config, whereMD, "-label", "staticskyRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = pxDataGet("stacktool_tosum.sql");
+    psString query = pxDataGet("staticskytool_todo.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -575 +445 @@
     }
 
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
-        psFree(whereClause);
-    }
-    psFree(where);
+    if (!psListLength(whereMD->list)) {
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        psFree(whereMD);
+        return false;
+    }
+
+    psString where = NULL;
+    psString whereClause = psDBGenerateWhereConditionSQL(whereMD, NULL);
+    psStringAppend(&where, "\n AND %s", whereClause);
+    psFree(whereClause);
+    psFree(whereMD);
 
     // treat limit == 0 as "no limit"
@@ -589 +464 @@
     }
 
-    if (!p_psDBRunQuery(config->dbh, query)) {
+    // the where clause is required and matches the WHERE hook format string
+    if (!p_psDBRunQueryF(config->dbh, query, where)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -611 +487 @@
     }
     if (!psArrayLength(output)) {
-        psTrace("stacktool", PS_LOG_INFO, "no rows found");
+        psTrace("staticskytool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
@@ -618 +494 @@
     if (psArrayLength(output)) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "staticskyResult", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -626 +502 @@
 
     psFree(output);
-
     return true;
 }
 
-
 static bool addresultMode(pxConfig *config)
 {
@@ -636 +510 @@
 
     // required
+    PXOPT_LOOKUP_S64(sky_id, config->args, "-sky_id", true, false);
     PXOPT_LOOKUP_S64(stack_id, config->args, "-stack_id", true, false);
 
     // optional
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
     PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
-    PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
-    PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
-    PXOPT_LOOKUP_F32(dtime_stack, config->args, "-dtime_stack", false, false);
-    PXOPT_LOOKUP_F32(dtime_match_mean, config->args, "-dtime_match_mean", false, false);
-    PXOPT_LOOKUP_F32(dtime_match_stdev, config->args, "-dtime_match_stdev", false, false);
-    PXOPT_LOOKUP_F32(dtime_initial, config->args, "-dtime_initial", false, false);
-    PXOPT_LOOKUP_F32(dtime_reject, config->args, "-dtime_reject", false, false);
-    PXOPT_LOOKUP_F32(dtime_final, config->args, "-dtime_final", false, false);
     PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false);
     PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
-    PXOPT_LOOKUP_F32(match_mean, config->args, "-match_mean", false, false);
-    PXOPT_LOOKUP_F32(match_stdev, config->args, "-match_stdev", false, false);
-    PXOPT_LOOKUP_F32(match_rms, config->args, "-match_rms", false, false);
-    PXOPT_LOOKUP_F32(stamps_mean, config->args, "-stamps_mean", false, false);
-    PXOPT_LOOKUP_F32(stamps_stdev, config->args, "-stamps_stdev", false, false);
-    PXOPT_LOOKUP_S32(stamps_min, config->args, "-stamps_min", false, false);
-    PXOPT_LOOKUP_S32(reject_images, config->args, "-reject_images", false, false);
-    PXOPT_LOOKUP_F32(reject_pix_mean, config->args, "-reject_pix_mean", false, false);
-    PXOPT_LOOKUP_F32(reject_pix_stdev, config->args, "-reject_pix_stdev", false, false);
     PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false);
     PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
@@ -668 +525 @@
     PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
 
+    // XXX not sure we need a transaction here...
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -673 +531 @@
     }
 
-    // XXX need to validate the stack_id here
-    // XXX instead of validiting it here we should just use forgein key
-    // constrants
-    if (!stackSumSkyfileInsert(config->dbh,
+    if (!staticskyResultInsert(config->dbh,
+                               sky_id,
                                stack_id,
-                               uri,
                                path_base,
-                               bg,
-                               bg_stdev,
-                               dtime_stack,
-                               dtime_match_mean,
-                               dtime_match_stdev,
-                               dtime_initial,
-                               dtime_reject,
-                               dtime_final,
                                dtime_phot,
                                dtime_script,
-                               match_mean,
-                               match_stdev,
-                               match_rms,
-                               stamps_mean,
-                               stamps_stdev,
-                               stamps_min,
-                               reject_images,
-                               reject_pix_mean,
-                               reject_pix_stdev,
                                sources,
                                hostname,
@@ -712 +550 @@
     }
 
-    if (fault == 0) {
-        if (!setstackRunState(config, stack_id, "full")) {
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            psError(PS_ERR_UNKNOWN, false, "failed to change stackRun's state");
-            return false;
-        }
-    }
-
     // point of no return
     if (!psDBCommit(config->dbh)) {
@@ -731 +559 @@
 }
 
-
+// XXX what is this used by?  what filters are needed?
 static bool resultMode(pxConfig *config)
 {
@@ -737 +565 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id", "stackRun.tess_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "stackRun.skycell_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-filter", "stackRun.filter", "LIKE");
-    PXOPT_COPY_STR(config->args, where, "-label", "stackRun.label", "==");
-    PXOPT_COPY_STR(config->args, where, "-data_group", "stackRun.data_group", "LIKE");
-    PXOPT_COPY_S16(config->args, where, "-fault", "stackSumSkyfile.fault", "==");
-
-//  The following three selectors are incompatible with the sql so omit them
-//    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpRun.warp_id", "==");
-//     PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
-//    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
+    PXOPT_COPY_S64(config->args, where, "-sky_id",     "staticskyRun.sky_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-label",      "staticskyRun.label", "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "staticskyRun.data_group", "LIKE");
+    PXOPT_COPY_S16(config->args, where, "-fault",      "staticskyResult.fault", "==");
 
     PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
@@ -755 +575 @@
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    psString query = pxDataGet("stacktool_sumskyfile.sql");
+    psString query = pxDataGet("staticskytool_result.sql");
     if (!query) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -801 +621 @@
     }
     if (!psArrayLength(output)) {
-        psTrace("stacktool", PS_LOG_INFO, "no rows found");
+        psTrace("staticskytool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
@@ -807 +627 @@
 
     if (psArrayLength(output)) {
-        if (!ippdbPrintMetadatas(stdout, output, "stackSumSkyfile", !simple)) {
+        if (!ippdbPrintMetadatas(stdout, output, "staticskyResult", !simple)) {
             psError(PS_ERR_UNKNOWN, false, "failed to print array");
             psFree(output);
@@ -819 +639 @@
 }
 
-
 static bool revertMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+# if (0)
     psMetadata *where = psMetadataAlloc();
     PXOPT_COPY_S64(config->args, where, "-stack_id", "stackSumSkyfile.stack_id", "==");
@@ -836 +656 @@
 
     // Delete product
-    psString delete = pxDataGet("stacktool_revertsumskyfile_delete.sql");
+    psString delete = pxDataGet("staticskytool_revertsumskyfile_delete.sql");
     if (!delete) {
         psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
@@ -857 +677 @@
 
     int numRows = psDBAffectedRows(config->dbh); // Number of row affected
-    psLogMsg("stacktool", PS_LOG_INFO, "Deleted %d rows", numRows);
+    psLogMsg("staticskytool", PS_LOG_INFO, "Deleted %d rows", numRows);
 
     psFree(where);
+# endif
 
     return true;
 }
 
-
-static bool updateresultState(pxConfig *config, psS64 stack_id, const char *state)
-{
-    PS_ASSERT_PTR_NON_NULL(state, false);
-
+# if (0)
+static bool updateresultState(pxConfig *config)
+{
     // check that state is a valid string value
     if (!pxIsValidState(state)) {
@@ -881 +700 @@
         return false;
     }
-
     return true;
 }
-
-static bool updateresultMode(pxConfig *config)
+# endif
+
+static bool updateresult(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, false);
 
+# if (0)
     PXOPT_LOOKUP_S16(fault, config->args, "-fault", true, false);
 
@@ -901 +721 @@
     psFree (where);
 
+# endif
     return true;
 }
@@ -906 +727 @@
 bool exportrunMode(pxConfig *config)
 {
+# if (0)
   typedef struct ExportTable {
     char tableName[80];
@@ -934 +756 @@
 
   ExportTable tables [] = {
-    {"stackRun", "stacktool_export_run.sql"},
-    {"stackInputSkyfile", "stacktool_export_input_skyfile.sql"},
-    {"stackSumSkyfile", "stacktool_export_sum_skyfile.sql"},
+    {"stackRun", "staticskytool_export_run.sql"},
+    {"stackInputSkyfile", "staticskytool_export_input_skyfile.sql"},
+    {"stackSumSkyfile", "staticskytool_export_sum_skyfile.sql"},
   };
 
@@ -999 +821 @@
     fclose (f);
 
+# endif
     return true;
 }
@@ -1004 +827 @@
 bool importrunMode(pxConfig *config)
 {
+# if (0)
   unsigned int nFail;
 
@@ -1072 +896 @@
   }
 
+# endif
   return true;
 }
+
+# if (0)
+        // now we need to loop over all requested filters and check that each is found
+        // XXX is this needed?  haven't we required that we only match the requested filters
+        // and that N unique filters are found?
+
+        psVector *found = psVectorAlloc(inputs->n, PS_TYPE_U8);
+        psVectorInit (found, 0);
+
+        psMetadataItem *filter = NULL;
+        psListIterator *iter = psListIteratorAlloc (filters->data.list, PS_LIST_HEAD, false);
+        while ((filter = psListGetAndIncrement(iter))) {
+
+            bool foundOne = false;
+
+            psAssert (filter->type == PS_DATA_STR, "filter is not a string?");
+            for (int j = 0; !foundOne && (j < inputs->n); j++) {
+                if (found->data.U8[j]) continue;
+                psMetadata *inputRow = inputs->data[j]; // Row from select
+
+                // pull out the skycell_id, tess_id, filter
+                psString inFilter = psMetadataLookupStr(&status, row, "filter");
+                psAssert (status);
+
+                found->data.U8[j] = true;
+                foundOne = true;
+            }
+            if (!foundOne) {
+                // this required filter was not found in the inputs, skip the entry
+                skip();
+            }
+        }           
+# endif
+

trunk/ippTools/src/staticskytoolConfig.c

@@ -52 +52 @@
     psMetadataAddF32(definebyqueryArgs,  PS_LIST_TAIL, "-select_good_frac_min", 0, "define min good_frac", 0.0);
     psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_label", PS_META_DUPLICATE_OK, "search by stackRun label (LIKE comparison, multiple OK)", NULL);
-    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple OK)", NULL);
+    psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-select_filter", PS_META_DUPLICATE_OK, "search by filter (LIKE comparison, multiple required)", NULL);
     psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-set_workdir", 0, "define workdir (required)", NULL);
     psMetadataAddStr(definebyqueryArgs,  PS_LIST_TAIL, "-set_label", 0, "define label", NULL);
@@ -89 +89 @@
     // -addresult
     psMetadata *addresultArgs = psMetadataAlloc();
-    psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-sky_id", 0, "define stack ID (required)", 0);
+    psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-sky_id", 0, "define sky ID (required)", 0);
+    psMetadataAddS64(addresultArgs, PS_LIST_TAIL, "-stack_id", 0, "define stack ID (required)", 0);
     psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-path_base", 0, "define base output location", 0);
     psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-dtime_phot", 0, "define photometry time", NAN);
@@ -96 +97 @@
     psMetadataAddStr(addresultArgs, PS_LIST_TAIL, "-hostname", 0, "define hostname", 0);
     psMetadataAddF32(addresultArgs, PS_LIST_TAIL, "-good_frac", 0, "define fraction of good pixels", NAN);
-    psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0)
+    psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-quality", 0, "set quality", 0);
     psMetadataAddS16(addresultArgs, PS_LIST_TAIL, "-fault", 0, "set fault code", 0);
 
@@ -102 +103 @@
     psMetadata *resultArgs= psMetadataAlloc();
     psMetadataAddS64(resultArgs, PS_LIST_TAIL, "-sky_id", 0, "search by staticsky ID", 0);
-    psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0);
-    psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0);
+    // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-tess_id", 0, "search by tess ID", 0);
+    // psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-skycell_id", 0, "search by skycell ID", 0);
     psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-label", 0, "search by label", NULL);
     psMetadataAddStr(resultArgs, PS_LIST_TAIL, "-data_group", 0, "search by data_group (LIKE comparison)", NULL);
@@ -109 +110 @@
     psMetadataAddU64(resultArgs, PS_LIST_TAIL, "-limit", 0, "limit result set to N items", 0);
     psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-simple", 0, "use the simple output format", false);
+    psMetadataAddBool(resultArgs, PS_LIST_TAIL, "-all", 0, "allow no search terms", false);
 
     // -revert
@@ -143 +145 @@
     PXOPT_ADD_MODE("-inputs",        "Get inputs",           STATICSKYTOOL_MODE_INPUTS,        inputsArgs);
     PXOPT_ADD_MODE("-todo",          "Get runs to do",       STATICSKYTOOL_MODE_TODO,          todoArgs);
-    PXOPT_ADD_MODE("-addresult",     "Add result of run",    STATICSKYTOOL_MODE_ADDRESULT,     addresultsArgs);
+    PXOPT_ADD_MODE("-addresult",     "Add result of run",    STATICSKYTOOL_MODE_ADDRESULT,     addresultArgs);
     PXOPT_ADD_MODE("-result",        "Get result of run",    STATICSKYTOOL_MODE_RESULT,        resultArgs);
     PXOPT_ADD_MODE("-revert",        "Revert failed run",    STATICSKYTOOL_MODE_REVERT,        revertArgs);