Changeset 35149 for branches/czw_branch/20130211/ippTools/src/difftool.c

Timestamp:

Feb 11, 2013, 4:50:12 PM (13 years ago)

Author:

watersc1

Message:

Changes that should not conflict with the trunk. This is the camera stage background continuity and the pattern.median example code.

Location:

branches/czw_branch/20130211/ippTools

Files:

• . (modified) (1 prop)
• src (modified) (1 prop)
• src/difftool.c (modified) (10 diffs)

branches/czw_branch/20130211/ippTools/src/difftool.c

@@ -41 +41 @@
 static bool revertdiffskyfileMode(pxConfig *config);
 static bool definepoprunMode(pxConfig *config);
-static bool definewarpstackMode(pxConfig *config);
+//static bool definewarpstackMode(pxConfig *config);
+static bool definewarpstackModeNew(pxConfig *config);
 static bool definewarpwarpMode(pxConfig *config);
 static bool definestackstackMode(pxConfig *config);
@@ -94 +95 @@
         MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
         MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
-        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,       definewarpstackMode);
+        MODECASE(DIFFTOOL_MODE_DEFINEWARPSTACK,       definewarpstackModeNew);
         MODECASE(DIFFTOOL_MODE_DEFINEWARPWARP,        definewarpwarpMode);
         MODECASE(DIFFTOOL_MODE_DEFINESTACKSTACK,      definestackstackMode);
@@ -1226 +1227 @@
 }
 
-
-static bool definewarpstackMode(pxConfig *config)
+static bool definewarpstackModeNew(pxConfig *config)
 {
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
+  PS_ASSERT_PTR_NON_NULL(config,false);
     psMetadata *expWhere = psMetadataAlloc();
     psMetadata *warp1Where = psMetadataAlloc(); // First set of restrictions on warp
-    psMetadata *warp2Where = psMetadataAlloc(); // Second set of restriction on warp
+    //    psMetadata *warp2Where = psMetadataAlloc(); // Second set of restriction on warp
     psMetadata *stackWhere = psMetadataAlloc();
 
@@ -1243 +1242 @@
     PXOPT_COPY_STR(config->args, warp1Where, "-tess_id", "warpRun.tess_id", "==");
     PXOPT_COPY_STR(config->args, warp1Where, "-data_group", "warpRun.data_group", "==");
-    PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "==");
-    PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "==");
-    PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "==");
-    PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "==");
-    PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "==");
-    PXOPT_COPY_F32(config->args, warp2Where,  "-good_frac", "warpSkyfile.good_frac", ">=");
+    //    PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "==");
+    //    PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "==");
+    //    PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "==");
+    //    PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "==");
+    //    PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "==");
+    PXOPT_COPY_F32(config->args, warp1Where,  "-good_frac", "warpSkyfile.good_frac", ">=");
     PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
     PXOPT_COPY_STR(config->args, stackWhere, "-stack_data_group", "stackRun.data_group", "==");
@@ -1268 +1267 @@
     PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
 
-    // find all things to queue
-    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    psString warp1Query = NULL;
-    psString warp2Query = NULL;
-    psString stackQuery = NULL;
-    psString expQuery = NULL;
-
+    psString newQuery = NULL;
+    psStringAppend(&newQuery, "select exp_id,warp_id,rawExp.filter,warpRun.label,warpRun.data_group,warpRun.tess_id,warpRun.data_group,");
+    psStringAppend(&newQuery, "warpSkyfile.skycell_id,stack_id,stackRun.skycell_id,stackRun.tess_id,stackRun.label,stackRun.data_group,diff_id ");
+    psStringAppend(&newQuery, "from warpRun JOIN fakeRun USING(fake_id) JOIN camRun USING(cam_id) JOIN chipRun USING(chip_id) JOIN rawExp USING(exp_id) ");
+    psStringAppend(&newQuery, "JOIN warpSkyfile USING(warp_id) JOIN stackRun USING(skycell_id,filter) ");
+    psStringAppend(&newQuery, "LEFT JOIN diffInputSkyfile ON (warp_id = diffInputSkyfile.warp1 AND stack_id = diffInputSkyfile.stack2 AND diffInputSkyfile.skycell_id = stackRun.skycell_id) WHERE 1");
+    
     if (psListLength(expWhere->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL);
-        psStringAppend(&expQuery, "\n AND %s", whereClause);
+        psStringAppend(&newQuery, "\n AND %s", whereClause);
         psFree(whereClause);
     } else {
-        expQuery = psStringCopy("\n");
+      //        newQuery = psStringCopy("\n");
     }
     psFree(expWhere);
     if (psListLength(warp1Where->list)) {
         psString whereClause = psDBGenerateWhereConditionSQL(warp1Where, NULL);
-        psStringAppend(&warp1Query, "\n AND %s", whereClause);
+        psStringAppend(&newQuery, "\n AND %s", whereClause);
         psFree(whereClause);
     } else {
-        warp1Query = psStringCopy("\n");
+      //        waQuery = psStringCopy("\n");
     }
     psFree(warp1Where);
-    if (psListLength(warp2Where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(warp2Where, NULL);
-        psStringAppend(&warp2Query, "\n AND %s", whereClause);
+    
+    // don't queue for exposures that have already been diff'd unless requested
+    //    psString diffQuery = NULL;
+    if (! (reRun || newTemplates) ) {
+        psStringAppend(&newQuery, "\nAND diff_id IS NULL");
+    } else {
+      //        diffQuery = psStringCopy("\n");
+    }
+
+    if (psListLength(stackWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
+        psStringAppend(&newQuery, "\nAND %s", whereClause);
         psFree(whereClause);
     } else {
-        warp2Query = psStringCopy("\n");
-    }
-    psFree(warp2Where);
-
-    // don't queue for exposures that have already been diff'd unless requested
-    psString diffQuery = NULL;
-    if (! (reRun || newTemplates) ) {
-        psStringAppend(&diffQuery, "\nAND diff_id IS NULL");
-    } else {
-        diffQuery = psStringCopy("\n");
-    }
-
-    if (psListLength(stackWhere->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
-        psStringAppend(&stackQuery, "\nAND %s", whereClause);
-        psFree(whereClause);
-    } else {
-        stackQuery = psStringCopy("");
+      //        stackQuery = psStringCopy("");
     }
     psFree(stackWhere);
 
-    psTrace("difftool", 1, query, warp1Query, warp2Query, diffQuery, expQuery, stackQuery);
-
+    //    psTrace("difftool", 1, query, warp1Query, warp2Query, diffQuery, expQuery, stackQuery);
+    fprintf(stderr,"%s",newQuery);
+    return(true);
+#if (0)    
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1329 +1317 @@
     }
 
+    
     if (!p_psDBRunQueryF(config->dbh, query, warp1Query, warp2Query, expQuery, diffQuery)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -1367 +1356 @@
         return true;
     }
-
-    if (pretend) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
-        psFree(output);
-        return true;
-    }
-
-    // create temporary table
-    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        return false;
-    }
-    psFree(warp1Query);
-    psFree(query);
-    query = NULL;
-
-    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");
-
     psArray *list = psArrayAllocEmpty(16); // List of runs, to print
     long numGood = 0;                   // Number of good rows added
@@ -1401 +1362 @@
         psMetadata *row = output->data[i]; // Output row from query
         bool mdok;                      // Status of MD lookup
-
-        // Take the first warp for each exposure.
-        // The list is sorted by exposure id and warp_id and the warps are in descending
-        // order.
-        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        if (exp_id == last_exp_id) {
-            continue;
-        }
-        last_exp_id = exp_id;
-
-        // clear temporary table
-        if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i);
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "skycell_count not found");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        psString filter = psMetadataLookupStr(&mdok, row, "filter");
-        if (!mdok) {
-            psError(PXTOOLS_ERR_PROG, false, "filter not found");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        psString warp_data_group = psMetadataLookupStr(&mdok, row, "data_group");
-        if (!mdok) {
-          psError(PXTOOLS_ERR_PROG, false, "warp data_group not found");
-          psFree(warp2Query);
-          psFree(stackQuery);
-          psFree(skycell_query);
-          if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-          }
-          return false;
-        }
-        if (!data_group && warp_data_group) {
-          data_group = warp_data_group;
-        }
-
-        if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warp2Query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psS64 num = psDBAffectedRows(config->dbh);
-
-        if (num == 0) {
-            psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for warp_id %" PRId64, warp_id);
-            continue;
-        }
-
-        if (!available && (num != skycell_count)) {
-            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" PRId64
-                    " but need %" PRId64, num, warp_id, skycell_count);
-            continue;
-        }
-
         // ok we've got one create the diffRun
         diffRunRow *run = diffRunRowAlloc(
@@ -1537 +1386 @@
                 note
         );
-
-        if (!diffRunInsertObject(config->dbh, run)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(run);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        run->diff_id = psDBLastInsertID(config->dbh);
-
-        psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id);
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(warp2Query);
-            psFree(stackQuery);
-            psFree(skycell_query);
-            psFree(query);
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-            return false;
-        }
-        psFree(query);
-        query = NULL;
         psStringAppend(&query,
                        "INSERT INTO diffInputSkyfile(diff_id, skycell_id, warp1, stack1, warp2, stack2, tess_id) SELECT diff_id, skycell_id, warp1, stack1, warp2, stack2, tess_id from skycellsToDiff");
@@ -1613 +1437 @@
 
     return true;
+#endif
 }
-
+#if (0)   
+static bool definewarpstackMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *expWhere = psMetadataAlloc();
+    psMetadata *warp1Where = psMetadataAlloc(); // First set of restrictions on warp
+    psMetadata *warp2Where = psMetadataAlloc(); // Second set of restriction on warp
+    psMetadata *stackWhere = psMetadataAlloc();
+
+    PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "==");
+    PXOPT_COPY_STR(config->args, expWhere, "-filter", "filter", "==");
+    PXOPT_COPY_STR(config->args, expWhere, "-comment", "comment", "LIKE");
+    PXOPT_COPY_S64(config->args, warp1Where, "-warp_id", "warpRun.warp_id", "==");
+    PXOPT_COPY_STR(config->args, warp1Where, "-warp_label", "warpRun.label", "==");
+    PXOPT_COPY_STR(config->args, warp1Where, "-tess_id", "warpRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, warp1Where, "-data_group", "warpRun.data_group", "==");
+    PXOPT_COPY_S64(config->args, warp2Where, "-warp_id", "warpRun.warp_id", "==");
+    PXOPT_COPY_STR(config->args, warp2Where, "-tess_id", "warpRun.tess_id", "==");
+    PXOPT_COPY_STR(config->args, warp2Where, "-data_group", "warpRun.data_group", "==");
+    PXOPT_COPY_STR(config->args, warp2Where, "-skycell_id", "warpSkyfile.skycell_id", "==");
+    PXOPT_COPY_STR(config->args, warp2Where, "-warp_label", "warpRun.label", "==");
+    PXOPT_COPY_F32(config->args, warp2Where,  "-good_frac", "warpSkyfile.good_frac", ">=");
+    PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");
+    PXOPT_COPY_STR(config->args, stackWhere, "-stack_data_group", "stackRun.data_group", "==");
+
+    PXOPT_LOOKUP_BOOL(bothways, config->args, "-bothways", false);
+
+    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false); // required option
+    PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false); // option
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false); // option
+    PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-set_registered", false, false);
+
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false);
+    PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
+    PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    // find all things to queue
+    psString query = pxDataGet("difftool_definewarpstack_part1.sql");
+    if (!query) {
+        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString warp1Query = NULL;
+    psString warp2Query = NULL;
+    psString stackQuery = NULL;
+    psString expQuery = NULL;
+
+    if (psListLength(expWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL);
+        psStringAppend(&expQuery, "\n AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        expQuery = psStringCopy("\n");
+    }
+    psFree(expWhere);
+    if (psListLength(warp1Where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(warp1Where, NULL);
+        psStringAppend(&warp1Query, "\n AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        warp1Query = psStringCopy("\n");
+    }
+    psFree(warp1Where);
+    if (psListLength(warp2Where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(warp2Where, NULL);
+        psStringAppend(&warp2Query, "\n AND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        warp2Query = psStringCopy("\n");
+    }
+    psFree(warp2Where);
+
+    // don't queue for exposures that have already been diff'd unless requested
+    psString diffQuery = NULL;
+    if (! (reRun || newTemplates) ) {
+        psStringAppend(&diffQuery, "\nAND diff_id IS NULL");
+    } else {
+        diffQuery = psStringCopy("\n");
+    }
+
+    if (psListLength(stackWhere->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
+        psStringAppend(&stackQuery, "\nAND %s", whereClause);
+        psFree(whereClause);
+    } else {
+        stackQuery = psStringCopy("");
+    }
+    psFree(stackWhere);
+
+    psTrace("difftool", 1, query, warp1Query, warp2Query, diffQuery, expQuery, stackQuery);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, query, warp1Query, warp2Query, expQuery, diffQuery)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psErrorCode err = psErrorCodeLast();
+        switch (err) {
+          case PS_ERR_DB_CLIENT:
+            psError(PXTOOLS_ERR_SYS, false, "database error");
+            break;
+          case PS_ERR_DB_SERVER:
+            psError(PXTOOLS_ERR_PROG, false, "database error");
+            break;
+          default:
+            psError(PXTOOLS_ERR_PROG, false, "unknown error");
+            break;
+        }
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("difftool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        if (!psDBCommit(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            return false;
+        }
+        return true;
+    }
+
+    if (pretend) {
+        // negative simple so the default is true
+        if (!ippdbPrintMetadatas(stdout, output, "diffRun", !simple)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+    // create temporary table
+    query = pxDataGet("difftool_definewarpstack_temp_create.sql");
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+        return false;
+    }
+    psFree(warp1Query);
+    psFree(query);
+    query = NULL;
+
+    psString skycell_query = pxDataGet("difftool_definewarpstack_part2.sql");
+
+    psArray *list = psArrayAllocEmpty(16); // List of runs, to print
+    long numGood = 0;                   // Number of good rows added
+    psS64 last_exp_id = 0;
+    for (long i = 0; i < output->n; i++) {
+        psMetadata *row = output->data[i]; // Output row from query
+        bool mdok;                      // Status of MD lookup
+
+        // Take the first warp for each exposure.
+        // The list is sorted by exposure id and warp_id and the warps are in descending
+        // order.
+        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        if (exp_id == last_exp_id) {
+            continue;
+        }
+        last_exp_id = exp_id;
+
+        // clear temporary table
+        if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i);
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "skycell_count not found");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psString filter = psMetadataLookupStr(&mdok, row, "filter");
+        if (!mdok) {
+            psError(PXTOOLS_ERR_PROG, false, "filter not found");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psString warp_data_group = psMetadataLookupStr(&mdok, row, "data_group");
+        if (!mdok) {
+          psError(PXTOOLS_ERR_PROG, false, "warp data_group not found");
+          psFree(warp2Query);
+          psFree(stackQuery);
+          psFree(skycell_query);
+          if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+          }
+          return false;
+        }
+        if (!data_group && warp_data_group) {
+          data_group = warp_data_group;
+        }
+
+        if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warp2Query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psS64 num = psDBAffectedRows(config->dbh);
+
+        if (num == 0) {
+            psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for warp_id %" PRId64, warp_id);
+            continue;
+        }
+
+        if (!available && (num != skycell_count)) {
+            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %" PRId64
+                    " but need %" PRId64, num, warp_id, skycell_count);
+            continue;
+        }
+
+        // ok we've got one create the diffRun
+        diffRunRow *run = diffRunRowAlloc(
+                0,            // ID
+                "reg",        // state
+                workdir,
+                label,
+                data_group ? data_group : label,
+                dist_group,
+                reduction,
+                NULL,         // dvodb
+                registered,
+                tess_id,
+                bothways,     // bothways (default is false)
+                true,         // exposure
+                0,            // magicked
+                NULL,         // software version
+                0,            // mask stat npix
+                NAN,          // static
+                NAN,          // dynamic
+                NAN,          // magic
+                NAN,          // advisory
+                IPP_DIFF_MODE_WARP_STACK,
+                note
+        );
+
+        if (!diffRunInsertObject(config->dbh, run)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(run);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        run->diff_id = psDBLastInsertID(config->dbh);
+
+        psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id);
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+        }
+            return false;
+        }
+        psFree(query);
+        query = NULL;
+        psStringAppend(&query,
+                       "INSERT INTO diffInputSkyfile(diff_id, skycell_id, warp1, stack1, warp2, stack2, tess_id) SELECT diff_id, skycell_id, warp1, stack1, warp2, stack2, tess_id from skycellsToDiff");
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(PS_ERR_UNKNOWN, false, "database error");
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+        psFree(query);
+        query = NULL;
+
+        if (!setdiffRunState(config, run->diff_id, "new", false)) {
+            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
+                run->diff_id);
+            psFree(warp2Query);
+            psFree(stackQuery);
+            psFree(skycell_query);
+            psFree(query);
+            if (!psDBRollback(config->dbh)) {
+                psError(PS_ERR_UNKNOWN, false, "database error");
+            }
+            return false;
+        }
+
+        psArrayAdd(list, list->n, run);
+        numGood++;
+    }
+    psFree(output);
+    psFree(warp2Query);
+    psFree(stackQuery);
+    psFree(skycell_query);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        psFree(list);
+        return false;
+    }
+
+    if (numGood && !diffRunPrintObjects(stdout, list, !simple)) {
+        psError(PS_ERR_UNKNOWN, false, "failed to print object");
+        psFree(list);
+        return false;
+    }
+    psFree(list);
+
+    return true;
+}
+#endif
 static bool definewarpwarpMode(pxConfig *config)
 {