Changeset 28199 for branches/pap/ippTools/src/bgtool.c

Timestamp:

Jun 2, 2010, 4:33:05 PM (16 years ago)

Author:

Paul Price

Message:

Adding first cut at bgtool, to implement background restoration pipeline.

Location:

branches/pap/ippTools/src

Files:

• . (modified) (1 prop)
• bgtool.c (modified) (41 diffs)

branches/pap/ippTools/src

@@ -39 +39 @@
 warptool
 staticskytool
+
+bgtool

@@ -39 +39 @@
 warptool
 staticskytool
+
+bgtool

branches/pap/ippTools/src/bgtool.c

@@ -1 +1 @@
 /*
- * warptool.c
+ * bgtool.c
  *
- * Copyright (C) 2006  Joshua Hoblitt
+ * Copyright (C) 2006-2010  Joshua Hoblitt, Paul Price
  *
  * This program is free software; you can redistribute it and/or modify it
@@ -29 +29 @@
 
 #include "pxtools.h"
-#include "warptool.h"
+#include "bgtool.h"
 
 static bool definechipMode(pxConfig *config);
 static bool updatechipMode(pxConfig *config);
 static bool tochipMode(pxConfig *config);
-static bool chipInputsMode(pxConfig *config);
+static bool chipinputsMode(pxConfig *config);
 static bool addchipMode(pxConfig *config);
 static bool chipMode(pxConfig *config);
@@ -42 +42 @@
 static bool updatewarpMode(pxConfig *config);
 static bool towarpMode(pxConfig *config);
-static bool warpInputsMode(pxConfig *config);
+static bool warpinputsMode(pxConfig *config);
 static bool addwarpMode(pxConfig *config);
 static bool warpMode(pxConfig *config);
@@ -56 +56 @@
 static bool importwarpMode(pxConfig *config);
 
+// Tables to import/export
+typedef struct {
+    const char *name;                   // Table name
+    void* (*parse)();                   // Parsing function
+    bool (*insert)();                   // Insertion function
+} tableData;
+static const tableData chipTables[] = {
+    { "chipBackgroundRun", (void*)&chipBackgroundRunObjectFromMetadata, &chipBackgroundRunInsertObject },
+    { "chipBackgroundImfile", (void*)&chipBackgroundImfileObjectFromMetadata, &chipBackgroundImfileInsertObject },
+    { NULL, NULL, NULL }
+};
+static const tableData warpTables[] = {
+    { "warpBackgroundRun", (void*)&warpBackgroundRunObjectFromMetadata, &warpBackgroundRunInsertObject },
+    { "warpBackgroundImfile", (void*)&warpBackgroundSkyfileObjectFromMetadata, &warpBackgroundSkyfileInsertObject },
+    { NULL, NULL, NULL }
+};
+
+
 # define MODECASE(caseName, func) \
     case caseName: \
@@ -77 +95 @@
         MODECASE(BGTOOL_MODE_UPDATECHIP,  updatechipMode);
         MODECASE(BGTOOL_MODE_TOCHIP,      tochipMode);
-        MODECASE(BGTOOL_MODE_CHIPINPUTS,  chipInputsMode);
+        MODECASE(BGTOOL_MODE_CHIPINPUTS,  chipinputsMode);
         MODECASE(BGTOOL_MODE_ADDCHIP,     addchipMode);
         MODECASE(BGTOOL_MODE_CHIP,        chipMode);
@@ -85 +103 @@
         MODECASE(BGTOOL_MODE_UPDATEWARP,  updatewarpMode);
         MODECASE(BGTOOL_MODE_TOWARP,      towarpMode);
-        MODECASE(BGTOOL_MODE_WARPINPUTS,  warpInputsMode);
+        MODECASE(BGTOOL_MODE_WARPINPUTS,  warpinputsMode);
         MODECASE(BGTOOL_MODE_ADDWARP,     addwarpMode);
         MODECASE(BGTOOL_MODE_WARP,        warpMode);
@@ -119 +137 @@
     exit(exit_status);
 }
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// General functions
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static bool exportTables(pxConfig *config, // Configuration (with DB handle)
+                         const char *filename, // Filename to which to write
+                         const tableData tables[], // Tables to export (NULL terminated)
+                         const psMetadata *where, // WHERE restrictions
+                         bool clean               // Write as cleaned?
+                         )
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    FILE *file = fopen(filename, "w");
+    if (!file) {
+        psError(PXTOOLS_ERR_SYS, true, "failed to open output file %s", filename);
+        return false;
+    }
+
+    if (!pxExportVersion(config, file)) {
+        psError(psErrorCodeLast(), false, "failed to write dbversion to output file %s", filename);
+        return false;
+    }
+
+    for (int i = 0; tables[i].name; i++) {
+        const char *name = tables[i].name; // Name of table
+        psString query = NULL;
+        psStringAppend(&query, "SELECT * FROM %s", name);
+
+        if (psListLength(where->list)) {
+            psString whereClause = psDBGenerateWhereSQL(where, NULL);
+            psStringAppend(&query, " WHERE %s", whereClause);
+            psFree(whereClause);
+        }
+
+        if (!p_psDBRunQuery(config->dbh, query)) {
+            psError(psErrorCodeLast(), false, "database error");
+            psFree(query);
+            return false;
+        }
+        psFree(query);
+
+        psArray *output = p_psDBFetchResult(config->dbh);
+        if (!output) {
+            psError(psErrorCodeLast(), false, "database error");
+            return false;
+        }
+        if (!psArrayLength(output)) {
+            psError(PXTOOLS_ERR_CONFIG, true, "no rows found");
+            psFree(output);
+            return false;
+        }
+
+        if (clean &&
+            (strcmp(name, "chipBackgroundRun") == 0 ||
+             strcmp(name, "warpBackgroundRun") == 0) &&
+            !pxSetStateCleaned(name, "state", output)) {
+            psFree(output);
+            psError(psErrorCodeLast(), false, "pxSetStateClean failed for table %s", name);
+            return false;
+        }
+
+        if (!ippdbPrintMetadatas(file, output, name, true)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+        psFree(output);
+    }
+    fclose(file);
+
+    return true;
+}
+
+static bool importTables(pxConfig *config, // Configuration
+                        const char *filename, //
+                        const tableData tables[] // Tables to read in
+                        )
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    unsigned int badLines = 0;          // Number of bad lines
+    psMetadata *input = psMetadataConfigRead(NULL, &badLines, filename, false); // Input file contents
+    if (!input) {
+        psError(psErrorCodeLast(), false, "Unable to parse input file %s", filename);
+        return false;
+    }
+    if (badLines > 0) {
+        psWarning("%d bad lines encountered when parsing %s", badLines, filename);
+    }
+
+    if (!pxCheckImportVersion(config, input)) {
+        psError(psErrorCodeLast(), false, "pxCheckImportVersion failed");
+        return false;
+    }
+
+    // Import primary table
+    for (int i = 0; tables[i].name; i++) {
+        const char *name = tables[i].name; // Name of table
+        psMetadataItem *item = psMetadataLookup(input, name); // Item from input
+        psAssert(item, "%s not in input", name);
+        psAssert(item->type == PS_DATA_METADATA_MULTI, "%s not MULTI type", name);
+        psAssert(psListLength(item->data.list) == 1, "%s has multiple entries", name);
+        psMetadataItem *entry = psListGet(item->data.list, PS_LIST_HEAD); // Entry of interest
+        void *data = tables[i].parse(entry);                             // Parsed entry
+        if (!data) {
+            psError(PXTOOLS_ERR_CONFIG, false, "Unable to parse entry %s", name);
+            psFree(input);
+            return false;
+        }
+        if (!tables[0].insert(config->dbh, data)) {
+            psError(psErrorCodeLast(), false, "Unable to insert entry %s", name);
+            psFree(input);
+            return false;
+        }
+    }
+
+    psFree(input);
+
+    return true;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Functions for chip stage
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 static bool definechipMode(pxConfig *config)
@@ -293 +437 @@
 
     psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.warp_id",   "==");
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id",   "==");
     PXOPT_COPY_STR(config->args, where, "-state",      "chipBackgroundRun.state",     "==");
     PXOPT_COPY_STR(config->args, where, "-data_group", "chipBackgroundRun.data_group","LIKE");
@@ -312 +456 @@
     }
 
-    psString query = psStringCopy("UPDATE chipBackgroundRun "
-                                  "JOIN chipBackgroundImfile USING(chip_bg_id) "
-                                  "JOIN chipRun USING(chip_id) "
-                                  "JOIN rawExp USING(exp_id)");
-
-    // pxUpdateRun gets parameters from config->args and updates
+    psString query = psStringCopy("UPDATE chipBackgroundRun JOIN chipBackgroundImfile USING(chip_bg_id)");
     bool result = pxUpdateRun(config, where, &query, "chipBackgroundRun", "chip_bg_id",
                               "chipBackgroundImfile", true);
@@ -352 +491 @@
     psFree(where);
 
-    psString limitString = psStringCopy("\nORDER BY priority DESC, warp_id");
+    psString limitString = psStringCopy("\nORDER BY priority DESC, chip_bg_id");
     if (limit) {
-        // We apply the limit to both sides of the UNION to avoid slow queries
-        // and to the query itself to satisfy the user's requested limit
         psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit));
         psStringAppend(&query, " %s", limitString);
@@ -394 +531 @@
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-static bool chipMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
+static bool chipinputsMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "chipProcessedImfile.class_id", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
@@ -435 +543 @@
 
     // find all rawImfiles matching the default query
-    psString query = pxDataGet("warptool_imfile.sql");
+    psString query = pxDataGet("bgtool_chipinputs.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
         return false;
     }
 
     if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun");
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
         psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
@@ -448 +556 @@
     psFree(where);
 
-    // treat limit == 0 as "no limit"
     if (limit) {
         psString limitString = psDBGenerateLimitSQL(limit);
@@ -456 +563 @@
 
     if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+        psError(psErrorCodeLast(), false, "database error");
         psFree(query);
         return false;
@@ -464 +571 @@
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
+        psError(psErrorCodeLast(), false, "database error");
         return false;
     }
     if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
     }
-
-    if (psArrayLength(output)) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "warpInputImfile", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
+    if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
     }
 
@@ -497 +591 @@
 
 
-static bool tooverlapMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
-
+static bool addchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true, false);
+    PXOPT_LOOKUP_STR(class_id, config->args, "-class_id", true, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
+
+    // optional
+
+    PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
+    PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
+    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false);
+    PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false);
+    PXOPT_LOOKUP_STR(ver_ppbackground, config->args, "-ver_ppbackground", false, false);
+    PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false);
+    PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
+    PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
+    PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_dynamic, config->args, "-maskfrac_dynamic", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_magic, config->args, "-maskfrac_magic", false, false);
+    PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false);
+
+    psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules);
+    ver_code = pxMergeCodeVersions(ver_code, ver_ppbackground);
+    ver_code = pxMergeCodeVersions(ver_code, ver_ppstats);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(PS_ERR_UNKNOWN, false, "database error");
+        return false;
+    }
+
+    if (!chipBackgroundImfileInsert(config->dbh, chip_bg_id, class_id, path_base, magicked, dtime_script,
+                                    hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
+                                    maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool chipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id",    "chipBackgroundRun.chip_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label",   "chipBackgroundRun.label", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-data_group",   "chipBackgroundRun.data_group", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-dist_group",   "chipBackgroundRun.data_group", "LIKE");
+
+    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
     // find all rawImfiles matching the default query
-    psString query = pxDataGet("warptool_tooverlap.sql");
+    psString query = pxDataGet("bgtool_chip.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString magicWhere = NULL;
+    if (!pxmagicAddWhere(config, &magicWhere, "chipBackgroundImfile")) {
+        psError(psErrorCodeLast(), false, "pxMagicAddWhere failed");
+        return false;
+    }
+    if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) {
+        psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed");
         return false;
     }
 
     if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun");
-        psStringAppend(&query, " AND %s", whereClause);
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
-    }
-    psFree(where);
-
-    // treat limit == 0 as "no limit"
+    } else if (!all && !magicWhere) {
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
+        return false;
+    }
+    if (magicWhere) {
+        psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere);
+    }
+    psFree(magicWhere);
+    psFree(where);
+
     if (limit) {
         psString limitString = psDBGenerateLimitSQL(limit);
@@ -530 +698 @@
 
     if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    if (psArrayLength(output)) {
+        if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundImfile", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(output);
+
+    return true;
+}
+
+
+static bool advancechipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "label", "==");
+
+    psString select = pxDataGet("bgtool_advancechip.sql");
+    if (!select) {
+        psError(psErrorCodeLast(), false, "failed to retrieve SQL statement");
+        return false;
+    }
+
+    psString selectWhere = psStringCopy("");
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereSQL(where, NULL);
+        psStringAppend(&selectWhere, "\n WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(select);
+        psFree(selectWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    psFree(select);
+    psFree(selectWhere);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+        bool status = true;             // Status of MD lookup
+        psS64 chip_bg_id = psMetadataLookupS64(&status, row, "chip_bg_id");
+        if (!status) {
+            psError(PXTOOLS_ERR_PROG, true, "failed to look up value for chip_bg_id");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psS64 magicked = psMetadataLookupS64(&status, row, "magicked");
+        if (!status) {
+            psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        if (!p_psDBRunQueryF(config->dbh,
+                             "UPDATE chipBackgroundRun "
+                             "SET state = 'full', magicked = %" PRId64 " "
+                             " WHERE chip_bg_id = %" PRId64,
+                             chip_bg_id, magicked)) {
+            psError(psErrorCodeLast(), false, "database error");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        psS64 numUpdated = psDBAffectedRows(config->dbh);
+        if (numUpdated != 1) {
+            psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row");
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool revertchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-class_id", "chipBackgroundImfile.class_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "chipBackgroundImfile.fault", "==");
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required");
+        return false;
+    }
+
+    psString query = pxDataGet("bgtool_revertchip.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
         psFree(query);
@@ -536 +871 @@
     psFree(query);
 
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    if (psArrayLength(output)) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-
-static bool addoverlapMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    PXOPT_LOOKUP_STR(mapfile, config->args, "-mapfile", false, false);
-    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false);
-    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
-
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    if (fault == 0) {
-        if (!parseAndInsertSkyCellMap(config, mapfile)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to inject mapfile: %s into the database", mapfile);
-            // rollback
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-    } else {
-        warpSkyCellMapInsert(config->dbh,
-            warp_id,
-            "faulted",   // skycell_id
-            "faulted",   // tess_id
-            "faulted",   // class_id
-            fault    // fault
-        );
-    }
-
-    // point of no return
-    if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-
-static bool revertoverlapMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyCellMap.warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyCellMap.skycell_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id",    "warpSkyCellMap.tess_id", "==");
-    pxAddLabelSearchArgs (config, where, "-label",     "warpRun.label", "==");
-    PXOPT_COPY_S16(config->args, where, "-fault",      "warpSkyCellMap.fault", "==");
-
-    if (!psListLength(where->list)
-        && !psMetadataLookupBool(NULL, config->args, "-all")) {
-        psFree(where);
-        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
-        return false;
-    }
-
-    int numDeleted;                     // Number deleted
-    {
-        psString query = pxDataGet("warptool_revertoverlap.sql");
-        if (!query) {
-            psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-
-        if (psListLength(where->list)) {
-            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-            psStringAppend(&query, " AND %s", whereClause);
-            psFree(whereClause);
-        }
-
-        if (!p_psDBRunQuery(config->dbh, query)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(query);
-            if (!psDBRollback(config->dbh)) {
-                psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return false;
-        }
-        psFree(query);
-
-        numDeleted = psDBAffectedRows(config->dbh);
-    }
-
-    psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkycellMap", numDeleted);
-
-    psFree(where);
-
-    return true;
-}
-
-
-
-static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile)
-{
-    unsigned int nFail = 0;
-    psMetadata *imfiles = psMetadataAlloc();
-    psMetadata *skycells = psMetadataConfigRead(NULL, &nFail, mapfile, false);
-    if (!skycells) {
-        psError(PS_ERR_UNKNOWN, false, "failed to parse mapfile: %s", mapfile);
-        return false;
-    }
-    if (nFail) {
-        psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing mapfile: %s", nFail, mapfile);
-        psFree(skycells);
-        psFree(imfiles);
-        return false;
-    }
-
-    psMetadataItem *item = NULL;
-    psMetadataIterator *iter = psMetadataIteratorAlloc(skycells, 0, NULL);
-    while ((item = psMetadataGetAndIncrement(iter))) {
-        if (item->type != PS_DATA_METADATA) {
-            psError(PS_ERR_UNKNOWN, false, "mapfile: %s is in the wrong format", mapfile);
-            psFree(iter);
-            psFree(skycells);
-            psFree(imfiles);
-            return false;
-        }
-
-        psMetadata *sc = item->data.md;
-        // this conversion isn't strictly nessicary but it's an easy way of
-        // validating the format
-        warpSkyCellMapRow *row = warpSkyCellMapObjectFromMetadata(sc);
-        if (!row) {
-            psError(PS_ERR_UNKNOWN, false, "failed to convert mapfile: %s metdata entry into a warpSkyCellMap object", mapfile);
-            psFree(iter);
-            psFree(skycells);
-            psFree(imfiles);
-            return false;
-        }
-        psMetadataAddS64(imfiles, PS_LIST_TAIL, row->skycell_id, PS_META_REPLACE, "", row->warp_id);
-
-        if (!warpSkyCellMapInsertObject(config->dbh, row)) {
-            psErrorCode err = psErrorCodeLast();
-            switch (err) {
-                case PS_ERR_DB_CLIENT:
-                    psError(PXTOOLS_ERR_SYS, false, "database error");
-                case PS_ERR_DB_SERVER:
-                    psError(PXTOOLS_ERR_PROG, false, "database error");
-                default:
-                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
-            }
-            psFree(row);
-            psFree(iter);
-            psFree(skycells);
-            psFree(imfiles);
-            return false;
-        }
-
-        psFree(row);
-    }
-    psFree(iter);
-
-    // create warp_skyfile_ids for the output skyfiles
-    psString query = "INSERT INTO warpImfile VALUES(%" PRId64 ", '%s', 0)";
-    iter = psMetadataIteratorAlloc(imfiles, 0, NULL);
-    while ((item = psMetadataGetAndIncrement(iter))) {
-        psString skycell_id = item->name;
-        psS64 warp_id = item->data.S64;
-
-        if (!p_psDBRunQueryF(config->dbh, query, warp_id, skycell_id)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            return false;
-        }
-    }
-    psFree(iter);
-    psFree(skycells);
-    psFree(imfiles);
-
-    return true;
-}
-
-
-static bool scmapMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-class_id", "class_id", "==");
+    int numDeleted = psDBAffectedRows(config->dbh);
+    psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d chipBackgroundImfiles", numDeleted);
+
+    return true;
+}
+
+static bool tocleanchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    // find all rawImfiles matching the default query
-    psString query = pxDataGet("warptool_scmap.sql");
+    psMetadata *where = psMetadataAlloc();
+    pxAddLabelSearchArgs(config, where, "-label", "chipBackgroundRun.label", "==");
+
+    psString query = pxDataGet("bgtool_tocleanchip.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, "warpSkyCellMap");
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (where && psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
         psStringAppend(&query, " AND %s", whereClause);
         psFree(whereClause);
@@ -777 +900 @@
     psFree(where);
 
-    // treat limit == 0 as "no limit"
     if (limit) {
         psString limitString = psDBGenerateLimitSQL(limit);
@@ -785 +907 @@
 
     if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+        psError(psErrorCodeLast(), false, "database error");
         psFree(query);
         return false;
@@ -793 +915 @@
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
+        psError(psErrorCodeLast(), false, "database error");
         return false;
     }
     if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
     }
 
-    if (psArrayLength(output)) {
+    // negative simple so the default is true
+    if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundRun", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+    psFree(output);
+
+    return true;
+}
+
+static bool cleanedchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chipBackgroundRun.chip_bg_id", "==");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
+    if (!pxIsValidCleanedState(state)) {
+        psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state);
+        return false;
+    }
+
+    if (!psListLength(where->list)) {
+        psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set.");
+        return false;
+    }
+
+    psString query = pxDataGet("bgtool_cleanedchip.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!p_psDBRunQueryF(config->dbh, query, state)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    psFree(query);
+
+    return true;
+}
+
+static bool exportchipMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(chip_bg_id, config->args, "-chip_bg_id", true,  false);
+    PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
+    PXOPT_LOOKUP_BOOL(clean,  config->args, "-clean", false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-chip_bg_id", "chip_bg_id", "==");
+
+    bool status = exportTables(config, outfile, chipTables, where, clean);
+
+    psFree(where);
+    return status;
+}
+
+static bool importchipMode(pxConfig *config)
+{
+    PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+
+    return importTables(config, infile, chipTables);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Functions for warp stage
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+static bool definewarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, NULL);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args,   where, "-warp_id",            "warpRun.warp_id",       "==");
+    PXOPT_COPY_S64(config->args,   where, "-exp_id",             "rawExp.exp_id",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-exp_name",           "rawExp.exp_name",       "==");
+    PXOPT_COPY_STR(config->args,   where, "-inst",               "rawExp.camera",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-telescope",          "rawExp.telescope",      "==");
+    PXOPT_COPY_TIME(config->args,  where, "-dateobs_begin",      "rawExp.dateobs",        ">=");
+    PXOPT_COPY_TIME(config->args,  where, "-dateobs_end",        "rawExp.dateobs",        "<=");
+    PXOPT_COPY_STR(config->args,   where, "-exp_tag",            "rawExp.exp_tag",        "==");
+    PXOPT_COPY_STR(config->args,   where, "-exp_type",           "rawExp.exp_type",       "==");
+    PXOPT_COPY_STR(config->args,   where, "-filelevel",          "rawExp.filelevel",      "==");
+    PXOPT_COPY_STR(config->args,   where, "-filter",             "rawExp.filter",         "==");
+    PXOPT_COPY_F64(config->args,   where, "-airmass_min",        "rawExp.airmass",        ">=");
+    PXOPT_COPY_F64(config->args,   where, "-airmass_max",        "rawExp.airmass",        "<");
+    PXOPT_COPY_RADEC(config->args, where, "-ra_min",             "rawExp.ra",             ">=");
+    PXOPT_COPY_RADEC(config->args, where, "-ra_max",             "rawExp.ra",             "<");
+    PXOPT_COPY_RADEC(config->args, where, "-decl_min",           "rawExp.decl",           ">=");
+    PXOPT_COPY_RADEC(config->args, where, "-decl_max",           "rawExp.decl",           "<");
+    PXOPT_COPY_F32(config->args,   where, "-exp_time_min",       "rawExp.exp_time",       ">=");
+    PXOPT_COPY_F32(config->args,   where, "-exp_time_max",       "rawExp.exp_time",       "<");
+    PXOPT_COPY_F32(config->args,   where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">=");
+    PXOPT_COPY_F32(config->args,   where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<");
+    PXOPT_COPY_F64(config->args,   where, "-bg_min",             "rawExp.bg",             ">=");
+    PXOPT_COPY_F64(config->args,   where, "-bg_max",             "rawExp.bg",             "<");
+    PXOPT_COPY_F64(config->args,   where, "-bg_stdev_min",       "rawExp.bg_stdev",       ">=");
+    PXOPT_COPY_F64(config->args,   where, "-bg_stdev_max",       "rawExp.bg_stdev",       "<");
+    PXOPT_COPY_F64(config->args,   where, "-bg_mean_stdev_min",  "rawExp.bg_mean_stdev",  ">=");
+    PXOPT_COPY_F64(config->args,   where, "-bg_mean_stdev_max",  "rawExp.bg_mean_stdev",  "<");
+    PXOPT_COPY_F64(config->args,   where, "-alt_min",            "rawExp.alt",            ">=");
+    PXOPT_COPY_F64(config->args,   where, "-alt_max",            "rawExp.alt",            "<");
+    PXOPT_COPY_F64(config->args,   where, "-az_min",             "rawExp.az",             ">=");
+    PXOPT_COPY_F64(config->args,   where, "-az_max",             "rawExp.az",             "<");
+    PXOPT_COPY_F32(config->args,   where, "-ccd_temp_min",       "rawExp.ccd_temp",       ">=");
+    PXOPT_COPY_F32(config->args,   where, "-ccd_temp_max",       "rawExp.ccd_temp",       "<");
+    PXOPT_COPY_F64(config->args,   where, "-posang_min",         "rawExp.posang",         ">=");
+    PXOPT_COPY_F64(config->args,   where, "-posang_max",         "rawExp.posang",         "<");
+    PXOPT_COPY_STR(config->args,   where, "-object",             "rawExp.object",         "==");
+    PXOPT_COPY_STR(config->args,   where, "-comment",            "rawExp.comment",        "LIKE");
+    PXOPT_COPY_STR(config->args,   where, "-obs_mode",           "rawExp.obs_mode",       "LIKE");
+    PXOPT_COPY_F32(config->args,   where, "-sun_angle_min",      "rawExp.sun_angle",      ">=");
+    PXOPT_COPY_F32(config->args,   where, "-sun_angle_max",      "rawExp.sun_angle",      "<");
+    pxAddLabelSearchArgs(config,   where, "-warp_label",         "warpRun.label",         "==");
+    pxAddLabelSearchArgs(config,   where, "-bg_label",           "chipBackgroundRun.label", "==");
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required");
+        return false;
+    }
+
+    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", true, false);
+    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
+    PXOPT_LOOKUP_STR(data_group, config->args, "-set_data_group", false, false);
+    PXOPT_LOOKUP_STR(dist_group, config->args, "-set_dist_group", false, false);
+    PXOPT_LOOKUP_STR(reduction, config->args, "-set_reduction", false, false);
+    PXOPT_LOOKUP_STR(note, config->args, "-set_note", false, false);
+    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+    PXOPT_LOOKUP_BOOL(pretend, config->args, "-pretend", false);
+
+    // Get warp runs to promote to warpBackgroundRun
+
+    psString query = pxDataGet("bgtool_definewarp.sql"); // Query to execute
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        psFree(where);
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, "AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh); // Matching rows
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return true;
+    }
+
+    if (pretend) {
         // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "warpSkyCellMap", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
-    }
-
+        if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psFree(output);
+        return true;
+    }
+
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *md = output->data[i];
+
+        psS64 chip_bg_id = psMetadataLookupS64(NULL, md, "chip_bg_id");
+
+        warpRunRow *row = warpRunObjectFromMetadata(md);
+        if (!row) {
+            psError(psErrorCodeLast(), false, "failed to convert metadata into fakeRun");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        if (!warpBackgroundRunInsert(config->dbh, 0, row->warp_id, chip_bg_id, "new",
+                                     workdir     ? workdir    : row->workdir,
+                                     label       ? label      : row->label,
+                                     data_group  ? data_group : row->data_group,
+                                     dist_group  ? dist_group : row->dist_group,
+                                     reduction   ? reduction  : row->reduction,
+                                     note        ? note       : row->note,
+                                     NULL)) {
+            psError(psErrorCodeLast(), false, "database error");
+            psFree(row);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psFree(row);
+    }
     psFree(output);
 
-    return true;
-}
-
-static bool towarpedMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+        return false;
+    }
+
+    return true;
+}
+
+static bool updatewarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id",   "==");
+    PXOPT_COPY_STR(config->args, where, "-state",      "warpBackgroundRun.state",     "==");
+    PXOPT_COPY_STR(config->args, where, "-data_group", "warpBackgroundRun.data_group","LIKE");
+    PXOPT_COPY_STR(config->args, where, "-dist_group", "warpBackgroundRun.dist_group","LIKE");
+    pxAddLabelSearchArgs(config,  where, "-label",     "warpBackgroundRun.label",     "LIKE");
+    PXOPT_COPY_TIME(config->args, where, "-registered_begin", "warpBackgroundRun.registered",  ">=");
+    PXOPT_COPY_TIME(config->args, where, "-registered_end",   "warpBackgroundRun.registered",  "<");
+
+    PXOPT_LOOKUP_BOOL(destreaked, config->args, "-destreaked", false);
+    if (destreaked) {
+        psMetadataAddS64(where, PS_LIST_TAIL, "warpBackgroundRun.magicked", PS_META_DUPLICATE_OK, ">", 0);
+    }
+
+    if (!psListLength(where->list)) {
+        psFree(where);
+        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
+        return false;
+    }
+
+    psString query = psStringCopy("UPDATE warpBackgroundRun JOIN warpBackgroundSkyfile USING(warp_bg_id)");
+    bool result = pxUpdateRun(config, where, &query, "warpBackgroundRun", "warp_bg_id",
+                              "warpBackgroundSkyfile", true);
+
+    psFree(query);
+    psFree(where);
+
+    return result;
+}
+
+static bool towarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
+    pxAddLabelSearchArgs (config, where, "-label", "warpBackgroundRun.label", "==");
 
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
-    // find all rawImfiles matching the default query
-    psString query = pxDataGet("warptool_towarped.sql");
+    psString query = pxDataGet("bgtool_towarp.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
         return false;
     }
@@ -851 +1233 @@
     psFree(where);
 
-    // treat limit == 0 as "no limit"
-    psString limitString = psStringCopy("\nORDER BY priority DESC, warp_id");
+    psString limitString = psStringCopy("\nORDER BY priority DESC, warp_bg_id");
     if (limit) {
-        // We apply the limit to both sides of the UNION to avoid slow queries
-        // and to the query itself to satisfy the user's requested limit
         psStringAppend(&limitString, "%s", psDBGenerateLimitSQL(limit));
         psStringAppend(&query, " %s", limitString);
@@ -861 +1240 @@
 
     if (!p_psDBRunQueryF(config->dbh, query, whereStr, limitString, whereStr,  limitString)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+        psError(psErrorCodeLast(), false, "database error");
         psFree(query);
         return false;
@@ -871 +1250 @@
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
+        psError(psErrorCodeLast(), false, "Unable to fetch result of query %s", query);
         return false;
     }
     if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
@@ -890 +1260 @@
 
     if (psArrayLength(output)) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
+        if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
             psFree(output);
             return false;
@@ -904 +1273 @@
 
 
-static bool addwarpedMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
+static bool warpinputsMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpProcessedSkyfile.skycell_id", "==");
+
+    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
+    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
+
+    // find all rawImfiles matching the default query
+    psString query = pxDataGet("bgtool_warpinputs.sql");
+    if (!query) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
+        psStringAppend(&query, " AND %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (limit) {
+        psString limitString = psDBGenerateLimitSQL(limit);
+        psStringAppend(&query, " %s", limitString);
+        psFree(limitString);
+    }
+
+    if (!p_psDBRunQuery(config->dbh, query)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(query);
+        return false;
+    }
+    psFree(query);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+    if (!ippdbPrintMetadatas(stdout, output, "chipBackgroundSkyfile", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
+    }
+
+    psFree(output);
+
+    return true;
+}
+
+
+static bool addwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true, false);
     PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
-    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
+    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", true, false);
 
     // optional
-    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
-    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
+
+    PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
+    PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
+    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
+    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
+    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
+    PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false);
+    PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false);
+    PXOPT_LOOKUP_STR(ver_ppbackground, config->args, "-ver_ppbackground", false, false);
+    PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false);
     PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
     PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
-    PXOPT_LOOKUP_F32(dtime_warp, config->args, "-dtime_warp", false, false);
-    PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
-    PXOPT_LOOKUP_S32(xmin, config->args, "-xmin", false, false);
-    PXOPT_LOOKUP_S32(xmax, config->args, "-xmax", false, false);
-    PXOPT_LOOKUP_S32(ymin, config->args, "-ymin", false, false);
-    PXOPT_LOOKUP_S32(ymax, config->args, "-ymax", false, false);
-    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
-    PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
-    PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
-
-    PXOPT_LOOKUP_STR(ver_pslib, config->args, "-ver_pslib", false, false);
-    PXOPT_LOOKUP_STR(ver_psmodules, config->args, "-ver_psmodules", false, false);
-    PXOPT_LOOKUP_STR(ver_psphot, config->args, "-ver_psphot", false, false);
-    PXOPT_LOOKUP_STR(ver_ppstats, config->args, "-ver_ppstats", false, false);
-    PXOPT_LOOKUP_STR(ver_pswarp, config->args, "-ver_pswarp", false, false);
-    PXOPT_LOOKUP_STR(ver_streaks, config->args, "-ver_streaks", false, false);
-
     PXOPT_LOOKUP_S32(maskfrac_npix, config->args, "-maskfrac_npix", false, false);
     PXOPT_LOOKUP_F32(maskfrac_static, config->args, "-maskfrac_static", false, false);
@@ -940 +1360 @@
     PXOPT_LOOKUP_F32(maskfrac_advisory, config->args, "-maskfrac_advisory", false, false);
 
-    psTrace("czw.test",1,"Received versions: pslib %s psmodules %s psphot %s ppstats %s pswarp %s streaks %s\n",
-            ver_pslib,ver_psmodules,ver_psphot,ver_ppstats,ver_pswarp,ver_streaks);
-    psString ver_code = NULL;
-    if ((ver_pslib)&&(ver_psmodules)) {
-      ver_code = pxMergeCodeVersions(ver_pslib,ver_psmodules);
-    }
-    if (ver_psphot) {
-      ver_code = pxMergeCodeVersions(ver_code,ver_psphot);
-    }
-    if (ver_ppstats) {
-      ver_code = pxMergeCodeVersions(ver_code,ver_ppstats);
-    }
-    if (ver_pswarp) {
-      ver_code = pxMergeCodeVersions(ver_code,ver_pswarp);
-    }
-    if (ver_streaks) {
-      ver_code = pxMergeCodeVersions(ver_code,ver_streaks);
-    }
-
-    // default values
-    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
-    PXOPT_LOOKUP_S16(quality, config->args, "-quality", false, false);
-
-
-
-    // we don't want to insert the last skyfile in a run but then not mark the
-    // run as 'stop'
+    psString ver_code = pxMergeCodeVersions(ver_pslib, ver_psmodules);
+    ver_code = pxMergeCodeVersions(ver_code, ver_ppbackground);
+    ver_code = pxMergeCodeVersions(ver_code, ver_ppstats);
+
     if (!psDBTransaction(config->dbh)) {
         psError(PS_ERR_UNKNOWN, false, "database error");
@@ -972 +1369 @@
     }
 
-    // XXX need to validate that this coresponds to an warpInputImfile
-    if (!warpSkyfileInsert(config->dbh,
-                           warp_id,
-                           skycell_id,
-                           tess_id,
-                           uri,
-                           path_base,
-                           "full",      // data_state
-                           bg,
-                           bg_stdev,
-                           dtime_warp,
-                           dtime_script,
-                           hostname,
-                           good_frac,
-                           xmin,
-                           xmax,
-                           ymin,
-                           ymax,
-                           fault,
-                           quality,
-                           magicked,
-                           ver_code,
-                           maskfrac_npix,
-                           maskfrac_static,
-                           maskfrac_dynamic,
-                           maskfrac_magic,
-                           maskfrac_advisory
-        )) {
+    if (!warpBackgroundSkyfileInsert(config->dbh, warp_bg_id, skycell_id, path_base, magicked, dtime_script,
+                                    hostname, quality, fault, ver_code, bg, bg_stdev, maskfrac_npix,
+                                    maskfrac_static, maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
+        psError(psErrorCodeLast(), false, "database error");
         if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    // point of no return
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+
     if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-
-static bool advancerunMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "label", "==");
-
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
-
-    psString query = pxDataGet("warptool_finished_run_select.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
-        return false;
-    }
-
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereSQL(where, NULL);
-        psStringAppend(&query, " %s", whereClause);
-        psFree(whereClause);
-    }
-    psFree(where);
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    query = pxDataGet("warptool_finish_run.sql");
-    for (long i = 0; i < psArrayLength(output); i++) {
-        psMetadata *row = output->data[i];
-
-        bool status;
-        psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id");
-
-        psString software_ver = NULL;
-        psS64 maskfrac_npix = 0;
-        psF32 maskfrac_static = 0;
-        psF32 maskfrac_dynamic = 0;
-        psF32 maskfrac_magic = 0;
-        psF32 maskfrac_advisory = 0;
-
-        // Calculate run level masking and software state
-        if (!pxCoalesceRunStatus(config,"warptool_coalesce_run.sql",warp_id,
-                                 &software_ver,&maskfrac_npix,
-                                 &maskfrac_static,&maskfrac_dynamic,
-                                 &maskfrac_magic,&maskfrac_advisory)) {
-          psError(PS_ERR_UNKNOWN, false, "failed to generate run level statistics");
-          psFree(output);
-          if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-          }
-          return(false);
-        }
-        // Set warpRun.software_ver to the appropriate value    if (
-        if (software_ver) {
-          if (!pxSetRunSoftware(config, "warpRun", "warp_id", warp_id, software_ver)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64,
-                    warp_id);
-            psFree(output);
-            if (!psDBRollback(config->dbh)) {
-              psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return(false);
-          }
-        }
-        // Set warpRun.maskfrac* to the appropriate values.
-        if (maskfrac_npix) {
-          if (!pxSetRunMaskfrac(config, "warpRun", "warp_id", warp_id, maskfrac_npix, maskfrac_static,
-                                maskfrac_dynamic, maskfrac_magic, maskfrac_advisory)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to set warpRun.software_ver for warp_id: %" PRId64,
-                    warp_id);
-            psFree(output);
-            if (!psDBRollback(config->dbh)) {
-              psError(PS_ERR_UNKNOWN, false, "database error");
-            }
-            return(false);
-          }
-        }
-
-
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id");
-            psFree(output);
-            psFree(query);
-            return false;
-        }
-        psS64 magicked = psMetadataLookupS64(&status, row, "magicked");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to look up value for magicked");
-            psFree(output);
-            psFree(query);
-            return false;
-        }
-        if (!p_psDBRunQueryF(config->dbh, query, magicked, warp_id)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(output);
-            psFree(query);
-            return false;
-        }
-
-        psS64 numUpdated = psDBAffectedRows(config->dbh);
-
-        if (numUpdated != 1) {
-            psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
-            psFree(query);
-            psFree(output);
-            return false;
-        }
-    }
-    psFree(output);
-    psFree(query);
-
-    return true;
-}
-
-bool warpCompletedRuns(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psString query = pxDataGet("warptool_finished_run_select.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retrieve SQL statement");
-        return false;
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    query = pxDataGet("warptool_finish_run.sql");
-    for (long i = 0; i < psArrayLength(output); i++) {
-        psMetadata *row = output->data[i];
-
-        bool status;
-        psS64 warp_id = psMetadataLookupS64(&status, row, "warp_id");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to look up value for warp_id");
-            psFree(output);
-            psFree(query);
-            return false;
-        }
-        psS32 magicked = psMetadataLookupS64(&status, row, "magicked");
-        if (!status) {
-            psError(PS_ERR_UNKNOWN, false, "failed to look up value for magicked");
-            psFree(output);
-            psFree(query);
-            return false;
-        }
-        if (!p_psDBRunQueryF(config->dbh, query, magicked, warp_id)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-            psFree(output);
-            psFree(query);
-            return false;
-        }
-
-        psS64 numUpdated = psDBAffectedRows(config->dbh);
-
-        if (numUpdated != 1) {
-            psError(PS_ERR_UNKNOWN, false, "should have affected 1 row");
-            psFree(query);
-            psFree(output);
-            return false;
-        }
-    }
-    psFree(output);
-    psFree(query);
-
-    return true;
-}
-
-static bool warpedMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-warp_skyfile_id", "warpImfile.warp_skyfile_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id",    "warpSkyfile.tess_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
-    PXOPT_COPY_S64(config->args, where, "-fake_id",    "fakeRun.fake_id", "==");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs",  ">=");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawExp.dateobs",  "<=");
-    PXOPT_COPY_STR(config->args, where, "-filter",    "rawExp.filter", "LIKE");
-    PXOPT_COPY_S64(config->args, where, "-magicked", "warpSkyfile.magicked", "==");
-    pxAddLabelSearchArgs (config, where, "-label",   "warpRun.label", "LIKE");
-    pxAddLabelSearchArgs (config, where, "-data_group",   "warpRun.data_group", "LIKE");
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool warpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id",    "warpBackgroundRun.warp_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label",   "warpBackgroundRun.label", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-data_group",   "warpBackgroundRun.data_group", "LIKE");
+    pxAddLabelSearchArgs(config, where, "-dist_group",   "warpBackgroundRun.data_group", "LIKE");
 
     PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
-
     PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
     PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
 
     // find all rawImfiles matching the default query
-    psString query = pxDataGet("warptool_warped.sql");
+    psString query = pxDataGet("bgtool_warp.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    // generate where strings for arguments that require extra processing
-    // beyond PXOPT_COPY*
-    psString where2 = NULL;
-    if (!pxmagicAddWhere(config, &where2, "warpSkyfile")) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    psString magicWhere = NULL;
+    if (!pxmagicAddWhere(config, &magicWhere, "warpBackgroundSkyfile")) {
         psError(psErrorCodeLast(), false, "pxMagicAddWhere failed");
         return false;
     }
-    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
+    if (!pxspaceAddWhere(config, &magicWhere, "rawExp")) {
         psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed");
         return false;
@@ -1262 +1423 @@
         psStringAppend(&query, " WHERE %s", whereClause);
         psFree(whereClause);
-    } else if (!all && !where2) {
+    } else if (!all && !magicWhere) {
         psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
         return false;
     }
-
-    if (where2) {
-        if (psListLength(where->list)) {
-            psStringAppend(&query, " %s", where2);
-        } else {
-            psStringAppend(&query, " WHERE 1 %s", where2);
-        }
-    }
-    psFree(where);
-
-    // treat limit == 0 as "no limit"
+    if (magicWhere) {
+        psStringAppend(&query, "%s %s", psListLength(where->list) ? "AND" : "WHERE", magicWhere);
+    }
+    psFree(magicWhere);
+    psFree(where);
+
     if (limit) {
         psString limitString = psDBGenerateLimitSQL(limit);
@@ -1284 +1440 @@
 
     if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+        psError(psErrorCodeLast(), false, "database error");
         psFree(query);
         return false;
@@ -1292 +1448 @@
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
+        psError(psErrorCodeLast(), false, "database error");
         return false;
     }
     if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
     }
-
     if (psArrayLength(output)) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "warpSkyfile", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
-    }
-
+        if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundSkyfile", !simple)) {
+            psError(psErrorCodeLast(), false, "failed to print array");
+            psFree(output);
+            return false;
+        }
+    }
     psFree(output);
 
@@ -1325 +1469 @@
 
 
-static bool revertwarpedMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id",    "warpSkyfile.tess_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-reduction",  "rawExp.reduction", "==");
-    pxAddLabelSearchArgs (config, where, "-label",     "warpRun.label", "==");
-    PXOPT_COPY_S16(config->args, where, "-fault",      "warpSkyfile.fault", "==");
-
-    if (!psListLength(where->list)
-        && !psMetadataLookupBool(NULL, config->args, "-all")) {
+static bool advancewarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "label", "==");
+
+    psString select = pxDataGet("bgtool_advancewarp.sql");
+    if (!select) {
+        psError(psErrorCodeLast(), false, "failed to retrieve SQL statement");
+        return false;
+    }
+
+    psString selectWhere = psStringCopy("");
+    if (psListLength(where->list)) {
+        psString whereClause = psDBGenerateWhereSQL(where, NULL);
+        psStringAppend(&selectWhere, "\n WHERE %s", whereClause);
+        psFree(whereClause);
+    }
+    psFree(where);
+
+    if (!psDBTransaction(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    if (!p_psDBRunQueryF(config->dbh, select, selectWhere)) {
+        psError(psErrorCodeLast(), false, "database error");
+        psFree(select);
+        psFree(selectWhere);
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    psFree(select);
+    psFree(selectWhere);
+
+    psArray *output = p_psDBFetchResult(config->dbh);
+    if (!output) {
+        psError(psErrorCodeLast(), false, "database error");
+        if (!psDBRollback(config->dbh)) {
+            psError(psErrorCodeLast(), false, "database error");
+        }
+        return false;
+    }
+    if (!psArrayLength(output)) {
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
+        psFree(output);
+        return true;
+    }
+
+    for (long i = 0; i < psArrayLength(output); i++) {
+        psMetadata *row = output->data[i];
+        bool status = true;             // Status of MD lookup
+        psS64 warp_bg_id = psMetadataLookupS64(&status, row, "warp_bg_id");
+        if (!status) {
+            psError(PXTOOLS_ERR_PROG, true, "failed to look up value for warp_bg_id");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+        psS64 magicked = psMetadataLookupS64(&status, row, "magicked");
+        if (!status) {
+            psError(PXTOOLS_ERR_PROG, true, "failed to look up value for magicked");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        if (!p_psDBRunQueryF(config->dbh,
+                             "UPDATE warpBackgroundRun "
+                             "SET state = 'full', magicked = %" PRId64 " "
+                             " WHERE warp_bg_id = %" PRId64,
+                             warp_bg_id, magicked)) {
+            psError(psErrorCodeLast(), false, "database error");
+            psFree(output);
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            return false;
+        }
+
+        psS64 numUpdated = psDBAffectedRows(config->dbh);
+        if (numUpdated != 1) {
+            psError(PXTOOLS_ERR_PROG, true, "should have affected 1 row");
+            if (!psDBRollback(config->dbh)) {
+                psError(psErrorCodeLast(), false, "database error");
+            }
+            psFree(output);
+            return false;
+        }
+    }
+    psFree(output);
+
+    if (!psDBCommit(config->dbh)) {
+        psError(psErrorCodeLast(), false, "database error");
+        return false;
+    }
+
+    return true;
+}
+
+static bool revertwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
+    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpBackgroundSkyfile.skycell_id", "==");
+    pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "==");
+    PXOPT_COPY_S16(config->args, where, "-fault", "warpBackgroundSkyfile.fault", "==");
+
+    if (!psListLength(where->list) && !psMetadataLookupBool(NULL, config->args, "-all")) {
         psFree(where);
-        psError(PXTOOLS_ERR_CONFIG, false, "search parameters are required");
-        return false;
-    }
-
-    psString query = pxDataGet("warptool_revertwarped_delete.sql");
+        psError(PXTOOLS_ERR_CONFIG, true, "search parameters are required");
+        return false;
+    }
+
+    psString query = pxDataGet("bgtool_revertwarp.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-    psString query_updated = pxDataGet("warptool_revertwarped_updated.sql");
-    if (!query_updated) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
         return false;
     }
@@ -1358 +1602 @@
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
         psStringAppend(&query, " AND %s", whereClause);
-        psStringAppend(&query_updated, " AND %s", whereClause);
         psFree(whereClause);
     }
@@ -1371 +1614 @@
 
     int numDeleted = psDBAffectedRows(config->dbh);
-
-    psLogMsg("warptool", PS_LOG_INFO, "Deleted %d warpSkyfiles", numDeleted);
-
-    // fix any faulted warpSkyfiles in data_state 'update'
-
-    if (!p_psDBRunQuery(config->dbh, query_updated)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query_updated);
-        return false;
-    }
-    psFree(query_updated);
-
-    int numUpdated = psDBAffectedRows(config->dbh);
-
-    psLogMsg("warptool", PS_LOG_INFO, "Updated %d warpSkyfiles", numUpdated);
-
-    return true;
-}
-
-
-static bool blockMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    PXOPT_LOOKUP_STR(label, config->args, "-label", true, false);
-
-    if (!warpMaskInsert(config->dbh, label)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-
-
-static bool maskedMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psString query = psStringCopy("SELECT * FROM warpMask");
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("warpool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    // negative simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "warpMask", !simple)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print array");
-        psFree(output);
-        return false;
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-static bool unblockMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    PXOPT_LOOKUP_STR(label, config->args, "-label", true, false);
-
-    char *query = "DELETE FROM warpMask WHERE label = '%s'";
-
-    if (!p_psDBRunQueryF(config->dbh, query, label)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-
-static bool pendingcleanuprunMode(pxConfig *config)
+    psLogMsg("bgtool", PS_LOG_INFO, "Deleted %d warpBackgroundSkyfiles", numDeleted);
+
+    return true;
+}
+
+static bool tocleanwarpMode(pxConfig *config)
 {
     PS_ASSERT_PTR_NON_NULL(config, NULL);
@@ -1468 +1627 @@
 
     psMetadata *where = psMetadataAlloc();
-    pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
-
-    psString query = pxDataGet("warptool_pendingcleanuprun.sql");
+    pxAddLabelSearchArgs(config, where, "-label", "warpBackgroundRun.label", "==");
+
+    psString query = pxDataGet("bgtool_tocleanwarp.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
         return false;
     }
@@ -1483 +1642 @@
     psFree(where);
 
-    // treat limit == 0 as "no limit"
     if (limit) {
         psString limitString = psDBGenerateLimitSQL(limit);
@@ -1491 +1649 @@
 
     if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+        psError(psErrorCodeLast(), false, "database error");
         psFree(query);
         return false;
@@ -1499 +1657 @@
     psArray *output = p_psDBFetchResult(config->dbh);
     if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
+        psError(psErrorCodeLast(), false, "database error");
         return false;
     }
     if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
+        psTrace("bgtool", PS_LOG_INFO, "no rows found");
         psFree(output);
         return true;
@@ -1509 +1667 @@
 
     // negative simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupRun", !simple)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print array");
-        psFree(output);
-        return false;
-    }
-
+    if (!ippdbPrintMetadatas(stdout, output, "warpBackgroundRun", !simple)) {
+        psError(psErrorCodeLast(), false, "failed to print array");
+        psFree(output);
+        return false;
+    }
     psFree(output);
 
@@ -1520 +1677 @@
 }
 
-
-static bool pendingcleanupwarpMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-
-    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
-    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-    pxAddLabelSearchArgs (config, where, "-label", "warpRun.label", "==");
-
-    psString query = pxDataGet("warptool_pendingcleanupskyfile.sql");
+static bool cleanedwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warpBackgroundRun.warp_bg_id", "==");
+
+    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);
+    if (!pxIsValidCleanedState(state)) {
+        psError(PXTOOLS_ERR_CONFIG, true, "Invalid state: %s", state);
+        return false;
+    }
+
+    if (!psListLength(where->list)) {
+        psError(PXTOOLS_ERR_CONFIG, true, "No search restrictions set.");
+        return false;
+    }
+
+    psString query = pxDataGet("bgtool_cleanedwarp.sql");
     if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (where && psListLength(where->list)) {
+        psError(psErrorCodeLast(), false, "failed to retreive SQL statement");
+        return false;
+    }
+
+    {
         psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
         psStringAppend(&query, " AND %s", whereClause);
@@ -1546 +1708 @@
     psFree(where);
 
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
+    if (!p_psDBRunQueryF(config->dbh, query, state)) {
+        psError(psErrorCodeLast(), false, "database error");
         return false;
     }
     psFree(query);
 
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    // negative simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupWarp", !simple)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print array");
-        psFree(output);
-        return false;
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-
-static bool donecleanupMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
-    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");
-
-    psString query = pxDataGet("warptool_donecleanup.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (where && psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " AND %s", whereClause);
-        psFree(whereClause);
-    }
-    psFree(where);
-
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    // negative simple so the default is true
-    if (!ippdbPrintMetadatas(stdout, output, "warpDoneCleanup", !simple)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print array");
-        psFree(output);
-        return false;
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-
-// update warpSkyfile.data_state to given value.
-// afterwards, if all skfyiles in the run have the new state, update the state for the run as well
-// shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile
-
-static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    // warp_id, skycell_id are required
-    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
-    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
-
-    psString query = pxDataGet("warptool_change_skyfile_data_state.sql");
-
-    if (!psDBTransaction(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    psString set_magicked_skyfile = psStringCopy("");
-    psString set_magicked_run = psStringCopy("");
-    if (!strcmp(data_state, "full")) {
-        // magicked is only an argument for for -tofullskyfile
-        PXOPT_LOOKUP_S64(magicked, config->args, "-set_magicked", false, false);
-        if (magicked) {
-            psStringAppend(&set_magicked_skyfile, "\n , warpSkyfile.magicked = %" PRId64, magicked);
-            psStringAppend(&set_magicked_run, "\n,  warpRun.magicked = %" PRId64, magicked);
-        }
-    } else if (!strcmp(data_state, "cleaned") || !strcmp(data_state, "purged")) {
-        // if magicked is currently nonzero set it to -1
-        // Set warpRun.magicked when the first skyfile is cleaned
-        psStringAppend(&set_magicked_skyfile, "\n, warpSkyfile.magicked = IF(warpSkyfile.magicked = 0, 0, -1), warpRun.magicked = IF(warpRun.magicked = 0, 0, -1)");
-    }
-
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_skyfile, warp_id, skycell_id)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        // rollback
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    psFree(query);
-    psFree(set_magicked_skyfile);
-
-    query = pxDataGet("warptool_change_run_state.sql");
-    if (!p_psDBRunQueryF(config->dbh, query, data_state, set_magicked_run, warp_id, data_state)) {
-        // rollback
-        if (!psDBRollback(config->dbh)) {
-            psError(PS_ERR_UNKNOWN, false, "database error");
-        }
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-    psFree(set_magicked_run);
-
-    if (!psDBCommit(config->dbh)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    return true;
-}
-static bool tocleanedskyfileMode(pxConfig *config)
-{
-    return change_skyfile_data_state(config, "cleaned", "goto_cleaned");
-}
-static bool tofullskyfileMode(pxConfig *config)
-{
-    return change_skyfile_data_state(config, "full", "update");
-}
-static bool topurgedskyfileMode(pxConfig *config)
-{
-    return change_skyfile_data_state(config, "purged", "goto_purged");
-}
-static bool toscrubbedskyfileMode(pxConfig *config)
-{
-     return change_skyfile_data_state(config, "scrubbed", "goto_scrubbed");
-}
-
-static bool updateskyfileMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    // warp_id, skycell_id, fault are required
-    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
-    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
-    PXOPT_LOOKUP_S16(fault, config->args, "-fault", false, false);
-    PXOPT_LOOKUP_STR(state, config->args, "-set_state", false, false);
-
-    if (!state) {
-      psString query = pxDataGet("warptool_updateskyfile.sql");
-
-      if (!p_psDBRunQueryF(config->dbh, query, fault, warp_id, skycell_id)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-      }
-      psFree(query);
-    }
-    else {
-      if (strcmp(state,"error_cleaned") == 0) {
-        change_skyfile_data_state(config,"error_cleaned","goto_cleaned");
-      }
-      else if (strcmp(state, "error_scrubbed") == 0) {
-        change_skyfile_data_state(config,"error_scrubbed","goto_scrubbed");
-      }
-      else if (strcmp(state, "error_purged") == 0) {
-        change_skyfile_data_state(config,"error_purged","goto_purged");
-      }
-      else {
-        psError(PS_ERR_UNKNOWN, false, "unhandled state given");
-        return(false);
-      }
-    }
-
-    return true;
-}
-
-bool exportrunMode(pxConfig *config)
-{
-  typedef struct ExportTable {
-    char tableName[80];
-    char sqlFilename[80];
-  } ExportTable;
-
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-
-    PXOPT_LOOKUP_S64(det_id,  config->args, "-warp_id", true,  false);
+    return true;
+}
+
+static bool exportwarpMode(pxConfig *config)
+{
+    PS_ASSERT_PTR_NON_NULL(config, false);
+
+    PXOPT_LOOKUP_S64(warp_bg_id, config->args, "-warp_bg_id", true,  false);
     PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
-    PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);
     PXOPT_LOOKUP_BOOL(clean,  config->args, "-clean", false);
 
-    FILE *f = fopen (outfile, "w");
-    if (f == NULL) {
-        psError(PS_ERR_UNKNOWN, false, "failed to open output file");
-        return false;
-    }
-
-    if (!pxExportVersion(config, f)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to write dbversion output file");
-        return false;
-    }
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
-
-    ExportTable tables [] = {
-      {"warpRun", "warptool_export_run.sql"},
-      {"warpImfile", "warptool_export_imfile.sql"},
-      {"warpSkyfile", "warptool_export_skyfile.sql"},
-      {"warpSkyCellMap", "warptool_export_skycell_map.sql"},
-    };
-
-    int numTables = sizeof(tables)/sizeof(tables[0]);
-
-    for (int i=0; i < numTables; i++) {
-      psString query = pxDataGet(tables[i].sqlFilename);
-      if (!query) {
-          psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-          return false;
-      }
-
-      if (where && psListLength(where->list)) {
-          psString whereClause = psDBGenerateWhereSQL(where, NULL);
-          psStringAppend(&query, " %s", whereClause);
-          psFree(whereClause);
-      }
-
-      // treat limit == 0 as "no limit"
-      if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-      }
-
-      if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-      }
-      psFree(query);
-
-      psArray *output = p_psDBFetchResult(config->dbh);
-      if (!output) {
-          psError(PS_ERR_UNKNOWN, false, "database error");
-          return false;
-      }
-      if (!psArrayLength(output)) {
-        psError(PS_ERR_UNKNOWN, true, "no rows found");
-        psFree(output);
-        return false;
-      }
-
-    if (clean) {
-        bool success = true;
-        if (!strcmp(tables[i].tableName, "warpRun")) {
-            success = pxSetStateCleaned("warpRun", "state", output);
-        } else if (!strcmp(tables[i].tableName, "warpSkyfile")) {
-            success = pxSetStateCleaned("warpSkyfile", "data_state", output);
-        }
-        if (!success) {
-            psFree(output);
-            psError(PS_ERR_UNKNOWN, false, "pxSetStateClean failed for table %s",  tables[i].tableName);
-            return false;
-        }
-    }
-
-      // we must write the export table in non-simple (true) format
-      if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
-        psError(PS_ERR_UNKNOWN, false, "failed to print array");
-        psFree(output);
-        return false;
-      }
-      psFree(output);
-    }
-
-    fclose (f);
-
-    return true;
-}
-
-bool importrunMode(pxConfig *config)
-{
-  unsigned int nFail;
-
-  int numImportTables = 3;
-
-  char tables[3] [80] = {"warpImfile", "warpSkyfile", "warpSkyCellMap"};
-
-  PS_ASSERT_PTR_NON_NULL(config, NULL);
-
-  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
-
-  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);
-
-#ifdef notdef
-  fprintf (stderr, "---- input ----\n");
-  psMetadataPrint (stderr, input, 1);
-#endif
-
-  if (!pxCheckImportVersion(config, input)) {
-      psError(PS_ERR_UNKNOWN, false, "pxCheckImportVersion failed");
-      return false;
-  }
-
-  psMetadataItem *item = psMetadataLookup (input, "warpRun");
-  psAssert (item, "entry not in input?");
-  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-
-  psMetadataItem *entry = psListGet (item->data.list, 0);
-  assert (entry);
-  assert (entry->type == PS_DATA_METADATA);
-  warpRunRow *warpRun = warpRunObjectFromMetadata (entry->data.md);
-  warpRunInsertObject (config->dbh, warpRun);
-
-  // fprintf (stdout, "---- warp run ----\n");
-  // psMetadataPrint (stderr, entry->data.md, 1);
-
-  for (int i = 0; i < numImportTables; i++) {
-    item = psMetadataLookup (input, tables[i]);
-    psAssert (item, "entry not in input?");
-    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
-
-    switch (i) {
-      case 0:
-        for (int i = 0; i < item->data.list->n; i++) {
-          entry = psListGet (item->data.list, i);
-          assert (entry);
-          assert (entry->type == PS_DATA_METADATA);
-          warpImfileRow *warpImfile = warpImfileObjectFromMetadata (entry->data.md);
-          warpImfileInsertObject (config->dbh, warpImfile);
-
-          // fprintf (stdout, "---- row %d ----\n", i);
-          // psMetadataPrint (stderr, entry->data.md, 1);
-        }
-        break;
-
-      case 1:
-        for (int i = 0; i < item->data.list->n; i++) {
-          entry = psListGet (item->data.list, i);
-          assert (entry);
-          assert (entry->type == PS_DATA_METADATA);
-          warpSkyfileRow *warpSkyfile = warpSkyfileObjectFromMetadata (entry->data.md);
-          warpSkyfileInsertObject (config->dbh, warpSkyfile);
-
-          // fprintf (stdout, "---- row %d ----\n", i);
-          // psMetadataPrint (stderr, entry->data.md, 1);
-        }
-        break;
-
-      case 2:
-        for (int i = 0; i < item->data.list->n; i++) {
-          entry = psListGet (item->data.list, i);
-          assert (entry);
-          assert (entry->type == PS_DATA_METADATA);
-          warpSkyCellMapRow *warpSkyCellMap = warpSkyCellMapObjectFromMetadata (entry->data.md);
-          warpSkyCellMapInsertObject (config->dbh, warpSkyCellMap);
-
-          // fprintf (stdout, "---- row %d ----\n", i);
-          // psMetadataPrint (stderr, entry->data.md, 1);
-        }
-        break;
-    }
-  }
-  return true;
-}
-
-static bool runstateMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpRun.warp_id", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
-    pxAddLabelSearchArgs (config, where, "-label",     "warpRun.label", "LIKE");
-
-//    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
-    PXOPT_LOOKUP_BOOL(no_magic, config->args, "-no_magic", false);
-
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
-    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-
-    psString query = pxDataGet("warptool_runstate.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " WHERE %s", whereClause);
-        psFree(whereClause);
-    } else {
-        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
-        return false;
-    }
-    psFree(where);
-
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    if (psArrayLength(output)) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "warpRunState", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-static bool listrunMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, false);
-
-    psMetadata *where = psMetadataAlloc();
-    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpRun.warp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-tess_id",    "warpRun.tess_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-state",      "warpRun.state", "==");
-    PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
-    PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
-    PXOPT_COPY_S64(config->args, where, "-fake_id",    "fakeRun.fake_id", "==");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs",  ">=");
-    PXOPT_COPY_TIME(config->args, where, "-dateobs_end",   "rawExp.dateobs",  "<=");
-    PXOPT_COPY_STR(config->args, where, "-filter",    "rawExp.filter", "LIKE");
-    PXOPT_COPY_S64(config->args, where, "-magicked", "warpRun.magicked", "==");
-    pxAddLabelSearchArgs (config, where, "-label",   "warpRun.label", "LIKE");
-    pxAddLabelSearchArgs (config, where, "-data_group",   "warpRun.data_group", "LIKE");
-    pxAddLabelSearchArgs (config, where, "-dist_group",   "warpRun.dist_group", "LIKE");
-
-    PXOPT_LOOKUP_BOOL(all, config->args, "-all", false);
-
-    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
-    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
-    PXOPT_LOOKUP_BOOL(pstamp_order, config->args, "-pstamp_order", false);
-
-    // find all rawImfiles matching the default query
-    psString query = pxDataGet("warptool_listrun.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    // generate where strings for arguments that require extra processing
-    // beyond PXOPT_COPY*
-    psString where2 = NULL;
-    if (!pxmagicAddWhere(config, &where2, "warpRun")) {
-        psError(psErrorCodeLast(), false, "pxMagicAddWhere failed");
-        return false;
-    }
-    if (!pxspaceAddWhere(config, &where2, "rawExp")) {
-        psError(psErrorCodeLast(), false, "pxSpaceAddWhere failed");
-        return false;
-    }
-
-    if (psListLength(where->list)) {
-        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
-        psStringAppend(&query, " WHERE %s", whereClause);
-        psFree(whereClause);
-    } else if (!all && !where2) {
-        psError(PXTOOLS_ERR_CONFIG, true, "search parameters or -all are required");
-        return false;
-    }
-
-    if (where2) {
-        if (psListLength(where->list)) {
-            psStringAppend(&query, " %s", where2);
-        } else {
-            psStringAppend(&query, " WHERE 1 %s", where2);
-        }
-    }
-    psFree(where);
-
-    if (pstamp_order) {
-        // put runs in order of exposure id with newest chip Runs first
-        // The postage stamp parser depends on this behavior
-        psStringAppend(&query, "\nORDER by exp_id, warp_id DESC");
-    }
-
-
-    // treat limit == 0 as "no limit"
-    if (limit) {
-        psString limitString = psDBGenerateLimitSQL(limit);
-        psStringAppend(&query, " %s", limitString);
-        psFree(limitString);
-    }
-
-    if (!p_psDBRunQuery(config->dbh, query)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        psFree(query);
-        return false;
-    }
-    psFree(query);
-
-    psArray *output = p_psDBFetchResult(config->dbh);
-    if (!output) {
-        psErrorCode err = psErrorCodeLast();
-        switch (err) {
-            case PS_ERR_DB_CLIENT:
-                psError(PXTOOLS_ERR_SYS, false, "database error");
-            case PS_ERR_DB_SERVER:
-                psError(PXTOOLS_ERR_PROG, false, "database error");
-            default:
-                psError(PXTOOLS_ERR_PROG, false, "unknown error");
-        }
-
-        return false;
-    }
-    if (!psArrayLength(output)) {
-        psTrace("warptool", PS_LOG_INFO, "no rows found");
-        psFree(output);
-        return true;
-    }
-
-    if (psArrayLength(output)) {
-        // negative simple so the default is true
-        if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) {
-            psError(PS_ERR_UNKNOWN, false, "failed to print array");
-            psFree(output);
-            return false;
-        }
-    }
-
-    psFree(output);
-
-    return true;
-}
-
-// a very specfic function to queue a cleaned warpSkyfile to be updated
-static bool setskyfiletoupdateMode(pxConfig *config)
-{
-    PS_ASSERT_PTR_NON_NULL(config, NULL);
-
-    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
-    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", false, false);
-    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
-
-    psString query = pxDataGet("warptool_setskyfiletoupdate.sql");
-    if (!query) {
-        psError(PXTOOLS_ERR_SYS, false, "failed to retreive SQL statement");
-        return false;
-    }
-
-    psString setHook = psStringCopy("");
-    if (label) {
-        psStringAppend(&setHook, "\n , warpRun.label = '%s'", label);
-    }
-
-    if (skycell_id) {
-        psStringAppend(&query, " AND (warpSkyfile.skycell_id = '%s')", skycell_id);
-    }
-
-    if (!p_psDBRunQueryF(config->dbh, query, setHook, warp_id)) {
-        psError(PS_ERR_UNKNOWN, false, "database error");
-        return false;
-    }
-
-    psFree(setHook);
-    psFree(query);
-
-    return true;
-}
+    psMetadata *where = psMetadataAlloc();
+    PXOPT_COPY_S64(config->args, where, "-warp_bg_id", "warp_bg_id", "==");
+
+    bool status = exportTables(config, outfile, warpTables, where, clean);
+
+    psFree(where);
+    return status;
+}
+
+static bool importwarpMode(pxConfig *config)
+{
+    PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);
+
+    return importTables(config, infile, warpTables);
+}

@@ -39 +39 @@
 warptool
 staticskytool
+
+bgtool