/*
 * warptool.c
 *
 * Copyright (C) 2006  Joshua Hoblitt
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * program; see the file COPYING. If not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#ifdef HAVB_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <ippdb.h>

#include "pxtools.h"
#include "warptool.h"
#include "pxwarp.h"

static psS64 definerunMode(pxConfig *config);
static bool definebyqueryMode(pxConfig *config);
static bool updaterunMode(pxConfig *config);
static bool expMode(pxConfig *config);
static bool imfileMode(pxConfig *config);
static bool tooverlapMode(pxConfig *config);
static bool addoverlapMode(pxConfig *config);
static bool scmapMode(pxConfig *config);
static bool towarpedMode(pxConfig *config);
static bool addwarpedMode(pxConfig *config);
static bool warpedMode(pxConfig *config);
static bool revertwarpedMode(pxConfig *config);
static bool blockMode(pxConfig *config);
static bool maskedMode(pxConfig *config);
static bool unblockMode(pxConfig *config);
static bool pendingcleanuprunMode(pxConfig *config);
static bool pendingcleanupwarpMode(pxConfig *config);
static bool donecleanupMode(pxConfig *config);
static bool tocleanedskyfileMode(pxConfig *config);
static bool topurgedskyfileMode(pxConfig *config);
static bool tofullskyfileMode(pxConfig *config);
static bool updateskyfileMode(pxConfig *config);

static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile);
static bool isValidMode(pxConfig *config, const char *mode);
bool warpCompletedRuns(pxConfig *config);

# define MODECASE(caseName, func) \
    case caseName: \
    if (!func(config)) { \
        goto FAIL; \
    } \
    break;

int main(int argc, char **argv)
{
    psLibInit(NULL);

    pxConfig *config = warptoolConfig(NULL, argc, argv);
    if (!config) {
        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
        goto FAIL;
    }

    switch (config->mode) {
        MODECASE(WARPTOOL_MODE_DEFINERUN,          definerunMode);
        MODECASE(WARPTOOL_MODE_DEFINEBYQUERY,      definebyqueryMode);
        MODECASE(WARPTOOL_MODE_UPDATERUN,          updaterunMode);
        MODECASE(WARPTOOL_MODE_EXP,                expMode);
        MODECASE(WARPTOOL_MODE_IMFILE,             imfileMode);
        MODECASE(WARPTOOL_MODE_TOOVERLAP,          tooverlapMode);
        MODECASE(WARPTOOL_MODE_ADDOVERLAP,         addoverlapMode);
        MODECASE(WARPTOOL_MODE_SCMAP,              scmapMode);
        MODECASE(WARPTOOL_MODE_TOWARPED,           towarpedMode);
        MODECASE(WARPTOOL_MODE_ADDWARPED,          addwarpedMode);
        MODECASE(WARPTOOL_MODE_WARPED,             warpedMode);
        MODECASE(WARPTOOL_MODE_REVERTWARPED,       revertwarpedMode);
        MODECASE(WARPTOOL_MODE_BLOCK,              blockMode);
        MODECASE(WARPTOOL_MODE_MASKED,             maskedMode);
        MODECASE(WARPTOOL_MODE_UNBLOCK,            unblockMode);
        MODECASE(WARPTOOL_MODE_PENDINGCLEANUPRUN,  pendingcleanuprunMode);
        MODECASE(WARPTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupwarpMode);
        MODECASE(WARPTOOL_MODE_DONECLEANUP,        donecleanupMode);
        MODECASE(WARPTOOL_MODE_TOCLEANEDSKYFILE,   tocleanedskyfileMode);
        MODECASE(WARPTOOL_MODE_TOPURGEDSKYFILE,    topurgedskyfileMode);
        MODECASE(WARPTOOL_MODE_TOFULLSKYFILE,      tofullskyfileMode);
        MODECASE(WARPTOOL_MODE_UPDATESKYFILE,      updateskyfileMode);

        default:
            psAbort("invalid option (this should not happen)");
    }

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(EXIT_SUCCESS);

FAIL:
    psErrorStackPrint(stderr, "\n");
    int exit_status = pxerrorGetExitStatus();

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(exit_status);
}


static psS64 definerunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_S64(fake_id, config->args, "-fake_id", true, false); // required
    PXOPT_LOOKUP_STR(mode, config->args, "-mode", true, false); // required
    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", false, false);
    PXOPT_LOOKUP_STR(end_stage, config->args, "-end_stage", false, false);
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // check mode
    if (mode && !isValidMode(config, mode)) {
        psError(PS_ERR_UNKNOWN, false, "invalid mode");
        return false;
    }

    warpRunRow *warpRun = warpRunRowAlloc(
            0,          // ID
            fake_id,
            mode,
            "new",      // state
            workdir,
            "dirty",    // workdir_state
            label,
            dvodb,
            tess_id,
            end_stage,
            registered,
            magiced
    );
    if (!warpRun) {
        psError(PS_ERR_UNKNOWN, false, "failed to alloc warpRun object");
        return true;
    }
    if (!warpRunInsertObject(config->dbh, warpRun)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(warpRun);
        return true;
    }

    // get the assigned warp_id
    psS64 warp_id = psDBLastInsertID(config->dbh);
    warpRun->warp_id = warp_id;

    if (!warpRunPrintObject(stdout, warpRun, !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print object");
            psFree(warpRun);
            return false;
    }

    psFree(warpRun);

    return warp_id;
}


static bool definebyqueryMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "==");
    PXOPT_COPY_S64(config->args, where, "-cam_id", "camRun.cam_id", "==");
    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
    PXOPT_COPY_S64(config->args, where, "-exp_id", "newExp.exp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "==");
    PXOPT_COPY_STR(config->args, where, "-inst", "camera", "==");
    PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "==");
    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">=");
    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<=");
    PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "==");
    PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "==");
    PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "==");
    PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "==");
    PXOPT_COPY_STR(config->args, where, "-filter", "filter", "==");
    PXOPT_COPY_F64(config->args, where, "-airmass_min", "airmass", ">=");
    PXOPT_COPY_F64(config->args, where, "-airmass_max", "airmass", "<");
    PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">=");
    PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<");
    PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">=");
    PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<");
    PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">=");
    PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<");
    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">=");
    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<");
    PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">=");
    PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<");
    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">=");
    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<");
    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">=");
    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<");
    PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">=");
    PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<");
    PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">=");
    PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<");
    PXOPT_COPY_F32(config->args, where, "-ccd_temp_min", "ccd_temp", ">=");
    PXOPT_COPY_F32(config->args, where, "-ccd_temp_max", "ccd_temp", "<");
    PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">=");
    PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<");
    PXOPT_COPY_STR(config->args, where, "-object", "object", "==");
    PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">=");
    PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<");
    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");

    if (!psListLength(where->list) &&
        !psMetadataLookupBool(NULL, config->args, "-all")) {
        psFree(where);
        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
        return false;
    }

    PXOPT_LOOKUP_STR(mode, config->args, "-set_mode", true, false); // required
    PXOPT_LOOKUP_STR(workdir, config->args, "-set_workdir", false, false);
    PXOPT_LOOKUP_STR(label, config->args, "-set_label", false, false);
    PXOPT_LOOKUP_STR(dvodb, config->args, "-set_dvodb", false, false);
    PXOPT_LOOKUP_STR(tess_id, config->args, "-set_tess_id", false, false);
    PXOPT_LOOKUP_STR(end_stage, config->args, "-set_end_stage", false, false);

    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    PXOPT_LOOKUP_BOOL(magiced, config->args, "-magiced", false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // check mode
    if (mode && !isValidMode(config, mode)) {
        psError(PS_ERR_UNKNOWN, false, "invalid mode");
        return false;
    }

    // find the exp_id of all the exposures that we want to queue up.
    psString query = pxDataGet("warptool_definebyquery.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        psFree(where);
        return false;
    }

    // use psDBGenerateWhereSQL because the SQL yields an intermediate table
    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereSQL(where, NULL);
        psStringAppend(&query, "%s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // would could do this "all in the database" if we didn't want the option
    // of changing the label/reduction/expgroup/dvodb/etc.  So we're pulling the
    // data out so we have the option of changing these values or leaving the
    // old values in place (i.e., passing the values through).

    // loop over our list of fakeRun rows
    for (long i = 0; i < psArrayLength(output); i++) {
        psMetadata *md = output->data[i];

        fakeRunRow *row = fakeRunObjectFromMetadata(md);
        if (!row) {
            psError(PS_ERR_UNKNOWN, false, "failed to convert metadata into fakeRun");
            psFree(output);
            return false;
        }

        // queue the exp
        if (!pxwarpQueueByFakeID(config,
                    row->fake_id,
                    workdir     ? workdir   : row->workdir,
                    label       ? label     : row->label,
                    dvodb       ? dvodb     : row->dvodb,
                    tess_id     ? tess_id   : row->tess_id,
                    end_stage   ? end_stage : row->end_stage))
          {
            psError(PS_ERR_UNKNOWN, false, "failed to trying to queue fake_id: %" PRId64, row->fake_id);
            psFree(row);
            psFree(output);
            return false;
        }
        psFree(row);
    }
    psFree(output);

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}

static bool updaterunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
    PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");
    PXOPT_COPY_S64(config->args, where, "-chip_id", "chip_id", "==");
    PXOPT_COPY_S64(config->args, where, "-exp_id", "exp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-exp_name", "exp_name", "==");
    PXOPT_COPY_STR(config->args, where, "-inst", "camera", "==");
    PXOPT_COPY_STR(config->args, where, "-telescope", "telescope", "==");
    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "dateobs", ">=");
    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "dateobs", "<=");
    PXOPT_COPY_STR(config->args, where, "-exp_tag", "exp_tag", "==");
    PXOPT_COPY_STR(config->args, where, "-exp_type", "exp_type", "==");
    PXOPT_COPY_STR(config->args, where, "-filelevel", "filelevel", "==");
    PXOPT_COPY_STR(config->args, where, "-reduction", "reduction", "==");
    PXOPT_COPY_STR(config->args, where, "-filter", "filter", "==");
    PXOPT_COPY_F32(config->args, where, "-airmass_min", "airmass", ">=");
    PXOPT_COPY_F32(config->args, where, "-airmass_max", "airmass", "<");
    PXOPT_COPY_F64(config->args, where, "-ra_min", "ra", ">=");
    PXOPT_COPY_F64(config->args, where, "-ra_max", "ra", "<");
    PXOPT_COPY_F64(config->args, where, "-decl_min", "decl", ">=");
    PXOPT_COPY_F64(config->args, where, "-decl_max", "decl", "<");
    PXOPT_COPY_F32(config->args, where, "-exp_time_min", "exp_time", ">=");
    PXOPT_COPY_F32(config->args, where, "-exp_time_max", "exp_time", "<");
    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "sat_pixel_frac", ">=");
    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "sat_pixel_frac", "<");
    PXOPT_COPY_F64(config->args, where, "-bg_min", "bt", ">=");
    PXOPT_COPY_F64(config->args, where, "-bg_max", "bt", "<");
    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "bg_stdev", ">=");
    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "bg_stdev", "<");
    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "bg_mean_stdev", ">=");
    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "bg_mean_stdev", "<");
    PXOPT_COPY_F64(config->args, where, "-alt_min", "alt", ">=");
    PXOPT_COPY_F64(config->args, where, "-alt_max", "alt", "<");
    PXOPT_COPY_F64(config->args, where, "-az_min", "az", ">=");
    PXOPT_COPY_F64(config->args, where, "-az_max", "az", "<");
    PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "ccd_temp", ">=");
    PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "ccd_temp", "<");
    PXOPT_COPY_F64(config->args, where, "-posang_min", "posang", ">=");
    PXOPT_COPY_F64(config->args, where, "-posang_max", "posang", "<");
    PXOPT_COPY_STR(config->args, where, "-object", "object", "==");
    PXOPT_COPY_F32(config->args, where, "-solang_min", "solang", ">=");
    PXOPT_COPY_F32(config->args, where, "-solang_max", "solang", "<");

    if (!psListLength(where->list)
        && !psMetadataLookupBool(NULL, config->args, "-all")) {
        psFree(where);
        where = NULL;
        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
        return false;
    }

    PXOPT_LOOKUP_STR(state, config->args, "-state", false, false);
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);

    if ((!state) && (!label)) {
        psError(PXTOOLS_ERR_DATA, false, "parameters (-state or -label) are required");
        psFree(where);
        return false;
    }

    if (state) {
        // set warpRun.state to state
        if (!pxwarpRunSetStateByQuery(config, where, state)) {
            psFree(where);
            return false;
        }
    }

    if (label) {
        // set chipRun.label to label
        if (!pxwarpRunSetLabelByQuery(config, where, label)) {
            psFree(where);
            return false;
        }
    }

    psFree(where);

    return true;
}


static bool expMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
    PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_exp.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool imfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
    PXOPT_COPY_S64(config->args, where, "-fake_id", "fake_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_imfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpInputImfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool tooverlapMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_tooverlap.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, "warpRun");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpRun", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool addoverlapMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_STR(mapfile, config->args, "-mapfile", false, false);
    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false);
    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (code == 0) {
        if (!parseAndInsertSkyCellMap(config, mapfile)) {
            psError(PS_ERR_UNKNOWN, false, "failed to inject mapfile: %s into the database", mapfile);
            // rollback
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
    } else {
        warpSkyCellMapInsert(config->dbh,
            warp_id,
            NULL,   // skycell_id
            NULL,   // tess_id
            NULL,   // class_id
            code    // fault
        );
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}


static bool parseAndInsertSkyCellMap(pxConfig *config, const char *mapfile)
{
    unsigned int nFail = 0;
    psMetadata *skycells = psMetadataConfigRead(NULL, &nFail, mapfile, false);
    if (!skycells) {
        psError(PS_ERR_UNKNOWN, false, "failed to parse mapfile: %s", mapfile);
        return false;
    }
    if (nFail) {
        psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing mapfile: %s", nFail, mapfile);
        psFree(skycells);
        return false;
    }

    psMetadataItem *item = NULL;
    psMetadataIterator *iter = psMetadataIteratorAlloc(skycells, 0, NULL);
    while ((item = psMetadataGetAndIncrement(iter))) {
        if (item->type != PS_DATA_METADATA) {
            psError(PS_ERR_UNKNOWN, false, "mapfile: %s is in the wrong format", mapfile);
            psFree(iter);
            psFree(skycells);
            return false;
        }

        psMetadata *sc = item->data.md;
        // this conversion isn't strictly nessicary but it's an easy way of
        // validating the format
        warpSkyCellMapRow *row = warpSkyCellMapObjectFromMetadata(sc);
        if (!row) {
            psError(PS_ERR_UNKNOWN, false, "failed to convert mapfile: %s metdata entry into a warpSkyCellMap object", mapfile);
            psFree(iter);
            psFree(skycells);
            return false;
        }

        if (!warpSkyCellMapInsertObject(config->dbh, row)) {
            psErrorCode err = psErrorCodeLast();
            switch (err) {
                case PS_ERR_DB_CLIENT:
                    psError(PXTOOLS_ERR_SYS, false, "database error");
                case PS_ERR_DB_SERVER:
                    psError(PXTOOLS_ERR_PROG, false, "database error");
                default:
                    psError(PXTOOLS_ERR_PROG, false, "unknown error");
            }
            psFree(row);
            psFree(iter);
            psFree(skycells);
            return false;
        }

        psFree(row);
    }
    psFree(iter);
    psFree(skycells);

    return true;
}


static bool scmapMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-skycell_id", "skycell_id", "==");
    PXOPT_COPY_STR(config->args, where, "-tess_id", "tess_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_scmap.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, "warpSkyCellMap");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpSkyCellMap", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}

static bool towarpedMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyCellMap.warp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_towarped.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpPendingSkyCell", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool addwarpedMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);

    // optional
    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", false, false);
    PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
    PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
    PXOPT_LOOKUP_F32(dtime_warp, config->args, "-dtime_warp", false, false);
    PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
    PXOPT_LOOKUP_S32(xmin, config->args, "-xmin", false, false);
    PXOPT_LOOKUP_S32(xmax, config->args, "-xmax", false, false);
    PXOPT_LOOKUP_S32(ymin, config->args, "-ymin", false, false);
    PXOPT_LOOKUP_S32(ymax, config->args, "-ymax", false, false);
    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
    PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);
    PXOPT_LOOKUP_BOOL(ignore, config->args, "-ignore", false);

    // default values
    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);

    // we don't want to insert the last skyfile in a run but then not mark the
    // run as 'stop'
    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    // XXX need to validate that this coresponds to an warpInputImfile
    if (!warpSkyfileInsert(config->dbh,
                           warp_id,
                           skycell_id,
                           tess_id,
                           uri,
                           path_base,
                           "full",      // data_state
                           bg,
                           bg_stdev,
                           dtime_warp,
                           dtime_script,
                           hostname,
                           good_frac,
                           xmin,
                           xmax,
                           ymin,
                           ymax,
                           ignore,
                           code
        )) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!warpCompletedRuns(config)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}

bool warpCompletedRuns(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // XXX this SQL has not been broken out to into seperate files as the MYSQL
    // < 5 & MYSQL 5 versions need to be kept in sync

#undef MYSQL5
#if MYSQL5
    // XXX at MySQL 4.1.21 (probably all of 4.1.x) chokes and dies on this
    // statement as it thinks it is trying to select from the table being
    // updated. The 4.1 manual says that nested sub-queries are explicited
    // allowed to do this with update statements as a temporary table is
    // created so that you are not actually selecting from the table you are
    // modifying.
    char *query =
        "UPDATE warpRun\n"
        "   SET warpRun.state = 'stop'\n"
        " WHERE\n"
        "   warpRun.warp_id =\n"
        "   (SELECT DISTINCT\n"
        "       warp_id\n"
        "   FROM\n"
        "       (SELECT DISTINCT\n"
        "           warpRun.warp_id,\n"
        "           warpSkyCellMap.warp_id as foo,\n"
        "           warpSkyfile.warp_id as bar\n"
        "       FROM warpRun\n"
        "       JOIN warpSkyCellMap\n"
        "           USING(warp_id)\n"
        "       LEFT JOIN warpSkyfile\n"
        "           USING(warp_id, skycell_id, tess_id)\n"
        "       WHERE\n"
        "           warpRun.state = 'new'\n"
        "       GROUP BY\n"
        "           warpRun.warp_id\n"
        "       HAVING\n"
        "       COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n"
        "       ) as Foo\n"
        "   )\n";

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
#else // if MYSQL5
{
    char *query =
        "CREATE TEMPORARY TABLE finished\n"
        " (warp_id INT, PRIMARY KEY(warp_id)) ENGINE=MEMORY\n";

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
}

{
    char *query =
        "INSERT INTO finished\n"
        " SELECT\n"
        "   warp_id\n"
        " FROM\n"
        "   (SELECT DISTINCT\n"
        "       warpRun.warp_id,\n"
        "       warpSkyCellMap.warp_id as foo,\n"
        "       warpSkyfile.warp_id as bar\n"
        "   FROM warpRun\n"
        "   JOIN warpSkyCellMap\n"
        "       USING(warp_id)\n"
        "   LEFT JOIN warpSkyfile\n"
        "       USING(warp_id, skycell_id)\n"
        "   WHERE\n"
        "       warpRun.state = 'new'\n"
        "   GROUP BY\n"
        "       warpRun.warp_id\n"
        "   HAVING\n"
        "       COUNT(warpSkyCellMap.warp_id) = COUNT(warpSkyfile.warp_id)\n"
        " ) as Foo \n";

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
}

{
    char *query =
        "UPDATE warpRun\n"
        "   SET warpRun.state = 'full'\n"
        " WHERE\n"
        "   warpRun.warp_id =\n"
        "   (SELECT DISTINCT\n"
        "       warp_id\n"
        "   FROM finished\n"
        "   )\n";

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
}
#endif // if MYSQL5

    return true;
}

static bool warpedMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-warp_id",    "warpSkyfile.warp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
    PXOPT_COPY_STR(config->args, where, "-tess_id",    "warpSkyfile.tess_id", "==");
    PXOPT_COPY_S64(config->args, where, "-exp_id",     "rawExp.exp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-exp_name",   "rawExp.exp_name", "==");
    PXOPT_COPY_S64(config->args, where, "-fake_id",    "fakeRun.fake_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // find all rawImfiles matching the default query
    psString query = pxDataGet("warptool_warped.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "warpSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool revertwarpedMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-warp_id", "warpSkyfile.warp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-skycell_id", "warpSkyfile.skycell_id", "==");
    PXOPT_COPY_STR(config->args, where, "-tess_id", "warpSkyfile.tess_id", "==");
    PXOPT_COPY_S64(config->args, where, "-fake_id", "fakeRun.fake_id", "==");
    PXOPT_COPY_S64(config->args, where, "-chip_id", "chipRun.chip_id", "==");
    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");

    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
    PXOPT_COPY_STR(config->args, where, "-label", "warpRun.label", "==");
    PXOPT_COPY_STR(config->args, where, "-inst", "rawExp.camera", "==");
    PXOPT_COPY_STR(config->args, where, "-telescope", "rawExp.telescope", "==");
    PXOPT_COPY_TIME(config->args, where, "-dateobs_begin", "rawExp.dateobs", ">=");
    PXOPT_COPY_TIME(config->args, where, "-dateobs_end", "rawExp.dateobs", "<=");
    PXOPT_COPY_STR(config->args, where, "-exp_tag", "rawExp.exp_tag", "==");
    PXOPT_COPY_STR(config->args, where, "-exp_type", "rawExp.exp_type", "==");
    PXOPT_COPY_STR(config->args, where, "-filelevel", "rawExp.filelevel", "==");
    PXOPT_COPY_STR(config->args, where, "-reduction", "rawExp.reduction", "==");
    PXOPT_COPY_STR(config->args, where, "-filter", "rawExp.filter", "==");

    PXOPT_COPY_F32(config->args, where, "-airmass_min", "rawExp.airmass", ">=");
    PXOPT_COPY_F32(config->args, where, "-airmass_max", "rawExp.airmass", "<");
    PXOPT_COPY_F64(config->args, where, "-ra_min", "rawExp.ra", ">=");
    PXOPT_COPY_F64(config->args, where, "-ra_max", "rawExp.ra", "<");
    PXOPT_COPY_F64(config->args, where, "-decl_min", "rawExp.decl", ">=");
    PXOPT_COPY_F64(config->args, where, "-decl_max", "rawExp.decl", "<");
    PXOPT_COPY_F32(config->args, where, "-exp_time_min", "rawExp.exp_time", ">=");
    PXOPT_COPY_F32(config->args, where, "-exp_time_max", "rawExp.exp_time", "<");
    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_min", "rawExp.sat_pixel_frac", ">=");
    PXOPT_COPY_F32(config->args, where, "-sat_pixel_frac_max", "rawExp.sat_pixel_frac", "<");
    PXOPT_COPY_F64(config->args, where, "-bg_min", "rawExp.bg", ">=");
    PXOPT_COPY_F64(config->args, where, "-bg_max", "rawExp.bg", "<");
    PXOPT_COPY_F64(config->args, where, "-bg_stdev_min", "rawExp.bg_stdev", ">=");
    PXOPT_COPY_F64(config->args, where, "-bg_stdev_max", "rawExp.bg_stdev", "<");
    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_min", "rawExp.bg_mean_stdev", ">=");
    PXOPT_COPY_F64(config->args, where, "-bg_mean_stdev_max", "rawExp.bg_mean_stdev", "<");
    PXOPT_COPY_F64(config->args, where, "-alt_min", "rawExp.alt", ">=");
    PXOPT_COPY_F64(config->args, where, "-alt_max", "rawExp.alt", "<");
    PXOPT_COPY_F64(config->args, where, "-az_min", "rawExp.az", ">=");
    PXOPT_COPY_F64(config->args, where, "-az_max", "rawExp.az", "<");
    PXOPT_COPY_F64(config->args, where, "-ccd_temp_min", "rawExp.ccd_temp", ">=");
    PXOPT_COPY_F64(config->args, where, "-ccd_temp_max", "rawExp.ccd_temp", "<");
    PXOPT_COPY_F64(config->args, where, "-posang_min", "rawExp.posang", ">=");
    PXOPT_COPY_F64(config->args, where, "-posang_max", "rawExp.posang", "<");
    PXOPT_COPY_STR(config->args, where, "-object", "rawExp.object", "==");
    PXOPT_COPY_F32(config->args, where, "-solang_min", "rawExp.solang", ">=");
    PXOPT_COPY_F32(config->args, where, "-solang_max", "rawExp.solang", "<");
    PXOPT_COPY_S16(config->args, where, "-code", "warpSkyfile.fault", "==");

    if (!psListLength(where->list)
        && !psMetadataLookupBool(NULL, config->args, "-all")) {
        psFree(where);
        psError(PXTOOLS_ERR_DATA, false, "search parameters are required");
        return false;
    }

    psString query = pxDataGet("warptool_revertwarped.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        psFree(where);
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    psFree(where);

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    return true;
}


static bool blockMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_STR(label, config->args, "-label", true, false);

    if (!warpMaskInsert(config->dbh, label)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}


static bool maskedMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psString query = psStringCopy("SELECT * FROM warpMask");

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warpool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "warpMask", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);

    return true;
}

static bool unblockMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_STR(label, config->args, "-label", true, false);

    char *query = "DELETE FROM warpMask WHERE label = '%s'";

    if (!p_psDBRunQuery(config->dbh, query, label)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}

static bool pendingcleanuprunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");

    psString query = pxDataGet("warptool_pendingcleanuprun.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupRun", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);

    return true;
}


static bool pendingcleanupwarpMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", false, false);
    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psMetadata *where = psMetadataAlloc();
    if (warp_id) {
        PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
    }
    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");

    psString query = pxDataGet("warptool_pendingcleanupskyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "warpPendingCleanupWarp", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);

    return true;
}


static bool donecleanupMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");

    psString query = pxDataGet("warptool_donecleanup.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("warptool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "warpDoneCleanup", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);

    return true;
}

static bool isValidMode(pxConfig *config, const char *mode)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(mode, false);

    // check that state is a valid string value
    if (!(
            (strncmp(mode, "warp", 5) == 0)
            || (strncmp(mode, "diff", 5) == 0)
            || (strncmp(mode, "stack", 6) == 0)
            || (strncmp(mode, "magic", 6) == 0)
        )
    ) {
        psError(PS_ERR_UNKNOWN, false,
                "invalid warpRun mode: %s", mode);
        return false;
    }

    return true;
}

// update warpSkyfile.data_state to given value.
// afterwards, if all skfyiles in the run have the new state, update the state for the run as well
// shared code for the modes -tocleanedskyfile -tofullskyfile -topurgedskyfile

static bool change_skyfile_data_state(pxConfig *config, psString data_state, psString run_state)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // warp_id, skycell_id are required
    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);

    psString query = pxDataGet("warptool_change_skyfile_data_state.sql");

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    // note only updates if warpRun.state = run_state
    if (!p_psDBRunQuery(config->dbh, query, data_state, warp_id, skycell_id, run_state)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        // rollback
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    psFree(query);

    query = pxDataGet("warptool_change_run_state.sql");
    if (!p_psDBRunQuery(config->dbh, query, data_state, warp_id, data_state)) {
        // rollback
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}
static bool tocleanedskyfileMode(pxConfig *config)
{
    return change_skyfile_data_state(config, "cleaned", "goto_cleaned");
}
static bool tofullskyfileMode(pxConfig *config)
{
    return change_skyfile_data_state(config, "full", "update");
}
static bool topurgedskyfileMode(pxConfig *config)
{
    return change_skyfile_data_state(config, "purged", "goto_purged");
}

static bool updateskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // warp_id, skycell_id, code are required
    PXOPT_LOOKUP_S64(warp_id, config->args, "-warp_id", true, false);
    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);

    psString query = pxDataGet("warptool_updateskyfile.sql");

    if (!p_psDBRunQuery(config->dbh, query, code, warp_id, skycell_id)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    psFree(query);

    return true;
}
