/*
 * difftool.c
 *
 * Copyright (C) 2007  Joshua Hoblitt
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * program; see the file COPYING. If not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#ifdef HAVB_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <ippdb.h>

#include "pxtools.h"
#include "difftool.h"

static bool definerunMode(pxConfig *config);
static bool updaterunMode(pxConfig *config);
static bool addinputskyfileMode(pxConfig *config);
static bool inputskyfileMode(pxConfig *config);
static bool todiffskyfileMode(pxConfig *config);
static bool adddiffskyfileMode(pxConfig *config);
static bool diffskyfileMode(pxConfig *config);
static bool revertdiffskyfileMode(pxConfig *config);
static bool definepoprunMode(pxConfig *config);
static bool definebyqueryMode(pxConfig *config);
static bool pendingcleanuprunMode(pxConfig *config);
static bool pendingcleanupskyfileMode(pxConfig *config);
static bool donecleanupMode(pxConfig *config);
static bool updatediffskyfileMode(pxConfig *config);
static bool exportrunMode(pxConfig *config);
static bool importrunMode(pxConfig *config);

static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state);
static bool diffRunComplete(pxConfig *config);

# define MODECASE(caseName, func) \
    case caseName: \
    if (!func(config)) { \
        goto FAIL; \
    } \
    break;

int main(int argc, char **argv)
{
    psLibInit(NULL);

    pxConfig *config = difftoolConfig(NULL, argc, argv);
    if (!config) {
        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
        goto FAIL;
    }

    switch (config->mode) {
        MODECASE(DIFFTOOL_MODE_DEFINERUN,             definerunMode);
        MODECASE(DIFFTOOL_MODE_UPDATERUN,             updaterunMode);
        MODECASE(DIFFTOOL_MODE_ADDINPUTSKYFILE,       addinputskyfileMode);
        MODECASE(DIFFTOOL_MODE_INPUTSKYFILE,          inputskyfileMode);
        MODECASE(DIFFTOOL_MODE_TODIFFSKYFILE,         todiffskyfileMode);
        MODECASE(DIFFTOOL_MODE_ADDDIFFSKYFILE,        adddiffskyfileMode);
        MODECASE(DIFFTOOL_MODE_DIFFSKYFILE,           diffskyfileMode);
        MODECASE(DIFFTOOL_MODE_REVERTDIFFSKYFILE,     revertdiffskyfileMode);
        MODECASE(DIFFTOOL_MODE_DEFINEPOPRUN,          definepoprunMode);
        MODECASE(DIFFTOOL_MODE_DEFINEBYQUERY,         definebyqueryMode);
        MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPRUN,     pendingcleanuprunMode);
        MODECASE(DIFFTOOL_MODE_PENDINGCLEANUPSKYFILE, pendingcleanupskyfileMode);
        MODECASE(DIFFTOOL_MODE_DONECLEANUP,           donecleanupMode);
        MODECASE(DIFFTOOL_MODE_UPDATEDIFFSKYFILE,     updatediffskyfileMode);
        MODECASE(DIFFTOOL_MODE_EXPORTRUN,             exportrunMode);
        MODECASE(DIFFTOOL_MODE_IMPORTRUN,             importrunMode);
        default:
            psAbort("invalid option (this should not happen)");
    }

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(EXIT_SUCCESS);

FAIL:
    psErrorStackPrint(stderr, "\n");
    int exit_status = pxerrorGetExitStatus();

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(exit_status);
}


static bool definerunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required options
    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);
    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false);
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);

    // default
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    diffRunRow *run = diffRunRowAlloc(
            0,          // ID
            "reg",      // state
            workdir,
            label,
            reduction,
            NULL,       // dvodb
            registered,
            tess_id,
            exp_id
    );
    if (!run) {
        psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object");
        return true;
    }
    if (!diffRunInsertObject(config->dbh, run)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(run);
        return true;
    }

    // get the assigned diff_id
    run->diff_id = psDBLastInsertID(config->dbh);

    if (!diffRunPrintObject(stdout, run, !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print object");
            psFree(run);
            return false;
    }

    psFree(run);

    return true;
}


static bool updaterunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required options
    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);

    if (state) {
        // set detRun.state to state
        return setdiffRunState(config, diff_id, state);
    }

    return true;
}


static bool addinputskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required options
    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);

    // optional
    PXOPT_LOOKUP_S64(stack1, config->args, "-stack1", false, false);
    PXOPT_LOOKUP_S64(stack2, config->args, "-stack2", false, false);
    PXOPT_LOOKUP_S64(warp1, config->args, "-warp1", false, false);
    PXOPT_LOOKUP_S64(warp2, config->args, "-warp2", false, false);
    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);

    // must provide either stack1 or warp1 but not BOTH
    if (!(stack1 || warp1)) {
        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
        return false;
    }
    if (stack1 && warp1) {
        psError(PS_ERR_UNKNOWN, true, "either -stack1 or -warp1 must be specified");
        return false;
    }
    // must provide either stack2 or warp2 but not BOTH
    if (!(stack2 || warp2)) {
        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
        return false;
    }
    if (stack2 && warp2) {
        psError(PS_ERR_UNKNOWN, true, "either -stack2 or -warp2 must be specified");
        return false;
    }

    // if a warp1 was provided we need to lookup the and tess_id from the diffRun
    psString tess_id = NULL;
    if (warp1) {
        if (!p_psDBRunQueryF(config->dbh, "SELECT * from diffRun WHERE diff_id = %" PRId64, diff_id)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }

        psArray *output = p_psDBFetchResult(config->dbh);
        if (!output) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }
        if (!psArrayLength(output)) {
            psError(PS_ERR_UNKNOWN, false, "diff_id %" PRId64 " not found", diff_id);
            psFree(output);
            return false;
        }

        diffRunRow *run = diffRunObjectFromMetadata(output->data[0]);
        tess_id = run->tess_id;
    }

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!diffInputSkyfileInsert(config->dbh,
            diff_id,
            skycell_id,
            warp1 ? warp1 : PS_MAX_S64, // defined or NULL
            stack1 ? stack1 : PS_MAX_S64, // defined or NULL
            warp2 ? warp2 : PS_MAX_S64, // defined or NULL
            stack2 ? stack2 : PS_MAX_S64, // defined or NULL
            tess_id,
            0                             // diff_skyfile_id
        )) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!p_psDBRunQueryF(config->dbh, "SELECT count(diff_id) FROM diffRun JOIN diffInputSkyfile USING(diff_id) WHERE diff_id = %" PRId64, diff_id)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "diff_id %" PRId64 " not found", diff_id);
        psFree(output);
        return false;
    }

    bool status;
    psS32 count = psMetadataLookupS32(&status, output->data[0], "count(diff_id)");

    if (count == 2) {
        if (!setdiffRunState(config, diff_id, "new")) {
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}


static bool inputskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();

    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diff_id", "==");
    PXOPT_COPY_S64(config->args, where, "-warp_id", "warp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
    PXOPT_COPY_STR(config->args, where,  "-tess_id", "tess_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    PXOPT_LOOKUP_BOOL(template, config->args, "-template", false);
    PXOPT_LOOKUP_BOOL(input, config->args, "-input", false);

    if (template && input) {
        // User apparently wants both, which is the default behaviour
        template = false;
        input = false;
    }

    // find all rawImfiles matching the default query
    psString query = pxDataGet("difftool_inputskyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    psString whereClause = NULL;
    if (psListLength(where->list)) {
        whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringPrepend(&whereClause, "\n AND ");
    }
    psFree(where);

    // Add condition to get only templates or only inputs
    {
        psString templateClause = NULL;
        if (template) {
            psStringAppend(&templateClause, " %s", " template != 0");
        } else if (input) {
            psStringAppend(&templateClause, " %s", " template = 0");
        }
        if (templateClause) {
            psStringAppend(&whereClause, "\n AND %s", templateClause);
        }
        psFree(templateClause);
    }


    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQueryF(config->dbh, query, whereClause, whereClause, whereClause, whereClause)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(whereClause);
        psFree(query);
        return false;
    }
    psFree(whereClause);
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "diffInputSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool todiffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffRun.diff_id", "==");
    PXOPT_COPY_STR(config->args, where,  "-label", "diffRun.label", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psString query = pxDataGet("difftool_todiffskyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool adddiffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false); // required
    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false);
    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);
    PXOPT_LOOKUP_STR(uri, config->args, "-uri", (code == 0), false);
    PXOPT_LOOKUP_STR(path_base, config->args, "-path_base", (code == 0), false);

    // optional
    PXOPT_LOOKUP_F64(bg, config->args, "-bg", false, false);
    PXOPT_LOOKUP_F64(bg_stdev, config->args, "-bg_stdev", false, false);
    PXOPT_LOOKUP_F32(dtime_diff, config->args, "-dtime_diff", false, false);
    PXOPT_LOOKUP_F32(dtime_match, config->args, "-dtime_match", false, false);
    PXOPT_LOOKUP_F32(dtime_phot, config->args, "-dtime_phot", false, false);
    PXOPT_LOOKUP_F32(dtime_script, config->args, "-dtime_script", false, false);
    PXOPT_LOOKUP_S32(stamps_num, config->args, "-stamps_num", false, false);
    PXOPT_LOOKUP_F32(stamps_mean, config->args, "-stamps_mean", false, false);
    PXOPT_LOOKUP_F32(stamps_rms, config->args, "-stamps_rms", false, false);
    PXOPT_LOOKUP_F32(norm, config->args, "-norm", false, false);
    PXOPT_LOOKUP_F32(bg_diff, config->args, "-bg_diff", false, false);
    PXOPT_LOOKUP_F32(kernel_x, config->args, "-kernel_x", false, false);
    PXOPT_LOOKUP_F32(kernel_y, config->args, "-kernel_y", false, false);
    PXOPT_LOOKUP_F32(kernel_xx, config->args, "-kernel_xx", false, false);
    PXOPT_LOOKUP_F32(kernel_xy, config->args, "-kernel_xy", false, false);
    PXOPT_LOOKUP_F32(kernel_yy, config->args, "-kernel_yy", false, false);
    PXOPT_LOOKUP_S32(sources, config->args, "-sources", false, false);
    PXOPT_LOOKUP_STR(hostname, config->args, "-hostname", false, false);
    PXOPT_LOOKUP_F32(good_frac, config->args, "-good_frac", false, false);

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!diffSkyfileInsert(config->dbh,
                           diff_id,
                           skycell_id,
                           uri,
                           path_base,
                           bg,
                           bg_stdev,
                           stamps_num,
                           stamps_mean,
                           stamps_rms,
                           norm,
                           bg_diff,
                           kernel_x,
                           kernel_y,
                           kernel_xx,
                           kernel_xy,
                           kernel_yy,
                           sources,
                           dtime_diff,
                           dtime_match,
                           dtime_phot,
                           dtime_script,
                           hostname,
                           good_frac,
                           code,
                           0        // magic_ds_id
          )) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!diffRunComplete(config)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }


    return true;
}


static bool diffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffSkyfile.diff_id", "==");
    PXOPT_COPY_STR(config->args, where, "-skycell_id", "diffInputSkyfile.skycell_id", "==");
    PXOPT_COPY_S64(config->args, where,  "-diff_skyfile_id", "diffInputSkyfile.diff_skyfile_id", "==");
    PXOPT_COPY_STR(config->args, where, "-tess_id", "diffSkyfile.tess_id", "==");
    PXOPT_COPY_S16(config->args, where, "-code", "diffSkyfile.fault", "==");
    PXOPT_COPY_S64(config->args, where, "-exp_id", "rawExp.exp_id", "==");
    PXOPT_COPY_STR(config->args, where, "-exp_name", "rawExp.exp_name", "==");
    PXOPT_COPY_STR(config->args, where, "-warp_id", "warpRun.warp_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psString query = pxDataGet("difftool_skyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " WHERE %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "diffSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool revertdiffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where,  "-diff_id", "diffSkyfile.diff_id", "==");
    PXOPT_COPY_STR(config->args, where,  "-label", "diffRun.label", "==");
    PXOPT_COPY_S16(config->args, where, "-code",     "fault", "==");

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(where);
        return false;
    }

    // Update state to 'new'
    {
        psString query = pxDataGet("difftool_revertdiffskyfile_update.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        if (psListLength(where->list)) {
            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
            psStringAppend(&query, " AND %s", whereClause);
            psFree(whereClause);
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psFree(query);

        psLogMsg("diftool", PS_LOG_INFO, "Updated %" PRIu64 " rows", psDBAffectedRows(config->dbh));
    }

    // Delete product
    {
        psString query = pxDataGet("difftool_revertdiffskyfile_delete.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        if (psListLength(where->list)) {
            psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
            psStringAppend(&query, " AND %s", whereClause);
            psFree(whereClause);
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(query);
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }
        psFree(query);

        psLogMsg("difftool", PS_LOG_INFO, "Deleted %" PRIu64 " rows", psDBAffectedRows(config->dbh));
    }

    psFree(where);

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}


static bool setdiffRunState(pxConfig *config, psS64 diff_id, const char *state)
{
    PS_ASSERT_PTR_NON_NULL(state, false);

    // check that state is a valid string value
    if (!pxIsValidState(state)) {
        psError(PS_ERR_UNKNOWN, false, "invalid diffRun state: %s", state);
        return false;
    }

    char *query = "UPDATE diffRun SET state = '%s' WHERE diff_id = %"PRId64;
    if (!p_psDBRunQueryF(config->dbh, query, state, diff_id)) {
        psError(PS_ERR_UNKNOWN, false,
                "failed to change state for diff_id %"PRId64, diff_id);
        return false;
    }

    return true;
}

// Generate a single populated run
static bool populatedrun(psArray *list, // List of runs, to print
                         const char *workdir, // Working directory
                         const char *skycell_id, // Skycell identifier
                         const char *tess_id, // Tessellation identifier
                         const char *label, // label
                         const char *reduction, // reduction
                         psS64 input_warp_id, // Warp identifier for input image, PS_MAX_S64 for none
                         psS64 input_stack_id, // Stack identifier for input image, PS_MAX_S64 for none
                         psS64 template_warp_id, // Warp identifier for template image, PS_MAX_S64 for none
                         psS64 template_stack_id, // Stack identifier for template image, PS_MAX_S64 for none
                         psS64 exp_id, // exposure id for input_warp_id (if defined)
                         pxConfig *config // Configuration
                         )
{
    PS_ASSERT_STRING_NON_EMPTY(workdir, false);
    PS_ASSERT_STRING_NON_EMPTY(skycell_id, false);
    PS_ASSERT_STRING_NON_EMPTY(tess_id, false);
    if ((input_warp_id == PS_MAX_S64 && input_stack_id == PS_MAX_S64) ||
        (input_warp_id != PS_MAX_S64 && input_stack_id != PS_MAX_S64)) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, input must be defined.");
        return false;
    }
    if ((template_warp_id == PS_MAX_S64 && template_stack_id == PS_MAX_S64) ||
        (template_warp_id != PS_MAX_S64 && template_stack_id != PS_MAX_S64)) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One, and only one, template must be defined.");
        return false;
    }

    // default
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);

    diffRunRow *run = diffRunRowAlloc(
            0,          // ID
            "reg",      // state
            workdir,
            label,
            reduction,
            NULL,       // dvodb
            registered,
            tess_id,
            exp_id
    );

    if (!run) {
        psError(PS_ERR_UNKNOWN, false, "failed to alloc diffRun object");
        return true;
    }

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!diffRunInsertObject(config->dbh, run)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(run);
        return true;
    }

    // get the assigned diff_id
    run->diff_id = psDBLastInsertID(config->dbh);

    // Template
    if (!diffInputSkyfileInsert(config->dbh,
            run->diff_id,
            skycell_id,
            input_warp_id,
            input_stack_id,
            template_warp_id,
            template_stack_id,
            tess_id,
            0
        )) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    char *query = "UPDATE diffRun SET state = 'new' WHERE diff_id = '%" PRId64 "'";
    if (!p_psDBRunQueryF(config->dbh, query, run->diff_id)) {
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        psError(PS_ERR_UNKNOWN, false,
                "failed to set state to run for diff_id %" PRId64, run->diff_id);
        return false;
    }

    // point of no return
    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (list) {
        psArrayAdd(list, list->n, run);
    }

    psFree(run);

    return true;
}


static bool definepoprunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options
    PXOPT_LOOKUP_STR(skycell_id, config->args, "-skycell_id", true, false); // required options
    PXOPT_LOOKUP_STR(tess_id, config->args, "-tess_id", true, false); // required options
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false);
    PXOPT_LOOKUP_S64(template_warp_id, config->args, "-template_warp_id", false, false);
    PXOPT_LOOKUP_S64(template_stack_id, config->args, "-template_stack_id", false, false);
    PXOPT_LOOKUP_S64(input_warp_id, config->args, "-input_warp_id", false, false);
    PXOPT_LOOKUP_S64(input_stack_id, config->args, "-input_stack_id", false, false);
    PXOPT_LOOKUP_S64(exp_id, config->args, "-exp_id", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    if (template_stack_id && template_warp_id) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one template can be defined.");
        return false;
    }
    if (!template_stack_id && !template_warp_id) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "No template has been defined (-template_stack_id or -template_warp_id)");
        return false;
    }

    if (input_stack_id && input_warp_id) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Only one input can be defined.");
        return false;
    }
    if (!input_stack_id && !input_warp_id) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "No input has been defined (-input_stack_id or -input_warp_id)");
        return false;
    }
    if (input_warp_id && !exp_id) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "-exp_id is required with -input_warp_id.");
        return false;
    }

    psArray *list = psArrayAllocEmpty(16); // List of runs, to print

    if (!populatedrun(list, workdir, skycell_id, tess_id, label, reduction,
                      input_warp_id ? input_warp_id : PS_MAX_S64,
                      input_stack_id ? input_stack_id : PS_MAX_S64,
                      template_warp_id ? template_warp_id : PS_MAX_S64,
                      template_stack_id ? template_stack_id : PS_MAX_S64,
                      exp_id ? exp_id : PS_MAX_S64,
                      config)) {
        psError(PS_ERR_UNKNOWN, false, "failed to create populated diffRun");
        psFree(list);
        return false;
    }

    if (!diffRunPrintObjects(stdout, list, !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print object");
            psFree(list);
            return false;
    }
    psFree(list);

    return true;
}


static bool definebyqueryMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *expWhere = psMetadataAlloc();
    psMetadata *warpWhere = psMetadataAlloc();
    psMetadata *stackWhere = psMetadataAlloc();

    PXOPT_COPY_S64(config->args, expWhere, "-exp_id", "exp_id", "==");
    PXOPT_COPY_S64(config->args, warpWhere, "-warp_id", "warpRun.warp_id", "==");
    PXOPT_COPY_STR(config->args, warpWhere, "-skycell_id", "warpSkyfile.skycell_id", "==");
    PXOPT_COPY_STR(config->args, warpWhere, "-tess_id", "warpRun.tess_id", "==");
    PXOPT_COPY_STR(config->args, warpWhere, "-filter", "rawExp.filter", "==");
    PXOPT_COPY_STR(config->args, warpWhere, "-warp_label", "warpRun.label", "==");
    PXOPT_COPY_F32(config->args, warpWhere,  "-good_frac", "warpSkyfile.good_frac", ">=");
    PXOPT_COPY_STR(config->args, stackWhere, "-stack_label", "stackRun.label", "==");

    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false); // required options
    PXOPT_LOOKUP_STR(reduction, config->args, "-reduction", false, false); // option
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false); // option
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);
    PXOPT_LOOKUP_BOOL(newTemplates, config->args, "-new-templates", false);
    PXOPT_LOOKUP_BOOL(reRun, config->args, "-rerun", false);
    PXOPT_LOOKUP_BOOL(available, config->args, "-available", false);

    // find all things to queue
    psString query = pxDataGet("difftool_definebyquery_part1.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    psString warpQuery = NULL;
    psString stackQuery = NULL;
    psString expQuery = NULL;

    if (psListLength(expWhere->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(expWhere, NULL);
        psStringAppend(&expQuery, "\n AND %s", whereClause);
        psFree(whereClause);
    } else {
        expQuery = psStringCopy("\n");
    }
    psFree(expWhere);
    if (psListLength(warpWhere->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(warpWhere, NULL);
        psStringAppend(&warpQuery, "\n AND %s", whereClause);
        psFree(whereClause);
    } else {
        warpQuery = psStringCopy("\n");
    }
    psFree(warpWhere);

    if (!available) {
        // diff what's available, even if warp run has some faults and is incomplete
        psStringAppend(&warpQuery, " AND warpRun.state = 'full'");
    }

    // don't queue for exposures that have already been diff'd unless requested
    psString diffQuery = NULL;
    if (! (reRun || newTemplates) ) {
        psStringAppend(&diffQuery, "\nAND diff_id IS NULL");
    } else {
        diffQuery = psStringCopy("\n");
    }

    if (psListLength(stackWhere->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(stackWhere, NULL);
        psStringAppend(&stackQuery, "\nAND %s", whereClause);
        psFree(whereClause);
    } else {
        stackQuery = psStringCopy("");
    }
    psFree(stackWhere);

    psTrace("difftool", 1, query, warpQuery, diffQuery, expQuery, stackQuery);


    if (!p_psDBRunQueryF(config->dbh, query, warpQuery, expQuery, diffQuery)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // create temporary table
    query = pxDataGet("difftool_definebyquery_temp_create.sql");
    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);
    query = NULL;

    psString skycell_query = pxDataGet("difftool_definebyquery_part2.sql");

    psArray *list = psArrayAllocEmpty(16); // List of runs, to print
    long numGood = 0;                   // Number of good rows added
    psS64 last_exp_id = 0;
    for (long i = 0; i < output->n; i++) {
        psMetadata *row = output->data[i]; // Output row from query
        bool mdok;                      // Status of MD lookup

        // Take the first warp for each exposure.
        // The list is sorted by exposure id and warp_id and the warps are in descending
        // order.
        psS64 exp_id = psMetadataLookupS64(&mdok, row, "exp_id");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "exp_id not found");
            return false;
        }
        if (exp_id == last_exp_id) {
            continue;
        }
        last_exp_id = exp_id;

        // clear temporary table
        if (!p_psDBRunQuery(config->dbh, "DELETE FROM skycellsToDiff")) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            return false;
        }
        psS64 warp_id = psMetadataLookupS64(&mdok, row, "warp_id");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "warp_id not found --- ignoring row %ld", i);
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            return false;
        }
        psS64 skycell_count = psMetadataLookupS64(&mdok, row, "skycell_count");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "skycell_count not found");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            return false;
        }
        psString tess_id = psMetadataLookupStr(&mdok, row, "tess_id");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "tess_id not found");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            return false;
        }
        psString filter = psMetadataLookupStr(&mdok, row, "filter");
        if (!mdok) {
            psError(PXTOOLS_ERR_PROG, false, "filter not found");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            return false;
        }
        if (!p_psDBRunQueryF(config->dbh, skycell_query, stackQuery, warp_id, filter, warpQuery)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            return false;
        }
        psS64 num = psDBAffectedRows(config->dbh);

        if (num == 0) {
            psTrace("difftool", PS_LOG_INFO, "no skycells with stack found for %" PRId64, warp_id);
            continue;
        }

        if (!available && (num != skycell_count)) {
            psTrace("difftool", PS_LOG_INFO, "%" PRId64 " skyfiles with stack found for warp_id %"
                PRId64 " need %" PRId64, num, skycell_count, warp_id);
            continue;
        }

        // ok we've got one create the diffRun
        diffRunRow *run = diffRunRowAlloc(
                0,          // ID
                "reg",      // state
                workdir,
                label,
                reduction,
                NULL,       // dvodb
                registered,
                tess_id,
                exp_id
        );

        if (!diffRunInsertObject(config->dbh, run)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(run);
            return true;
        }
        run->diff_id = psDBLastInsertID(config->dbh);

        psStringAppend(&query, "UPDATE skycellsToDiff SET diff_id = %" PRId64, run->diff_id);
        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            psFree(query);
            return false;
        }
        psFree(query);
        query = NULL;
        psStringAppend(&query, "INSERT INTO diffInputSkyfile SELECT * from skycellsToDiff");
        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            psFree(query);
            return false;
        }
        psFree(query);
        query = NULL;

        if (!setdiffRunState(config, run->diff_id, "new")) {
            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
                run->diff_id);
            psFree(warpQuery);
            psFree(stackQuery);
            psFree(skycell_query);
            psFree(query);
            return false;
        }

        psArrayAdd(list, list->n, run);
        numGood++;
    }
    psFree(output);
    psFree(warpQuery);
    psFree(stackQuery);
    psFree(skycell_query);

    if (!diffRunPrintObjects(stdout, list, !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print object");
        psFree(list);
        return false;
    }
    psFree(list);

    return true;
}

static bool pendingcleanuprunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");

    psString query = pxDataGet("difftool_pendingcleanuprun.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "diffPendingCleanupRun", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);

    return true;
}


static bool pendingcleanupskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", false, false);
    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psMetadata *where = psMetadataAlloc();
    if (diff_id) {
        PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
    }
    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");

    psString query = pxDataGet("difftool_pendingcleanupskyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "diffPendingCleanupSkyfile", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);
    return true;
}


static bool donecleanupMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_STR(config->args, where, "-label", "label", "==");

    psString query = pxDataGet("difftool_donecleanup.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (where && psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // negative simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "diffDoneCleanup", !simple)) {
        psError(PS_ERR_UNKNOWN, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);
    return true;
}

static bool updatediffskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_S16(code, config->args, "-code", true, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-diff_id",   "diff_id",   "==");

    if (!pxSetFaultCode(config->dbh, "diffSkyfile", where, code)) {
        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
        psFree (where);
        return false;
    }
    psFree (where);

    return true;
}

static bool diffRunComplete(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // look for completed diffRuns
    psString query = pxDataGet("difftool_completed_runs.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("difftool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }
    for (long i = 0; i < psArrayLength(output); i++) {
        psMetadata *row = output->data[i];

        psS64 diff_id = psMetadataLookupS64(NULL, row, "diff_id");

        // set diffRun.state to 'stop'
        if (!setdiffRunState(config, diff_id, "full")) {
            psError(PS_ERR_UNKNOWN, false, "failed to change diffRun.state for diff_id: %" PRId64,
                diff_id);
            psFree(output);
            return false;
        }
    }

    return true;
}

bool exportrunMode(pxConfig *config)
{
  typedef struct ExportTable {
    char tableName[80];
    char sqlFilename[80];
  } ExportTable;
  
  int numExportFiles = 3;

  PS_ASSERT_PTR_NON_NULL(config, NULL);

  PXOPT_LOOKUP_S64(det_id, config->args, "-diff_id", true,  false);
  PXOPT_LOOKUP_STR(outfile, config->args, "-outfile", true,  false);
  PXOPT_LOOKUP_U64(limit,   config->args, "-limit",   false, false);

  FILE *f = fopen (outfile, "w");
  if (f == NULL) {
    psError(PS_ERR_UNKNOWN, false, "failed to open output file");
    return false;
  }

  psMetadata *where = psMetadataAlloc();
  PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");

  ExportTable tables [] = {
    {"diffRun", "difftool_export_run.sql"},
    {"diffInputSkyfile", "difftool_export_input_skyfile.sql"},
    {"diffSkyfile", "difftool_export_skyfile.sql"},
  };

  for (int i=0; i < numExportFiles; i++) {
    psString query = pxDataGet(tables[i].sqlFilename);
    if (!query) {
      psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
      return false;
    }

    if (where && psListLength(where->list)) {
      psString whereClause = psDBGenerateWhereSQL(where, NULL);
      psStringAppend(&query, " %s", whereClause);
      psFree(whereClause);
    }

    // treat limit == 0 as "no limit"
    if (limit) {
      psString limitString = psDBGenerateLimitSQL(limit);
      psStringAppend(&query, " %s", limitString);
      psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
      psError(PS_ERR_UNKNOWN, false, "database error");
      psFree(query);
      return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
      psError(PS_ERR_UNKNOWN, false, "database error");
      return false;
    }
    if (!psArrayLength(output)) {
      psTrace("regtool", PS_LOG_INFO, "no rows found");
      psFree(output);
      return true;
    }

    // we must write the export table in non-simple (true) format
    if (!ippdbPrintMetadatas(f, output, tables[i].tableName, true)) {
      psError(PS_ERR_UNKNOWN, false, "failed to print array");
      psFree(output);
      return false;
    }
    psFree(output);
  }

  fclose (f);

  return true;
}

bool importrunMode(pxConfig *config)
{
  unsigned int nFail;
  
  int numImportTables = 2;
  
  char tables[2] [80] = {"diffInputSkyfile", "diffSkyfile"};

  PS_ASSERT_PTR_NON_NULL(config, NULL);
  
  PXOPT_LOOKUP_STR(infile, config->args, "-infile", true,  false);

  psMetadata *input = psMetadataConfigRead (NULL, &nFail, infile, false);

  fprintf (stdout, "---- input ----\n");
  psMetadataPrint (stderr, input, 1);

  psMetadataItem *item = psMetadataLookup (input, "diffRun");
  psAssert (item, "entry not in input?");
  psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");

  psMetadataItem *entry = psListGet (item->data.list, 0);
  assert (entry);
  assert (entry->type == PS_DATA_METADATA);
  diffRunRow *diffRun = diffRunObjectFromMetadata (entry->data.md);
  diffRunInsertObject (config->dbh, diffRun);

  // fprintf (stdout, "---- diff run ----\n");
  // psMetadataPrint (stderr, entry->data.md, 1);

  for (int i = 0; i < numImportTables; i++) {
    item = psMetadataLookup (input, tables[i]);
    psAssert (item, "entry not in input?");
    psAssert (item->type == PS_DATA_METADATA_MULTI, "entry not multi?");
  
    switch (i) {
      case 0:
        for (int i = 0; i < item->data.list->n; i++) {
          entry = psListGet (item->data.list, i);
          assert (entry);
          assert (entry->type == PS_DATA_METADATA);
          diffInputSkyfileRow *diffInputSkyfile = diffInputSkyfileObjectFromMetadata (entry->data.md);
          diffInputSkyfileInsertObject (config->dbh, diffInputSkyfile);

          // fprintf (stdout, "---- row %d ----\n", i);
          // psMetadataPrint (stderr, entry->data.md, 1);
        }
        break;
        
      case 1:
        for (int i = 0; i < item->data.list->n; i++) {
          entry = psListGet (item->data.list, i);
          assert (entry);
          assert (entry->type == PS_DATA_METADATA);
          diffSkyfileRow *diffSkyfile = diffSkyfileObjectFromMetadata (entry->data.md);
          diffSkyfileInsertObject (config->dbh, diffSkyfile);

          // fprintf (stdout, "---- row %d ----\n", i);
          // psMetadataPrint (stderr, entry->data.md, 1);
        }
        break;
    }
  }

  return true;
}
