/*
 * magictool.c
 *
 * Copyright (C) 2006-2007  Joshua Hoblitt
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * program; see the file COPYING. If not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#ifdef HAVB_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <math.h>
#include <ippdb.h>

#include "pxtools.h"
#include "magictool.h"

static bool definebyqueryMode(pxConfig *config);
static psS64 definerunMode(pxConfig *config);
static bool updaterunMode(pxConfig *config);
static bool addinputskyfileMode(pxConfig *config);
static bool inputskyfileMode(pxConfig *config);
static bool totreeMode(pxConfig *config);
static bool inputtreeMode(pxConfig *config);
static bool reverttreeMode(pxConfig *config);
static bool toprocessMode(pxConfig *config);
static bool addresultMode(pxConfig *config);
static bool revertnodeMode(pxConfig *config);
static bool inputsMode(pxConfig *config);
static bool tomaskMode(pxConfig *config);
static bool addmaskMode(pxConfig *config);
static bool revertmaskMode(pxConfig *config);
static bool maskMode(pxConfig *config);

static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state);
static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename);

# define MODECASE(caseName, func) \
    case caseName: \
    if (!func(config)) { \
        goto FAIL; \
    } \
    break;

int main(int argc, char **argv)
{
    psLibInit(NULL);

    pxConfig *config = magictoolConfig(NULL, argc, argv);
    if (!config) {
        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
        goto FAIL;
    }

    switch (config->mode) {
        MODECASE(MAGICTOOL_MODE_DEFINEBYQUERY,  definebyqueryMode);
        MODECASE(MAGICTOOL_MODE_DEFINERUN,      definerunMode);
        MODECASE(MAGICTOOL_MODE_UPDATERUN,      updaterunMode);
        MODECASE(MAGICTOOL_MODE_ADDINPUTSKYFILE,addinputskyfileMode);
        MODECASE(MAGICTOOL_MODE_INPUTSKYFILE,   inputskyfileMode);
        MODECASE(MAGICTOOL_MODE_TOTREE,         totreeMode);
        MODECASE(MAGICTOOL_MODE_INPUTTREE,      inputtreeMode);
        MODECASE(MAGICTOOL_MODE_REVERTTREE,     reverttreeMode);
        MODECASE(MAGICTOOL_MODE_TOPROCESS,      toprocessMode);
        MODECASE(MAGICTOOL_MODE_ADDRESULT,      addresultMode);
        MODECASE(MAGICTOOL_MODE_REVERTNODE,     revertnodeMode);
        MODECASE(MAGICTOOL_MODE_INPUTS,         inputsMode);
        MODECASE(MAGICTOOL_MODE_TOMASK,         tomaskMode);
        MODECASE(MAGICTOOL_MODE_ADDMASK,        addmaskMode);
        MODECASE(MAGICTOOL_MODE_REVERTMASK,     revertmaskMode);
        MODECASE(MAGICTOOL_MODE_MASK,           maskMode);
        default:
            psAbort("invalid option (this should not happen)");
    }

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(EXIT_SUCCESS);

FAIL:
    psErrorStackPrint(stderr, "\n");
    int exit_status = pxerrorGetExitStatus();

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(exit_status);
}

static bool definebyqueryMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", false, false);
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);

    // create warped skycells temp table
    {
        psString query = pxDataGet("magictool_create_tmp_warpcomplete.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            return false;
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            return false;
        }
    }

    // find warped skycells
    {
        psString query = pxDataGet("magictool_find_complete_warpruns.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            return false;
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(query);
            return false;
        }
        psFree(query);
    }

    // find the diff_id's of the warped skycells
    {
        psString query = pxDataGet("magictool_find_complete_diffed_exposures.sql");
        if (!query) {
            psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
            return false;
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(query);
            return false;
        }
        psFree(query);
    }
    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
          case PS_ERR_DB_CLIENT:
            psError(PXTOOLS_ERR_SYS, false, "database error");
          case PS_ERR_DB_SERVER:
            psError(PXTOOLS_ERR_PROG, false, "database error");
          default:
            psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magictool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // pre-allocate enough hash buckets for 1 unique warp_id per row
    psHash *groups = psHashAlloc(psArrayLength(output));
    // iterate over result set, and group by warp_id
    for (long i = 0; i < psArrayLength(output); i++) {
        // lookup the key we're grouping by
        bool status = false;
        psS64 warp_id = psMetadataLookupS64(&status, output->data[i], "warp_id");
        if (!status) {
            psAbort("failed to lookup value for warp_id");
        }

        psString key = psDBIntToString(warp_id);
        // look to see if there is a bin for this key
        psArray *groupBin = psHashLookup(groups, key);
        if (!groupBin) {
            // if not, create one
            groupBin = psArrayAlloc(0);
            psHashAdd(groups, key, groupBin);
        }
        psFree(key);

        // add this row to the bin
        psArrayAdd(groupBin, 0, output->data[i]);
        psFree(groupBin);
    }
    psFree(output);

    // create a magic run
    psArray *grouped =  psHashToArray(groups);
    psFree(groups);

    // iterate over array of grouped warp_ids
    for (long i = 0; i < psArrayLength(grouped); i++) {
        psArray *group = grouped->data[i];

        // create a new magicRun for this group
        magicRunRow *run = magicRunRowAlloc(
            0,                          // ID
            "reg",                      // state
            workdir,                    // workdir
            "dirty",                    // workdir_state
            label,                      // label
            dvodb,                      // dvodb
            registered,                 // registered
            0                           // fault
            );
        if (!run) {
            psAbort("failed to alloc magicRun object");
        }
        if (!magicRunInsertObject(config->dbh, run)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(run);
            psFree(grouped);
            return false;
        }

        // get the assigned warp_id
        psS64 magic_id = psDBLastInsertID(config->dbh);

        // insert all rows in this bin into the new magicRun
        for (long j = 0; j < psArrayLength(group); j++) {
            psMetadata *row = group->data[j];

            bool status = false;
            psS64 diff_id = psMetadataLookupS64(&status, row, "diff_id");
            if (!status) {
                psAbort("failed to lookup value for diff_id");
            }
            psString node = psMetadataLookupStr(&status, row, "skycell_id");
            if (!status) {
                psAbort("failed to lookup value for diff_id");
            }

            if (!magicInputSkyfileInsert(config->dbh,
                                         magic_id,
                                         diff_id,
                                         node)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
                psFree(grouped);
                return false;
            }
        }

        // set magicRun.state to 'run'
        return setmagicRunState(config, magic_id, "run");
    }
    psFree(grouped);

    return true;
}

static psS64 definerunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required
    PXOPT_LOOKUP_STR(workdir, config->args, "-workdir", true, false);

    // optional
    PXOPT_LOOKUP_STR(label, config->args, "-label", false, false);
    PXOPT_LOOKUP_STR(dvodb, config->args, "-dvodb", false, false);
    PXOPT_LOOKUP_TIME(registered, config->args, "-registered", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    magicRunRow *run = magicRunRowAlloc(
            0,          // ID
            "reg",      // state
            workdir,
            "dirty",    // workdir_state
            label,
            dvodb,
            registered,
            0
    );

    if (!run) {
        psError(PS_ERR_UNKNOWN, false, "failed to alloc magicRun object");
        return false;
    }
    if (!magicRunInsertObject(config->dbh, run)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(run);
        return false;
    }

    // get the assigned warp_id
    psS64 magic_id = psDBLastInsertID(config->dbh);
    run->magic_id = magic_id;

    if (!magicRunPrintObject(stdout, run, !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print object");
            psFree(run);
            return false;
    }

    psFree(run);

    return magic_id;
}


static bool updaterunMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required
    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
    PXOPT_LOOKUP_STR(state, config->args, "-state", true, false);

    if (state) {
        // set detRun.state to state
        return setmagicRunState(config, magic_id, state);
    }

    return true;
}


static bool addinputskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required
    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
    PXOPT_LOOKUP_S64(diff_id, config->args, "-diff_id", true, false);
    PXOPT_LOOKUP_STR(node, config->args, "-node", true, false);

    magicInputSkyfileInsert(
            config->dbh,
            magic_id,
            diff_id,
            node
    );

    return true;
}


static bool inputskyfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    PXOPT_COPY_S64(config->args, where, "-diff_id", "diff_id", "==");
    PXOPT_COPY_STR(config->args, where, "-node", "node", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // find all rawImfiles matching the default query
    psString query = pxDataGet("magictool_inputskyfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, "magicInputSkyfile");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magictool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "magicInputSkyfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}

static bool totreeMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // look for "inputs" that need to processed
    psString query = pxDataGet("magictool_totree.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magictool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "totree", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}

static bool inputtreeMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required
    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);

    // Optional values
    PXOPT_LOOKUP_STR(dep_file, config->args, "-dep_file", false, false);

    // default values
    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);

    if (code > 0) {
        char *query = "UPDATE magicRun SET fault = %d, state = 'stop' WHERE magic_id = %" PRId64;
        if (!p_psDBRunQuery(config->dbh, query, code, magic_id)) {
            psError(PS_ERR_UNKNOWN, false,
                    "failed to set fault for magic_id %" PRId64, magic_id);
            return false;
        }
        return true;
    }

    if (!parseAndInsertNodeDeps(config, magic_id, dep_file)) {
        psError(PS_ERR_UNKNOWN, false, "failed to parse file");
        return false;
    }

    return true;
}

static bool reverttreeMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    PXOPT_COPY_S16(config->args, where, "-code", "fault", "==");

    psString query = psStringCopy("UPDATE magicRun SET fault = 0, state = 'run' WHERE fault != 0");

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "failed to revert");
        return false;
    }
    return true;
}


static bool inputsMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    PXOPT_COPY_STR(config->args, where, "-node", "node", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psString query = pxDataGet("magictool_inputs.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magictool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "magicNode", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}

bool findBaseNodes(void *arg, pxNode *node)
{
    bool status = false;
    psS64 done = psMetadataLookupS64(&status, node->data, "done");
    if (!status) {
        psAbort("failed to lookup value for done column");
    }

    if ((!pxNodeHasChildren(node)) && (!done)) {
        // if this node has no child and it's not 'done', then push it's data
        // onto the void *array
        psArrayAdd((psArray *)arg, 0, node->data);
        return false;
    }

    return true;
}


bool findReadyNodes(void *arg, pxNode *node)
{
    if (!node) {
        // It's not there --- must have failed
        return false;
    }

    if (psMetadataLookupBool(NULL, node->data, "done")) {
        // It's already done
        return true;
    }

    if (pxNodeHasChildren(node)) {
        psListIterator *iter = psListIteratorAlloc(node->children, 0, false);
        psMetadata *work = psMetadataCopy(NULL, node->data);
        psMetadataRemoveKey(work, "dep");
        psMetadataRemoveKey(work, "done");
        pxNode *child = NULL;
        while ((child = psListGetAndIncrement(iter))) {
            psMetadata *data = child->data;

            bool status = false;
            psS32 done = psMetadataLookupS32(&status, data, "done");
            if (!status) {
                psAbort("failed to lookup value for done column");
            }
            psS16 bad = psMetadataLookupS16(&status, data, "bad");
            if (!status) {
                psAbort("failed to lookup value for bad column");
            }

            if (!done || bad) {
                // if a child isn't "done", give up on this node and continue
                // to crawl the tree
                psFree(iter);
                psFree(work);
                return true;
            }
        }
        psFree(iter);
        // if all this nodes children are done, then push it's data onto the
        // void *array
        psArrayAdd((psArray *)arg, 0, work);
        psFree(work);
        return false;
    }

    return true;
}


static bool toprocessMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");

    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // look for "inputs" that need to processed
    psString query = pxDataGet("magictool_toprocess_inputs.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    psString whereClause = NULL;
    if (psListLength(where->list)) {
        whereClause = psDBGenerateWhereSQL(where, NULL);
        psStringAppend(&query, " %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magictool", PS_LOG_INFO, "no rows found");
        // psFree(output);
        // return true;
    }

    // look for tree nodes that need to be processed
    query = pxDataGet("magictool_toprocess_tree.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (whereClause) {
        psStringAppend(&query, " %s", whereClause);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *magicTree = p_psDBFetchResult(config->dbh);
    if (!magicTree) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(magicTree)) {
        psTrace("magictool", PS_LOG_INFO, "no rows found");
        psFree(magicTree);
        return true;
    }

    psHash *forest = psHashAlloc(psArrayLength(magicTree));

    // convert the array of metadata into a pxTree structure
    for (long i = 0; i < psArrayLength(magicTree); i++) {
        bool status;
        psString node = psMetadataLookupStr(&status, magicTree->data[i], "node");
        if (!status) {
            psAbort("failed to lookup value for node column");
        }

        psString dep = psMetadataLookupStr(&status, magicTree->data[i], "dep");
        if (!status) {
            psAbort("failed to lookup value for dep column");
        }

        pxTreeBuilder(forest, node, dep, magicTree->data[i]);

    }
    psFree(magicTree);

    // find the root of the tree
    pxNode *root = psHashLookup(forest, "root");
    psFree(forest);
    // pxTreePrint(stdout, root);

    // crawl through the tree and looking for nodes with children that are all
    // "done"
    pxTreeCrawl(root, findReadyNodes, output);
    psFree(root);

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "magicMe", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);
    psFree(whereClause);

    return true;
}


static bool addresultMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required
    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);
    PXOPT_LOOKUP_STR(node, config->args, "-node", true, false);

    // optional
    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);

    // default values
    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);

    if (!magicNodeResultInsert(config->dbh,
                               magic_id,
                               node,
                               uri,
                               code
        )) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}

static bool revertnodeMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    PXOPT_COPY_STR(config->args, where, "-node", "node", "==");
    PXOPT_COPY_S16(config->args, where, "-code", "fault", "==");

    psString query = psStringCopy("DELETE FROM magicNodeResult WHERE fault != 0");

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "failed to revert");
        return false;
    }
    return true;
}


static bool tomaskMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    // look for "inputs" that need to processed
    psString query = pxDataGet("magictool_tomask.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magictool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "tomask", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool addmaskMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // required
    PXOPT_LOOKUP_S64(magic_id, config->args, "-magic_id", true, false);

    // optional
    PXOPT_LOOKUP_STR(uri, config->args, "-uri", false, false);
    PXOPT_LOOKUP_S32(streaks, config->args, "-streaks", false, false);

    // default values
    PXOPT_LOOKUP_S16(code, config->args, "-code", false, false);

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    if (!magicMaskInsert(config->dbh,
                         magic_id,
                         uri,
                         streaks,
                         code
        )) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        return false;
    }

    // Set the magicRun state
    psString query = pxDataGet("magictool_addmask.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        return false;
    }

    // manually add constraint
    psStringAppend(&query, " AND magic_id = %" PRId64, magic_id);

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        if (!psDBRollback(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
        }
        return false;
    }
    psFree(query);

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}

static bool revertmaskMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_id", "magic_id", "==");
    PXOPT_COPY_S16(config->args, where, "-code", "fault", "==");

    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    // Set to "run"
    {
        psString query = psStringCopy("UPDATE magicRun JOIN magicMask USING(magic_id) "
                                      "SET magicRun.state = 'run' WHERE magicMask.fault != 0");

        if (psListLength(where->list)) {
            psString whereClause = psDBGenerateWhereConditionSQL(where, "magicMask");
            psStringAppend(&query, " AND %s", whereClause);
            psFree(whereClause);
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "failed to revert");
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        psFree(query);
    }

    // Delete failed attempt at mask
    {
        psString query = psStringCopy("DELETE FROM magicMask WHERE fault != 0");

        if (psListLength(where->list)) {
            psString whereClause = psDBGenerateWhereConditionSQL(where, "magicMask");
            psStringAppend(&query, " AND %s", whereClause);
            psFree(whereClause);
        }

        if (!p_psDBRunQuery(config->dbh, query)) {
            psError(PS_ERR_UNKNOWN, false, "failed to revert");
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            return false;
        }

        psFree(query);
    }

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    psFree(where);

    return true;
}


static bool maskMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psMetadata *where = psMetadataAlloc();
    PXOPT_COPY_S64(config->args, where, "-magic_id", "magicRun.magic_id", "==");

    PXOPT_LOOKUP_U64(limit, config->args, "-limit", false, false);
    PXOPT_LOOKUP_BOOL(simple, config->args, "-simple", false);

    psString query = pxDataGet("magictool_mask.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (psListLength(where->list)) {
        psString whereClause = psDBGenerateWhereConditionSQL(where, NULL);
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }
    psFree(where);

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("magictool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "magicMask", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool setmagicRunState(pxConfig *config, psS64 magic_id, const char *state)
{
    PS_ASSERT_PTR_NON_NULL(state, false);

    // check that state is a valid string value
    if (!(
            (strncmp(state, "run", 4) == 0)
            || (strncmp(state, "stop", 5) == 0)
            || (strncmp(state, "reg", 4) == 0)
        )
    ) {
        psError(PS_ERR_UNKNOWN, false,
                "invalid magicRun state: %s", state);
        return false;
    }

    char *query = "UPDATE magicRun SET state = '%s' WHERE magic_id = %" PRId64;
    if (!p_psDBRunQuery(config->dbh, query, state, magic_id)) {
        psError(PS_ERR_UNKNOWN, false,
                "failed to change state for magic_id %" PRId64, magic_id);
        return false;
    }

    return true;
}

static bool parseAndInsertNodeDeps(pxConfig *config, psS64 magic_id, const char *filename)
{
    unsigned int nFail = 0;
    psMetadata *deps = psMetadataConfigRead(NULL, &nFail, filename, false);
    if (!deps) {
        psError(PS_ERR_UNKNOWN, false, "failed to parse file: %s", filename);
        return false;
    }
    if (nFail) {
        psError(PS_ERR_UNKNOWN, false, "there were %d errors parsing file: %s", nFail, filename);
        psFree(deps);
        return false;
    }

    psMetadataItem *item = NULL;
    psMetadataIterator *iter = psMetadataIteratorAlloc(deps, 0, NULL);
    while ((item = psMetadataGetAndIncrement(iter))) {
        if (item->type != PS_DATA_STRING) {
            psError(PS_ERR_UNKNOWN, false, "file: %s is in the wrong format", filename);
            psFree(iter);
            psFree(deps);
            return false;
        }

        char *name = item->name;
        char *dependsOn = item->data.str;

        if (!magicTreeInsert(
                config->dbh,
                magic_id,
                name,
                dependsOn
            )) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(iter);
            psFree(deps);
            return false;
        }
    }
    psFree(iter);
    psFree(deps);

    return true;
}
