/*
 * regtool.c
 *
 * Copyright (C) 2006  Joshua Hoblitt
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License as published by the Free
 * Software Foundation; version 2 of the License.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
 * more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * program; see the file COPYING. If not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdlib.h>
#include <stdint.h>
#include <math.h>

#include "pxtools.h"
#include "pxdata.h"
#include "regtool.h"
#include "chiptool.h"

static bool pendingimfileMode(pxConfig *config);
static bool addprocessedimfileMode(pxConfig *config);
static bool processedimfileMode(pxConfig *config);
static bool updateprocessedimfileMode(pxConfig *config);

static bool pendingexpMode(pxConfig *config);
static bool addprocessedexpMode(pxConfig *config);
static bool processedexpMode(pxConfig *config);
static bool updateprocessedexpMode(pxConfig *config);


// static guidePendingExpRow *newToP1PendingExp(newExpRow *newExp);
//static chipPendingExpRow *newTochipPendingExp(pxConfig *config, newExpRow *newExp);
//static chipPendingImfileRow *rawImfileTochipPendingImfile(pxConfig *config, psS64 chip_id, rawImfileRow *rawImfile);

static rawExpRow *newToRawExp(pxConfig *config, newExpRow *exp);
static rawImfileRow *newToRawImfile(pxConfig *config, newImfileRow *exp);
//static psU32 mapCodeStrToInt(const char *codeStr);


# define MODECASE(caseName, func) \
    case caseName: \
    if (!func(config)) { \
        goto FAIL; \
    } \
    break;

int main(int argc, char **argv)
{
    psLibInit(NULL);

    pxConfig *config = regtoolConfig(NULL, argc, argv);
    if (!config) {
        psError(PXTOOLS_ERR_CONFIG, false, "failed to configure");
        goto FAIL;
    }

    switch (config->mode) {
        MODECASE(REGTOOL_MODE_PENDINGIMFILE,         pendingimfileMode);
        MODECASE(REGTOOL_MODE_ADDPROCESSEDIMFILE,    addprocessedimfileMode);
        MODECASE(REGTOOL_MODE_PROCESSEDIMFILE,       processedimfileMode);
        MODECASE(REGTOOL_MODE_UPDATEPROCESSEDIMFILE, updateprocessedimfileMode);
        MODECASE(REGTOOL_MODE_PENDINGEXP,            pendingexpMode);
        MODECASE(REGTOOL_MODE_ADDPROCESSEDEXP,       addprocessedexpMode);
        MODECASE(REGTOOL_MODE_PROCESSEDEXP,          processedexpMode);
        MODECASE(REGTOOL_MODE_UPDATEPROCESSEDEXP,    updateprocessedexpMode);
        default:
            psAbort("invalid option (this should not happen)");
    }

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(EXIT_SUCCESS);

FAIL:
    psErrorStackPrint(stderr, "\n");
    int exit_status = pxerrorGetExitStatus();

    psFree(config);
    pmConfigDone();
    psLibFinalize();

    exit(exit_status);
}


static bool pendingimfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    // select newImfiles that:
    // exp_tag is in newExp
    // don't have their exp_tag in rawExp 
    // XXX having the same exp_tag in newExp and raw*Exp is probably an error
    // that should be checked for

    psString query = pxDataGet("regtool_pendingimfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
	// XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
        psError(PXTOOLS_ERR_PROG, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }
        
        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("regtool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    simple = psMetadataLookupBool(&status, config->args, "-simple");
    if (!status) {
	psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -simple");
	return false;
    }

    // negate simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "regPendingImfile", !simple)) {
        psError(PXTOOLS_ERR_PROG, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);

    return true;
}


static bool addprocessedimfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    psString query = pxDataGet("regtool_find_unprocessed_imfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    {
        // build a query to search by exp_tag, class, class_id
        psMetadata *where = psMetadataAlloc();
        bool status = false;
        psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag");
            psFree(query);
            return false;
        }
        if (exp_tag) {
            if (!psMetadataAddStr(where, PS_LIST_TAIL, "exp_tag", 0, "==", exp_tag)) {
                psError(PS_ERR_UNKNOWN, false, "failed to add item exp_tag");
                psFree(where);
                psFree(query);
                return false;
            }
        }

        psString class_id = psMetadataLookupStr(&status, config->args, "-class_id");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -class_id");
            psFree(query);
            return false;
        }
        if (class_id) {
            if (!psMetadataAddStr(where, PS_LIST_TAIL, "class_id", 0, "==", class_id)) {
                psError(PS_ERR_UNKNOWN, false, "failed to add item class_id");
                psFree(where);
                psFree(query);
                return false;
            }
        }

        // there's not
        psString whereClaus = psDBGenerateWhereSQL(where, NULL);
        psFree(where);
        if (whereClaus) {
            psStringAppend(&query, " %s", whereClaus);
            psFree(whereClaus);
        }
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psError(PS_ERR_UNKNOWN, false, "no pending newImfile rows found");
        psFree(output);
        return false;
    }

    // insert 'newImfile's into rawImfile
    if (psArrayLength(output) > 0) {
        // start a transaction so we don't end up half of the imfiles we were
        // trying to update uninserted
        if (!psDBTransaction(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(output);
            return false;
        }

        for (long i = 0; i < psArrayLength(output); i++) {
            // convert newImfile metadata -> newImfile object
            newImfileRow *object = newImfileObjectFromMetadata(output->data[i]);
            // convert newImfile object -> rawImfile object
            rawImfileRow *imfile = newToRawImfile(config, object); 
            if (!imfile) {
                // rollback
                if (!psDBRollback(config->dbh)) {
                    psError(PS_ERR_UNKNOWN, false, "database error");
                }
                psError(PS_ERR_UNKNOWN, false, "failed to create a new rawImfile row");
                psFree(object);
                psFree(output);
                return false;
            }
            // insert the rawImfile object into the database
            if (!rawImfileInsertObject(config->dbh, imfile)) {
                // rollback
                if (!psDBRollback(config->dbh)) {
                    psError(PS_ERR_UNKNOWN, false, "database error");
                }
                psError(PS_ERR_UNKNOWN, false, "failed to insert row into the database");
                psFree(imfile);
                psFree(object);
                psFree(output);
                return false;
            }
            psFree(imfile);
            // remove the neImfile object from the database
            if (!newImfileDeleteObject(config->dbh, object)) {
                // rollback
                if (!psDBRollback(config->dbh)) {
                    psError(PS_ERR_UNKNOWN, false, "database error");
                }
                psError(PS_ERR_UNKNOWN, false, "failed to delete row from the database");
                psFree(object);
                psFree(output);
                return false;
            }
            psFree(object);
        }

        // point of no return for rawImfile
        if (!psDBCommit(config->dbh)) {
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool processedimfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    bool faulted = psMetadataLookupU64(&status, config->args, "-faulted");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -faulted");
        return false;
    }

    psString query = pxDataGet("regtool_processedimfile.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (config->where) {
        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawImfile");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    if (faulted) {
        // list only faulted rows
        psStringAppend(&query, " %s", "AND rawImfile.fault != 0");
    } else {
        // don't list faulted rows
        psStringAppend(&query, " %s", "AND rawImfile.fault = 0");
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("regtool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            return false;
        }
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "rawImfile", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool updateprocessedimfileMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
        return false;
    }
    if (code == INT16_MAX) {
        psError(PS_ERR_UNKNOWN, true, "-code is required");
        return false;
    }

    if (!pxSetFaultCode(config->dbh, "rawImfile", config->where, code)) {
        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
        return false;
    }

    return true;
}


static bool pendingexpMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    // return only exps that:
    // are not in rawExp 
    // have ALL of their imfiles in rawImfile (by count)
    // and have no associated imfiles left in newImfile
    psString query = pxDataGet("regtool_pendingexp.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
	// XXX PS_EXIT_PROG_ERROR (incorrect SQL) or SYS_ERROR (database comms)
        psError(PXTOOLS_ERR_PROG, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("regtool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    simple = psMetadataLookupBool(&status, config->args, "-simple");
    if (!status) {
	psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -simple");
	return false;
    }

    // negate simple so the default is true
    if (!ippdbPrintMetadatas(stdout, output, "regPendingExp", !simple)) {
        psError(PXTOOLS_ERR_PROG, false, "failed to print array");
        psFree(output);
        return false;
    }

    psFree(output);

    return true;
}


static bool addprocessedexpMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    // make sure that the exp_tag(s) are ready to be updated based on:
    // exp_tag is not in rawExp
    // exp_tag is not in newImfile
    // that the correct count of imfiles is in rawImfile
   
    bool status = false;
    psString exp_tag = psMetadataLookupStr(&status, config->args, "-exp_tag");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_tag");
        return false;
    }
    if (!exp_tag) {
        psError(PS_ERR_UNKNOWN, true, "-exp_tag is required");
        return false;
    }

    bool detrend = psMetadataLookupBool(&status, config->args, "-detrend");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -detrend");
        return false;
    }

    psString query = pxDataGet("regtool_find_unprocessed_exp.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (!p_psDBRunQuery(config->dbh, query, exp_tag)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(query);
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("regtool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    // start a transaction so we don't end up with an exp in both rawExp &
    // newExp
    if (!psDBTransaction(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        psFree(output);
        return false;
    }

    // insert the exp into rawExp
    for (long i = 0; psArrayLength(output) > i; i++) {
        psMetadata *row = output->data[i];
        // convert metadata into a newExp object
        newExpRow *newExp = newExpObjectFromMetadata(row);
        // convert newExp object into a rawExp object
        rawExpRow *rawExp = newToRawExp(config, newExp);
        if (!rawExp) {
            // rollback
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            psError(PS_ERR_UNKNOWN, false, "failed to convert newExp to rawExp");
            psFree(newExp);
            psFree(output);
            return false;
        }

        // insert the rawExp object into the database
        if (!rawExpInsertObject(config->dbh, rawExp)) {
            // rollback
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(rawExp);
            psFree(newExp);
            psFree(output);
            return false;
        }
        psFree(rawExp);

        // delete the newExp object from the database
        if (!newExpDeleteObject(config->dbh, newExp)) {
            // rollback
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            psError(PS_ERR_UNKNOWN, false, "database error");
            psFree(newExp);
            psFree(output);
            return false;
        }

        psFree(newExp);

        // if this is a detrend image don't put it in the chip queue (and we're
        // done)
        if (detrend) {
            continue;
        }

        // insert an entry into the chipPendingExp table
        if (!chipQueueExpTag(config, exp_tag, NULL, NULL, NULL, NULL, NULL)) {
            // rollback
            if (!psDBRollback(config->dbh)) {
                psError(PS_ERR_UNKNOWN, false, "database error");
            }
            psError(PS_ERR_UNKNOWN, false, "failed to queue chipPendingExp");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    if (!psDBCommit(config->dbh)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }

    return true;
}


static bool processedexpMode(pxConfig *config)
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    bool status = false;
    psU64 limit = psMetadataLookupU64(&status, config->args, "-limit");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -limit");
        return false;
    }

    bool faulted = psMetadataLookupU64(&status, config->args, "-faulted");
    if (!status) {
        psError(PXTOOLS_ERR_PROG, false, "failed to lookup value for -faulted");
        return false;
    }

    psString query = pxDataGet("regtool_processedexp.sql");
    if (!query) {
        psError(PXTOOLS_ERR_DATA, false, "failed to retreive SQL statement");
        return false;
    }

    if (config->where) {
        psString whereClause = psDBGenerateWhereConditionSQL(config->where, "rawExp");
        psStringAppend(&query, " AND %s", whereClause);
        psFree(whereClause);
    }

    if (faulted) {
        // list only faulted rows
        psStringAppend(&query, " %s", "AND rawExp.fault != 0");
    } else {
        // don't list faulted rows
        psStringAppend(&query, " %s", "AND rawExp.fault = 0");
    }

    // treat limit == 0 as "no limit"
    if (limit) {
        psString limitString = psDBGenerateLimitSQL(limit);
        psStringAppend(&query, " %s", limitString);
        psFree(limitString);
    }

    if (!p_psDBRunQuery(config->dbh, query)) {
        psError(PS_ERR_UNKNOWN, false, "database error");
        return false;
    }
    psFree(query);

    psArray *output = p_psDBFetchResult(config->dbh);
    if (!output) {
        psErrorCode err = psErrorCodeLast();
        switch (err) {
            case PS_ERR_DB_CLIENT:
                psError(PXTOOLS_ERR_SYS, false, "database error");
            case PS_ERR_DB_SERVER:
                psError(PXTOOLS_ERR_PROG, false, "database error");
            default:
                psError(PXTOOLS_ERR_PROG, false, "unknown error");
        }

        return false;
    }
    if (!psArrayLength(output)) {
        psTrace("regtool", PS_LOG_INFO, "no rows found");
        psFree(output);
        return true;
    }

    bool simple = false;
    {
        bool status = false;
        simple = psMetadataLookupBool(&status, config->args, "-simple");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -simple");
            return false;
        }
    }

    if (psArrayLength(output)) {
        // negative simple so the default is true
        if (!ippdbPrintMetadatas(stdout, output, "rawExp", !simple)) {
            psError(PS_ERR_UNKNOWN, false, "failed to print array");
            psFree(output);
            return false;
        }
    }

    psFree(output);

    return true;
}


static bool updateprocessedexpMode(pxConfig *config)
{
    bool status = false;
    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
        return false;
    }
    if (code == INT16_MAX) {
        psError(PS_ERR_UNKNOWN, true, "-code is required");
        return false;
    }

    if (!pxSetFaultCode(config->dbh, "rawExp", config->where, code)) {
        psError(PS_ERR_UNKNOWN, false, "failed to set set fault flag");
        return false;
    }

    return true;
}


# if 0
static guidePendingExpRow *newToP1PendingExp(newExpRow *newExp)
{
    return guidePendingExpRowAlloc(
        newExp->exp_tag,
        newExp->camera,
        newExp->telescope,
        newExp->exp_type,
        newExp->imfiles,
        "my filter",
        0.1, // airmass
        0.2, // ra
        0.3, // dec
        0.4, // exp time
        0.5, // background
        "my recipe",
        0xff // XXX calc version number
    );
}
#endif


#if 0
static chipPendingExpRow *newTochipPendingExp(pxConfig *config, newExpRow *exp)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(exp, NULL);

    // optional
    bool status = false;
    psString label = psMetadataLookupStr(&status, config->args, "-label");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -label");
        return false;
    }

    chipPendingExpRow *chipExp = chipPendingExpRowAlloc(
            0,              // chip ID, assigned by db
            exp->exp_tag,
            0xdeadbeef,     // guide version
            label,
            "my recipe",    // recipe
            NULL,           // expgroup
            NULL            // dvodb
    );

    return chipExp;
}


static chipPendingImfileRow *rawImfileTochipPendingImfile(pxConfig *config, psS64 chip_id, rawImfileRow *rawImfile)
{
    return chipPendingImfileRowAlloc(
            chip_id,
            rawImfile->class_id,
            rawImfile->uri
    );
}
#endif


static rawExpRow *newToRawExp(pxConfig *config, newExpRow *exp)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(exp, NULL);

    bool status = false;
    // optional
    psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type");
        return false;
    }

    psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel");
        return false;
    }
    if (!filelevel) {
        psError(PS_ERR_UNKNOWN, true, "-filelevel is required");
        return false;
    }

    psString filter = psMetadataLookupStr(&status, config->args, "-filter");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter");
        return false;
    }

    psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass");
        return false;
    }

    psF64 ra = psMetadataLookupF64(&status, config->args, "-ra");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra");
        return false;
    }

    psF64 decl = psMetadataLookupF64(&status, config->args, "-decl");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl");
        return false;
    }

    psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time");
        return false;
    }

    psF32 sat_pixel_frac = psMetadataLookupF32(&status, config->args, "-sat_pixel_frac");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sat_pixel_frac");
        return false;
    }

    psF64 bg = psMetadataLookupF64(&status, config->args, "-bg");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg");
        return false;
    }

    psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev");
        return false;
    }

    psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev");
        return false;
    }

    psF64 alt = psMetadataLookupF64(&status, config->args, "-alt");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -alt");
        return false;
    }

    psF64 az = psMetadataLookupF64(&status, config->args, "-az");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -az");
        return false;
    }

    psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-ccd_temp");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp");
        return false;
    }

    psF64 posang = psMetadataLookupF32(&status, config->args, "-posang");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang");
        return false;
    }

    psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1");
        return false;
    }
    psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2");
        return false;
    }
    psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3");
        return false;
    }
    psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4");
        return false;
    }
    psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5");
        return false;
    }

    psString object = psMetadataLookupStr(&status, config->args, "-object");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -object");
        return false;
    }

    psTime *dateobs = NULL;
    {
        psString dateobsStr = psMetadataLookupStr(&status, config->args, "-dateobs");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -dateobs");
            return false;
        }
        // pass through NULL as this is an optional field
        if (dateobsStr) {
            dateobs = psTimeFromISO(dateobsStr, PS_TIME_UTC);
        } else {
            dateobs = NULL;
        }
    }

    // default
    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
        return false;
    }

    rawExpRow *raw = rawExpRowAlloc(
        exp->exp_tag,
        exp->exp_id,
        exp->camera,
        exp->telescope,
        dateobs,
        exp_type,
        exp->imfiles,
        filelevel,
        exp->workdir,
        filter,
        airmass,
        ra,
        decl,
        exp_time,
        sat_pixel_frac,
        bg,
        bg_stdev,
        bg_mean_stdev,
        alt,
        az,
        ccd_temp,
        posang,
	user_1,
	user_2,
	user_3,
	user_4,
	user_5,
        object,
        0.0,
        code
    );

    if (dateobs) {
        psFree(dateobs);
    }

    return raw;
}


static rawImfileRow *newToRawImfile(pxConfig *config, newImfileRow *imfile)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(exp, NULL);

    bool status = false;
    // optional
    psString exp_type = psMetadataLookupStr(&status, config->args, "-exp_type");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_type");
        return false;
    }

    psString filelevel = psMetadataLookupStr(&status, config->args, "-filelevel");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filelevel");
        return false;
    }

    psString filter = psMetadataLookupStr(&status, config->args, "-filter");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -filter");
        return false;
    }

    psF32 airmass = psMetadataLookupF32(&status, config->args, "-airmass");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -airmass");
        return false;
    }

    psF64 ra = psMetadataLookupF64(&status, config->args, "-ra");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ra");
        return false;
    }

    psF64 decl = psMetadataLookupF64(&status, config->args, "-decl");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -decl");
        return false;
    }

    psF32 exp_time = psMetadataLookupF32(&status, config->args, "-exp_time");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -exp_time");
        return false;
    }

    psF32 sat_pixel_frac = psMetadataLookupF32(&status, config->args, "-sat_pixel_frac");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -sat_pixel_frac");
        return false;
    }

    psF64 bg = psMetadataLookupF64(&status, config->args, "-bg");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg");
        return false;
    }

    psF64 bg_stdev = psMetadataLookupF64(&status, config->args, "-bg_stdev");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_stdev");
        return false;
    }

    psF64 bg_mean_stdev = psMetadataLookupF64(&status, config->args, "-bg_mean_stdev");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -bg_mean_stdev");
        return false;
    }

    psF64 alt = psMetadataLookupF64(&status, config->args, "-alt");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -alt");
        return false;
    }

    psF64 az = psMetadataLookupF64(&status, config->args, "-az");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -az");
        return false;
    }

    psF32 ccd_temp = psMetadataLookupF32(&status, config->args, "-ccd_temp");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -ccd_temp");
        return false;
    }

    psF64 posang = psMetadataLookupF32(&status, config->args, "-posang");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -posang");
        return false;
    }

    psF64 user_1 = psMetadataLookupF64(&status, config->args, "-user_1");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_1");
        return false;
    }
    psF64 user_2 = psMetadataLookupF64(&status, config->args, "-user_2");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_2");
        return false;
    }
    psF64 user_3 = psMetadataLookupF64(&status, config->args, "-user_3");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_3");
        return false;
    }
    psF64 user_4 = psMetadataLookupF64(&status, config->args, "-user_4");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_4");
        return false;
    }
    psF64 user_5 = psMetadataLookupF64(&status, config->args, "-user_5");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -user_5");
        return false;
    }

    psString object = psMetadataLookupStr(&status, config->args, "-object");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -object");
        return false;
    }

    psTime *dateobs = NULL;
    {
        psString dateobsStr = psMetadataLookupStr(&status, config->args, "-dateobs");
        if (!status) {
            psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -dateobs");
            return false;
        }
        // pass through NULL as this is an optional field
        if (dateobsStr) {
            dateobs = psTimeFromISO(dateobsStr, PS_TIME_UTC);
        } else {
            dateobs = NULL;
        }
    }

    psS16 code = psMetadataLookupS16(&status, config->args, "-code");
    if (!status) {
        psError(PS_ERR_UNKNOWN, false, "failed to lookup value for -code");
        return false;
    }

    rawImfileRow *raw = rawImfileRowAlloc(
        imfile->exp_tag,
        imfile->class_id,
        imfile->uri,
        exp_type,
        filelevel,
        filter,
        airmass,
        ra,
        decl,
        exp_time,
        sat_pixel_frac,
        bg,
        bg_stdev,
        bg_mean_stdev,
        alt,
        az,
        ccd_temp,
        posang,
	user_1,
	user_2,
	user_3,
	user_4,
	user_5,
        object,
        dateobs,
        code
    );

    psFree(dateobs);

    return raw;
}
