#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <pslib.h>

#include "pmConfig.h"
#include "pmConcepts.h"
#include "pmHDU.h"
#include "pmFPA.h"
#include "pmFPAMaskWeight.h"
#include "pmFPAConstruct.h"
#include "pmFPAview.h"
#include "pmFPAfile.h"
#include "pmFPAfile.h"
#include "pmFPACopy.h"
#include "pmFPARead.h"
#include "pmFPAWrite.h"
#include "pmPeaks.h"
#include "pmMoments.h"
#include "pmModel.h"
#include "pmSource.h"
#include "pmSourceIO.h"
#include "pmGrowthCurve.h"
#include "pmPSF.h"
#include "pmPSF_IO.h"
#include "pmFPA_JPEG.h"
#include "pmDetrendDB.h"
#include "pmConceptsPhotcode.h"

// define an input-type pmFPAfile, bind to the optional fpa if supplied
pmFPAfile *pmFPAfileDefineInput(pmConfig *config, pmFPA *fpa, const char *name)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    bool status;
    char *type;

    // select the FILERULES from config->camera
    psMetadata *filerules = psMetadataLookupPtr (&status, config->camera, "FILERULES");
    if (filerules == NULL) {
        psError(PS_ERR_IO, true, "Can't find FILERULES in the CAMERA configuration!");
        return NULL;
    }

    // select the name from the FILERULES
    // check for alias name (type == STR, name is aliased name)
    const char *realname = psMetadataLookupStr (&status, filerules, name);
    if (!realname || strlen(realname) == 0) {
        realname = name;
    }

    psMetadata *data = psMetadataLookupPtr (&status, filerules, realname);
    if (data == NULL) {
        psError(PS_ERR_IO, true, "Can't find file concept %s!", name);
        return NULL;
    }

    pmFPAfile *file = pmFPAfileAlloc ();

    // save the name of this pmFPAfile
    file->name = psStringCopy (name);

    file->filerule = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "FILENAME.RULE"));
    file->filextra = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "FILENAME.XTRA"));
    file->extrule  = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "EXTNAME.RULE"));
    file->extxtra  = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "EXTNAME.XTRA"));

    type = psMetadataLookupStr (&status, data, "FILE.TYPE");
    file->type = pmFPAfileTypeFromString(type);
    if (file->type == PM_FPA_FILE_NONE) {
        psError(PS_ERR_IO, true, "FILE.TYPE is not defined for %s\n", name);
        return NULL;
    }

    file->mode = PM_FPA_MODE_READ;
    file->fileLevel = PM_FPA_LEVEL_NONE; // the fileLevel depends on the input data

    file->dataLevel = pmFPALevelFromName(psMetadataLookupStr (&status, data, "DATA.LEVEL"));
    if (file->dataLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "DATA.LEVEL is not set for %s\n", name);
        return NULL;
    }
    // default is to free the data after use (after written out)
    // this can be overridden for pmFPAfiles used as carriers as well
    file->freeLevel = file->dataLevel;

    if (fpa != NULL) {
        file->fpa = psMemIncrRefCounter(fpa);
        file->camera = psMemIncrRefCounter((psMetadata *)fpa->camera);
    }

    // XXX ppImage and similar require the added file to be unique
    // XXX ppFocus wants to override the selection with the new selection
    // XXX require programs like ppFocus to remove existing files by hand
    if (!psMetadataAddPtr (config->files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN, "", file)) {
        psError (PS_ERR_IO, false, "could not add %s to config files", name);
        return NULL;
    }
    psFree (file);
    return (file);
}

// Check the various levels for file operations to make sure that they are suitable based on the format
static void adjustLevels(pmFPAfile *file, // File for which to adjust levels
                         const psMetadata *format // Format to use to adjust levels
                        )
{
    assert(file);
    assert(format);


    return;
}

// define a pmFPAfile, bind to the optional fpa if supplied
pmFPAfile *pmFPAfileDefineOutput(pmConfig *config, pmFPA *fpa, const char *name)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    bool status;

    // select the FILERULES from the camera config
    psMetadata *filerules = psMetadataLookupPtr (&status, config->camera, "FILERULES");
    if (filerules == NULL) {
        psError(PS_ERR_IO, true, "Can't find FILERULES in the CAMERA configuration!");
        return NULL;
    }

    // select the name from the FILERULES
    // check for alias name (type == STR, name is aliased name)
    const char *realname = psMetadataLookupStr (&status, filerules, name);
    if (!realname || strlen(realname) == 0) {
        realname = name;
    }

    psMetadata *data = psMetadataLookupPtr (&status, filerules, realname);
    if (data == NULL) {
        psError(PS_ERR_IO, true, "Can't find file concept %s!", name);
        return NULL;
    }

    pmFPAfile *file = pmFPAfileAlloc ();

    // save the name of this pmFPAfile
    file->name = psStringCopy (name);

    file->filerule = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "FILENAME.RULE"));
    file->filextra = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "FILENAME.XTRA"));
    file->extrule  = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "EXTNAME.RULE"));
    file->extxtra  = psMemIncrRefCounter(psMetadataLookupStr (&status, data, "EXTNAME.XTRA"));

    const char *type = psMetadataLookupStr (&status, data, "FILE.TYPE");
    file->type = pmFPAfileTypeFromString(type);
    if (file->type == PM_FPA_FILE_NONE) {
        psError(PS_ERR_IO, true, "FILE.TYPE is not defined for %s\n", name);
        return NULL;
    }

    file->mode = PM_FPA_MODE_WRITE;
    file->save = false;
    char *save = psMetadataLookupStr (&status, data, "FILE.SAVE");
    if (save != NULL) {
        if (!strcasecmp (save, "TRUE"))     {
            file->save = true;
        } else if (!strcasecmp (save, "FALSE"))     {
            file->save = false;
        } else {
            psError(PS_ERR_IO, true, "Illegal value \"%s\" for FILE.SAVE for %s", save, name);
            return NULL;
        }
    }

    file->camera = psMemIncrRefCounter(config->camera);
    const char *formatName = psMetadataLookupStr (&status, data, "FILE.FORMAT");
    psMetadata *format = NULL;          // Camera format to use
    if (formatName && strcasecmp(formatName, "NONE") != 0) {
        // select the format list from the selected camera
        psMetadata *formats = psMetadataLookupMetadata(&status, file->camera, "FORMATS"); // List of formats

        // select the desired format
        format = psMetadataLookupMetadata(&status, formats, formatName);
        if (!format) {
            psError(PS_ERR_IO, false, "format %s for %s not defined", formatName, name);
            psFree(file);
            return NULL;
        }
        file->format = psMemIncrRefCounter(format);
        #if 0

        file->src = psMemIncrRefCounter(fpa);
        file->fpa = pmFPAConstruct(file->camera);
        #endif

    } else {
        format = config->format;
        #if 0

        file->fpa = psMemIncrRefCounter(fpa);
        #endif

    }
    #if 1
    file->fpa = psMemIncrRefCounter(fpa);
    #endif


    file->fileLevel = pmFPAPHULevel(format);
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "Unable to determine file level for %s\n", name);
        return NULL;
    }

    file->dataLevel = pmFPALevelFromName(psMetadataLookupStr(&status, data, "DATA.LEVEL"));
    if (file->dataLevel == PM_FPA_LEVEL_NONE) {
        psError(PS_ERR_IO, true, "DATA.LEVEL is not set for %s\n", name);
        return NULL;
    }
    // default is to free the data after use (after written out)
    // this can be overridden for pmFPAfiles used as carriers as well
    file->freeLevel = file->dataLevel;

    pmFPALevel extLevel = pmFPAExtensionsLevel(format); // Level for extensions
    if (extLevel != PM_FPA_LEVEL_NONE) {
        if (extLevel < file->dataLevel) {
            psWarning("Level for extensions is higher than desired data level --- adjusting.\n");
            file->dataLevel = extLevel;
        }
        if (extLevel < file->freeLevel) {
            psWarning("Level for extensions is higher than desired free level --- adjusting.\n");
            file->freeLevel = extLevel;
        }
    }
    adjustLevels(file, format);


    // add argument-supplied OUTPUT name to this file
    char *outname = psMetadataLookupStr(&status, config->arguments, "OUTPUT");
    psMetadataAddStr(file->names, PS_LIST_TAIL, "OUTPUT", PS_META_NO_REPLACE, "", outname);

    // place the resulting file in the config system
    psMetadataAddPtr (config->files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN, "", file);
    psFree (file); // we free this copy of file, but 'files' still has a copy
    return (file); // the returned value is a view into the version on 'files'
}


// search for argname on the config->argument list
// construct an FPA based on the files in this list (must represent a single FPA)
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filename and bind it to this FPA
// save the pmFPAfile on config->files
// return the pmFPAfile (a view to the one saved on config->files)
pmFPAfile *pmFPAfileDefineFromArgs (bool *found, pmConfig *config, const char *filename, const char *argname)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(argname, NULL);

    bool status;
    pmFPA *fpa = NULL;
    psFits *fits = NULL;
    pmFPAfile *file = NULL;
    psMetadata *phu = NULL;
    psMetadata *format = NULL;

    if (found && *found)
        return NULL;

    // we search the argument data for the named fileset (argname)
    psArray *infiles = psMetadataLookupPtr(&status, config->arguments, argname);
    if (!status) {
        return NULL;
    }
    if (infiles->n < 1) {
        psError(PS_ERR_IO, false, "Found n == %ld files in %s in arguments\n", infiles->n, argname);
        return NULL;
    }

    // this function is implicitly an INPUT operation: do not create the file
    psString realName = pmConfigConvertFilename (infiles->data[0], config, false);
    if (!realName) {
        psError(PS_ERR_IO, false, "Failed to convert file name %s\n", (char *) infiles->data[0]);
        return NULL;
    }

    // load the header of the first image
    fits = psFitsOpen (realName, "r");
    if (!fits) {
        psError(PS_ERR_IO, false, "Failed to open file %s\n", realName);
        psFree (realName);
        return NULL;
    }
    phu = psFitsReadHeader (NULL, fits);
    if (!phu) {
        psError(PS_ERR_IO, false, "Failed to read file header %s\n", realName);
        psFree (realName);
        return NULL;
    }
    psFitsClose(fits);

    // determine the current format from the header
    // determine camera if not specified already
    format = pmConfigCameraFormatFromHeader (config, phu);
    if (!format) {
        psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", realName);
        psFree(phu);
        psFree (realName);
        return NULL;
    }

    // build the template fpa, set up the basic view
    fpa = pmFPAConstruct (config->camera);
    if (!fpa) {
        psError(PS_ERR_IO, false, "Failed to construct FPA from %s", realName);
        psFree (realName);
        return NULL;
    }
    psFree (realName);

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    file = pmFPAfileDefineInput (config, fpa, filename);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }
    psFree (format);
    file->format = psMemIncrRefCounter(format);

    // adjust the rules to identify these files in the file->names data
    psFree (file->filerule);
    psFree (file->filextra);
    file->filerule = psStringCopy ("@FILES");
    file->filextra = psStringCopy ("{CHIP.NAME}.{CELL.NAME}");

    // find the matching fileLevel
    psMetadata *filemenu = psMetadataLookupPtr (&status, format, "FILE");
    if (!filemenu) {
        psError (PS_ERR_IO, true, "missing FILE in FORMAT");
        return NULL;
    }
    char *levelName = psMetadataLookupStr (&status, filemenu, "PHU");
    if (!levelName) {
        psError (PS_ERR_IO, true, "missing PHU in FILE in FORMAT");
        return NULL;
    }
    file->fileLevel = pmFPALevelFromName (levelName);
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError (PS_ERR_IO, true, "unknown level");
        return NULL;
    }

    // examine the list of input files and validate their cameras
    // associated each filename with an element of the FPA
    // save the association on file->names
    for (int i = 0; i < infiles->n; i++) {
        if (i > 0) {
            // this function is implicitly an INPUT operation: do not create the file
            psString realName = pmConfigConvertFilename (infiles->data[i], config, false);
            if (!realName) {
                psError(PS_ERR_IO, false, "Failed to convert file name %s", (char *) infiles->data[i]);
                return NULL;
            }
            fits = psFitsOpen (realName, "r");
            if (!fits) {
                psError(PS_ERR_IO, false, "Failed to open file %s\n", realName);
                psFree (realName);
                return NULL;
            }
            phu = psFitsReadHeader (NULL, fits);
            if (!phu) {
                psError(PS_ERR_IO, false, "Failed to read file header %s", realName);
                psFree (realName);
                psFitsClose (fits);
                return NULL;
            }
            bool valid = false;
            if (!pmConfigValidateCameraFormat (&valid, format, phu)) {
                psError (PS_ERR_UNKNOWN, false, "Error in config scripts\n");
                psFree (realName);
                psFitsClose (fits);
                return NULL;
            }
            if (!valid) {
                psError(PS_ERR_IO, false, "file %s is not from the required camera", realName);
                psFree (realName);
                psFitsClose (fits);
                return NULL;
            }
            psFree(realName);
            psFitsClose (fits);
        }

        // set the view to the corresponding entry for this phu
        pmFPAview *view = pmFPAAddSourceFromHeader (fpa, phu, format);
        if (!view) {
            psError(PS_ERR_IO, false, "Unable to determine source for %s", file->name);
            psFree(phu);
            psFree (fpa);
            psFree (format);
            return NULL;
        }

        // associate the filename with the FPA element
        char *name = pmFPAfileNameFromRule (file->filextra, file, view);

        // save the name association in the pmFPAfile structure
        psMetadataAddStr (file->names, PS_LIST_TAIL, name, 0, "", infiles->data[i]);

        psFree (view);
        psFree (name);
        psFree (phu);
    }
    psFree (fpa);
    if (found)
        *found = true;
    return file;
}

// search for argname on the config->argument list
// construct an FPA based on the files in this list (must represent a single FPA)
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filename and bind it to this FPA
// save the pmFPAfile on config->files
// return the pmFPAfile (a view to the one saved on config->files)
pmFPAfile *pmFPAfileBindFromArgs (bool *found, pmFPAfile *input, pmConfig *config, const char *filename, const char *argname)
{
    PS_ASSERT_PTR_NON_NULL(input, NULL);
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(argname, NULL);

    bool status;
    psFits *fits = NULL;
    pmFPAfile *file = NULL;
    psMetadata *phu = NULL;

    if (found && *found)
        return NULL;

    // we search the argument data for the named fileset (argname)
    psArray *infiles = psMetadataLookupPtr(&status, config->arguments, argname);
    if (!status) {
        // this is not an error: this just means no matching argument was supplied
        return NULL;
    }
    if (infiles->n < 1) {
        psError(PS_ERR_IO, false, "Found n == %ld files in %s in arguments\n", infiles->n, argname);
        return NULL;
    }

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    file = pmFPAfileDefineInput (config, input->fpa, filename);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        psFree(phu);
        return NULL;
    }
    // XXX temporary hack : force PSF input onto filename
    // XXX EAM : I need to define a PSF container which specifies the chip/cell element
    // XXX we could use the flat metadata style IF we restrict it to SIMPLE cameras
    // XXX (ie, one readout per cell, one cell per chip, one chip per fpa)
    if (file->type == PM_FPA_FILE_PSF) {
        file->filerule = psStringCopy (infiles->data[0]);
        return file;
    }

    // set derived values
    file->fileLevel = input->fileLevel;
    psFree (file->format);
    file->format = psMemIncrRefCounter(input->format);

    // define the rule to identify these files in the file->names data
    psFree (file->filerule);
    psFree (file->filextra);
    file->filerule = psStringCopy ("@FILES");
    file->filextra = psStringCopy ("{CHIP.NAME}.{CELL.NAME}");

    // examine the list of input files and validate their cameras
    // associated each filename with an element of the FPA
    // save the association on file->names
    for (int i = 0; i < infiles->n; i++) {
        // this function is implicitly an INPUT operation: do not create the file
        psString realName = pmConfigConvertFilename (infiles->data[i], config, false);
        if (!realName) {
            psError(PS_ERR_IO, false, "Failed to convert file name %s", (char *) infiles->data[i]);
            return NULL;
        }
        fits = psFitsOpen (realName, "r");
        if (!fits) {
            psError(PS_ERR_IO, false, "Failed to open file %s\n", realName);
            psFree (realName);
            return NULL;
        }
        phu = psFitsReadHeader (NULL, fits);
        if (!phu) {
            psError(PS_ERR_IO, false, "Failed to read file header %s", realName);
            psFree (realName);
            psFitsClose (fits);
            return NULL;
        }
        bool valid = false;
        if (!pmConfigValidateCameraFormat (&valid, input->format, phu)) {
            psError (PS_ERR_UNKNOWN, false, "Error in config scripts\n");
            psFree (realName);
            psFitsClose (fits);
            return NULL;
        }
        if (!valid) {
            psError(PS_ERR_IO, false, "specified data file %s does not match format of supplied INPUT\n", realName);
            psFree (realName);
            psFitsClose (fits);
            return NULL;
        }
        psFree(realName);
        psFitsClose (fits);

        // set the view to the corresponding entry for this phu
        pmFPAview *view = pmFPAIdentifySourceFromHeader (input->fpa, phu, input->format);
        if (!view) {
            psError(PS_ERR_IO, false, "Unable to determine source for %s", file->name);
            psFree(phu);
            return NULL;
        }

        // associate the filename with the FPA element
        char *name = pmFPAfileNameFromRule (file->filextra, file, view);

        // save the name association in the pmFPAfile structure
        psMetadataAddStr (file->names, PS_LIST_TAIL, name, 0, "", infiles->data[i]);

        psFree (view);
        psFree (name);
        psFree (phu);
    }
    if (found)
        *found = true;
    return file;
}

// search for argname on the config->argument list
// construct an FPA based on the files in this list (each represents the same FPA)
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filenames and bind them to the FPAs
// save the pmFPAfiles on config->files
// return the pmFPAfiles (a view to the one saved on config->files)
pmFPAfile *pmFPAfileDefineSingleFromArgs (bool *found, pmConfig *config, const char *filename,
        const char *argname, int entry)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(argname, NULL);

    bool status;
    pmFPA *fpa = NULL;
    psFits *fits = NULL;
    pmFPAfile *file = NULL;
    psMetadata *phu = NULL;
    psMetadata *format = NULL;

    if (found && *found)
        return NULL;

    // we search the argument data for the named fileset (argname)
    psArray *infiles = psMetadataLookupPtr(&status, config->arguments, argname);
    if (!status) {
        psTrace("psModules.camera", 5, "Failed to find %s in argument list", argname);
        return NULL;
    }
    if (infiles->n <= entry) {
        psError(PS_ERR_IO, false, "only %ld files in %s in argument, entry %d requested\n", infiles->n, argname, entry);
        return NULL;
    }

    // examine the list of input files and validate their cameras
    // associated each filename with an element of the FPA
    // save the association on file->names
    fits = psFitsOpen (infiles->data[entry], "r");
    phu = psFitsReadHeader (NULL, fits);
    psFitsClose (fits);

    // on first call to this function, config->camera is not set.
    // later calls will give an error if the cameras do not match
    format = pmConfigCameraFormatFromHeader (config, phu);
    if (!format) {
        psError(PS_ERR_IO, false, "Failed to read CCD format from %s\n", (char *)infiles->data[0]);
        psFree(phu);
        return NULL;
    }

    // build the template fpa, set up the basic view
    fpa = pmFPAConstruct (config->camera);
    if (!fpa) {
        psError(PS_ERR_IO, false, "Failed to construct FPA from %s", (char *)infiles->data[0]);
        return NULL;
    }

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    // we need a variable name here... (but in filerule)
    file = pmFPAfileDefineInput (config, fpa, filename);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }
    psFree (file->format);
    file->format = format;

    // adjust the rules to identify these files in the file->names data
    psFree (file->filerule);
    psFree (file->filextra);
    file->filerule = psStringCopy ("@FILES");
    file->filextra = psStringCopy ("{CHIP.NAME}.{CELL.NAME}");

    // find the matching fileLevel
    psMetadata *filemenu = psMetadataLookupPtr (&status, format, "FILE");
    if (!filemenu) {
        psError (PS_ERR_IO, true, "missing FILE in FORMAT");
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }
    char *levelName = psMetadataLookupStr (&status, filemenu, "PHU");
    if (!levelName) {
        psError (PS_ERR_IO, true, "missing PHU in FILE in FORMAT");
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }
    file->fileLevel = pmFPALevelFromName (levelName);
    if (file->fileLevel == PM_FPA_LEVEL_NONE) {
        psError (PS_ERR_IO, true, "unknown level");
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }

    // set the view to the corresponding entry for this phu
    pmFPAview *view = pmFPAAddSourceFromHeader (fpa, phu, format);
    if (!view) {
        psError(PS_ERR_IO, false, "Unable to determine source for %s", file->name);
        psFree(phu);
        psFree(fpa);
        psFree(format);
        return NULL;
    }

    // associate the filename with the FPA element
    char *name = pmFPAfileNameFromRule (file->filextra, file, view);

    // save the name association in the pmFPAfile structure
    psMetadataAddStr (file->names, PS_LIST_TAIL, name, 0, "", infiles->data[entry]);

    psFree (view);
    psFree (name);
    psFree (phu);
    psFree (fpa);

    if (found)
        *found = true;
    return file;
}

// define the named pmFPAfile from the camera->config
// only valid for pmFPAfile->mode = READ
pmFPAfile *pmFPAfileDefineFromConf (bool *found, pmConfig *config, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);

    if (found && *found)
        return NULL;

    // a camera config is needed (as source of file rule)
    if (config->camera == NULL) {
        psError(PS_ERR_IO, true, "camera is not defined");
        return NULL;
    }

    // build the template fpa, set up the basic view
    pmFPA *fpa = pmFPAConstruct (config->camera);
    if (!fpa) {
        psError(PS_ERR_IO, false, "Failed to construct FPA for %s", filename);
        return NULL;
    }

    // load the given filerule (from config->camera) and bind it to the fpa
    // the returned file is just a view to the entry on config->files
    pmFPAfile *file = pmFPAfileDefineInput (config, fpa, filename);
    psFree (fpa);
    if (!file) {
        psError(PS_ERR_IO, false, "file %s not defined\n", filename);
        return NULL;
    }

    // image names may not come from file->names
    if (!strcasecmp (file->filerule, "@FILES")) {
        psError(PS_ERR_IO, true, "supplied filerule uses illegal value @FILES");
        // XXX remove the file from config->files
        psFree(file);
        return NULL;
    }

    // image names may come from the detrend database
    if (!strcasecmp (file->filerule, "@DETDB")) {
        psTrace ("pmFPAfile", 5, "requiring use of detrend database source\n");
        // don't free the file here: it is left on config->files
        // to be used optionally by pmFPAfileDefineFromDetDB (or others)
        return NULL;
    }

    // Prepend the global path to the file rule
    // this function is implicitly an INPUT operation: do not create the file
    psString tmpName = pmConfigConvertFilename (file->filerule, config, false);
    psFree (file->filerule);
    file->filerule = tmpName;

    if (found)
        *found = true;
    return file;
}

// construct an FPA based on the supplied config->camera
// built the association between the FPA elements (CHIP/CELL) and the files
// define the pmFPAfile filename and bind it to this FPA
// save the pmFPAfile on config->files
// return the pmFPAfile (a view to the one saved on config->files)
pmFPAfile *pmFPAfileDefineFromDetDB (bool *found, pmConfig *config, const char *filename,
                                     pmFPA *input, pmDetrendType type)
{
    PS_ASSERT_PTR_NON_NULL(input, NULL);
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->camera, NULL);
    PS_ASSERT_PTR_NON_NULL(config->files, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);

    bool status;
    pmFPA *fpa = NULL;
    pmFPAfile *file = NULL;

    if (found && *found)
        return NULL;

    // a camera config is needed (as source of file rule)
    if (config->camera == NULL) {
        psError(PS_ERR_IO, true, "camera is not defined");
        return NULL;
    }
    // a camera config is needed (as source of file rule)
    if (config->cameraName == NULL) {
        psAbort ("pmFPAfileDefine", "camera defined but not cameraName!");
    }

    // find or define a pmFPAfile with this name
    file = psMetadataLookupPtr (NULL, config->files, filename);
    if (!file) {
        // build the template fpa, set up the basic view
        fpa = pmFPAConstruct (config->camera);
        if (!fpa) {
            psError(PS_ERR_IO, false, "Failed to construct FPA for %s", filename);
            return NULL;
        }
        // load the given filerule (from config->camera) and bind it to the fpa
        // the returned file is just a view to the entry on config->files
        file = pmFPAfileDefineInput (config, fpa, filename);
        if (!file) {
            psError(PS_ERR_IO, false, "file %s not defined\n", filename);
            psFree (fpa);
            return NULL;
        }
    }

    // we are constructing a detselect command of the form:
    //   detselect -search -inst (camera) -type (type) -time (time) [others]
    // camera, type, and time are derived from pmFPA *input, other options are
    // added if specified for the particular detrend type by the DETREND.CONSTRAINTS
    // note that the filter-dependent choices are set for ppImage in ppImageParseCamera

    // Get the time from FPA.TIME
    psTime *time = psMetadataLookupPtr(NULL, input->concepts, "FPA.TIME");
    if (time->sec == 0 && time->nsec == 0) {
        psLogMsg ("psModules.camera", PS_LOG_WARN, "FPA.TIME has not been set.\n");
    }
    pmDetrendSelectOptions *options = pmDetrendSelectOptionsAlloc(config->cameraName, *time, type);

    // add additional constraints based on the type defined in the PPIMAGE recipe
    // use PPIMAGE or DETREND or ???
    psMetadata *recipe  = psMetadataLookupPtr (&status, config->recipes, "PPIMAGE");
    if (!status)
        psAbort ("psModules", "PPIMAGE recipe not found");
    psMetadata *detConstraints = psMetadataLookupPtr (&status, recipe, "DETREND.CONSTRAINTS");
    if (!status)
        psAbort ("psModules", "DETREND.CONSTRAINTS not found");
    // XXX allow this and just skip?

    psString typeName = pmDetrendTypeToString (type);
    psMetadata *constraints = psMetadataLookupPtr (&status, detConstraints, typeName);
    if (!status) {
        psAbort ("psModules", "DETREND.CONSTRAINTS for type %s not found", typeName);
    }
    psFree(typeName);

    // loop over the constraints and include in the detselect options
    psMetadataIterator *iter = psMetadataIteratorAlloc (constraints, PS_LIST_HEAD, NULL);
    psMetadataItem *item = NULL;
    while ((item = psMetadataGetAndIncrement (iter)) != NULL) {
        if (item->type != PS_DATA_STRING) {
            psAbort ("psModules", "invalid type for DETREND.CONSTRAINT element");
        }
        char *option  = item->name;     // item->name must correspond to a valid detselect option
        char *concept = item->data.V;

        if (!strcasecmp (option, "filter")) {
            options->filter = psMetadataLookupPtr (&status, input->concepts, concept);
            if (!status)
                psAbort ("psModules", "failed to find filter (concept %s)", concept);

        }
        if (!strcasecmp (option, "exptime")) {
            options->exptime = psMetadataLookupF32 (&status, input->concepts, concept);
            if (!status)
                psAbort ("psModules", "exptime not found (concept %s)", concept);
        }
        if (!strcasecmp (option, "airmass")) {
            options->airmass = psMetadataLookupF32 (&status, input->concepts, concept);
            if (!status)
                psAbort ("psModules", "airmass not found (concept %s)", concept);
        }
        # if (0)
            if (!strcasecmp (option, "dettemp")) {
                options->dettemp = psMetadataLookupF32 (&status, input->concepts, concept);
                if (!status)
                    psAbort ("psModules", "dettemp not found (concept %s)", concept);
            }
        if (!strcasecmp (option, "version")) {
            // version is applied as a literal string
            options->version = psMetadataLookupF32 (&status, input->concepts, concept);
            if (!status)
                psAbort ("psModules", "version not found (concept %s)", concept);
        }
        if (!strcasecmp (option, "twilight")) {
            // XXX determine the twilight time based on concept defining the time-of-day
            // XXX need to include twilight time somehow (uses lookup based on a time value)
            options->twilight = psMetadataLookupF32 (&status, input->concepts, concept);
            if (!status)
                psAbort ("psModules", "twilight not found (concept %s)", concept);
        }
        # endif

    }
    psFree(iter);

    // search for existing detrend data (detID)
    pmDetrendSelectResults *results = pmDetrendSelect (options, config);
    if (!results) {
        psError (PS_ERR_IO, false, "no matching detrend data");
        return NULL;
    }

    // replace the existing value of filextra with the selected detID
    psFree (file->filextra);
    file->filextra  = psStringCopy (results->detID);

    // set the fileLevel based on the detDB results
    // XXX this is not supplied by the db.  use the config value for now.
    // file->fileLevel = results->level;

    psFree (results);
    psFree (options);

    if (found)
        *found = true;
    return file;
}

// create a new output pmFPAfile based on an existing FPA
// only valid for pmFPAfile->mode == WRITE (or internal?)
pmFPAfile *pmFPAfileDefineFromFPA (pmConfig *config, pmFPA *src, int xBin, int yBin, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(src, false);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);

    // Need to look up the format
    bool mdok;                          // Status of MD lookup
    psMetadata *formats = psMetadataLookupMetadata(&mdok, src->camera, "FORMATS"); // The FORMATS
    if (!mdok || !formats) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find FORMATS in camera configuration %s.\n",
                config->cameraName);
        return NULL;
    }
    psMetadata *format = psMetadataLookupMetadata(&mdok, formats, config->formatName); // The format
    if (!mdok || !format) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find camera format %s within camera %s.\n",
                config->formatName, config->cameraName);
        return NULL;
    }

    pmFPA *fpa = pmFPAConstruct(src->camera);
    pmFPAfile *file = pmFPAfileDefineOutput (config, fpa, filename);
    if (!file) {
        psErrorStackPrint(stderr, "file %s not defined\n", filename);
        return NULL;
    }
    file->src = src; // inherit output elements from this source pmFPA
    file->xBin = xBin;
    file->yBin = yBin;
    psFree (file->format);
    file->format = psMemIncrRefCounter(format);

    psFree (fpa);
    return file;
}

// create a new output pmFPAfile based on an existing FPA
// only valid for pmFPAfile->mode == WRITE (or internal?)
pmFPAfile *pmFPAfileDefineNewCamera (pmConfig *config, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);

    pmFPAfile *file = pmFPAfileDefineOutput (config, NULL, filename);
    if (!file) {
        psErrorStackPrint(stderr, "file %s not defined\n", filename);
        return NULL;
    }
    if (!file->camera) {
        psErrorStackPrint(stderr, "file %s does not define a new camera\n", filename);
        return NULL;
    }
    file->fpa = pmFPAConstruct(file->camera);

    return file;
}

pmFPAfile *pmFPAfileDefineChipMosaic(pmConfig *config, pmFPA *src, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(src, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(config->cameraName, NULL);
    PS_ASSERT_STRING_NON_EMPTY(config->formatName, NULL);

    if (config->cameraName[0] == '_' &&
            strcmp(config->cameraName + strlen(config->cameraName) - 5, "-CHIP") == 0) {
        // The input camera has already been mosaicked to this level
        pmFPAfile *file = pmFPAfileDefineOutput(config, NULL, filename);
        psFree (file->camera);
        psFree (file->format);
        psFree (file->fpa);
        file->camera = psMemIncrRefCounter(config->camera);
        file->format = psMemIncrRefCounter(config->format);
        file->fpa = pmFPAConstruct(file->camera);
        return file;
    }

    // Find the correct camera configuration
    bool mdok;                          // Status of MD lookup
    psMetadata *cameras = psMetadataLookupMetadata(&mdok, config->site, "CAMERAS"); // Camera configurations
    if (!mdok || !cameras) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find CAMERAS in the site configuration.\n");
        return NULL;
    }
    psString name = NULL;               // Name of the new (automatically-generated) camera configuration
    psStringAppend(&name, "_%s-CHIP", config->cameraName);
    psMetadata *camera = psMetadataLookupMetadata(&mdok, cameras, name); // Camera configuration of interest
    if (!mdok || !camera) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find automatically generated "
                "camera configuration %s in site configuration.\n", name);
        psFree(name);
        return NULL;
    }
    psFree(name);

    // Need to look up the format of the same name, but under the mosaic camera
    psMetadata *formats = psMetadataLookupMetadata(&mdok, camera, "FORMATS"); // The FORMATS
    if (!mdok || !formats) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find FORMATS in camera configuration %s.\n",
                config->cameraName);
        return NULL;
    }
    psMetadata *format = psMetadataLookupMetadata(&mdok, formats, config->formatName); // The format
    if (!mdok || !format) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find camera format %s within camera %s.\n",
                config->formatName, config->cameraName);
        return NULL;
    }

    pmFPA *fpa = pmFPAConstruct(camera);
    if (!fpa) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate mosaicked camera.\n");
        return NULL;
    }
    pmFPAfile *file = pmFPAfileDefineOutput(config, fpa, filename);
    if (!file) {
        psErrorStackPrint(stderr, "file %s not defined\n", filename);
        return NULL;
    }
    file->src = src; // inherit output elements from this source pmFPA
    file->mosaicLevel = PM_FPA_LEVEL_CHIP; // don't do any I/O on this at a lower level

    psFree (file->format);
    file->format = psMemIncrRefCounter(format);

    psFree(fpa);

    return file;
}

pmFPAfile *pmFPAfileDefineFPAMosaic(pmConfig *config, pmFPA *src, const char *filename)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(src, NULL);
    PS_ASSERT_STRING_NON_EMPTY(filename, NULL);
    PS_ASSERT_STRING_NON_EMPTY(config->cameraName, NULL);

    if (config->cameraName[0] == '_' &&
            strcmp(config->cameraName + strlen(config->cameraName) - 4 , "-FPA") == 0) {
        // The input camera has already been mosaicked to this level
        pmFPAfile *file = pmFPAfileDefineOutput(config, NULL, filename);
        psFree (file->camera);
        psFree (file->format);
        psFree (file->fpa);
        file->camera = psMemIncrRefCounter(config->camera);
        file->format = psMemIncrRefCounter(config->format);
        file->fpa = pmFPAConstruct(file->camera);
        return file;
    }

    // Find the correct camera configuration
    bool mdok;                          // Status of MD lookup
    psMetadata *cameras = psMetadataLookupMetadata(&mdok, config->site, "CAMERAS"); // Camera configurations
    if (!mdok || !cameras) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find CAMERAS in the site configuration.\n");
        return NULL;
    }
    psString original;                  // Name of the original camera configuration
    psString name = NULL;               // Name of the new (automatically-generated) camera configuration
    if (config->cameraName[0] == '_' &&
            strcmp(config->cameraName + strlen(config->cameraName) - 5 , "-CHIP") == 0) {
        // It's a chip mosaic; we need to get the original name
        original = psStringNCopy(config->cameraName + 1, strlen(config->cameraName) - 6);
    } else {
        original = psMemIncrRefCounter(config->cameraName);
    }
    psStringAppend(&name, "_%s-FPA", original);
    psFree(original);
    psMetadata *camera = psMetadataLookupMetadata(&mdok, cameras, name); // Camera configuration of interest
    if (!mdok || !camera) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find automatically generated "
                "camera configuration %s in site configuration.\n", name);
        psFree(name);
        return NULL;
    }
    psFree(name);

    // Need to look up the format of the same name, but under the mosaic camera
    psMetadata *formats = psMetadataLookupMetadata(&mdok, camera, "FORMATS"); // The FORMATS
    if (!mdok || !formats) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find FORMATS in camera configuration %s.\n",
                config->cameraName);
        return NULL;
    }
    psMetadata *format = psMetadataLookupMetadata(&mdok, formats, config->formatName); // The format
    if (!mdok || !format) {
        psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find camera format %s within camera %s.\n",
                config->formatName, config->cameraName);
        return NULL;
    }

    pmFPA *fpa = pmFPAConstruct(camera);
    if (!fpa) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to generate mosaicked camera.\n");
        return NULL;
    }
    pmFPAfile *file = pmFPAfileDefineOutput(config, fpa, filename);
    if (!file) {
        psErrorStackPrint(stderr, "file %s not defined\n", filename);
        return NULL;
    }
    file->src = src; // inherit output elements from this source pmFPA
    file->mosaicLevel = PM_FPA_LEVEL_FPA; // don't do any I/O on this at a lower level
    psFree (file->format);
    file->format = psMemIncrRefCounter(format);

    psFree(fpa);

    return file;
}

// create a file with the given name, assign it type "INTERNAL", and supply it with an image
// of the requested dimensions. (image only, mask and weight are ignored)
pmReadout *pmFPAfileDefineInternal (psMetadata *files, const char *name, int Nx, int Ny, int type)
{
    PS_ASSERT_PTR_NON_NULL(files, false);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    pmReadout *readout = pmReadoutAlloc(NULL);
    readout->image = psImageAlloc(Nx, Ny, type);

    // I want an image from the
    pmFPAfile *file = pmFPAfileAlloc();
    file->mode = PM_FPA_MODE_INTERNAL;
    file->name = psStringCopy (name);

    file->readout = readout;
    psMetadataAddPtr(files, PS_LIST_TAIL, name, PS_DATA_UNKNOWN, "", file);
    psFree(file);
    // we free this copy of file, but 'files' still has a copy

    return readout;
}

bool pmFPAfileDropInternal(psMetadata *files, const char *name)
{
    PS_ASSERT_PTR_NON_NULL(files, false);
    PS_ASSERT_STRING_NON_EMPTY(name, NULL);

    bool status = false;

    pmFPAfile *file = psMetadataLookupPtr (&status, files, name);
    if (!status) {
        psTrace("psModules.camera", 6, "Internal File %s not in file list", name);
        return true;
    }
    if (file == NULL) {
        psError(PS_ERR_IO, true, "file %s is NULL", name);
        return false;
    }
    if (file->mode != PM_FPA_MODE_INTERNAL) {
        psTrace("psModules.camera", 6, "FPA File %s not Internal, not dropping", name);
        return true;
    }

    psTrace("psModules.camera", 6, "dropping Internal FPA File %s", name);
    psMetadataRemoveKey (files, name);
    return true;
}

