/** @file  pmConfig.h
 *
 *  @author PAP, IfA
 *
 *  @version $Revision: 1.43 $ $Name: not supported by cvs2svn $
 *  @date $Date: 2006-09-15 09:49:01 $
 *
 *  Copyright 2004 Maui High Performance Computing Center, University of Hawaii
 *
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <libgen.h>
#include <assert.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <glob.h>
#include <pslib.h>
#include "pmConfig.h"

#define PS_SITE "PS_SITE"         // Name of the environment variable containing the site config file
#define PS_DEFAULT_SITE ".ipprc"  // Default site config file

static psArray *configPath = NULL;

static void configFree(pmConfig *config)
{
    psFree(config->site);
    psFree(config->files);
    psFree(config->camera);
    psFree(config->recipes);
    psFree(config->recipesSource);
    psFree(config->arguments);
    psFree(config->database);
}

pmConfig *pmConfigAlloc(int *argc, char **argv)
{
    pmConfig *config = psAlloc(sizeof(pmConfig));
    (void)psMemSetDeallocator(config, (psFreeFunc)configFree);

    // Initialise
    config->site = NULL;
    config->camera = NULL;
    config->recipes = psMetadataAlloc();
    config->recipesRead = P_PM_RECIPE_SOURCE_NONE;
    config->recipesSource = psMetadataAlloc();
    config->arguments = psMetadataAlloc();
    config->database = NULL;
    config->argc = argc;
    config->argv = argv;

    // the file structure is used to carry pmFPAfiles
    config->files = psMetadataAlloc ();
    return config;
}

void pmConfigSet(const char *path)
{
    pmConfigDone();

    psList *list = psStringSplit(path, ":", false);
    configPath = psListToArray(list);
    psFree(list);
}

void pmConfigDone(void)
{
    psFree(configPath);
    configPath = NULL;
}


/** readConfig
 *
 * This function attempts to open the file specified in the parameter list
 * parse it into metadata, then return it as a psMetaData table.
 *
 */
bool readConfig(
    psMetadata **config,                // Config to output
    const char *name,                   // Name of file
    const char *description)            // Description of file
{
    assert(config);
    assert(name && strlen(name) > 0);
    assert(description && strlen(description) > 0);

    char *realName = NULL;
    unsigned int numBadLines = 0;
    struct stat filestat;

    psTrace("psModules.config", 3, "Loading %s configuration from file %s\n",
            description, name);

    uid_t uid = getuid();
    gid_t gid = getgid();

    // we try: name, path[0]/name, path[1]/name, ...
    // find the first existing entry in the path (starting with the bare name)
    realName = psStringCopy (name);
    psTrace (__func__, 5, "trying %s\n", realName);

    int status = stat (realName, &filestat);
    if (status == 0) {
        if ((uid == filestat.st_uid) && (filestat.st_mode & S_IRUSR))
            goto found;
        if ((gid == filestat.st_gid) && (filestat.st_mode & S_IRGRP))
            goto found;
        if (filestat.st_mode & S_IROTH)
            goto found;
    }
    psFree (realName);

    if (configPath == NULL) {
        psError(PS_ERR_IO, true, "Cannot find %s configuration file (%s) in path\n", description, name);
        return false;
    }

    for (int i = 0; i < configPath->n; i++) {
        realName = psStringCopy (configPath->data[i]);
        psStringAppend (&realName, "/%s", name);
        psTrace (__func__, 5, "trying %s\n", realName);

        status = stat (realName, &filestat);
        if (status == 0) {
            if ((uid == filestat.st_uid) && (filestat.st_mode & S_IRUSR))
                goto found;
            if ((gid == filestat.st_gid) && (filestat.st_mode & S_IRGRP))
                goto found;
            if (filestat.st_mode & S_IROTH)
                goto found;
        }
        psFree (realName);
    }

    psError(PS_ERR_IO, true, "Cannot find %s configuration file %s in path\n", description, name);
    return false;

found:
    *config = psMetadataConfigRead(NULL, &numBadLines, realName, true);
    if (numBadLines > 0) {
        psError(PS_ERR_IO, false, "%d bad lines in %s configuration file (%s)",
                numBadLines, description, realName);
        psFree (realName);

        return false;
    }
    if (!*config) {
        psError(PS_ERR_IO, true, "Unable to read %s configuration from %s",
                description, realName);
        psFree (realName);
        return false;
    }

    psFree (realName);
    return true;
}


/******************************************************************************
pmConfigRead(**site, **camera, **recipe, *argc, **argv, char *recipeName)
 
XXX: The log/trace command line options (as processed by psArgumentVerbosity)
must take precedence override the values set here.  This must be, somehow,
coded.
 
XXX: Must load camera and recipe configuration if specified in the command
 line.
 *****************************************************************************/
pmConfig *pmConfigRead(
    int *argc,
    char **argv)
{
    PS_ASSERT_PTR_NON_NULL(argc, NULL);
    PS_ASSERT_INT_POSITIVE(*argc, NULL);
    PS_ASSERT_PTR_NON_NULL(argv, NULL);

    pmConfig *config = pmConfigAlloc(argc, argv); // The configuration, containing site, camera and recipes

    //
    // The following section of code attempts to determine which file is
    // the configuration file.  At the end of this code block, the siteName
    // variable will contain the name of the configuration file.
    //
    char *siteName = NULL;
    //
    // First, try command line
    //
    psS32 argNum = psArgumentGet(*config->argc, config->argv, "-site");
    if (argNum != 0) {
        //
        // We remove the "-site" argument from argv.  Then
        // we look for the next argument, which should be the filename, and
        // remove it as well.
        //
        psArgumentRemove(argNum, config->argc, config->argv);
        if (argNum >= *config->argc) {
            psLogMsg(__func__, PS_LOG_WARN,
                     "-site command-line switch provided without the required filename --- ignored.\n");
        } else {
            siteName = psStringCopy(config->argv[argNum]);
            psArgumentRemove(argNum, config->argc, config->argv);
        }
    }
    //
    // Next, try environment variable
    //
    if (!siteName) {
        siteName = getenv(PS_SITE);
        if (siteName) {
            siteName = psStringCopy (siteName);
        }
    }

    //
    // Last chance is ~/.ipprc
    //
    if (!siteName) {
        char *home = getenv("HOME");
        siteName = psStringCopy(home);
        psStringAppend(&siteName, "/%s", PS_DEFAULT_SITE);
    }


    // We have the configuration filename; now we read and parse the config
    // file and store in psMetadata struct site.
    //

    if (!readConfig(&config->site, siteName, "site")) {
        psFree(config);
        return NULL;
    }

    // define the config-file search path (configPath).  Ensure that
    // it contains the directory where we found the config file in
    // the first place
    if (configPath) {
        pmConfigDone();
    }

    psString siteNameDir = psStringCopy(dirname(siteName));
    psFree(siteName);

    psString path = psMetadataLookupStr(NULL, config->site, "PATH");
    if (path) {
        psStringAppend(&path, ":%s", siteNameDir);
    } else {
        path = psMemIncrRefCounter(siteNameDir);
    }
    pmConfigSet(path);
    psFree(siteNameDir);

    // Next, we do a similar thing for the camera configuration file.  The
    // file is read and parsed into psMetadata struct "camera".
    //
    argNum = psArgumentGet(*config->argc, config->argv, "-camera");
    if (argNum > 0) {
        psArgumentRemove(argNum, config->argc, config->argv);
        if (argNum >= *config->argc) {
            psLogMsg(__func__, PS_LOG_WARN,
                     "-camera command-line switch provided without the required camera or filename --- "
                     "ignored.\n");
        } else {
            char *cameraFile = config->argv[argNum]; // The camera configuration file to read

            // look for a symbolic camera name in the CAMERAS metadata
            // otherwise fall back to trying the -camera option as a literal
            // filename
            bool mdok = true;           // Status of MD lookup
            psMetadata *cameras = psMetadataLookupMD(&mdok, config->site, "CAMERAS");
            if (mdok && cameras) {
                char *newFile = psMetadataLookupStr(NULL, cameras, cameraFile); // The filename
                if (newFile) {
                    cameraFile = newFile;
                }
            } else {
                psError(PS_ERR_IO, false, "Unable to find CAMERAS in site configuration.\n");
                psFree(config);
                return NULL;
            }

            readConfig(&config->camera, cameraFile, "camera");
            psArgumentRemove(argNum, config->argc, config->argv);
        }
    }

    // Load the recipes from the camera file, if appropriate
    if(!pmConfigReadRecipes(config)) {
        psError(PS_ERR_IO, false, "Failed to read recipes from camera file");
        psFree(config);
        return NULL;
    }

    //
    // We now have the config, camera, and recipe files parsed and stored in
    // metadata.  Now, we can look into the site configuration and do
    // the required stuff.
    //
    bool mdok = true;   // Status of MD lookup result

    //
    // If TIME is specified in the configuration file, then we must initialize
    // with a call to psTimeInitialize.
    //
    psString timeName = psMetadataLookupStr(&mdok, config->site, "TIME");
    if (mdok && timeName) {
        psTrace("psModules.config", 7, "Initialising psTime with file %s\n", timeName);
        // XXX: PAP had a call to psLibInit is PRODUCTION not set.  Why?
        psTimeInitialize(timeName);
    }


    //
    // If LOGLEVEL is specified in the configuration file, then we must initialize
    // with a call to psLogSetLevel().
    //
    int logLevel = psMetadataLookupS32(&mdok, config->site, "LOGLEVEL");
    if (mdok && logLevel >= 0) {
        psTrace("psModules.config", 7, "Setting log level to %d\n", logLevel);
        psLogSetLevel(logLevel);
    }


    //
    // If LOGFORMAT is specified in the configuration file, then we must initialize
    // with a call to psLogSetFormat().
    //
    psString logFormat = psMetadataLookupStr(&mdok, config->site, "LOGFORMAT");
    if (mdok && logFormat) {
        psTrace("psModules.config", 7, "Setting log format to %s\n", logFormat);
        psLogSetFormat(logFormat);
    }


    argNum = psArgumentGet(*config->argc, config->argv, "-log");
    if (argNum > 0) {
        psArgumentRemove(argNum, config->argc, config->argv);
        if (argNum >= *config->argc) {
            psLogMsg(__func__, PS_LOG_WARN,
                     "-log command-line switch provided without the required log destination --- ignored.\n");
        } else {
            if (!psLogSetDestination(psMessageDestination(config->argv[argNum]))) {
                psLogMsg(__func__, PS_LOG_WARN, "Unable to set log destination to %s\n",
                         config->argv[argNum]);
            }
            psArgumentRemove(argNum, config->argc, config->argv);
        }
    } else {
        //
        // If LOGDEST is specified in the configuration file, then we must initialize
        // with a call to psLogSetDestination().
        // XXX: This is not spec'ed in the SDRS.
        //
        psString logDest = psMetadataLookupStr(&mdok, config->site, "LOGDEST");
        if (mdok && logDest && strlen(logDest) > 0) {
            // XXX: Only stdout and stderr are provided for now; this section should be
            // expanded in the future to do files, and perhaps even sockets.
            if (!psLogSetDestination(psMessageDestination(logDest))) {
                psLogMsg(__func__, PS_LOG_WARN, "Unable to set log destination to %s\n",
                         config->argv[argNum]);
            }
        }
    }

    //
    // If TRACE is specified in the configuration file, then we must initialize
    // with a call to psTraceSetLevel().
    // XXX: This is not spec'ed in the SDRS.
    //
    psMetadata *trace = psMetadataLookupMD(&mdok, config->site, "TRACE");
    if (mdok && trace) {
        psMetadataIterator *traceIter = psMetadataIteratorAlloc(trace, PS_LIST_HEAD, NULL); // Iterator
        psMetadataItem *traceItem = NULL; // Item from MD iteration
        while ((traceItem = psMetadataGetAndIncrement(traceIter))) {
            if (traceItem->type != PS_DATA_S32) {
                psLogMsg(__func__, PS_LOG_WARN, "The level for trace component %s is not of type S32 (%x)\n",
                         traceItem->name, traceItem->type);
                continue;
            }
            psTrace("psModules.config", 7, "Setting trace level for %s to %d\n", traceItem->name, traceItem->data.S32);
            psTraceSetLevel(traceItem->name, traceItem->data.S32);
        }
        psFree(traceIter);
    }
    //
    // If TRACEFORMAT is specified in the configuration file, then we must
    // initialize with a call to psTraceSetFormat().
    //
    psString traceFormat = psMetadataLookupStr(&mdok, config->site, "TRACEFORMAT");
    if (mdok && traceFormat) {
        psTrace("psModules.config", 7, "Setting trace format to %s\n", traceFormat);
        psTraceSetFormat(traceFormat);
    }

    //
    // If LOGDEST is specified in the configuration file, then we must initialize
    // with a call to psLogSetDestination().
    // XXX: This is not spec'ed in the SDRS.
    //
    psString traceDest = psMetadataLookupStr(&mdok, config->site, "TRACEDEST");
    if (mdok && traceDest && strlen(traceDest) > 0) {
        psTrace("psModules.config", 7, "Setting trace destination to %s\n", traceDest);
        // XXX: Only stdout and stderr are provided for now; this section should be
        // expanded in the future to do files, and perhaps even sockets.
        if (!psTraceSetDestination(psMessageDestination(traceDest))) {
            psLogMsg(__func__, PS_LOG_WARN, "Unable to set trace destination to %s\n",
                     config->argv[argNum]);
        }
    }

    //
    // Allow command line options to override defaults for logging.
    // XXX: Is it appropriate to use the ArgVerbosity function for this?
    //   A: it removes the options from the command line.
    //   B: will the pmConfigRead function always be called on initialization.
    //
    psS32 saveLogLevel = psLogGetLevel();
    psArgumentVerbosity(config->argc, config->argv);
    // XXX: substitute the string for the default log level for "2".
    if (2 == psLogGetLevel()) {
        psLogSetLevel(saveLogLevel);
    }

    psErrorClear();   // we may have failed to find some items in the metadata

    return config;
}


bool pmConfigValidateCameraFormat(
    const psMetadata *cameraFormat,
    const psMetadata *header)
{
    PS_ASSERT_PTR_NON_NULL(cameraFormat, false);
    PS_ASSERT_PTR_NON_NULL(header, false);

    // Read the rule for that camera format
    bool mdStatus = true;
    psMetadata *rule = psMetadataLookupMD(&mdStatus, cameraFormat, "RULE");
    if (! mdStatus || ! rule) {
        psLogMsg(__func__, PS_LOG_WARN, "Unable to read rule for camera.\n");
        return false;
    }

    // Apply the rules
    psMetadataIterator *ruleIter = psMetadataIteratorAlloc(rule, PS_LIST_HEAD, NULL); // Rule iterator
    psMetadataItem *ruleItem = NULL;    // Item from the metadata
    while ((ruleItem = psMetadataGetAndIncrement(ruleIter))) {
        // Check for the existence of the rule
        psMetadataItem *headerItem = psMetadataLookup((psMetadata*)header, ruleItem->name);
        if (! headerItem) {
            // It doesn't have a required header keyword, so it's not it
            psFree(ruleIter);
            psTrace("psModules.config", 5, "Can't find %s\n", ruleItem->name);
            return false;
        }

        // Check to see if the rule works
        if (! psMetadataItemCompare(headerItem, ruleItem)) {
            psTrace("psModules.config", 5, "%s doesn't match.\n", ruleItem->name);
            psFree(ruleIter);
            return false;
        }
    } // Iterating through the RULEs

    psFree(ruleIter);
    return true;
}


static bool formatFromHeader(psMetadata **format, // Format to return
                             psMetadata *camera, // Camera configuration
                             const psMetadata *header, // FITS header
                             const char *cameraName // Name of camera
                            )
{
    assert(format);
    assert(camera);
    assert(header);
    assert(cameraName && strlen(cameraName) > 0);

    bool result = false;                // Did we find the first match?

    // Read the list of formats
    bool mdok = true;                   // Status of MD lookup
    psMetadata *formats = psMetadataLookupMD(&mdok, camera, "FORMATS"); // List of formats
    if (!mdok || !formats) {
        psLogMsg(__func__, PS_LOG_WARN, "Unable to find list of FORMATS in camera %s --- ignored\n",
                 cameraName);
        return false;
    }
    // Iterate over the formats
    psMetadataIterator *formatsIter = psMetadataIteratorAlloc(formats, PS_LIST_HEAD, NULL);
    psMetadataItem *formatsItem = NULL; // Item from formats
    while ((formatsItem = psMetadataGetAndIncrement(formatsIter))) {
        if (formatsItem->type != PS_DATA_STRING) {
            psLogMsg(__func__, PS_LOG_WARN, "In camera %s, camera format %s is not of type STR --- "
                     "ignored.\n", cameraName, formatsItem->name);
            continue;
        }
        psTrace("psModules.config", 5, "Reading camera format for %s...\n", formatsItem->name);
        psMetadata *testFormat = NULL;  // Format to test against what we've got
        if (!readConfig(&testFormat, formatsItem->data.V, formatsItem->name)) {
            psLogMsg(__func__, PS_LOG_WARN, "Trouble reading reading camera format %s --- ignored.\n",
                     formatsItem->name);
            psFree(testFormat);
            continue;
        }

        if (pmConfigValidateCameraFormat(testFormat, header)) {
            if (!*format) {
                psLogMsg(__func__, PS_LOG_INFO, "Camera %s, format %s matches header.\n", cameraName,
                         formatsItem->name);
                *format = psMemIncrRefCounter(testFormat);
                result = true;
            } else {
                psLogMsg(__func__, PS_LOG_WARN, "Camera %s, format %s also matches header --- ignored.\n",
                         cameraName, formatsItem->name);
            }
        }
        psFree(testFormat);
    }
    psFree(formatsIter);

    return result;
}

// Work out what camera we have, based on the FITS header and a set of rules specified in the IPP
// configuration; return the camera configuration and format
psMetadata *pmConfigCameraFormatFromHeader(
    pmConfig *config,                   // The configuration
    const psMetadata *header            // The FITS header
)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(header, NULL);

    psMetadata *format = NULL;          // The winning format
    bool mdok = false;                  // Metadata lookup status

    // If we don't know what sort of camera we have, we try all that we know
    if (! config->camera) {
        psMetadata *cameras = psMetadataLookupMD(&mdok, config->site, "CAMERAS");
        if (! mdok) {
            psError(PS_ERR_IO, true, "Unable to find CAMERAS in the configuration.");
            return false;
        }

        // Iterate over the cameras
        psMetadataIterator *camerasIter = psMetadataIteratorAlloc(cameras, PS_LIST_HEAD, NULL);
        psMetadataItem *camerasItem = NULL; // Item from the metadata
        while ((camerasItem = psMetadataGetAndIncrement(camerasIter))) {
            // Open the camera information
            psTrace("psModules.config", 3, "Inspecting camera %s (%s)\n", camerasItem->name, camerasItem->comment);
            if (camerasItem->type != PS_DATA_STRING) {
                psLogMsg(__func__, PS_LOG_WARN, "Camera configuration for %s in CAMERAS is not of type STR "
                         "--- ignored.\n", camerasItem->name);
                continue;
            }

            psTrace("psModules.config", 5, "Reading camera configuration for %s...\n", camerasItem->name);
            psMetadata *testCamera = NULL; // Camera to test against what we've got:

            if (!readConfig(&testCamera, camerasItem->data.V, camerasItem->name)) {
                psLogMsg(__func__, PS_LOG_WARN, "Trouble reading reading camera configuration %s --- "
                         "ignored.\n", camerasItem->name);
                psFree(testCamera);
                continue;
            }

            if (formatFromHeader(&format, testCamera, header, camerasItem->name)) {
                config->camera = psMemIncrRefCounter(testCamera);
            }
            psFree(testCamera);
        } // Done looking at all cameras
        psFree(camerasIter);

        if (! config->camera) {
            psError(PS_ERR_IO, false, "Unable to find a camera that matches input FITS header!");
            return NULL;
        }

        // Now we have the camera, we can read the recipes
        pmConfigReadRecipes(config);

        return format;
    }

    // Otherwise, try the specific camera
    if (!formatFromHeader(&format, config->camera, header, "specified camera")) {
        psError(PS_ERR_IO, true, "Unable to find a format with the specified camera that matches the "
                "given header.\n");
        return NULL;
    }
    return format;
}

// Work out what camera we have, based on the FITS header and a set of rules specified in the IPP
// configuration; return the camera configuration and format
psMetadata *pmConfigCameraByName(
    pmConfig *config,                   // The configuration
    const char *cameraName   // The camera name header
)
{
    PS_ASSERT_PTR_NON_NULL(cameraName, NULL);
    PS_ASSERT_PTR_NON_NULL(config, NULL);

    psMetadata *cameras = psMetadataLookupMD(NULL, config->site, "CAMERAS");
    if (!cameras) {
        psError(PS_ERR_IO, true, "Unable to find CAMERAS in the configuration.");
        return NULL;
    }

    char *cameraPath = psMetadataLookupStr(NULL, cameras, cameraName);
    if (!cameraPath) {
        psError(PS_ERR_IO, true, "Unable to find requested CAMERA in the configuration.");
        return NULL;
    }

    psMetadata *camera = NULL; // Camera to test against what we've got:

    if (!readConfig(&camera, cameraPath, cameraName)) {
        psLogMsg(__func__, PS_LOG_WARN, "Trouble reading reading camera configuration %s", cameraName);
        psFree(camera);
        return NULL;
    }

    return camera;
}


// Load the recipes
static bool loadRecipes(pmConfig *config, // The configuration into which to read the recipes
                        psMetadata *source, // The source configuration, from which to read the filenames
                        p_pmRecipeSource sourceType, // The source type
                        const char *sourceName // The name of the source, for error messages
                       )
{
    assert(config);

    if (!source) {
        #if 0
        psLogMsg(__func__, PS_LOG_WARN, "The %s has not been read --- cannot read recipes from this "
                 "location.\n", sourceName);
        #endif

        config->recipesRead &= ~sourceType;
        return false;
    }

    bool mdok = true;                   // Status of MD lookup
    psMetadata *recipes = psMetadataLookupMD(&mdok, source, "RECIPES"); // The list of recipes
    if (!mdok || !recipes) {
        psLogMsg(__func__, PS_LOG_WARN, "RECIPES in the %s is not of type METADATA --- ignored\n",
                 sourceName);
        config->recipesRead &= ~sourceType;
        return false;
    }

    // Copy the filenames to the target from the "RECIPES" in the source.
    // We could use psMetadataCopy for this, but it's better to check that everything's of the correct type.
    // If it's not of the correct type, we can tell the user which file it's in, so they can find it easier.
    {
        psMetadataIterator *recipesIter = psMetadataIteratorAlloc(recipes, PS_LIST_HEAD, NULL); // Iterator
        psMetadataItem *fileItem = NULL;    // MD item containing the filename, from recipe iteration
        while ((fileItem = psMetadataGetAndIncrement(recipesIter)))
        {
            if (fileItem->type != PS_DATA_STRING) {
                psLogMsg(__func__, PS_LOG_WARN, "Recipe %s from %s is not of type STR --- ignored.\n",
                         fileItem->name, sourceName);
                continue;
            }

            // Check to see if it's currently defined
            int check = psMetadataLookupS32(&mdok, config->recipesSource, fileItem->name);
            if (mdok && check > sourceType) {
                // It's already defined with a higher priority
                continue;
            }
            psString comment = psStringCopy("Recipe added at ");
            psStringAppend(&comment, "%s from %s", sourceName, (char*)fileItem->data.V);
            psMetadataAddS32(config->recipesSource, PS_LIST_TAIL, fileItem->name,
                             PS_META_REPLACE, comment, sourceType);
            psFree(comment);

            // Read the recipe
            psMetadata *recipe = NULL;      // Recipe from file
            if (readConfig(&recipe, fileItem->data.V, "recipe")) {
                psMetadataAdd(config->recipes, PS_LIST_TAIL, fileItem->name,
                              PS_DATA_METADATA | PS_META_REPLACE, fileItem->comment, recipe);
            } else {
                psError(PS_ERR_IO, false, "Failed to read recipe");

                psFree(recipe);  // Drop reference
                psFree(recipesIter);

                return false;
            }
            psFree(recipe);                 // Drop reference
        }
        psFree(recipesIter);
    }
    config->recipesRead |= sourceType;

    {
        // Having read that, we now need to check to see if any symbolic links need to be resolved
        psMetadataIterator *recipesIter = psMetadataIteratorAlloc(config->recipesSource, PS_LIST_HEAD, NULL);
        psMetadataItem *sourceItem = NULL;  // Item containing source, from iteration
        while ((sourceItem = psMetadataGetAndIncrement(recipesIter))) {
            assert(sourceItem->type == PS_TYPE_S32); // It should be this type: we put it in ourselves
            if (sourceItem->data.S32 == P_PM_RECIPE_SOURCE_SYMBOLIC) {
                const char *linkName = sourceItem->comment; // The name of the link
                psMetadata *linkSource = psMetadataLookupMD(&mdok, config->recipes, linkName); // The source
                if (!mdok || !linkSource) {
                    // Can't yet resolve it
                    continue;
                }
                psString comment = NULL;
                psStringAppend(&comment, "Symbolic link from %s", linkName);
                psMetadataAdd(config->recipes, PS_LIST_TAIL, sourceItem->name,
                              PS_DATA_METADATA | PS_META_REPLACE, comment, linkSource);
                psFree(comment);
            }
        }
        psFree(recipesIter);
    }
    return true;
}


bool pmConfigReadRecipes(pmConfig *config
                        )
{
    PS_ASSERT_PTR_NON_NULL(config, false);

    if (!config->recipes) {
        config->recipes = psMetadataAlloc();
    }

    bool mdok = true;                   // Status of MD lookup

    // Read the recipe file names from the site configuration and camera configuration
    if (config->site && !(config->recipesRead & P_PM_RECIPE_SOURCE_SITE)) {
        if(!loadRecipes(config, config->site, P_PM_RECIPE_SOURCE_SITE, "site configuration")) {
            psError(PS_ERR_IO, false, "Failed to read recipes from site config");
            #if 0     // see comment at end of routine

            return false;
            #endif

        }
    }
    if (config->camera && !(config->recipesRead & P_PM_RECIPE_SOURCE_CAMERA)) {
        if (!loadRecipes(config, config->camera, P_PM_RECIPE_SOURCE_CAMERA, "camera configuration")) {
            psError(PS_ERR_IO, false, "Failed to read recipes from camera config");
            #if 0     // see comment at end of routine

            return false;
            #endif

        }
    }

    if (!(config->recipesRead & P_PM_RECIPE_SOURCE_CL)) {
        // Go through the command-line arguments
        int argNum; // Argument number for "-recipe"
        while ((argNum = psArgumentGet(*config->argc, config->argv, "-recipe")) > 0) {
            psArgumentRemove(argNum, config->argc, config->argv);
            if (argNum + 1 >= *config->argc) {
                psLogMsg(__func__, PS_LOG_WARN, "-recipe command-line switch provided without the "
                         "required recipe and source --- ignored.\n");
                if (argNum == *config->argc) {
                    // Remove the single last argument (we required two, they gave us one)
                    psArgumentRemove(argNum, config->argc, config->argv);
                }
                continue;
            }

            const char *recipeName = psStringCopy(config->argv[argNum]); // Name of the recipe
            psArgumentRemove(argNum, config->argc, config->argv);
            const char *recipeSource = psStringCopy(config->argv[argNum]); // Source of the recipe
            psArgumentRemove(argNum, config->argc, config->argv);

            // Command-line has the highest priority, so we don't have to check to see if it's already there

            // Is the source a symbolic reference?
            psMetadata *extant = psMetadataLookupMD(&mdok, config->recipes, recipeSource); // Does it exist?
            if (mdok && extant) {
                psString comment = psStringCopy("Recipe added from command line as symbolic link to ");
                psStringAppend(&comment, "%s", recipeSource);
                psMetadataAdd(config->recipes, PS_LIST_TAIL, recipeName, PS_DATA_METADATA | PS_META_REPLACE,
                              comment, extant);
                psFree(comment);
                // Put the source in the comment, so we can retrieve it again if we need it
                psMetadataAddS32(config->recipesSource, PS_LIST_TAIL, recipeName, PS_META_REPLACE,
                                 recipeSource, P_PM_RECIPE_SOURCE_SYMBOLIC);

            } else if (access(recipeSource, R_OK) == 0) {
                // The source is a file
                psMetadata *recipe = NULL;      // Recipe from file
                if (readConfig(&recipe, recipeSource, "recipe")) {
                    psString comment = psStringCopy("Recipe added at command line from ");
                    psStringAppend(&comment, "%s", recipeSource);
                    psMetadataAdd(config->recipes, PS_LIST_TAIL, recipeName,
                                  PS_DATA_METADATA | PS_META_REPLACE, comment, recipe);
                    psFree(comment);
                    psFree(recipe);                 // Drop reference
                    psMetadataAddS32(config->recipesSource, PS_LIST_TAIL, recipeName, PS_META_REPLACE,
                                     "Recipe added from command line", P_PM_RECIPE_SOURCE_CL);
                }
            } else {
                // Assume it's a symbolic reference to something that's not yet read in
                psMetadataAddS32(config->recipesSource, PS_LIST_TAIL, recipeName, PS_META_REPLACE,
                                 recipeSource, P_PM_RECIPE_SOURCE_SYMBOLIC);
            }
            psFree(recipeName);
            psFree(recipeSource);
        } // Iterating through the command-line arguments
        config->recipesRead |= P_PM_RECIPE_SOURCE_CL;
    }
    /*
     * We may have seen real errors, but we also get false status returned by valid conditions (e.g. asking a
     * file for a recipe when none was provided).  For now we'll never signal an error, but this should
     * be reconsidered ASAP.
     */
    psErrorClear();
    return true;
}

/******************************************************************************
pmConfigDB(*site)
 
XXX: this should allow the option of having NO database server, if chosen by config
XXX: What should we use for the Database namespace in the call to psDBInit()?
This is currently NULL.
 *****************************************************************************/

psDB *pmConfigDB(
    pmConfig *config                    // Configuration
)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->site, NULL);

    #ifdef OMIT_PSDB

    return NULL;
    #else

    psBool mdStatus01 = false;
    psBool mdStatus02 = false;
    psBool mdStatus03 = false;

    // XXX leaky strings
    psString dbServer = psMetadataLookupStr(&mdStatus01, config->site, "DBSERVER");
    psString dbUsername = psMetadataLookupStr(&mdStatus02, config->site, "DBUSER");
    psString dbPassword = psMetadataLookupStr(&mdStatus03, config->site, "DBPASSWORD");
    psString dbName = psMetadataLookupStr(&mdStatus01, config->site, "DBNAME");
    if (!(mdStatus01 & mdStatus02 & mdStatus03)) {
        psLogMsg(__func__, PS_LOG_WARN, "Could not determine database server name, userID, and password from site metadata.\n");
        return(NULL);
    }

    return(psDBInit(dbServer, dbUsername, dbPassword, dbName));
    #endif
}


bool pmConfigConformHeader(psMetadata *header, // Header to conform
                           const psMetadata *format // Camera format
                          )
{
    PS_ASSERT_PTR_NON_NULL(header, false);
    PS_ASSERT_PTR_NON_NULL(format, false);

    bool mdok = true;                   // Status of MD lookup
    psMetadata *rules = psMetadataLookupMD(&mdok, format, "RULE"); // How to identify this format
    if (!mdok || !rules) {
        psError(PS_ERR_IO, true, "Unable to find RULE in camera format.\n");
        return false;
    }

    psMetadataIterator *rulesIter = psMetadataIteratorAlloc(rules, PS_LIST_HEAD, NULL); // Iterator for rules
    psMetadataItem *rulesItem = NULL;   // Item from iteration
    while ((rulesItem = psMetadataGetAndIncrement(rulesIter))) {
        psMetadataItem *newItem = psMetadataItemCopy(rulesItem); // Copy of item
        psMetadataAddItem(header, newItem, PS_LIST_TAIL, PS_META_REPLACE);
        psFree(newItem);                // Drop reference
    }
    psFree(rulesIter);

    return true;
}

// given the 'file' and 'list' words, find the arguments associated with these words
// and interpret them as lists of files.  return an array of the resulting filenames.
psArray *pmConfigFileSets(pmConfig *config, char *file, char *list)
{
    PS_ASSERT_PTR_NON_NULL(config, NULL);
    PS_ASSERT_PTR_NON_NULL(config->argc, NULL);
    PS_ASSERT_INT_NONNEGATIVE(*config->argc, NULL);
    PS_ASSERT_PTR_NON_NULL(config->argv, NULL);
    PS_ASSERT_PTR_NON_NULL(file, NULL);
    PS_ASSERT_INT_POSITIVE(strlen(file), NULL);
    PS_ASSERT_PTR_NON_NULL(list, NULL);
    PS_ASSERT_INT_POSITIVE(strlen(list), NULL);

    int Narg;

    // we load all input files onto a psArray, to be parsed later
    psArray *input = psArrayAlloc (16);
    input->n = 0;

    // load the list of filenames the supplied file
    // maybe a comma-separated list of words
    // each word may be a glob: "file*.fits"
    if ((Narg = psArgumentGet (*config->argc, config->argv, file))) {

        // select the word after 'file' and split by comma
        psArgumentRemove (Narg, config->argc, config->argv);
        psArray *words = psStringSplitArray (config->argv[Narg], ",", true);
        psArgumentRemove (Narg, config->argc, config->argv);

        // parse the word as a glob
        glob_t globList;
        for (int i = 0; i < words->n; i++) {
            globList.gl_offs = 0;
            glob (words->data[i], 0, NULL, &globList);

            if (globList.gl_pathc == 0) {
                psError(PS_ERR_IO, true, "No match for %s", (char *)words->data[i]);
                return input;
            }

            for (int j = 0; j < globList.gl_pathc; j++) {
                char *filename = psStringCopy (globList.gl_pathv[j]);
                psArrayAdd (input, 16, filename);
                psFree (filename);
            }
        }
        psFree (words);
    }

    // load the list from the supplied text file
    if ((Narg = psArgumentGet (*config->argc, config->argv, list))) {
        int nItems;
        char line[1024]; // XXX limits the list lines to 1024 chars
        char word[1024];
        char *filename;

        psArgumentRemove (Narg, config->argc, config->argv);
        FILE *f = fopen (config->argv[Narg], "r");
        if (f == NULL) {
            psAbort ("psphot", "unable to open specified list file");
        }
        while (fgets (line, 1024, f) != NULL) {
            nItems = sscanf (line, "%s", word);
            switch (nItems) {
            case 0:
                break;
            case 1:
                filename = psStringCopy (word);
                psArrayAdd (input, 16, filename);
                psFree (filename);
                break;
            default:
                // rigid format, no comments allowed?
                psAbort ("pmConfig", "error parsing input list file");
                break;
            }
        }
        psArgumentRemove (Narg, config->argc, config->argv);
    }

    return input;
}

bool pmConfigFileSetsMD (psMetadata *metadata, pmConfig *config, char *name, char *file, char *list)
{
    PS_ASSERT_PTR_NON_NULL(metadata, false);
    PS_ASSERT_PTR_NON_NULL(config, false);
    PS_ASSERT_PTR_NON_NULL(name, false);
    PS_ASSERT_INT_POSITIVE(strlen(name), false);
    PS_ASSERT_PTR_NON_NULL(file, false);
    PS_ASSERT_INT_POSITIVE(strlen(file), false);
    PS_ASSERT_PTR_NON_NULL(list, false);
    PS_ASSERT_INT_POSITIVE(strlen(list), false);

    psErrorClear();   // pmConfigFileSets may or may not call psError, so
    // if files->n == 0 we'll want to call psError(..., false, ...)
    psArray *files = pmConfigFileSets(config, file, list);
    if (files->n == 0) {
        // psError(PS_ERR_IO, false, "pmConfigFileSets failed to find %s in the argument list", name);
        psFree (files);
        return false;
    }

    psMetadataAddPtr(metadata, PS_LIST_TAIL, name,  PS_DATA_ARRAY, "", files);
    psFree (files);
    return true;
}
