/** @file  psMetadata.c
 *
 *
 *  @brief Contains metadata structures, enumerations and functions prototypes.
 *
 *  This file defines metadata item, metadata type, metadata flags, metadata containers, and function
 *  prototypes necessary creating psLib metadata APIs
 *
 *  @ingroup Metadata
 *
 *  @author Robert DeSonia, MHPCC
 *  @author Ross Harman, MHPCC
 *
 *  @version $Revision: 1.148 $ $Name: not supported by cvs2svn $
 *  @date $Date: 2007-01-09 22:38:53 $
 *
 *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
 */

#ifdef HAVE_CONFIG_H
# include "config.h"
#endif

/******************************************************************************/
/*  INCLUDE FILES                                                             */
/******************************************************************************/
#include <stdio.h>
#include <stdarg.h>
#include <string.h>

#include "fitsio.h"
#include "psType.h"
#include "psMemory.h"
#include "psError.h"
#include "psAbort.h"
#include "psList.h"
#include "psHash.h"
#include "psVector.h"
#include "psMetadata.h"
#include "psLookupTable.h"
#include "psString.h"

#include "psAssert.h"
#include "psLogMsg.h"
#include "psTrace.h"

/******************************************************************************/
/*  DEFINE STATEMENTS                                                         */
/******************************************************************************/

/** Maximum size of a string */
#define MAX_STRING_LENGTH 1024

/******************************************************************************/
/*  TYPE DEFINITIONS                                                          */
/******************************************************************************/

// None

/*****************************************************************************/
/*  GLOBAL VARIABLES                                                         */
/*****************************************************************************/

// None

/*****************************************************************************/
/*  FILE STATIC VARIABLES                                                    */
/*****************************************************************************/

static psS32 metadataId = 0;

/*****************************************************************************/
/*  FUNCTION IMPLEMENTATION - LOCAL                                          */
/*****************************************************************************/

static psMetadataItem* makeMetaMulti(psHash* table,
                                     const char* key,
                                     psMetadataItem* existing)
{

    if (existing != NULL && existing->type == PS_DATA_METADATA_MULTI) {
        return existing;
    }

    // move any existing entry into a psList
    psList* newList = psListAlloc(existing);

    psMetadataItem* item = psMetadataItemAlloc(key,
                           PS_DATA_METADATA_MULTI,
                           "",
                           newList);

    if (existing != NULL) {
        psHashRemove(table,key); // take out the old entry
    }

    psHashAdd(table, key, item); // put in the new entry
    psMemDecrRefCounter(item); // get rid of extra reference

    // free local references of newly allocated items.
    psFree(newList);

    return item;
}

static void metadataItemFree(psMetadataItem* metadataItem)
{
    psDataType type;

    type = metadataItem->type;


    psMemDecrRefCounter(metadataItem->name);
    psMemDecrRefCounter(metadataItem->comment);

    if(!PS_DATA_IS_PRIMITIVE(type)) {
        psFree(metadataItem->data.V);
    }
}

static void metadataIteratorFree(psMetadataIterator* iter)
{
    psFree(iter->iter);

    if (iter->regex != NULL) {
        regfree(iter->regex);
        psFree(iter->regex);
    }
}

static void metadataFree(psMetadata* metadata)
{
    psFree(metadata->list);
    psFree(metadata->hash);
}

/*****************************************************************************/
/* FUNCTION IMPLEMENTATION - PUBLIC                                          */
/*****************************************************************************/

psMetadataItem* psMetadataItemAlloc(const char *name,
                                    psDataType type,
                                    const char *comment,
                                    ...)
{
    va_list argPtr;
    psMetadataItem* metadataItem = NULL;

    // Get the variable list parameters to pass to allocation function
    va_start(argPtr, comment);

    // Call metadata item allocation
    metadataItem = psMetadataItemAllocV(name, type, comment, argPtr);

    // Clean up stack after variable arguement has been used
    va_end(argPtr);

    return metadataItem;
}

bool psMemCheckMetadataItem(psPtr ptr)
{
    PS_ASSERT_PTR(ptr, false);
    return ( psMemGetDeallocator(ptr) == (psFreeFunc)metadataItemFree );
}


#define METADATAITEM_ALLOC_TYPE(NAME,TYPE,METATYPE) \
psMetadataItem* psMetadataItemAlloc##NAME(const char* name, \
        const char* comment, \
        TYPE value) \
{ \
    return psMetadataItemAlloc(name, METATYPE, comment, value); \
}

METADATAITEM_ALLOC_TYPE(Str,const char*,PS_DATA_STRING)
METADATAITEM_ALLOC_TYPE(F32,psF32,PS_DATA_F32)
METADATAITEM_ALLOC_TYPE(F64,psF64,PS_DATA_F64)
METADATAITEM_ALLOC_TYPE(S8,psS8,PS_DATA_S8)
METADATAITEM_ALLOC_TYPE(S16,psS16,PS_DATA_S16)
METADATAITEM_ALLOC_TYPE(S32,psS32,PS_DATA_S32)
METADATAITEM_ALLOC_TYPE(S64,psS64,PS_DATA_S64)
METADATAITEM_ALLOC_TYPE(U8,psU8,PS_DATA_U8)
METADATAITEM_ALLOC_TYPE(U16,psU16,PS_DATA_U16)
METADATAITEM_ALLOC_TYPE(U32,psU32,PS_DATA_U32)
METADATAITEM_ALLOC_TYPE(U64,psU64,PS_DATA_U64)
METADATAITEM_ALLOC_TYPE(Bool,psBool,PS_DATA_BOOL)

psMetadataItem* psMetadataItemAllocPtr(const char *name,
                                       psDataType type,
                                       const char *comment,
                                       psPtr value)
{
    return (psMetadataItemAlloc(name, type, comment, value));
}


psMetadataItem* psMetadataItemAllocV(const char *name,
                                     psDataType type,
                                     const char *comment,
                                     va_list argPtr)
{
    psMetadataItem* metadataItem = NULL;


    PS_ASSERT_PTR_NON_NULL(name,NULL);

    // Allocate metadata item
    metadataItem = (psMetadataItem*) psAlloc(sizeof(psMetadataItem));
    metadataItem->data.V = NULL;

    // Set deallocator
    psMemSetDeallocator(metadataItem, (psFreeFunc) metadataItemFree);

    // Allocate and set metadata item comment
    if (comment == NULL) {
        // Per SDRS, null isn't allowed, must use "" instead
        metadataItem->comment = psStringCopy("");
    } else {
        metadataItem->comment = psStringCopy(comment);
    }

    // Set metadata item unique id
    *(psS32 *)(&metadataItem->id) = ++metadataId;

    // Set metadata item type
    metadataItem->type = type & PS_METADATA_TYPE_MASK;

    // Allocate and set metadata item name
    metadataItem->name = psStringCopy(name);

    // Set metadata item value
    switch(metadataItem->type) {
    case PS_DATA_BOOL:
        metadataItem->data.B = (psBool)va_arg(argPtr, psS32);
        break;
    case PS_DATA_S8:
        metadataItem->data.S8 = (psS8)va_arg(argPtr, psS32);
        break;
    case PS_DATA_S16:
        metadataItem->data.S16 = (psS16)va_arg(argPtr, psS32);
        break;
    case PS_DATA_S32:
        metadataItem->data.S32 = (psS32)va_arg(argPtr, psS32);
        break;
    case PS_DATA_S64:
        metadataItem->data.S64 = (psS64)va_arg(argPtr, psS64);
        break;
    case PS_DATA_U8:
        metadataItem->data.U8 = (psU8)va_arg(argPtr, psU32);
        break;
    case PS_DATA_U16:
        metadataItem->data.U16 = (psU16)va_arg(argPtr, psU32);
        break;
    case PS_DATA_U32:
        metadataItem->data.U32 = (psU32)va_arg(argPtr, psU32);
        break;
    case PS_DATA_U64:
        metadataItem->data.U64 = (psU64)va_arg(argPtr, psU64);
        break;
    case PS_DATA_F32:
        metadataItem->data.F32 = (psF32)va_arg(argPtr, psF64);
        break;
    case PS_DATA_F64:
        metadataItem->data.F64 = (psF64)va_arg(argPtr, psF64);
        break;
    case PS_DATA_STRING:
        // Perform copy of input strings
        {
            char *string = va_arg(argPtr, char *);
            metadataItem->data.str =
                string ? psStringNCopy(string, MAX_STRING_LENGTH)
                : NULL;
        }
        break;
    case     PS_DATA_ARRAY:                     // psArray
    case     PS_DATA_BITSET:                    // psBitSet
    case     PS_DATA_CUBE:                      // psCube
    case     PS_DATA_FITS:                      // psFits
    case     PS_DATA_HASH:                      // psHash
    case     PS_DATA_HISTOGRAM:                 // psHistogram
    case     PS_DATA_IMAGE:                     // psImage
    case     PS_DATA_KERNEL:                    // psKernel
    case     PS_DATA_LIST:                      // psList
    case     PS_DATA_LOOKUPTABLE:               // psLookupTable
    case     PS_DATA_METADATA:                  // psMetadata
    case     PS_DATA_METADATAITEM:              // psMetadataItem
    case     PS_DATA_MINIMIZATION:              // psMinimization
    case     PS_DATA_PIXELS:                    // psPixels
    case     PS_DATA_PLANE:                     // psPlane
    case     PS_DATA_PLANEDISTORT:              // psPlaneDistort
    case     PS_DATA_PLANETRANSFORM:            // psPlaneTransform
    case     PS_DATA_POLYNOMIAL1D:              // psPolynomial1D
    case     PS_DATA_POLYNOMIAL2D:              // psPolynomial2D
    case     PS_DATA_POLYNOMIAL3D:              // psPolynomial3D
    case     PS_DATA_POLYNOMIAL4D:              // psPolynomial4D
    case     PS_DATA_PROJECTION:                // psProjection
    case     PS_DATA_REGION:                  // psRegion
    case     PS_DATA_SCALAR:                    // psScalar
    case     PS_DATA_SPHERE:                    // psSphere
    case     PS_DATA_SPHEREROT:                 // psSphereTransform
    case     PS_DATA_SPLINE1D:                  // psSpline1D
    case     PS_DATA_STATS:                     // psStats
    case     PS_DATA_TIME:                      // psTime
    case     PS_DATA_VECTOR:                    // psVector
    case     PS_DATA_UNKNOWN:                   // "other"
    case     PS_DATA_METADATA_MULTI:
        // Copy of input data not performed due to variability of data types
        metadataItem->data.V = psMemIncrRefCounter(va_arg(argPtr, psPtr));
        break;
    default:
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, _("Specified psDataType, %d, is not supported."), type);
        psFree(metadataItem);
        metadataItem = NULL;
    }

    return metadataItem;
}

psMetadata* psMetadataAlloc(void)
{
    psList* list = NULL;
    psHash* hash = NULL;
    psMetadata* metadata = NULL;

    // Allocate metadata
    metadata = (psMetadata*) psAlloc(sizeof(psMetadata));
    // Set deallocator
    psMemSetDeallocator(metadata, (psFreeFunc) metadataFree);

    // Allocate metadata's internal containers
    list = (psList*) psListAlloc(NULL);
    hash = (psHash*) psHashAlloc(10);

    metadata->list = list;
    metadata->hash = hash;

    return metadata;
}


bool psMemCheckMetadata(psPtr ptr)
{
    PS_ASSERT_PTR(ptr, false);
    return ( psMemGetDeallocator(ptr) == (psFreeFunc)metadataFree );
}


psMetadataItem *psMetadataItemCopy(const psMetadataItem *in)
{
    PS_ASSERT_PTR_NON_NULL(in,NULL);

    psMetadataItem *newItem = NULL;     // New metadata item, to be returned

    #define PS_METADATA_ITEM_COPY_CASE(NAME,TYPE) \
case PS_DATA_##NAME: \
    newItem = psMetadataItemAlloc(in->name, PS_DATA_##NAME, in->comment, in->data.TYPE); \
    break; \

    switch (in->type) {
        // Simple types
        PS_METADATA_ITEM_COPY_CASE(BOOL,B);
        PS_METADATA_ITEM_COPY_CASE(S8,S8);
        PS_METADATA_ITEM_COPY_CASE(S16,S16);
        PS_METADATA_ITEM_COPY_CASE(S32,S32);
        PS_METADATA_ITEM_COPY_CASE(S64,S64);
        PS_METADATA_ITEM_COPY_CASE(U8,U8);
        PS_METADATA_ITEM_COPY_CASE(U16,U16);
        PS_METADATA_ITEM_COPY_CASE(U32,U32);
        PS_METADATA_ITEM_COPY_CASE(U64,U64);
        PS_METADATA_ITEM_COPY_CASE(F32,F32);
        PS_METADATA_ITEM_COPY_CASE(F64,F64);
        PS_METADATA_ITEM_COPY_CASE(STRING,V); // This will copy the string, not point at it.
    case PS_DATA_VECTOR: {
            PS_ASSERT_PTR_NON_NULL(in->data.V, NULL);
            psVector *vecCopy = psVectorCopy(NULL, (psVector*)(in->data.V),
                                             ((psVector*)(in->data.V))->type.type);
            newItem = psMetadataItemAlloc(in->name, PS_DATA_VECTOR, in->comment, vecCopy);
            psFree(vecCopy);    // Drop reference
            break;
        }
    case PS_DATA_TIME: {
            psTime *timeCopy = NULL;
            // pass through NULL values
            if (in->data.V) {
                timeCopy = psTimeCopy( (psTime*)(in->data.V) );
                if (timeCopy == NULL) {
                    psError(PS_ERR_BAD_PARAMETER_NULL, false,
                            "Error copying time.  Time skipped.\n");
                }
            }
            newItem = psMetadataItemAlloc(in->name, PS_DATA_TIME, in->comment, timeCopy);
            if (timeCopy) {
                psFree(timeCopy);   // Drop reference
            }
            break;
        }
    case PS_DATA_METADATA: {
            // Metadata: copy the next level and stuff that in too
            psMetadata *metadata = psMetadataCopy(NULL, in->data.md);
            newItem = psMetadataItemAlloc(in->name, PS_DATA_METADATA, in->comment, metadata);
            psFree(metadata);       // Drop reference
            break;
        }
    case PS_DATA_REGION: {
            psRegion *region = in->data.V; // The region
            psRegion *new = psRegionAlloc(region->x0, region->x1, region->y0, region->y1); // Copy of the region
            newItem = psMetadataItemAlloc(in->name, PS_DATA_REGION, in->comment, new);
            psFree(new);                  // Drop reference
            break;
        }
    default:
        // Other kinds of pointers
        psLogMsg(__func__, PS_LOG_WARN, "Copying a pointer in the metadata item: %x\n", in->type);
        newItem = psMetadataItemAlloc(in->name, in->type, in->comment, in->data.V);
        break;
    }

    return newItem;
}

psMetadata *psMetadataCopy(psMetadata *out,
                           const psMetadata *in)
{

    if (in == NULL) {
        psError(PS_ERR_BAD_PARAMETER_NULL, true, "psMetadataCopy: 'in' cannot be NULL");
        return NULL;
    }

    //    PS_ASSERT_PTR_NON_NULL(in->list,NULL);
    if (in->list == NULL) {
        psError(PS_ERR_BAD_PARAMETER_NULL, true, "psMetadataCopy: 'in' has NULL list");
        return NULL;
    }
    bool errorOccurred = false;
    bool outAlloced = false;
    if (out ==  NULL) {
        out = psMetadataAlloc();
        outAlloced = true;
    }
    psMetadataIterator *iter = psMetadataIteratorAlloc(in, PS_LIST_HEAD, NULL);
    psMetadataItem *inItem = NULL;
    while ((inItem = psMetadataGetAndIncrement(iter))) {
        // Need to look for MULTI, which won't be picked up using the iterator.
        psMetadataItem *multiCheckItem = psMetadataLookup(in, inItem->name);
        unsigned int flag = PS_META_REPLACE; // Flag to indicate MULTI; otherwise, replace
        if (multiCheckItem->type == PS_DATA_METADATA_MULTI) {
            psTrace("psLib.types", 10, "MULTI: %s (%s)\n", inItem->name, inItem->comment);
            flag = PS_META_DUPLICATE_OK;
        }
        psTrace("psLib.types", 5, "Copying %s (%s)...\n", inItem->name, inItem->comment);

        // Copy the item and add it on
        psMetadataItem *newItem = psMetadataItemCopy(inItem); // Copied item
        if (!psMetadataAddItem(out, newItem, PS_LIST_TAIL, flag)) {
            psErrorStackPrint(stderr, "Error copying %s (%s) in the metadata\n", inItem->name,
                              inItem->comment);
            if (outAlloced) {
                psFree(out);
                psFree(newItem);
                psFree(iter);
                return NULL;
            } else {
                errorOccurred = true;
            }
        }
        psFree(newItem);                // Drop reference
    }
    psFree(iter);

    if (errorOccurred) {
        return NULL;
    }

    return out;
}

// may need to extend this to change the keyname in the copy
bool psMetadataItemTransfer(psMetadata *out,
                            const psMetadata *in,
                            const char *key)
{
    if (in == NULL) {
        psError(PS_ERR_BAD_PARAMETER_NULL, true,
                "psMetadataItemTransfer:  'in' cannot be NULL.\n");
        return false;
    }
    if (out == NULL) {
        psError(PS_ERR_BAD_PARAMETER_NULL, true,
                "psMetadataItemTransfer:  'out' cannot be NULL.\n");
        return false;
    }
    if (key == NULL) {
        psError(PS_ERR_BAD_PARAMETER_NULL, true,
                "psMetadataItemTransfer:  'key' cannot be NULL.\n");
        return false;
    }

    psMetadataItem *item = psMetadataLookup(in, key);
    if (!item) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "psMetadataItemTransfer:  Could not find 'key' in psMetadata 'in'.\n");
        return false;
    }
    if (!psMetadataAddItem(out, item, PS_LIST_TAIL, PS_META_REPLACE) ) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "psMetadataItemTransfer:  Could not add item to psMetadata 'out'.\n");
        return false;
    }

    return true;
}


bool psMetadataAddItem(psMetadata *md,
                       const psMetadataItem *item,
                       int location,
                       psS32 flags)
{
    char * key = NULL;
    psHash *mdTable = NULL;
    psList *mdList = NULL;
    psMetadataItem *existingEntry = NULL;

    PS_ASSERT_PTR_NON_NULL(md,false);
    PS_ASSERT_PTR_NON_NULL(md->hash,false);
    PS_ASSERT_PTR_NON_NULL(md->list,false);
    PS_ASSERT_PTR_NON_NULL(item,false);
    PS_ASSERT_PTR_NON_NULL(item->name,false);

    mdTable = md->hash;
    mdList = md->list;
    key = item->name;

    // See if key is already in table
    existingEntry = (psMetadataItem*)psHashLookup(mdTable, key);

    if (item->type == PS_DATA_METADATA_MULTI) {
        // the incoming entry is PS_DATA_METADATA_MULTI

        //Shouldn't have a second reference to the same MULTI in a single Metadata!
        if (item == existingEntry) {
            psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                    "Cannot have 2 references to the same MULTI in a single Metadata!");
            return false;
        }

        // force the hash entry to be PS_DATA_METADATA_MULTI
        existingEntry = makeMetaMulti(mdTable,key,existingEntry);

        // add all the items in the incoming entry to metadata
        psList* list = item->data.list;
        if (list != NULL) {
            psListIterator* iter = psListIteratorAlloc(list,PS_LIST_HEAD,true);
            psMetadataItem* listItem;
            while ((listItem=(psMetadataItem*)psListGetAndIncrement(iter)) != NULL) {
                psMetadataAddItem(md,listItem,location,flags);
            }
            psFree(iter);
        }

        return true; // all done.
    }

    // how the item is added to the hash depends on prior existence, flags, etc.
    if(existingEntry != NULL) { // prior existence
        if (existingEntry->type == PS_DATA_METADATA_MULTI || (flags & PS_META_DUPLICATE_OK) != 0) {
            // duplicate entries allowed - add another entry.

            // make sure the existing entry is PS_DATA_METADATA_MULTI
            existingEntry = makeMetaMulti(mdTable,key,existingEntry);

            // add to the hash's list of duplicate entries
            if (! psListAdd(existingEntry->data.list, PS_LIST_TAIL, (psPtr)item) ) {
                psError(PS_ERR_UNKNOWN, false,
                        _("Failed to add metadata item, %s, to metadata collection list."),
                        key);
                return false;
            }
        } else if ((flags & PS_META_REPLACE) != 0) {
            // replace entry instead of creating a duplicate entry.

            // remove the existing entry from metadata
            psMetadataRemoveKey(md, key);

            // treat as if new (added to list below)
            /*  The following is unneeded.  HashAdd can't return false with non-null inputs.
                        if(!psHashAdd(mdTable, key, (psPtr)item)) {
                            psError(PS_ERR_UNKNOWN,false,
                                _("Failed to add metadata item, %s, to items table."),key);
                            return false;
                        }
            */
            psHashAdd(mdTable, key, (psPtr)item);

            // Generate warning that item has been replaced
            //            psLogMsg(__func__,PS_LOG_WARN,"Metadata item %s has been replaced",
            //                     existingEntry->name);
        } else {
            // default is to error on duplicate entry.
            if ((flags & PS_META_NO_REPLACE) != 0) {
                return true;
            }
            psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                    _("Duplicate metadata item name: %s is not allowed.  "
                      "Use a psMetadataFlags option to allow such action."), item->name);
            return false;
        }
    } else {
        // OK, this is a new item.

        // Node doesn't exist - Add new metadata item to metadata collection's hash
        /*  The following is unneeded.  HashAdd can't return false with non-null inputs.

                if(!psHashAdd(mdTable, key, (psPtr)item)) {
                    psError(PS_ERR_UNKNOWN,false,
                            _("Failed to add metadata item, %s, to items table."),key);
                    return false;
                }
        */
        psHashAdd(mdTable, key, (psPtr)item);
        // Create a multi, if required
        if (flags & PS_META_DUPLICATE_OK) {
            makeMetaMulti(mdTable,key,(psMetadataItem*)item); // Casting away const!
        }
    }

    if(!psListAdd(mdList, location, (psPtr)item)) {
        psError(PS_ERR_UNKNOWN, false,
                _("Failed to add metadata item, %s, to metadata collection list."), key);
        return false;
    }

    return true;
}

bool psMetadataAdd(psMetadata *md,
                   long location,
                   const char *name,
                   int format,
                   const char *comment,...)
{
    PS_ASSERT_PTR_NON_NULL(md,false);
    PS_ASSERT_PTR_NON_NULL(md->list,false);
    va_list argPtr;

    va_start(argPtr, comment);
    psBool result = psMetadataAddV(md,location,name,format,comment,argPtr);
    va_end(argPtr);

    return result;
}

bool psMetadataAddV(psMetadata *md,
                    long location,
                    const char *name,
                    int format,
                    const char *comment,
                    va_list list)
{
    PS_ASSERT_PTR_NON_NULL(md,false);
    PS_ASSERT_PTR_NON_NULL(md->list,false);
    psMetadataItem* metadataItem = NULL;

    metadataItem = psMetadataItemAllocV(name, format & PS_METADATA_TYPE_MASK, comment, list);

    if (!psMetadataAddItem(md, metadataItem, location, format & PS_METADATA_FLAGS_MASK)) {
        psError(PS_ERR_UNKNOWN,false,_("Failed to add metadata item to metadata collection list."));
        psFree(metadataItem);
        return false;
    }
    // Decrement reference count, since the metadata item is now in metadata collection and no longer needed
    psFree(metadataItem);

    return true;
}

#define METADATA_ADD_TYPE(NAME,TYPE,METATYPE) \
bool psMetadataAdd##NAME(psMetadata* md, long where, const char* name, \
                         int format, const char* comment, TYPE value) { \
    return psMetadataAdd(md, where, name, format | METATYPE, comment, value); \
}

METADATA_ADD_TYPE(Bool,psBool,PS_DATA_BOOL)
METADATA_ADD_TYPE(S8,psS8,PS_DATA_S8)
METADATA_ADD_TYPE(S16,psS16,PS_DATA_S16)
METADATA_ADD_TYPE(S32,psS32,PS_DATA_S32)
METADATA_ADD_TYPE(S64,psS64,PS_DATA_S64)
METADATA_ADD_TYPE(U8,psU8,PS_DATA_U8)
METADATA_ADD_TYPE(U16,psU16,PS_DATA_U16)
METADATA_ADD_TYPE(U32,psU32,PS_DATA_U32)
METADATA_ADD_TYPE(U64,psU64,PS_DATA_U64)
METADATA_ADD_TYPE(F32,psF32,PS_DATA_F32)
METADATA_ADD_TYPE(F64,psF64,PS_DATA_F64)
METADATA_ADD_TYPE(List,psList*,PS_DATA_LIST)
METADATA_ADD_TYPE(Str,const char*,PS_DATA_STRING)
METADATA_ADD_TYPE(Vector,psVector*,PS_DATA_VECTOR)
METADATA_ADD_TYPE(Image,psImage*,PS_DATA_IMAGE)
METADATA_ADD_TYPE(Hash,psHash*,PS_DATA_HASH)
METADATA_ADD_TYPE(LookupTable,psLookupTable*,PS_DATA_LOOKUPTABLE)
METADATA_ADD_TYPE(Metadata,psMetadata*,PS_DATA_METADATA)
METADATA_ADD_TYPE(Array,psArray*,PS_DATA_ARRAY)
METADATA_ADD_TYPE(Time,psTime*,PS_DATA_TIME)
METADATA_ADD_TYPE(Unknown,psPtr,PS_DATA_UNKNOWN)

bool psMetadataAddPtr(psMetadata* md, long where, const char* name,
                      psDataType type, const char* comment, psPtr value)
{
    return psMetadataAdd(md, where, name, type, comment, value);
}



// Remove by key name
bool psMetadataRemoveKey(psMetadata *md,
                         const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md, false);

    PS_ASSERT_PTR_NON_NULL(md->list, false);
    psList* mdList = md->list;

    PS_ASSERT_PTR_NON_NULL(md->hash,false);
    psHash* mdTable = md->hash;

    PS_ASSERT_PTR_NON_NULL(key, false);
    PS_ASSERT_INT_POSITIVE(strlen(key), false);

    psMetadataItem* entry = psHashLookup(mdTable,key);
    if (entry == NULL) {
        psError(PS_ERR_UNKNOWN, false,
                _("Failed to remove metadata item, %s, from metadata table."), key);
        return false;
    }
    if (entry->type == PS_DATA_METADATA_MULTI) {
        psMetadataItem* listItem;
        psListIterator* iter = psListIteratorAlloc(
                                   entry->data.list,
                                   PS_LIST_HEAD,true);
        while ((listItem=psListGetAndIncrement(iter)) != NULL) {
            psListRemoveData(mdList, listItem);
        }
        psFree(iter);
        psHashRemove(mdTable,key);
    } else {
        psListRemoveData(mdList, entry);
        psHashRemove(mdTable, key);
    }

    return true;
}


// Remove by index
bool psMetadataRemoveIndex(psMetadata *md,
                           long location)
{
    PS_ASSERT_PTR_NON_NULL(md, false);

    PS_ASSERT_PTR_NON_NULL(md->list, false);
    psList* mdList = md->list;

    PS_ASSERT_PTR_NON_NULL(md->hash,false);
    psHash* mdTable = md->hash;

    psMetadataItem* entry = psListGet(mdList, location);
    if (entry == NULL) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                _("Could not find metadata item at index %ld."), location);
        return false;
    }
    const char *key = entry->name;

    if (key == NULL) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                _("Failed to remove metadata item, at index %ld, from metadata list."),
                location);
        return false;
    }

    psMetadataItem* tableItem = psHashLookup(mdTable, key);
    if (tableItem == NULL) {
        psError(PS_ERR_UNKNOWN, false,
                _("Failed to remove metadata item, %s, from metadata table."), key);
        return false;
    }

    if (tableItem->type == PS_DATA_METADATA_MULTI) {
        // multiple entries with same key, remove just the specified one
        psListRemoveData(tableItem->data.list, entry);
    } else {
        //Tested below.  psHashRemove can't return false here.
        psHashRemove(mdTable, key);
    }
    psListRemove(mdList, location);

    return true;
}

psMetadataItem* psMetadataLookup(const psMetadata *md,
                                 const char *key)
{
    psHash* mdTable = NULL;
    psMetadataItem* entry = NULL;


    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->hash,NULL);
    PS_ASSERT_PTR_NON_NULL(key,NULL);

    mdTable = md->hash;
    entry = (psMetadataItem*)psHashLookup(mdTable, key);

    return entry;
}

void* psMetadataLookupPtr(bool *status,
                          const psMetadata *md,
                          const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    psMetadataItem *metadataItem = NULL;

    if (status) {
        *status = true;
    }

    metadataItem = psMetadataLookup(md, key);
    if(metadataItem == NULL) {
        if (status) {
            *status = false;
        }
        return NULL;
    }
    if (metadataItem->type == PS_DATA_METADATA_MULTI) {
        // if multiple keys found, use the first.
        //        metadataItem = (psMetadataItem*)((metadataItem->data.list)->head);
        metadataItem = (psMetadataItem*)(metadataItem->data.list->head->data);
        if (status) {
            *status = true;
        }
    }

    if(PS_DATA_IS_PRIMITIVE(metadataItem->type)) {
        if (status) {
            *status = false;
        }
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                _("Specified psDataType, %d, is not supported."),
                metadataItem->type);
        return NULL;
    } else {
        return metadataItem->data.V;
    }
}

#define psMetadataLookupNumTYPE(TYPE) \
ps##TYPE psMetadataLookup##TYPE(bool *status, const psMetadata *md, const char *key) \
{ \
    psMetadataItem *metadataItem = NULL; \
    ps##TYPE value = 0; \
    \
    if (status) {  \
        *status = true; \
    } \
    \
    metadataItem = psMetadataLookup(md, key); \
    if(metadataItem == NULL) { \
        if (status) {  \
            *status = false; \
        } \
        return 0; \
    } \
    if (metadataItem->type == PS_DATA_METADATA_MULTI) { \
        /* if multiple keys found, use the first. */ \
        metadataItem = (psMetadataItem*)((metadataItem->data.list)->head); \
    } \
    \
    switch (metadataItem->type) { \
    case PS_DATA_S8: \
        value = (ps##TYPE)metadataItem->data.S8; \
        break; \
    case PS_DATA_S16: \
        value = (ps##TYPE)metadataItem->data.S16; \
        break; \
    case PS_DATA_S32: \
        value = (ps##TYPE)metadataItem->data.S32; \
        break; \
    case PS_DATA_S64: \
        value = (ps##TYPE)metadataItem->data.S64; \
        break; \
    case PS_DATA_U8: \
        value = (ps##TYPE)metadataItem->data.U8; \
        break; \
    case PS_DATA_U16: \
        value = (ps##TYPE)metadataItem->data.U16; \
        break; \
    case PS_DATA_U32: \
        value = (ps##TYPE)metadataItem->data.U32; \
        break; \
    case PS_DATA_U64: \
        value = (ps##TYPE)metadataItem->data.U64; \
        break; \
    case PS_DATA_F32: \
        value = (ps##TYPE)metadataItem->data.F32; \
        break; \
    case PS_DATA_F64: \
        value = (ps##TYPE)metadataItem->data.F64; \
        break; \
    case PS_DATA_BOOL: \
        if (metadataItem->data.B) { \
            value = 1; \
        } \
        break; \
    default: \
        /* if you get to this point, the value is not a number. */ \
        if (status) {  \
            *status = false; \
        }  \
        break; \
    } \
    \
    /* psFree(metadataItem); currently, the lookup doesn't increment the ref count */ \
    return value; \
}

psMetadataLookupNumTYPE(F32)
psMetadataLookupNumTYPE(F64)
psMetadataLookupNumTYPE(S8)
psMetadataLookupNumTYPE(S16)
psMetadataLookupNumTYPE(S32)
psMetadataLookupNumTYPE(S64)
psMetadataLookupNumTYPE(U8)
psMetadataLookupNumTYPE(U16)
psMetadataLookupNumTYPE(U32)
psMetadataLookupNumTYPE(U64)
psMetadataLookupNumTYPE(Bool)

psMetadataItem* psMetadataGet(const psMetadata *md,
                              int location)
{
    psMetadataItem* entry = NULL;

    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);

    entry = (psMetadataItem*) psListGet(md->list, location);
    if (entry == NULL) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, _("Could not find metadata item at index %d."), location);
        return NULL;
    }

    return entry;
}

// XXX should md be const?
psMetadataIterator* psMetadataIteratorAlloc(const psMetadata* md,
        long location,
        const char* regex)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);

    psMetadataIterator* newIter = psAlloc(sizeof(psMetadataIterator));
    psMemSetDeallocator(newIter, (psFreeFunc) metadataIteratorFree);

    newIter->iter = psListIteratorAlloc(md->list, location, false);

    if (regex) {
        newIter->regex = psAlloc(sizeof(regex_t));
        int regRtn = regcomp(newIter->regex, regex, 0);
        if (regRtn != 0) {
            char errMsg[256];
            regerror(regRtn, newIter->regex, errMsg, 256);
            psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                    _("Specified regular expression is invalid.  %s."),
                    errMsg);
            psFree(newIter);
            return NULL;
        }
        if (!psMetadataIteratorSet(newIter, location)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to set location=%ld for metadata iterator.\n", location);
            psFree(newIter);
            return NULL;
        }
    } else {
        newIter->regex = NULL;
    }

    return newIter;
}

bool psMetadataIteratorSet(psMetadataIterator* iterator,
                           long location)
{
    PS_ASSERT_PTR_NON_NULL(iterator, NULL);
    psListIterator *listIter = iterator->iter;
    PS_ASSERT_PTR_NON_NULL(listIter, NULL);

    regex_t *regex = iterator->regex;

    // handle trivial case where no regex subsetting is required.
    if (regex == NULL) {
        return psListIteratorSet(listIter, location);
    }

    // If there's a regex, then we need to count into the list.
    // We count ONLY those entries that match the regex

    // Here we count in from the tail
    if (location < 0) {
        psListIteratorSet(listIter, PS_LIST_TAIL);
        psMetadataItem *item;           // Item from iteration
        int match = 0;                  // Match number
        while (listIter->cursor && (item = listIter->cursor->data)) {
            if (regexec(regex, item->name, 0, NULL, 0) == 0) {
                // this key is a match
                match--;
                if (match == location) {
                    break;
                }
            }
            (void)psListGetAndDecrement(listIter);
        }
        return (match == location);
    }

    // Here we count in from the head
    psListIteratorSet(listIter, PS_LIST_HEAD);
    psMetadataItem *item;               // Item from iteration
    int match = -1;                     // Match number
    while (listIter->cursor && (item = listIter->cursor->data)) {
        if (regexec(regex, item->name, 0, NULL, 0) == 0) {
            // this key is a match
            match++;
            if (match == location) {
                break;
            }
        }
        (void)psListGetAndIncrement(listIter);
    }
    return (match == location);
}

psMetadataItem* psMetadataGetAndIncrement(psMetadataIterator* iterator)
{
    PS_ASSERT_PTR_NON_NULL(iterator,NULL);
    psListIterator* listIter = iterator->iter;
    PS_ASSERT_PTR_NON_NULL(listIter,NULL);

    regex_t* regex = iterator->regex;

    // handle trivial case where no regex subsetting is required.
    if (regex == NULL) {
        return (psMetadataItem*)psListGetAndIncrement(listIter);
    }

    // Iterate until we find something matching the regex
    psMetadataItem *newItem;            // New MD item from iteration
    while ((newItem = psListGetAndIncrement(listIter))) {
        if (regexec(regex, newItem->name, 0, NULL, 0) == 0) {
            // this key is a match
            break;
        }
    }
    return newItem;
}

psMetadataItem* psMetadataGetAndDecrement(psMetadataIterator* iterator)
{
    PS_ASSERT_PTR_NON_NULL(iterator, NULL);
    psListIterator* listIter = iterator->iter;
    PS_ASSERT_PTR_NON_NULL(listIter, NULL);

    regex_t* regex = iterator->regex;

    // handle trivial case where no regex subsetting is required.
    if (regex == NULL) {
        return (psMetadataItem*)psListGetAndDecrement(listIter);
    }

    // Iterate until we find something matching the regex
    psMetadataItem *newItem;            // New MD item from iteration
    while ((newItem = psListGetAndDecrement(listIter))) {
        if (regexec(regex, newItem->name, 0, NULL, 0) == 0) {
            // this key is a match
            break;
        }
    }
    return newItem;
}

psMetadata *psMetadataLookupMetadata(bool *status,
                                     const psMetadata *md,
                                     const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    psMetadataItem *item = psMetadataLookup((psMetadata*)md, key); // The metadata with instruments
    psMetadata *value = NULL;  // The value to return
    if (!item) {
        // The given key isn't in the metadata
        if (status) {
            *status = false;
        } else {
            psError(PS_ERR_IO, true, "Couldn't find %s in the metadata.\n", key);
        }
    } else if (item->type != PS_DATA_METADATA) {
        // The value at the key isn't metadata
        if (status) {
            *status = false;
        } else {
            psLogMsg(__func__, PS_LOG_WARN, "%s isn't of type PS_DATA_META, as expected.\n", key);
        }
        //        value = NULL;
    } else {
        // We have the requested metadata
        if (status) {
            *status = true;
        }
        value = item->data.md; // The requested metadata
    }
    return value;
}

psTime *psMetadataLookupTime(bool *status,
                             const psMetadata *md,
                             const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);

    psMetadataItem *item = psMetadataLookup((psMetadata*)md, key);
    if (!item) {
        // The given key isn't in the metadata
        if (status) {
            *status = false;
        } else {
            psError(PS_ERR_IO, true, "Couldn't find %s in the metadata.\n", key);
        }
        return NULL;
    }

    if (item->type != PS_DATA_TIME) {
        // The value at the key isn't metadata
        if (status) {
            *status = false;
        } else {
            psLogMsg(__func__, PS_LOG_WARN, "%s isn't of type PS_DATA_TIME, as expected.\n", key);
        }
        return NULL;
    }

    // We have the requested metadata
    if (status) {
        *status = true;
    }

    return item->data.V;
}


psString psMetadataLookupStr(bool *status,
                             const psMetadata *md,
                             const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    psMetadataItem *item = psMetadataLookup((psMetadata*)md, key); // The metadata with instruments
    //    char *value = NULL;   // The value to return
    psString value = NULL;
    if (!item) {
        // The given key isn't in the metadata
        if (status) {
            *status = false;
        } else {
            psError(PS_ERR_IO, true, "Couldn't find %s in the metadata.\n", key);
        }
    } else if (item->type != PS_DATA_STRING) {
        // The value at the key isn't of the desired type
        if (status) {
            *status = false;
        } else {
            psLogMsg(__func__, PS_LOG_WARN, "%s isn't of type PS_DATA_STRING, as expected.\n", key);
        }
        //        value = NULL;
    } else {
        // We have the requested metadata
        if (status) {
            *status = true;
        }
        value = item->data.V;
    }
    return value;
}


bool psMetadataPrint(FILE *fd,
                     const psMetadata *md,
                     int level)
{
    PS_ASSERT_PTR_NON_NULL(md,false);
    if (fd == NULL) {
        fd = stdout;
    } else {
        if ( fprintf(fd, "\n") < 0 ) {
            psError(PS_ERR_IO, true,
                    "Invalid file pointer in psMetadataPrint.  Could not write to fd.\n");
            return false;
        }
    }

    psErrorClear();   // we're going to _append_ errors to the stack
    bool noErrors = true;  // have see seen any errors?

    // Casting away const --- the addition of an iterator should not be considered an invasion of "const".
    psMetadataIterator *iter = psMetadataIteratorAlloc(md, PS_LIST_HEAD, NULL); // Iterator
    psMetadataItem *item = NULL; // Item from metadata
    while ( (item = psMetadataGetAndIncrement(iter)) ) {
        //So that unsupported types are not printed here, adding a switch statement
        switch(item->type) {
        case PS_DATA_STRING:
        case PS_DATA_BOOL:
        case PS_DATA_S8:
        case PS_DATA_S16:
        case PS_DATA_S32:
        case PS_DATA_S64:
        case PS_DATA_U8:
        case PS_DATA_U16:
        case PS_DATA_U32:
        case PS_DATA_U64:
        case PS_DATA_F32:
        case PS_DATA_F64:
        case PS_DATA_METADATA:
        case PS_DATA_REGION:
        case PS_DATA_LIST:
        case PS_DATA_TIME:
        case PS_DATA_VECTOR:
            // Indent...
            //                fprintf(fd, "\n");
            for (int i = 0; i < level; i++) {
                fprintf(fd, "  ");
            }
            fprintf(fd, "%s", item->name);
            if (item->comment && strlen(item->comment) > 0) {
                fprintf(fd, " (%s)", item->comment);
            }
            fprintf(fd, ": ");
            break;
        case PS_DATA_UNKNOWN:
        default:
            break;
        }
        switch (item->type) {
        case PS_DATA_STRING:
            fprintf(fd, "%s\n", item->data.str);
            break;
        case PS_DATA_BOOL:
            if (item->data.B) {
                fprintf(fd, "True\n");
            } else {
                fprintf(fd, "False\n");
            }
            break;
        case PS_DATA_S8:
            fprintf(fd, "%d\n", item->data.S8);
            break;
        case PS_DATA_S16:
            fprintf(fd, "%d\n", item->data.S16);
            break;
        case PS_DATA_S32:
            fprintf(fd, "%d\n", item->data.S32);
            break;
        case PS_DATA_S64:
            fprintf(fd, "%jd\n", item->data.S64);
            break;
        case PS_DATA_U8:
            fprintf(fd, "%u\n", item->data.U8);
            break;
        case PS_DATA_U16:
            fprintf(fd, "%u\n", item->data.U16);
            break;
        case PS_DATA_U32:
            fprintf(fd, "%u\n", item->data.U32);
            break;
        case PS_DATA_U64:
            fprintf(fd, "%ju\n", item->data.U64);
            break;
        case PS_DATA_F32:
            fprintf(fd, "%f\n", item->data.F32);
            break;
        case PS_DATA_F64:
            fprintf(fd, "%f\n", item->data.F64);
            break;
        case PS_DATA_METADATA:
            //            fprintf(fd, "\n");
            psMetadataPrint(fd, item->data.V, level + 1);
            for (int i = 0; i < level; i++) {
                fprintf(fd, "  ");
            }
            fprintf(fd, "%s  -- END\n", item->name);
            break;
        case PS_DATA_REGION: {
                psString region = psRegionToString(*(psRegion*)item->data.V);
                fprintf(fd, "%s\n", region);
                psFree(region);
                break;
            }
        case PS_DATA_LIST:
            fprintf(fd, "<a list of size %ld>\n", ((psList*)item->data.V)->n);
            break;
        case PS_DATA_TIME: {
                psString time = psTimeToISO(item->data.V);
                fprintf(fd, "%s\n", time);
                psFree(time);
                break;
            }
        case PS_DATA_VECTOR: {
                psVector *vector = item->data.V;
                switch (vector->type.type) {
                case PS_DATA_U8:
                    fprintf(fd, "U8  ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%u ", vector->data.U8[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_U16:
                    fprintf(fd, "U16  ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%u ", vector->data.U16[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_U32:
                    fprintf(fd, "U32  ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%u ", vector->data.U32[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_U64:
                    fprintf(fd, "U64  ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%ju ", vector->data.U64[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_S8:
                    fprintf(fd, "S8  ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%d ", vector->data.S8[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_S16:
                    fprintf(fd, "S16  ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%d ", vector->data.S16[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_S32:
                    fprintf(fd, "S32  ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%d ", vector->data.S32[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_S64:
                    fprintf(fd, "S64  ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%jd ", vector->data.S64[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_F32:
                    fprintf(fd, "F32 ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%f ", vector->data.F32[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_F64:
                    fprintf(fd, "F64 ");
                    for (int i = 0; i < vector->n; i++) {
                        fprintf(fd, "%f ", vector->data.F64[i]);
                    }
                    fprintf(fd, "\n");
                    break;
                case PS_DATA_UNKNOWN:
                default:
                    psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                            _("Specified psDataType, %d, is not supported."), vector->type.type);
                    psFree(iter);
                    fprintf(fd, "<Unsupported type>\n");
                    return false;
                }
                //                fprintf(fd, "\n");
                break;
            }
        default:
            //            fprintf(fd, "\n");
            psError(PS_ERR_IO, false, "Non-printable metadata type for \"%s\": %x\n",
                    item->name, item->type);
            noErrors = false;
            break;
        }
        //        fprintf(fd, "\n");
    }
    //    fprintf(fd, "\n");
    psFree(iter);

    return noErrors;
}

psPolynomial1D *psPolynomial1DfromMetadata(const psMetadata *folder)
{
    PS_ASSERT_PTR_NON_NULL(folder, NULL);
    bool status;
    char keyword[80];

    // get polynomial orders
    int nXorder = psMetadataLookupS32 (&status, folder, "NORDER_X");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial1D in metadata is missing NORDER_X");
        return NULL;
    }
    // how many polynomial coeffs are expected?
    int nElementsExpected = psMetadataLookupS32 (&status, folder, "NELEMENTS");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial1D in metadata is missing NELEMENTS");
        return NULL;
    }

    psPolynomial1D *poly = psPolynomial1DAlloc (PS_POLYNOMIAL_ORD, nXorder);

    int nElements = 0;
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        sprintf (keyword, "VAL_X%02d", nx);
        poly->coeff[nx] = psMetadataLookupF64 (&status, folder, keyword);
        if (!status) {
            // an undefined component implies the component was masked
            // this is symmetrical with the 1DtoMD function
            poly->mask[nx] = 1;
            poly->coeff[nx] = 0;
            poly->coeffErr[nx] = 0;
        } else {
            poly->mask[nx] = 0;
            nElements ++;
        }
        sprintf (keyword, "ERR_X%02d", nx);
        poly->coeffErr[nx] = psMetadataLookupF64 (&status, folder, keyword);
    }
    if (nElements != nElementsExpected) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "psPolynomial1D in metadata does not have the correct number of coefficients: "
                "%d found vs %d expected", nElements, nElementsExpected);
        psFree(poly);
        return NULL;
    }
    return (poly);
}

// XXX : these may need F64, or %g format for output
bool psPolynomial1DtoMetadata(psMetadata *md,
                              const psPolynomial1D *poly,
                              const char *format,
                              ...)
{
    PS_ASSERT_PTR_NON_NULL(md, false);
    PS_ASSERT_PTR_NON_NULL(poly, false);
    //XXX:  Current implementation only supports ordinary polynomials.
    if (poly->type != PS_POLYNOMIAL_ORD)
        return false;

    int Nbyte;
    char tmp;
    char *root;
    va_list argp;

    va_start (argp, format);
    Nbyte = vsnprintf (&tmp, 0, format, argp);
    va_end (argp);

    if (Nbyte <= 0)
        return false;

    va_start (argp, format);
    root = (char *) psAlloc (Nbyte + 1);
    memset (root, 0, Nbyte + 1);
    vsnprintf (root, Nbyte + 1, format, argp);
    va_end (argp);

    psMetadata *folder = psMetadataAlloc ();

    // specify the polynomial orders
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_X", PS_DATA_S32, "number of x orders", poly->nX);

    char namespace[80];
    char namespace_err[80];
    int nElements = 0;   // count the number of unmasked elements

    // place polynomial entries on folder
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        if (poly->mask[nx] == 0) {
            sprintf(namespace, "VAL_X%02d", nx);
            sprintf(namespace_err, "ERR_X%02d", nx);
            psMetadataAdd (folder, PS_LIST_TAIL, namespace, PS_DATA_F64,
                           "polynomial coefficient", poly->coeff[nx]);
            psMetadataAdd (folder, PS_LIST_TAIL, namespace_err, PS_DATA_F64,
                           "polynomial coefficient error", poly->coeffErr[nx]);
            nElements ++;
        }
    }
    psMetadataAdd (folder, PS_LIST_TAIL, "NELEMENTS", PS_DATA_S32, "number of unmasked coeffs", nElements);
    psMetadataAdd (md, PS_LIST_TAIL, root, PS_DATA_METADATA, "folder for 1D polynomial", folder);
    psFree (root);
    psFree(folder);
    return true;
}

psPolynomial2D *psPolynomial2DfromMetadata(const psMetadata *folder)
{
    PS_ASSERT_PTR_NON_NULL(folder, NULL);
    bool status;
    char keyword[80];

    // get polynomial orders
    int nXorder = psMetadataLookupS32 (&status, folder, "NORDER_X");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial2D in metadata is missing NORDER_X");
        return NULL;
    }
    int nYorder = psMetadataLookupS32 (&status, folder, "NORDER_Y");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial2D in metadata is missing NORDER_Y");
        return NULL;
    }
    // how many polynomial coeffs are expected?
    int nElementsExpected = psMetadataLookupS32 (&status, folder, "NELEMENTS");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial2D in metadata is missing NELEMENTS");
        return NULL;
    }

    psPolynomial2D *poly = psPolynomial2DAlloc (PS_POLYNOMIAL_ORD, nXorder, nYorder);

    int nElements = 0;
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            sprintf (keyword, "VAL_X%02d_Y%02d", nx, ny);
            poly->coeff[nx][ny] = psMetadataLookupF64 (&status, folder, keyword);
            if (!status) {
                // an undefined component implies the component was masked
                // this is symmetrical with the 2DtoMD function
                poly->mask[nx][ny] = 1;
                poly->coeff[nx][ny] = 0;
                poly->coeffErr[nx][ny] = 0;
            } else {
                poly->mask[nx][ny] = 0;
                nElements ++;
            }
            sprintf (keyword, "ERR_X%02d_Y%02d", nx, ny);
            poly->coeffErr[nx][ny] = psMetadataLookupF64 (&status, folder, keyword);
        }
    }
    if (nElements != nElementsExpected) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "psPolynomial2D in metadata does not have the correct number of coefficients: "
                "%d found vs %d expected", nElements, nElementsExpected);
        psFree(poly);
        return NULL;
    }
    return (poly);
}

// XXX : these may need F64, or %g format for output
bool psPolynomial2DtoMetadata (psMetadata *md,
                               const psPolynomial2D *poly,
                               const char *format,
                               ...)
{
    PS_ASSERT_PTR_NON_NULL(md, false);
    PS_ASSERT_PTR_NON_NULL(poly, false);
    //XXX:  Current implementation only supports ordinary polynomials.
    if (poly->type != PS_POLYNOMIAL_ORD)
        return false;

    // XXX I'm puzzled by this test.  a polynomial of 0 order with a value of 0 is a
    // perfectly valid polynomial and can be written out.  a polynomial with all elements
    // masked still carries information.
    //Make sure polynomial isn't 0, completely empty
    //if (poly->nX == 0 && poly->nY == 0 && poly->coeff[0][0] == 0)
    //return false;

    int Nbyte;
    char tmp;
    char *root;
    va_list argp;

    va_start (argp, format);
    Nbyte = vsnprintf (&tmp, 0, format, argp);
    va_end (argp);

    if (Nbyte <= 0)
        return false;

    va_start (argp, format);
    root = (char *) psAlloc (Nbyte + 1);
    memset (root, 0, Nbyte + 1);
    vsnprintf (root, Nbyte + 1, format, argp);
    va_end (argp);

    psMetadata *folder = psMetadataAlloc ();

    // specify the polynomial orders
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_X", PS_DATA_S32, "number of x orders", poly->nX);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Y", PS_DATA_S32, "number of y orders", poly->nY);

    char namespace[80];
    char namespace_err[80];
    int nElements = 0;   // count the number of unmasked elements

    // place polynomial entries on folder
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            if (poly->mask[nx][ny] == 0) {
                sprintf(namespace, "VAL_X%02d_Y%02d", nx, ny);
                sprintf(namespace_err, "ERR_X%02d_Y%02d", nx, ny);
                psMetadataAdd (folder, PS_LIST_TAIL, namespace, PS_DATA_F64,
                               "polynomial coefficient", poly->coeff[nx][ny]);
                psMetadataAdd (folder, PS_LIST_TAIL, namespace_err, PS_DATA_F64,
                               "polynomial coefficient error", poly->coeffErr[nx][ny]);
                nElements ++;
            }
        }
    }
    psMetadataAdd (folder, PS_LIST_TAIL, "NELEMENTS", PS_DATA_S32, "number of unmasked coeffs", nElements);
    psMetadataAdd (md, PS_LIST_TAIL, root, PS_DATA_METADATA, "folder for 2D polynomial", folder);
    psFree (root);
    psFree(folder);
    return true;
}

psPolynomial3D *psPolynomial3DfromMetadata (const psMetadata *folder)
{
    PS_ASSERT_PTR_NON_NULL(folder, NULL);

    bool status;
    char keyword[80];

    // get polynomial orders
    int nXorder = psMetadataLookupS32 (&status, folder, "NORDER_X");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial3D in metadata is missing NORDER_X");
        return NULL;
    }
    int nYorder = psMetadataLookupS32 (&status, folder, "NORDER_Y");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial3D in metadata is missing NORDER_Y");
        return NULL;
    }
    int nZorder = psMetadataLookupS32 (&status, folder, "NORDER_Z");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial3D in metadata is missing NORDER_Z");
        return NULL;
    }
    // how many polynomial coeffs are expected?
    int nElementsExpected = psMetadataLookupS32 (&status, folder, "NELEMENTS");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial3D in metadata is missing NELEMENTS");
        return NULL;
    }

    psPolynomial3D *poly = psPolynomial3DAlloc (PS_POLYNOMIAL_ORD, nXorder, nYorder, nZorder);

    int nElements = 0;
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            for (int nz = 0; nz < poly->nZ + 1; nz++) {
                sprintf (keyword, "VAL_X%02d_Y%02d_Z%02d", nx, ny, nz);
                poly->coeff[nx][ny][nz] = psMetadataLookupF64 (&status, folder, keyword);
                if (!status) {
                    // an undefined component implies the component was masked
                    // this is symmetrical with the 3DtoMD function
                    poly->mask[nx][ny][nz] = 1;
                    poly->coeff[nx][ny][nz] = 0;
                    poly->coeffErr[nx][ny][nz] = 0;
                } else {
                    poly->mask[nx][ny][nz] = 0;
                    nElements ++;
                }
                sprintf (keyword, "ERR_X%02d_Y%02d_Z%02d", nx, ny, nz);
                poly->coeffErr[nx][ny][nz] = psMetadataLookupF64 (&status, folder, keyword);
            }
        }
    }
    if (nElements != nElementsExpected) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "psPolynomial3D in metadata does not have the correct number of coefficients: "
                "%d found vs %d expected", nElements, nElementsExpected);
        psFree(poly);
        return NULL;
    }
    return (poly);
}

bool psPolynomial3DtoMetadata (psMetadata *md,
                               const psPolynomial3D *poly,
                               const char *format,
                               ...)
{
    PS_ASSERT_PTR_NON_NULL(md, false);
    PS_ASSERT_PTR_NON_NULL(poly, false);
    //XXX:  Current implementation only supports ordinary polynomials.
    if (poly->type != PS_POLYNOMIAL_ORD)
        return false;

    int Nbyte;
    char tmp;
    char *root;
    va_list argp;

    va_start (argp, format);
    Nbyte = vsnprintf (&tmp, 0, format, argp);
    va_end (argp);

    if (Nbyte <= 0)
        return false;

    va_start (argp, format);
    root = (char *) psAlloc (Nbyte + 1);
    memset (root, 0, Nbyte + 1);
    vsnprintf (root, Nbyte + 1, format, argp);
    va_end (argp);

    psMetadata *folder = psMetadataAlloc ();

    // specify the polynomial orders
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_X", PS_DATA_S32, "number of x orders", poly->nX);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Y", PS_DATA_S32, "number of y orders", poly->nY);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Z", PS_DATA_S32, "number of z orders", poly->nZ);

    char namespace[80];
    char namespace_err[80];
    int nElements = 0;   // count the number of unmasked elements

    // place polynomial entries on folder
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            for (int nz = 0; nz < poly->nZ + 1; nz++) {
                if (poly->mask[nx][ny][nz] == 0) {
                    sprintf(namespace, "VAL_X%02d_Y%02d_Z%02d", nx, ny, nz);
                    sprintf(namespace_err, "ERR_X%02d_Y%02d_Z%02d", nx, ny, nz);
                    psMetadataAdd (folder, PS_LIST_TAIL, namespace,
                                   PS_DATA_F64, "polynomial coefficient",
                                   poly->coeff[nx][ny][nz], nx, ny, nz);
                    psMetadataAdd (folder, PS_LIST_TAIL, namespace_err,
                                   PS_DATA_F64, "polynomial coeffficient error",
                                   poly->coeffErr[nx][ny][nz], nx, ny, nz);
                    nElements ++;
                }
            }
        }
    }
    psMetadataAdd (folder, PS_LIST_TAIL, "NELEMENTS", PS_DATA_S32, "number of unmasked coeffs", nElements);
    psMetadataAdd (md, PS_LIST_TAIL, root, PS_DATA_METADATA, "folder for 3D polynomial", folder);
    psFree(root);
    psFree(folder);
    return true;
}

psPolynomial4D *psPolynomial4DfromMetadata(const psMetadata *folder)
{
    PS_ASSERT_PTR_NON_NULL(folder, NULL);

    bool status;
    char keyword[80];

    int nXorder = psMetadataLookupS32 (&status, folder, "NORDER_X");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial4D in metadata is missing NORDER_X");
        return NULL;
    }
    int nYorder = psMetadataLookupS32 (&status, folder, "NORDER_Y");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial4D in metadata is missing NORDER_Y");
        return NULL;
    }
    int nZorder = psMetadataLookupS32 (&status, folder, "NORDER_Z");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial4D in metadata is missing NORDER_Z");
        return NULL;
    }
    int nTorder = psMetadataLookupS32 (&status, folder, "NORDER_T");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial4D in metadata is missing NORDER_T");
        return NULL;
    }
    // how many polynomial coeffs are expected?
    int nElementsExpected = psMetadataLookupS32 (&status, folder, "NELEMENTS");
    if (!status) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, "psPolynomial4D in metadata is missing NELEMENTS");
        return NULL;
    }

    psPolynomial4D *poly = psPolynomial4DAlloc(PS_POLYNOMIAL_ORD, nXorder, nYorder, nZorder, nTorder);

    int nElements = 0;
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            for (int nz = 0; nz < poly->nZ + 1; nz++) {
                for (int nt = 0; nt < poly->nT + 1; nt++) {
                    sprintf (keyword, "VAL_X%02d_Y%02d_Z%02d_T%02d", nx, ny, nz, nt);
                    poly->coeff[nx][ny][nz][nt] = psMetadataLookupF64 (&status, folder, keyword);
                    if (!status) {
                        // an undefined component implies the component was masked
                        // this is symmetrical with the 4DtoMD function
                        poly->mask[nx][ny][nz][nt] = 1;
                        poly->coeff[nx][ny][nz][nt] = 0;
                        poly->coeffErr[nx][ny][nz][nt] = 0;
                    } else {
                        poly->mask[nx][ny][nz][nt] = 0;
                        nElements ++;
                    }
                    sprintf (keyword, "ERR_X%02d_Y%02d_Z%02d_T%02d", nx, ny, nz, nt);
                    poly->coeffErr[nx][ny][nz][nt] = psMetadataLookupF64 (&status, folder, keyword);
                }
            }
        }
    }
    if (nElements != nElementsExpected) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                "psPolynomial4D in metadata does not have the correct number of coefficients: "
                "%d found vs %d expected", nElements, nElementsExpected);
        psFree(poly);
        return NULL;
    }
    return (poly);
}

bool psPolynomial4DtoMetadata (psMetadata *md,
                               const psPolynomial4D *poly,
                               const char *format,
                               ...)
{
    PS_ASSERT_PTR_NON_NULL(md, false);
    PS_ASSERT_PTR_NON_NULL(poly, false);
    //XXX:  Current implementation only supports ordinary polynomials.
    if (poly->type != PS_POLYNOMIAL_ORD)
        return false;

    int Nbyte;
    char tmp;
    char *root;
    va_list argp;

    va_start (argp, format);
    Nbyte = vsnprintf (&tmp, 0, format, argp);
    va_end (argp);

    if (Nbyte <= 0)
        return false;

    va_start (argp, format);
    root = (char *) psAlloc (Nbyte + 1);
    memset (root, 0, Nbyte + 1);
    vsnprintf (root, Nbyte + 1, format, argp);
    va_end (argp);

    psMetadata *folder = psMetadataAlloc ();

    // specify the polynomial orders
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_X", PS_DATA_S32, "number of x orders", poly->nX);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Y", PS_DATA_S32, "number of y orders", poly->nY);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Z", PS_DATA_S32, "number of z orders", poly->nZ);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_T", PS_DATA_S32, "number of t orders", poly->nT);

    char namespace[80];
    char namespace_err[80];
    int nElements = 0;   // count the number of unmasked elements

    // place polynomial entries on folder
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            for (int nz = 0; nz < poly->nZ + 1; nz++) {
                for (int nt = 0; nt < poly->nT + 1; nt++) {
                    if (poly->mask[nx][ny][nz][nt] == 0) {
                        sprintf(namespace, "VAL_X%02d_Y%02d_Z%02d_T%02d", nx, ny, nz, nt);
                        sprintf(namespace_err, "ERR_X%02d_Y%02d_Z%02d_T%02d", nx, ny, nz, nt);
                        psMetadataAdd (folder, PS_LIST_TAIL, namespace,
                                       PS_DATA_F64, "polynomial coefficient",
                                       poly->coeff[nx][ny][nz][nt], nx, ny, nz, nt);
                        psMetadataAdd (folder, PS_LIST_TAIL, namespace_err,
                                       PS_DATA_F64, "polynomial coeffficient error",
                                       poly->coeffErr[nx][ny][nz][nt], nx, ny, nz, nt);
                        nElements ++;
                    }
                }
            }
        }
    }
    psMetadataAdd (folder, PS_LIST_TAIL, "NELEMENTS", PS_DATA_S32, "number of unmasked coeffs", nElements);
    psMetadataAdd (md, PS_LIST_TAIL, root, PS_DATA_METADATA, "folder for 4D polynomial", folder);
    psFree(root);
    psFree(folder);
    return true;
}
