/** @file  psMetadata.c
 *
 *
 *  @brief Contains metadata structures, enumerations and functions prototypes.
 *
 *  This file defines metadata item, metadata type, metadata flags, metadata containers, and function
 *  prototypes necessary creating psLib metadata APIs
 *
 *  @ingroup Metadata
 *
 *  @author Robert DeSonia, MHPCC
 *  @author Ross Harman, MHPCC
 *
 *  @version $Revision: 1.104 $ $Name: not supported by cvs2svn $
 *  @date $Date: 2006-04-21 20:47:39 $
 *
 *  Copyright 2004-2005 Maui High Performance Computing Center, University of Hawaii
 */

/******************************************************************************/
/*  INCLUDE FILES                                                             */
/******************************************************************************/
#include<stdio.h>
#include<stdarg.h>
#include<string.h>

#include "fitsio.h"
#include "psType.h"
#include "psMemory.h"
#include "psError.h"
#include "psAbort.h"
#include "psList.h"
#include "psHash.h"
#include "psVector.h"
#include "psMetadata.h"
#include "psLookupTable.h"
#include "psString.h"
#include "psErrorText.h"
#include "psConstants.h"
#include "psLogMsg.h"
#include "psTrace.h"

/******************************************************************************/
/*  DEFINE STATEMENTS                                                         */
/******************************************************************************/

/** Maximum size of a string */
#define MAX_STRING_LENGTH 1024

/******************************************************************************/
/*  TYPE DEFINITIONS                                                          */
/******************************************************************************/

// None

/*****************************************************************************/
/*  GLOBAL VARIABLES                                                         */
/*****************************************************************************/

// None

/*****************************************************************************/
/*  FILE STATIC VARIABLES                                                    */
/*****************************************************************************/

static psS32 metadataId = 0;

/*****************************************************************************/
/*  FUNCTION IMPLEMENTATION - LOCAL                                          */
/*****************************************************************************/

static psMetadataItem* makeMetaMulti(psHash* table,
                                     const char* key,
                                     psMetadataItem* existing)
{

    if (existing != NULL && existing->type == PS_DATA_METADATA_MULTI) {
        return existing;
    }

    // move any existing entry into a psList
    psList* newList = psListAlloc(existing);

    psMetadataItem* item = psMetadataItemAlloc(key,
                           PS_DATA_METADATA_MULTI,
                           "",
                           newList);

    if (existing != NULL) {
        psHashRemove(table,key); // take out the old entry
    }

    psHashAdd(table, key, item); // put in the new entry
    psMemDecrRefCounter(item); // get rid of extra reference

    // free local references of newly allocated items.
    psFree(newList);

    return item;
}

static void metadataItemFree(psMetadataItem* metadataItem)
{
    psDataType type;

    type = metadataItem->type;

    if (metadataItem == NULL) {
        return;
    }

    psMemDecrRefCounter(metadataItem->name);
    psMemDecrRefCounter(metadataItem->comment);

    if(!PS_DATA_IS_PRIMITIVE(type)) {
        psFree(metadataItem->data.V);
    }
}

static void metadataIteratorFree(psMetadataIterator* iter)
{
    if (iter == NULL) {
        return;
    }
    psFree(iter->iter);

    if (iter->regex != NULL) {
        regfree(iter->regex);
    }
}

static void metadataFree(psMetadata* metadata)
{
    if (metadata == NULL) {
        return;
    }
    psFree(metadata->list);
    psFree(metadata->hash);

}

/*****************************************************************************/
/* FUNCTION IMPLEMENTATION - PUBLIC                                          */
/*****************************************************************************/

psMetadataItem* psMetadataItemAlloc(const char *name,
                                    psDataType type,
                                    const char *comment,
                                    ...)
{
    va_list argPtr;
    psMetadataItem* metadataItem = NULL;

    // Get the variable list parameters to pass to allocation function
    va_start(argPtr, comment);

    // Call metadata item allocation
    metadataItem = psMetadataItemAllocV(name, type, comment, argPtr);

    // Clean up stack after variable arguement has been used
    va_end(argPtr);

    return metadataItem;
}

bool psMemCheckMetadataItem(psPtr ptr)
{
    return ( psMemGetDeallocator(ptr) == (psFreeFunc)metadataItemFree );
}


#define METADATAITEM_ALLOC_TYPE(NAME,TYPE,METATYPE) \
psMetadataItem* psMetadataItemAlloc##NAME(const char* name, \
        const char* comment, \
        TYPE value) \
{ \
    return psMetadataItemAlloc(name, METATYPE, comment, value); \
}

METADATAITEM_ALLOC_TYPE(Str,const char*,PS_DATA_STRING)
METADATAITEM_ALLOC_TYPE(F32,psF32,PS_DATA_F32)
METADATAITEM_ALLOC_TYPE(F64,psF64,PS_DATA_F64)
METADATAITEM_ALLOC_TYPE(S8,psS8,PS_DATA_S8)
METADATAITEM_ALLOC_TYPE(S16,psS16,PS_DATA_S16)
METADATAITEM_ALLOC_TYPE(S32,psS32,PS_DATA_S32)
METADATAITEM_ALLOC_TYPE(U8,psU8,PS_DATA_U8)
METADATAITEM_ALLOC_TYPE(U16,psU16,PS_DATA_U16)
METADATAITEM_ALLOC_TYPE(U32,psU32,PS_DATA_U32)
METADATAITEM_ALLOC_TYPE(Bool,psBool,PS_DATA_BOOL)

psMetadataItem* psMetadataItemAllocPtr(const char *name,
                                       psDataType type,
                                       const char *comment,
                                       psPtr value)
{
    return (psMetadataItemAlloc(name, type, comment, value));
}


psMetadataItem* psMetadataItemAllocV(const char *name,
                                     psDataType type,
                                     const char *comment,
                                     va_list argPtr)
{
    psMetadataItem* metadataItem = NULL;


    PS_ASSERT_PTR_NON_NULL(name,NULL);

    // Allocate metadata item
    metadataItem = (psMetadataItem*) psAlloc(sizeof(psMetadataItem));
    metadataItem->data.V = NULL;

    // Set deallocator
    psMemSetDeallocator(metadataItem, (psFreeFunc) metadataItemFree);

    // Allocate and set metadata item comment
    if (comment == NULL) {
        // Per SDRS, null isn't allowed, must use "" instead
        metadataItem->comment = psStringCopy("");
    } else {
        metadataItem->comment = psStringCopy(comment);
    }

    // Set metadata item unique id
    *(psS32 *)(&metadataItem->id) = ++metadataId;

    // Set metadata item type
    metadataItem->type = type & PS_METADATA_TYPE_MASK;

    // Allocate and set metadata item name
    metadataItem->name = (char *)psAlloc(sizeof(char) * MAX_STRING_LENGTH);
    vsnprintf(metadataItem->name, MAX_STRING_LENGTH, name, argPtr);

    // Set metadata item value
    switch(metadataItem->type) {
    case PS_DATA_BOOL:
        metadataItem->data.B = (psBool)va_arg(argPtr, psS32);
        break;
    case PS_DATA_S8:
        metadataItem->data.S8 = (psS8)va_arg(argPtr, psS32);
        break;
    case PS_DATA_S16:
        metadataItem->data.S16 = (psS16)va_arg(argPtr, psS32);
        break;
    case PS_DATA_S32:
        metadataItem->data.S32 = (psS32)va_arg(argPtr, psS32);
        break;
    case PS_DATA_U8:
        metadataItem->data.U8 = (psU8)va_arg(argPtr, psU32);
        break;
    case PS_DATA_U16:
        metadataItem->data.U16 = (psU16)va_arg(argPtr, psU32);
        break;
    case PS_DATA_U32:
        metadataItem->data.U32 = (psU32)va_arg(argPtr, psU32);
        break;
    case PS_DATA_F32:
        metadataItem->data.F32 = (psF32)va_arg(argPtr, psF64);
        break;
    case PS_DATA_F64:
        metadataItem->data.F64 = (psF64)va_arg(argPtr, psF64);
        break;
    case PS_DATA_STRING:
        // Perform copy of input strings
        {
            char *string = va_arg(argPtr, char *);
            metadataItem->data.V =
                string ? psStringNCopy(string, MAX_STRING_LENGTH)
                : NULL;
        }
        break;
    case     PS_DATA_ARRAY:                     // psArray
    case     PS_DATA_BITSET:                    // psBitSet
    case     PS_DATA_CUBE:                      // psCube
    case     PS_DATA_FITS:                      // psFits
    case     PS_DATA_HASH:                      // psHash
    case     PS_DATA_HISTOGRAM:                 // psHistogram
    case     PS_DATA_IMAGE:                     // psImage
    case     PS_DATA_KERNEL:                    // psKernel
    case     PS_DATA_LIST:                      // psList
    case     PS_DATA_LOOKUPTABLE:               // psLookupTable
    case     PS_DATA_METADATA:                  // psMetadata
    case     PS_DATA_METADATAITEM:              // psMetadataItem
    case     PS_DATA_MINIMIZATION:              // psMinimization
    case     PS_DATA_PIXELS:                    // psPixels
    case     PS_DATA_PLANE:                     // psPlane
    case     PS_DATA_PLANEDISTORT:              // psPlaneDistort
    case     PS_DATA_PLANETRANSFORM:            // psPlaneTransform
    case     PS_DATA_POLYNOMIAL1D:              // psPolynomial1D
    case     PS_DATA_POLYNOMIAL2D:              // psPolynomial2D
    case     PS_DATA_POLYNOMIAL3D:              // psPolynomial3D
    case     PS_DATA_POLYNOMIAL4D:              // psPolynomial4D
    case     PS_DATA_PROJECTION:                // psProjection
    case     PS_DATA_REGION:                  // psRegion
    case     PS_DATA_SCALAR:                    // psScalar
    case     PS_DATA_SPHERE:                    // psSphere
    case     PS_DATA_SPHEREROT:                 // psSphereTransform
    case     PS_DATA_SPLINE1D:                  // psSpline1D
    case     PS_DATA_STATS:                     // psStats
    case     PS_DATA_TIME:                      // psTime
    case     PS_DATA_VECTOR:                    // psVector
    case     PS_DATA_UNKNOWN:                   // "other"
    case     PS_DATA_METADATA_MULTI:
        // Copy of input data not performed due to variability of data types
        metadataItem->data.V = psMemIncrRefCounter(va_arg(argPtr, psPtr));
        break;
    default:
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, PS_ERRORTEXT_psMetadata_METATYPE_INVALID, type);
        psFree(metadataItem);
        metadataItem = NULL;
    }

    return metadataItem;
}

psMetadata* psMetadataAlloc(void)
{
    psList* list = NULL;
    psHash* hash = NULL;
    psMetadata* metadata = NULL;

    // Allocate metadata
    metadata = (psMetadata*) psAlloc(sizeof(psMetadata));
    // Set deallocator
    psMemSetDeallocator(metadata, (psFreeFunc) metadataFree);

    // Allocate metadata's internal containers
    list = (psList*) psListAlloc(NULL);
    hash = (psHash*) psHashAlloc(10);

    metadata->list = list;
    metadata->hash = hash;

    return metadata;
}


bool psMemCheckMetadata(psPtr ptr)
{
    return ( psMemGetDeallocator(ptr) == (psFreeFunc)metadataFree );
}


psMetadataItem *psMetadataItemCopy(const psMetadataItem *in
                                  )
{
    PS_ASSERT_PTR_NON_NULL(in,NULL);

    psMetadataItem *newItem = NULL;     // New metadata item, to be returned

    #define PS_METADATA_ITEM_COPY_CASE(NAME,TYPE) \
case PS_DATA_##NAME: \
    newItem = psMetadataItemAlloc(in->name, PS_DATA_##NAME, in->comment, in->data.TYPE); \
    break; \

    switch (in->type) {
        // Simple types
        PS_METADATA_ITEM_COPY_CASE(BOOL,B);
        PS_METADATA_ITEM_COPY_CASE(S8,S8);
        PS_METADATA_ITEM_COPY_CASE(S16,S16);
        PS_METADATA_ITEM_COPY_CASE(S32,S32);
        PS_METADATA_ITEM_COPY_CASE(U8,U8);
        PS_METADATA_ITEM_COPY_CASE(U16,U16);
        PS_METADATA_ITEM_COPY_CASE(U32,U32);
        PS_METADATA_ITEM_COPY_CASE(F32,F32);
        PS_METADATA_ITEM_COPY_CASE(F64,F64);
        PS_METADATA_ITEM_COPY_CASE(STRING,V); // This will copy the string, not point at it.
    case PS_DATA_VECTOR: {
            psVector *vecCopy = psVectorCopy(NULL, (psVector*)(in->data.V),
                                             ((psVector*)(in->data.V))->type.type);
            if (vecCopy == NULL) {
                psError(PS_ERR_BAD_PARAMETER_NULL, false,
                        "Error copying vector.  Vector skipped.\n");
            } else {
                newItem = psMetadataItemAlloc(in->name, PS_DATA_VECTOR, in->comment, vecCopy);
                psFree(vecCopy);    // Drop reference
            }
            break;
        }
    case PS_DATA_TIME: {
            psTime *timeCopy = p_psTimeCopy( (psTime*)(in->data.V) );
            if (timeCopy == NULL) {
                psError(PS_ERR_BAD_PARAMETER_NULL, false,
                        "Error copying time.  Time skipped.\n");
            } else {
                newItem = psMetadataItemAlloc(in->name, PS_DATA_TIME, in->comment, timeCopy);
                psFree(timeCopy);   // Drop reference
            }
            break;
        }
    case PS_DATA_METADATA: {
            // Metadata: copy the next level and stuff that in too
            psMetadata *metadata = psMetadataCopy(NULL, in->data.md);
            newItem = psMetadataItemAlloc(in->name, PS_DATA_METADATA, in->comment, metadata);
            psFree(metadata);       // Drop reference
            break;
        }
    case PS_DATA_REGION: {
            psRegion *region = in->data.V; // The region
            psRegion *new = psRegionAlloc(region->x0, region->x1, region->y0, region->y1); // Copy of the region
            newItem = psMetadataItemAlloc(in->name, PS_DATA_REGION, in->comment, new);
            psFree(new);                  // Drop reference
            break;
        }
    default:
        // Other kinds of pointers
        psLogMsg(__func__, PS_LOG_WARN, "Copying a pointer in the metadata item: %x\n", in->type);
        newItem = psMetadataItemAlloc(in->name, in->type, in->comment, in->data.V);
        break;
    }

    return newItem;
}

psMetadata *psMetadataCopy(psMetadata *out,
                           const psMetadata *in)
{
    PS_ASSERT_PTR_NON_NULL(in,NULL);
    PS_ASSERT_PTR_NON_NULL(in->list,NULL);
    if (out ==  NULL) {
        out = psMetadataAlloc();
    }
    psMetadataIterator *iter = psMetadataIteratorAlloc(*(psMetadata**)&in, PS_LIST_HEAD, NULL);
    psMetadataItem *inItem = NULL;
    while ((inItem = psMetadataGetAndIncrement(iter))) {
        // Need to look for MULTI, which won't be picked up using the iterator.
        psMetadataItem *multiCheckItem = psMetadataLookup(in, inItem->name);
        unsigned int flag = PS_META_REPLACE; // Flag to indicate MULTI; otherwise, replace
        if (multiCheckItem->type == PS_DATA_METADATA_MULTI) {
            psTrace(__func__, 10, "MULTI: %s (%s)\n", inItem->name, inItem->comment);
            flag = PS_META_DUPLICATE_OK;
        }
        psTrace(__func__, 5, "Copying %s (%s)...\n", inItem->name, inItem->comment);

        // Copy the item and add it on
        psMetadataItem *newItem = psMetadataItemCopy(inItem); // Copied item
        if (!psMetadataAddItem(out, newItem, PS_LIST_TAIL, flag)) {
            psErrorStackPrint(stderr, "Error copying %s (%s) in the metadata\n", inItem->name,
                              inItem->comment);
        }
        psFree(newItem);                // Drop reference
    }
    psFree(iter);

    return out;
}

bool psMetadataAddItem(psMetadata *md,
                       const psMetadataItem *item,
                       int location,
                       psS32 flags)
{
    char * key = NULL;
    psHash *mdTable = NULL;
    psList *mdList = NULL;
    psMetadataItem *existingEntry = NULL;

    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->hash,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    PS_ASSERT_PTR_NON_NULL(item,NULL);
    PS_ASSERT_PTR_NON_NULL(item->name,NULL);

    mdTable = md->hash;
    mdList = md->list;
    key = item->name;

    // See if key is already in table
    existingEntry = (psMetadataItem*)psHashLookup(mdTable, key);

    if (item->type == PS_DATA_METADATA_MULTI) {
        // the incoming entry is PS_DATA_METADATA_MULTI

        // force the hash entry to be PS_DATA_METADATA_MULTI
        existingEntry = makeMetaMulti(mdTable,key,existingEntry);

        // add all the items in the incoming entry to metadata
        psList* list = item->data.list;
        if (list != NULL) {
            psListIterator* iter = psListIteratorAlloc(list,PS_LIST_HEAD,true);
            psMetadataItem* listItem;
            while ((listItem=(psMetadataItem*)psListGetAndIncrement(iter)) != NULL) {
                psMetadataAddItem(md,listItem,location,flags);
            }
            psFree(iter);
        }

        return true; // all done.
    }

    // how the item is added to the hash depends on prior existence, flags, etc.
    if(existingEntry != NULL) { // prior existence
        if (existingEntry->type == PS_DATA_METADATA_MULTI || (flags & PS_META_DUPLICATE_OK) != 0) {
            // duplicate entries allowed - add another entry.

            // make sure the existing entry is PS_DATA_METADATA_MULTI
            existingEntry = makeMetaMulti(mdTable,key,existingEntry);

            // add to the hash's list of duplicate entries
            if (! psListAdd(existingEntry->data.list, PS_LIST_TAIL, (psPtr)item) ) {
                psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_COLLECTION_FAILED,key);
                return false;
            }
        } else if ((flags & PS_META_REPLACE) != 0) {
            // replace entry instead of creating a duplicate entry.

            // remove the existing entry from metadata
            psMetadataRemove(md,0,key);

            // treat as if new (added to list below)
            if(!psHashAdd(mdTable, key, (psPtr)item)) {
                psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_TABLE_FAILED,key);
                return false;
            }

            // Generate warning that item has been replaced
            //            psLogMsg(__func__,PS_LOG_WARN,"Metadata item %s has been replaced",
            //                     existingEntry->name);
        } else {
            // default is to error on duplicate entry.
            if ((flags & PS_META_NO_REPLACE) != 0) {
                return true;
            }
            psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                    PS_ERRORTEXT_psMetadata_DUPLICATE_NOT_ALLOWED);
            return false;
        }
    } else {
        // OK, this is a new item.

        // Node doesn't exist - Add new metadata item to metadata collection's hash
        if(!psHashAdd(mdTable, key, (psPtr)item)) {
            psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_TABLE_FAILED,key);
            return false;
        }
    }

    if(!psListAdd(mdList, location, (psPtr)item)) {
        psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_COLLECTION_FAILED,key);
        return false;
    }

    return true;
}

bool psMetadataAdd(psMetadata *md,
                   long location,
                   const char *name,
                   int format,
                   const char *comment,...)
{
    PS_ASSERT_PTR_NON_NULL(md,false);
    PS_ASSERT_PTR_NON_NULL(md->list,false);
    va_list argPtr;

    va_start(argPtr, comment);
    psBool result = psMetadataAddV(md,location,name,format,comment,argPtr);
    va_end(argPtr);

    return result;
}

bool psMetadataAddV(psMetadata *md,
                    long location,
                    const char *name,
                    int format,
                    const char *comment,
                    va_list list)
{
    PS_ASSERT_PTR_NON_NULL(md,false);
    PS_ASSERT_PTR_NON_NULL(md->list,false);
    psMetadataItem* metadataItem = NULL;

    metadataItem = psMetadataItemAllocV(name, format & PS_METADATA_TYPE_MASK, comment, list);

    if (!psMetadataAddItem(md, metadataItem, location, format & PS_METADATA_FLAGS_MASK)) {
        psError(PS_ERR_UNKNOWN,false,PS_ERRORTEXT_psMetadata_ADD_FAILED);
        psFree(metadataItem);
        return false;
    }
    // Decrement reference count, since the metadata item is now in metadata collection and no longer needed
    psFree(metadataItem);

    return true;
}

#define METADATA_ADD_TYPE(NAME,TYPE,METATYPE) \
psBool psMetadataAdd##NAME(psMetadata* md, long where, const char* name, \
                           int format, const char* comment, TYPE value) { \
    return psMetadataAdd(md,where,name, format | METATYPE,comment,value); \
}

METADATA_ADD_TYPE(Bool,psBool,PS_DATA_BOOL)
METADATA_ADD_TYPE(S8,psS8,PS_DATA_S8)
METADATA_ADD_TYPE(S16,psS16,PS_DATA_S16)
METADATA_ADD_TYPE(S32,psS32,PS_DATA_S32)
METADATA_ADD_TYPE(U8,psU8,PS_DATA_U8)
METADATA_ADD_TYPE(U16,psU16,PS_DATA_U16)
METADATA_ADD_TYPE(U32,psU32,PS_DATA_U32)
METADATA_ADD_TYPE(F32,psF32,PS_DATA_F32)
METADATA_ADD_TYPE(F64,psF64,PS_DATA_F64)
METADATA_ADD_TYPE(List,psList*,PS_DATA_LIST)
//METADATA_ADD_TYPE(Str,const char*,PS_DATA_STRING)
METADATA_ADD_TYPE(Vector,psVector*,PS_DATA_VECTOR)
METADATA_ADD_TYPE(Image,psImage*,PS_DATA_IMAGE)
METADATA_ADD_TYPE(Hash,psHash*,PS_DATA_HASH)
METADATA_ADD_TYPE(LookupTable,psLookupTable*,PS_DATA_LOOKUPTABLE)
METADATA_ADD_TYPE(Unknown,void*,PS_DATA_UNKNOWN)
METADATA_ADD_TYPE(Metadata,psMetadata*,PS_DATA_METADATA)
METADATA_ADD_TYPE(Array,psArray*,PS_DATA_ARRAY)
METADATA_ADD_TYPE(Time,psTime*,PS_DATA_TIME)

bool psMetadataAddPtr(psMetadata *md,
                      long location,
                      const char *name,
                      psDataType type,
                      const char *comment,
                      psPtr value)
{
    PS_ASSERT_PTR_NON_NULL(md,false);
    PS_ASSERT_PTR_NON_NULL(md->list,false);
    PS_ASSERT_PTR_NON_NULL(value,false);
    return (psMetadataAdd(md, location, name, type, comment, value) );

}

bool psMetadataAddStr(psMetadata *md,
                      long location,
                      const char *name,
                      int format,
                      const char *comment,
                      const char* value)
{
    PS_ASSERT_PTR_NON_NULL(md,false);
    PS_ASSERT_PTR_NON_NULL(md->list,false);
    PS_ASSERT_PTR_NON_NULL(value,false);
    return (psMetadataAdd(md, location, name, format | PS_DATA_STRING, comment, value) );

}

bool psMetadataRemove(psMetadata *md,
                      long where,
                      const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);

    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    psList* mdList = md->list;

    PS_ASSERT_PTR_NON_NULL(md->hash,NULL);
    psHash* mdTable = md->hash;

    // Select removal by key or index
    if (key != NULL) {
        // Remove by key name
        psMetadataItem* entry = psHashLookup(mdTable,key);
        if (entry == NULL) {
            psError(PS_ERR_UNKNOWN, false, PS_ERRORTEXT_psMetadata_REMOVE_TABLE_FAILED, key);
            return false;
        }
        if (entry->type == PS_DATA_METADATA_MULTI) {
            psMetadataItem* listItem;
            psListIterator* iter = psListIteratorAlloc(
                                       entry->data.list,
                                       PS_LIST_HEAD,true);
            while ((listItem=psListGetAndIncrement(iter)) != NULL) {
                psListRemoveData(mdList, listItem);
            }
            psFree(iter);
            psHashRemove(mdTable,key);

        } else {
            psListRemoveData(mdList, entry);
            psHashRemove(mdTable, key);
        }
    } else {
        // Remove by index
        psMetadataItem* entry = psListGet(mdList, where);
        if (entry == NULL) {
            psError(PS_ERR_BAD_PARAMETER_VALUE, true, PS_ERRORTEXT_psMetadata_FIND_INDEX_FAILED, where);
            return false;
        }
        key = entry->name;

        if (key == NULL) {
            psError(PS_ERR_BAD_PARAMETER_VALUE, true, PS_ERRORTEXT_psMetadata_REMOVE_LIST_INDEX_FAILED, where);
            return false;
        }

        psMetadataItem* tableItem = psHashLookup(mdTable, key);
        if (tableItem == NULL) {
            psError(PS_ERR_UNKNOWN, false, PS_ERRORTEXT_psMetadata_REMOVE_TABLE_FAILED, key);
            return false;
        }

        if (tableItem->type == PS_DATA_METADATA_MULTI) {
            // multiple entries with same key, remove just the specified one
            psListRemoveData(tableItem->data.list, entry);
        } else {
            if (!psHashRemove(mdTable, key)) {
                psError(PS_ERR_UNKNOWN, false, PS_ERRORTEXT_psMetadata_REMOVE_TABLE_FAILED, key);
                return false;
            }
        }
        psListRemove(mdList, where);
    }

    return true;
}

bool psMetadataRemoveKey(psMetadata *md,
                         const char *key)
{
    return ( psMetadataRemove(md, PS_LIST_HEAD, key) );
}

bool psMetadataRemoveIndex(psMetadata *md,
                           int location)
{
    return ( psMetadataRemove(md, location, NULL) );
}

psMetadataItem* psMetadataLookup(const psMetadata *md,
                                 const char *key)
{
    psHash* mdTable = NULL;
    psMetadataItem* entry = NULL;


    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->hash,NULL);
    PS_ASSERT_PTR_NON_NULL(key,NULL);

    mdTable = md->hash;
    entry = (psMetadataItem*)psHashLookup(mdTable, key);

    return entry;
}

void* psMetadataLookupPtr(bool *status,
                          const psMetadata *md,
                          const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    psMetadataItem *metadataItem = NULL;

    if (status) {
        *status = true;
    }

    metadataItem = psMetadataLookup(md, key);
    if(metadataItem == NULL) {
        if (status) {
            *status = false;
        }
        return NULL;
    }
    if (metadataItem->type == PS_DATA_METADATA_MULTI) {
        // if multiple keys found, use the first.
        metadataItem = (psMetadataItem*)((metadataItem->data.list)->head);
    }

    if(PS_DATA_IS_PRIMITIVE(metadataItem->type)) {
        if (status) {
            *status = false;
        }
        psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                PS_ERRORTEXT_psMetadata_METATYPE_INVALID,
                metadataItem->type);
        return NULL;
    } else {
        return metadataItem->data.V;
    }
}

#define psMetadataLookupNumTYPE(TYPE) \
ps##TYPE psMetadataLookup##TYPE(bool *status, const psMetadata *md, const char *key) \
{ \
    psMetadataItem *metadataItem = NULL; \
    ps##TYPE value = 0; \
    \
    if (status) { \
        *status = true; \
    } \
    \
    metadataItem = psMetadataLookup(md, key); \
    if(metadataItem == NULL) { \
        if (status) { \
            *status = false; \
        } \
        return 0; \
    } \
    if (metadataItem->type == PS_DATA_METADATA_MULTI) { \
        /* if multiple keys found, use the first. */ \
        metadataItem = (psMetadataItem*)((metadataItem->data.list)->head); \
    } \
    \
    switch (metadataItem->type) { \
    case PS_DATA_S8: \
        value = (ps##TYPE)metadataItem->data.S8; \
        break; \
    case PS_DATA_S16: \
        value = (ps##TYPE)metadataItem->data.S16; \
        break; \
    case PS_DATA_S32: \
        value = (ps##TYPE)metadataItem->data.S32; \
        break; \
    case PS_DATA_U8: \
        value = (ps##TYPE)metadataItem->data.U8; \
        break; \
    case PS_DATA_U16: \
        value = (ps##TYPE)metadataItem->data.U16; \
        break; \
    case PS_DATA_U32: \
        value = (ps##TYPE)metadataItem->data.U32; \
        break; \
    case PS_DATA_F32: \
        value = (ps##TYPE)metadataItem->data.F32; \
        break; \
    case PS_DATA_F64: \
        value = (ps##TYPE)metadataItem->data.F64; \
        break; \
    case PS_DATA_BOOL: \
        if (metadataItem->data.B) { \
            value = 1; \
        } \
        break; \
    default: \
        /* if you get to this point, the value is not a number. */ \
        if (status) { \
            *status = false; \
        } \
        break; \
    } \
    \
    /* psFree(metadataItem); currently, the lookup doesn't increment the ref count */ \
    return value; \
}

psMetadataLookupNumTYPE(F32)
psMetadataLookupNumTYPE(F64)
psMetadataLookupNumTYPE(S8)
psMetadataLookupNumTYPE(S16)
psMetadataLookupNumTYPE(S32)
psMetadataLookupNumTYPE(U8)
psMetadataLookupNumTYPE(U16)
psMetadataLookupNumTYPE(U32)
psMetadataLookupNumTYPE(Bool)

psMetadataItem* psMetadataGet(const psMetadata *md,
                              int location)
{
    psMetadataItem* entry = NULL;

    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);

    entry = (psMetadataItem*) psListGet(md->list, location);
    if (entry == NULL) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, true, PS_ERRORTEXT_psMetadata_FIND_INDEX_FAILED, location);
        return NULL;
    }

    return entry;
}

psMetadataIterator* psMetadataIteratorAlloc(psMetadata* md,
        long location,
        const char* regex)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);

    psMetadataIterator* newIter = psAlloc(sizeof(psMetadataIterator));
    newIter->regex = NULL;
    newIter->iter = NULL;

    // Set deallocator
    psMemSetDeallocator(newIter, (psFreeFunc) metadataIteratorFree);

    if (regex == NULL) {
        newIter->iter = psListIteratorAlloc(md->list, location, false);
        return newIter;
    } else {
        int regRtn = regcomp(newIter->regex,regex,0);
        if (regRtn != 0) {
            char errMsg[256];
            regerror(regRtn, newIter->regex, errMsg, 256);
            regfree(newIter->regex);
            psError(PS_ERR_BAD_PARAMETER_VALUE, true,
                    PS_ERRORTEXT_psMetadata_REGEX_INVALID,
                    errMsg);
            psFree(newIter);
            return NULL;
        }
    }

    psMetadataIteratorSet(newIter, location); // XXX: do we error if no match is found?

    return newIter;
}

bool psMetadataIteratorSet(psMetadataIterator* iterator,
                           long location)
{
    int match;
    psMetadataItem* cursor;

    PS_ASSERT_PTR_NON_NULL(iterator,NULL);

    psListIterator* iter = iterator->iter;
    PS_ASSERT_PTR_NON_NULL(iterator->iter,NULL);

    regex_t* regex = iterator->regex;

    // handle trivial case where no regex subsetting is required.
    if (regex == NULL) {
        return psListIteratorSet(iter,location);
    }

    if (location < 0) {
        // match from the tail
        match = 0;
        psListIteratorSet(iter,PS_LIST_TAIL);
        while ( (cursor=(psMetadataItem*)iter->cursor) != NULL) {
            if (regexec(regex, cursor->name, 0, NULL, 0) == 0) {
                // this key is a match
                match--;
                if (match == location) {
                    break;
                }
            }
            (void)psListGetAndDecrement(iter);
        }
        return (match == location);
    }

    // find the n-th match from the head
    match = -1;
    psListIteratorSet(iter,PS_LIST_HEAD);
    while ( (cursor=(psMetadataItem*)iter->cursor) != NULL) {
        if (regexec(regex, cursor->name, 0, NULL, 0) == 0) {
            // this key is a match
            match++;
            if (match == location) {
                break;
            }
        }
        (void)psListGetAndIncrement(iter);
    }
    return (match == location);
}

psMetadataItem* psMetadataGetAndIncrement(psMetadataIterator* iterator)
{
    psMetadataItem* oldValue;

    PS_ASSERT_PTR_NON_NULL(iterator,NULL);

    psListIterator* iter = iterator->iter;
    PS_ASSERT_PTR_NON_NULL(iterator->iter,NULL);

    regex_t* regex = iterator->regex;

    // handle trivial case where no regex subsetting is required.
    if (regex == NULL) {
        return (psMetadataItem*)psListGetAndIncrement(iter);
    }

    oldValue = (psMetadataItem*)iter->cursor;

    while (psListGetAndIncrement(iter) != NULL) {
        if (iter->cursor != NULL &&
                regexec(regex, ((psMetadataItem*)iter->cursor)->name, 0, NULL, 0) == 0) {
            // this key is a match
            break;
        }
    }
    return oldValue;
}

psMetadataItem* psMetadataGetAndDecrement(psMetadataIterator* iterator)
{
    psMetadataItem* oldValue;

    PS_ASSERT_PTR_NON_NULL(iterator,NULL);

    psListIterator* iter = iterator->iter;
    PS_ASSERT_PTR_NON_NULL(iterator->iter,NULL);

    regex_t* regex = iterator->regex;

    // handle trivial case where no regex subsetting is required.
    if (regex == NULL) {
        return (psMetadataItem*)psListGetAndDecrement(iter);
    }

    oldValue = (psMetadataItem*)iter->cursor;

    while (psListGetAndDecrement(iter) != NULL) {
        if (iter->cursor != NULL &&
                regexec(regex, ((psMetadataItem*)iter->cursor)->name, 0, NULL, 0) == 0) {
            // this key is a match
            break;
        }
    }
    return oldValue;
}

psMetadata *psMetadataLookupMD(bool *status,
                               const psMetadata *md,
                               const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    psMetadataItem *item = psMetadataLookup((psMetadata*)md, key); // The metadata with instruments
    psMetadata *value = NULL;  // The value to return
    if (!item) {
        // The given key isn't in the metadata
        if (status) {
            *status = false;
        } else {
            psError(PS_ERR_IO, true, "Couldn't find %s in the metadata.\n", key);
        }
    } else if (item->type != PS_DATA_METADATA) {
        // The value at the key isn't metadata
        if (status) {
            *status = false;
        } else {
            psLogMsg(__func__, PS_LOG_WARN, "%s isn't of type PS_DATA_META, as expected.\n", key);
        }
        value = NULL;
    } else {
        // We have the requested metadata
        if (status) {
            *status = true;
        }
        value = item->data.md; // The requested metadata
    }
    return value;
}


psString psMetadataLookupStr(bool *status,
                             const psMetadata *md,
                             const char *key)
{
    PS_ASSERT_PTR_NON_NULL(md,NULL);
    PS_ASSERT_PTR_NON_NULL(md->list,NULL);
    psMetadataItem *item = psMetadataLookup((psMetadata*)md, key); // The metadata with instruments
    //    char *value = NULL;   // The value to return
    psString value = NULL;
    if (!item) {
        // The given key isn't in the metadata
        if (status) {
            *status = false;
        } else {
            psError(PS_ERR_IO, true, "Couldn't find %s in the metadata.\n", key);
        }
    } else if (item->type != PS_DATA_STRING) {
        // The value at the key isn't of the desired type
        if (status) {
            *status = false;
        } else {
            psLogMsg(__func__, PS_LOG_WARN, "%s isn't of type PS_DATA_STRING, as expected.\n", key);
        }
        value = NULL;
    } else {
        // We have the requested metadata
        if (status) {
            *status = true;
        }
        value = item->data.V;
    }
    return value;
}


void psMetadataPrint(psMetadata *md,
                     int level)
{
    PS_ASSERT_PTR_NON_NULL(md,);
    psMetadataIterator *iter = psMetadataIteratorAlloc(md, PS_LIST_HEAD, NULL); // Iterator
    psMetadataItem *item = NULL; // Item from metadata
    while ( (item = psMetadataGetAndIncrement(iter)) ) {
        // Indent...
        for (int i = 0; i < level; i++) {
            printf(" ");
        }
        printf("%s", item->name);
        if (item->comment && strlen(item->comment) > 0) {
            printf(" (%s)", item->comment);
        }
        printf(": ");
        switch (item->type) {
        case PS_DATA_STRING:
            printf("%s", (char*)(item->data.V));
            break;
        case PS_DATA_BOOL:
            if (item->data.B) {
                printf("True");
            } else {
                printf("False");
            }
            break;
        case PS_DATA_S8:
            printf("%d", item->data.S8);
            break;
        case PS_DATA_S16:
            printf("%d", item->data.S16);
            break;
        case PS_DATA_S32:
            printf("%d", item->data.S32);
            break;
        case PS_DATA_U8:
            printf("%u", item->data.U8);
            break;
        case PS_DATA_U16:
            printf("%u", item->data.U16);
            break;
        case PS_DATA_U32:
            printf("%u", item->data.U32);
            break;
        case PS_DATA_F32:
            printf("%f", item->data.F32);
            break;
        case PS_DATA_F64:
            printf("%f", item->data.F64);
            break;
        case PS_DATA_METADATA:
            printf("\n");
            psMetadataPrint(item->data.V, level + 1);
            break;
        case PS_DATA_REGION: {
                psString region = psRegionToString(*(psRegion*)item->data.V);
                printf("%s", region);
                psFree(region);
                break;
            }
        case PS_DATA_LIST:
            printf("<a list of unknown contents>");
            break;
        case PS_DATA_TIME: {
                psString time = psTimeToISO(item->data.V);
                printf("%s", time);
                psFree(time);
                break;
            }
        default:
            printf("\n");
            psError(PS_ERR_IO, false, "Non-printable metadata type: %x\n", item->type);
        }
        printf("\n");
    }
    psFree(iter);

    return;
}

psPolynomial2D *psPolynomial2DfromMD (psMetadata *folder)
{

    bool status;
    char keyword[80];

    // get polynomial orders
    // XXX add status failures tests
    int nXorder = psMetadataLookupS32 (&status, folder, "NORDER_X");
    int nYorder = psMetadataLookupS32 (&status, folder, "NORDER_Y");

    psPolynomial2D *poly = psPolynomial2DAlloc (PS_POLYNOMIAL_ORD, nXorder, nYorder);

    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            sprintf (keyword, "VAL_X%02d_Y%02d", nx, ny);
            poly->coeff[nx][ny] = psMetadataLookupF64 (&status, folder, keyword);
            if (!status)
                poly->mask[nx][ny] = 1;

            sprintf (keyword, "ERR_X%02d_Y%02d", nx, ny);
            poly->coeffErr[nx][ny] = psMetadataLookupF64 (&status, folder, keyword);
        }
    }
    return (poly);
}

// XXX : these may need F64, or %g format for output
bool psPolynomial2DtoMD (psMetadata *md, psPolynomial2D *poly, char *format, ...)
{

    int Nbyte;
    char tmp;
    char *root;
    va_list argp;

    va_start (argp, format);
    Nbyte = vsnprintf (&tmp, 0, format, argp);
    va_end (argp);

    if (!Nbyte)
        return false;

    va_start (argp, format);
    root = (char *) psAlloc (Nbyte + 1);
    memset (root, 0, Nbyte + 1);
    vsnprintf (root, Nbyte + 1, format, argp);
    va_end (argp);

    psMetadata *folder = psMetadataAlloc ();
    psMetadataAdd (md, PS_LIST_TAIL, root, PS_DATA_METADATA, "folder for 2D polynomial", folder);
    psFree (root);

    // specify the polynomial orders
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_X", PS_DATA_S32, "number of x orders", poly->nX);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Y", PS_DATA_S32, "number of y orders", poly->nY);

    // place polynomial entries on folder
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            if (poly->mask[nx][ny])
                continue;
            psMetadataAdd (folder, PS_LIST_TAIL, "VAL_X%02d_Y%02d", PS_DATA_F64, "polynomial coefficient", poly->coeff[nx][ny], nx, ny);
            psMetadataAdd (folder, PS_LIST_TAIL, "ERR_X%02d_Y%02d", PS_DATA_F64, "polynomial coefficient error", poly->coeffErr[nx][ny], nx, ny);
        }
    }
    return true;
}

psPolynomial3D *psPolynomial3DfromMD (psMetadata *folder)
{

    bool status;
    char keyword[80];

    // get polynomial orders
    // XXX add status failures tests
    int nXorder = psMetadataLookupS32 (&status, folder, "NORDER_X");
    int nYorder = psMetadataLookupS32 (&status, folder, "NORDER_Y");
    int nZorder = psMetadataLookupS32 (&status, folder, "NORDER_Z");

    psPolynomial3D *poly = psPolynomial3DAlloc (PS_POLYNOMIAL_ORD, nXorder, nYorder, nZorder);

    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            for (int nz = 0; nz < poly->nZ + 1; nz++) {
                sprintf (keyword, "VAL_X%02d_Y%02d_Z%02d", nx, ny, nz);
                poly->coeff[nx][ny][nz] = psMetadataLookupF64 (&status, folder, keyword);
                if (!status)
                    poly->mask[nx][ny][nz] = 1;

                sprintf (keyword, "ERR_X%02d_Y%02d_Z%02d", nx, ny, nz);
                poly->coeffErr[nx][ny][nz] = psMetadataLookupF64 (&status, folder, keyword);
            }
        }
    }
    return (poly);
}

bool psPolynomial3DtoMD (psMetadata *md, psPolynomial3D *poly, char *format, ...)
{

    int Nbyte;
    char tmp;
    char *root;
    va_list argp;

    va_start (argp, format);
    Nbyte = vsnprintf (&tmp, 0, format, argp);
    va_end (argp);

    if (!Nbyte)
        return false;

    va_start (argp, format);
    root = (char *) psAlloc (Nbyte + 1);
    memset (root, 0, Nbyte + 1);
    vsnprintf (root, Nbyte + 1, format, argp);
    va_end (argp);

    psMetadata *folder = psMetadataAlloc ();
    psMetadataAdd (md, PS_LIST_TAIL, root, PS_DATA_METADATA, "folder for 3D polynomial", folder);
    psFree (root);

    // specify the polynomial orders
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_X", PS_DATA_S32, "number of x orders", poly->nX);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Y", PS_DATA_S32, "number of y orders", poly->nY);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Z", PS_DATA_S32, "number of z orders", poly->nZ);

    // place polynomial entries on folder
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            for (int nz = 0; nz < poly->nZ + 1; nz++) {
                if (poly->mask[nx][ny][nz])
                    continue;
                psMetadataAdd (folder, PS_LIST_TAIL, "VAL_X%02d_Y%02d_Z%02d", PS_DATA_F64, "polynomial coefficient", poly->coeff[nx][ny][nz], nx, ny, nz);
                psMetadataAdd (folder, PS_LIST_TAIL, "ERR_X%02d_Y%02d_Z%02d", PS_DATA_F64, "polynomial coeffficient error", poly->coeffErr[nx][ny][nz], nx, ny, nz);
            }
        }
    }
    return true;
}


psPolynomial4D *psPolynomial4DfromMD (psMetadata *folder)
{

    bool status;
    char keyword[80];

    // get polynomial orders
    // XXX add status failures tests
    int nXorder = psMetadataLookupS32 (&status, folder, "NORDER_X");
    int nYorder = psMetadataLookupS32 (&status, folder, "NORDER_Y");
    int nZorder = psMetadataLookupS32 (&status, folder, "NORDER_Z");
    int nTorder = psMetadataLookupS32 (&status, folder, "NORDER_T");

    psPolynomial4D *poly = psPolynomial4DAlloc (PS_POLYNOMIAL_ORD, nXorder, nYorder, nZorder, nTorder);

    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            for (int nz = 0; nz < poly->nZ + 1; nz++) {
                for (int nt = 0; nt < poly->nT + 1; nt++) {
                    sprintf (keyword, "VAL_X%02d_Y%02d_Z%02d_T%02d", nx, ny, nz, nt);
                    poly->coeff[nx][ny][nz][nt] = psMetadataLookupF64 (&status, folder, keyword);
                    if (!status)
                        poly->mask[nx][ny][nz][nt] = 1;

                    sprintf (keyword, "ERR_X%02d_Y%02d_Z%02d_T%02d", nx, ny, nz, nt);
                    poly->coeffErr[nx][ny][nz][nt] = psMetadataLookupF64 (&status, folder, keyword);
                }
            }
        }
    }
    return (poly);
}

bool psPolynomial4DtoMD (psMetadata *md, psPolynomial4D *poly, char *format, ...)
{

    int Nbyte;
    char tmp;
    char *root;
    va_list argp;

    va_start (argp, format);
    Nbyte = vsnprintf (&tmp, 0, format, argp);
    va_end (argp);

    if (!Nbyte)
        return false;

    va_start (argp, format);
    root = (char *) psAlloc (Nbyte + 1);
    memset (root, 0, Nbyte + 1);
    vsnprintf (root, Nbyte + 1, format, argp);
    va_end (argp);

    psMetadata *folder = psMetadataAlloc ();
    psMetadataAdd (md, PS_LIST_TAIL, root, PS_DATA_METADATA, "folder for 4D polynomial", folder);
    psFree (root);

    // specify the polynomial orders
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_X", PS_DATA_S32, "number of x orders", poly->nX);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Y", PS_DATA_S32, "number of y orders", poly->nY);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_Z", PS_DATA_S32, "number of z orders", poly->nZ);
    psMetadataAdd (folder, PS_LIST_TAIL, "NORDER_T", PS_DATA_S32, "number of z orders", poly->nT);

    // place polynomial entries on folder
    for (int nx = 0; nx < poly->nX + 1; nx++) {
        for (int ny = 0; ny < poly->nY + 1; ny++) {
            for (int nz = 0; nz < poly->nZ + 1; nz++) {
                for (int nt = 0; nt < poly->nT + 1; nt++) {
                    if (poly->mask[nx][ny][nz][nt])
                        continue;
                    psMetadataAdd (folder, PS_LIST_TAIL, "VAL_X%02d_Y%02d_Z%02d_T%02d", PS_DATA_F64, "polynomial coefficient", poly->coeff[nx][ny][nz][nt], nx, ny, nz, nt);
                    psMetadataAdd (folder, PS_LIST_TAIL, "ERR_X%02d_Y%02d_Z%02d_T%02d", PS_DATA_F64, "polynomial coeffficient error", poly->coeffErr[nx][ny][nz][nt], nx, ny, nz, nt);
                }
            }
        }
    }
    return true;
}
