#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <libgen.h>
#include <pslib.h>
#include <psmodules.h>
#include <ppStats.h>

#include "ppStack.h"

//#define TESTING

#define WCS_TOLERANCE 0.001             // Tolerance for WCS
#define PIXELS_BUFFER 1024              // Initial size of pixel lists

// Here follows lists of files for activation/deactivation at various stages.  Each must be NULL-terminated.

// Files required in preparation for convolution
static char *prepareFiles[] = { "PPSTACK.INPUT.PSF", "PPSTACK.INPUT.SOURCES", "PPSTACK.TARGET.PSF", NULL };

// Files required for the convolution
static char *convolveFiles[] = { "PPSTACK.INPUT", "PPSTACK.INPUT.MASK", "PPSTACK.INPUT.WEIGHT", NULL };

// Output files for the combination
static char *combineFiles[] = { "PPSTACK.OUTPUT", "PPSTACK.OUTPUT.MASK", "PPSTACK.OUTPUT.WEIGHT",
                                "PPSTACK.OUTPUT.JPEG1", "PPSTACK.OUTPUT.JPEG2", NULL };

// Files for photometry
static char *photFiles[] = { "PSPHOT.INPUT", "PSPHOT.OUTPUT", "PSPHOT.RESID", "PSPHOT.BACKMDL",
                             "PSPHOT.BACKMDL.STDEV", "PSPHOT.BACKGND", "PSPHOT.BACKSUB",
                             "SOURCE.PLOT.MOMENTS", "SOURCE.PLOT.PSFMODEL", "SOURCE.PLOT.APRESID",
                             "PSPHOT.INPUT.CMF", NULL };

static void memDump(const char *name)
{
    return;

    static int num = 0;                 // Counter, to make files unique and give an idea of sequence

    psString filename = NULL;           // Name of file
    psStringAppend(&filename, "memdump_%s_%03d.txt", name, num);
    FILE *memFile = fopen(filename, "w");
    psFree(filename);

    psMemBlock **leaks = NULL;
    int numLeaks = psMemCheckLeaks(0, &leaks, NULL, true);
    fprintf(memFile, "# MemBlock Size Source\n");
    unsigned long total = 0;            // Total memory used
    for (int i = 0; i < numLeaks; i++) {
        psMemBlock *mb = leaks[i];
        fprintf(memFile, "%12lu\t%12zd\t%s:%d\n", mb->id, mb->userMemorySize,
                mb->file, mb->lineno);
        total += mb->userMemorySize;
    }
    fclose(memFile);
    psFree(leaks);

    fprintf(stderr, "Memdump %s %d: Memory use: %ld, sbrk: %p\n", name, num, total, sbrk(0));
    num++;
}



// Activate/deactivate a list of files
static void fileActivation(pmConfig *config, // Configuration
                           char **files, // Files to turn on/off
                           bool state   // Activation state
    )
{
    assert(config);
    assert(files);

    for (int i = 0; files[i] != NULL; i++) {
        pmFPAfileActivate(config->files, state, files[i]);
    }
    return;
}

// Activate/deactivate a single element for a list
static void fileActivationSingle(pmConfig *config, // Configuration
                                 char **files, // Files to turn on/off
                                 bool state,   // Activation state
                                 int num // Number of file in sequence
                                 )
{
    assert(config);
    assert(files);

    for (int i = 0; files[i] != NULL; i++) {
        pmFPAfileActivateSingle(config->files, state, files[i], num); // Activated file
    }
    return;
}

// Iterate down the hierarchy, loading files; we can get away with this because we're working on skycells
static pmFPAview *filesIterateDown(pmConfig *config // Configuration
                                  )
{
    assert(config);

    pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy
    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
        return NULL;
    }
    view->chip = 0;
    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
        return NULL;
    }
    view->cell = 0;
    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
        return NULL;
    }
    view->readout = 0;
    if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) {
        return NULL;
    }
    return view;
}

// Iterate up the hierarchy, writing files; we can get away with this because we're working on skycells
static bool filesIterateUp(pmConfig *config // Configuration
                           )
{
    assert(config);

    pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy
    view->chip = view->cell = view->readout = 0;
    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
        return false;
    }
    view->readout = -1;
    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
        return false;
    }
    view->cell = -1;
    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
        return false;
    }
    view->chip = -1;
    if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) {
        return false;
    }
    psFree(view);
    return true;
}

// Write an image to a FITS file
static bool writeImage(const char *name, // Name of image
                       psMetadata *header, // Header
                       const psImage *image, // Image
                       pmConfig *config // Configuration
                       )
{
    assert(name);
    assert(image);

    psString resolved = pmConfigConvertFilename(name, config, true, true); // Resolved file name
    psFits *fits = psFitsOpen(resolved, "w");
    if (!fits) {
        psError(PS_ERR_IO, false, "Unable to open FITS file %s to write image.", resolved);
        psFree(resolved);
        return false;
    }
    if (!psFitsWriteImage(fits, header, image, 0, NULL)) {
        psError(PS_ERR_IO, false, "Unable to write FITS image %s.", resolved);
        psFitsClose(fits);
        psFree(resolved);
        return false;
    }
    psFitsClose(fits);
    psFree(resolved);
    return true;
}


bool ppStackLoop(pmConfig *config)
{
    assert(config);

    psTimerStart("PPSTACK_TOTAL");

    psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // ppStack recipe
    psAssert(recipe, "We've thrown an error on this before.");

    bool mdok;                          // Status of MD lookup
    bool tempDelete = psMetadataLookupBool(&mdok, recipe, "TEMP.DELETE"); // Delete temporary files?
    const char *tempDir = psMetadataLookupStr(NULL, recipe, "TEMP.DIR"); // Directory for temporary images
    if (!tempDir) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find TEMP.DIR in recipe");
        return false;
    }

    psString outputName = psStringCopy(psMetadataLookupStr(NULL, config->arguments, "OUTPUT")); // Name for temporary files
    const char *tempName = basename(outputName);
    if (!tempName) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to construct basename for temporary files.");
        return false;
    }

    const char *tempImage = psMetadataLookupStr(NULL, recipe, "TEMP.IMAGE"); // Suffix for temporary images
    const char *tempMask = psMetadataLookupStr(NULL, recipe, "TEMP.MASK"); // Suffix for temporary masks
    const char *tempWeight = psMetadataLookupStr(NULL, recipe, "TEMP.WEIGHT"); // Suffix for temp weight maps
    if (!tempImage || !tempMask || !tempWeight) {
        psError(PS_ERR_BAD_PARAMETER_VALUE, false,
                "Unable to find TEMP.IMAGE, TEMP.MASK and TEMP.WEIGHT in recipe");
        return false;
    }

    float threshold = psMetadataLookupF32(NULL, recipe, "THRESHOLD.MASK"); // Threshold for mask deconvolution
    float imageRej = psMetadataLookupF32(NULL, recipe, "IMAGE.REJ"); // Maximum fraction of image to reject
                                                                     // before rejecting entire image
    float poorFrac = psMetadataLookupF32(&mdok, recipe, "POOR.FRACTION"); // Fraction for "poor"

    const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics
    psMetadata *stats = NULL;           // Container for statistics
    FILE *statsFile = NULL;             // File stream for statistics
    if (statsName && strlen(statsName) > 0) {
        psString resolved = pmConfigConvertFilename(statsName, config, true, true); // Resolved filename
        statsFile = fopen(resolved, "w");
        if (!statsFile) {
            psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved);
            psFree(resolved);
            return false;
        } else {
            stats = psMetadataAlloc();
        }
        psFree(resolved);
    }

    pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSTACK.OUTPUT"); // Output file
    if (!output) {
        psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find output data!");
        return false;
    }
    int num = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of inputs

    psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe
    int overlap = 2 * psMetadataLookupS32(NULL, ppsub,
                                          "KERNEL.SIZE"); // Overlap by kernel size between consecutive scans

    if (!pmConfigMaskSetBits(NULL, NULL, config)) {
        psError(PS_ERR_UNKNOWN, false, "Unable to determine mask value.");
        return false;
    }

    memDump("start");

    // Preparation iteration: Load the sources, and get a target PSF model
    psTrace("ppStack", 1, "Determining target PSF....\n");
    psArray *globalSources = NULL;      // Global list of sources for matching (haveSources = TRUE)
    psArray *indSources = psArrayAlloc(num); // Individual lists of sources for matching (haveSources = FALSE)
    pmPSF *targetPSF = NULL;            // Target PSF
    bool haveSources = psMetadataLookupBool(NULL, config->arguments, "HAVE.SOURCES"); // Global sources?
    if (psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) {
        pmFPAfileActivate(config->files, false, NULL);
        fileActivation(config, prepareFiles, true);
        pmFPAview *view = filesIterateDown(config);
        if (!view) {
            return false;
        }

        // Generate list of PSFs
        psMetadataIterator *fileIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD,
                                                               "^PPSTACK.INPUT$"); // Iterator
        psMetadataItem *fileItem; // Item from iteration
        psArray *psfs = psArrayAlloc(num); // PSFs for PSF envelope
        psArray *sourceLists = NULL;    // Source lists for merging sources from multiple readouts
        int numCols = 0, numRows = 0;   // Size of image
        int index = 0;                  // Index for file
        while ((fileItem = psMetadataGetAndIncrement(fileIter))) {
            assert(fileItem->type == PS_DATA_UNKNOWN);
            pmFPAfile *inputFile = fileItem->data.V; // An input file
            pmChip *chip = pmFPAviewThisChip(view, inputFile->fpa); // The chip: holds the PSF
            pmPSF *psf = psMetadataLookupPtr(NULL, chip->analysis, "PSPHOT.PSF"); // PSF
            if (!psf) {
                psError(PS_ERR_UNKNOWN, false, "Unable to find PSF.");
                psFree(view);
                psFree(globalSources);
                psFree(indSources);
                psFree(sourceLists);
                psFree(fileIter);
                psFree(psfs);
                return false;
            }
            psfs->data[index] = psMemIncrRefCounter(psf);
            psMetadataRemoveKey(chip->analysis, "PSPHOT.PSF");

            pmCell *cell = pmFPAviewThisCell(view, inputFile->fpa); // Cell of interest
            pmHDU *hdu = pmHDUFromCell(cell);
            assert(hdu && hdu->header);
            int naxis1 = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); // Number of columns
            int naxis2 = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); // Number of rows
            if (naxis1 <= 0 || naxis2 <= 0) {
                psError(PS_ERR_UNKNOWN, false, "Unable to determine size of image from PSF.");
                psFree(view);
                psFree(globalSources);
                psFree(indSources);
                psFree(sourceLists);
                psFree(fileIter);
                psFree(psfs);
                return false;
            }
            if (numCols == 0 && numRows == 0) {
                numCols = naxis1;
                numRows = naxis2;
            }

            if (!haveSources) {
                pmReadout *ro = pmFPAviewThisReadout(view, inputFile->fpa); // Readout with sources
                psArray *sources = psMetadataLookupPtr(NULL, ro->analysis, "PSPHOT.SOURCES"); // Sources
                if (!sources) {
                    psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find sources in readout.");
                    return NULL;
                }
                indSources->data[index] = psMemIncrRefCounter(sources);


                // Calculate zero points if we don't trust the source lists
                if (psMetadataLookupBool(NULL, recipe, "ZP")) {
                    psTrace("ppStack", 2, "Photometering input %d of %d....\n", index, num);
                    pmFPAfileActivate(config->files, false, NULL);
                    fileActivationSingle(config, convolveFiles, true, index);
                    pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT",
                                                            index);// File of interest
                    pmFPAview *view = filesIterateDown(config);
                    if (!view) {
                        psFree(globalSources);
                        psFree(indSources);
                        psFree(targetPSF);
                        return false;
                    }

                    pmReadout *ro = pmFPAviewThisReadout(view, file->fpa); // Readout of interest

                    if (!ppStackInputPhotometry(ro, sources, config)) {
                        psError(PS_ERR_UNKNOWN, false, "Unable to do photometry on input sources");
                        psFree(globalSources);
                        psFree(indSources);
                        psFree(targetPSF);
                        return false;
                    }

                    psFree(view);
                    if (!filesIterateUp(config)) {
                        psFree(globalSources);
                        psFree(indSources);
                        psFree(targetPSF);
                        return false;
                    }
                    pmFPAfileActivate(config->files, false, NULL);
                    fileActivation(config, prepareFiles, true);
                }


#ifdef TESTING
                ppStackSourcesPrint(sources);
#endif
                sourceLists = ppStackSourceListAdd(sourceLists, sources, config);
                if (!sourceLists) {
                    psError(PS_ERR_UNKNOWN, false, "Unable to add sources to list.");
                    psFree(view);
                    psFree(globalSources);
                    psFree(indSources);
                    psFree(fileIter);
                    psFree(psfs);
                    return false;
                }
            }

            index++;
        }
        psFree(fileIter);

        targetPSF = ppStackPSF(config, numCols, numRows, psfs);
        psFree(psfs);
        if (!targetPSF) {
            psError(PS_ERR_UNKNOWN, false, "Unable to determine output PSF.");
            psFree(globalSources);
            psFree(indSources);
            psFree(view);
            return false;
        }

        pmChip *outChip = pmFPAviewThisChip(view, output->fpa); // Output chip
        psMetadataAddPtr(outChip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN,
                         "Target PSF", targetPSF);
        outChip->data_exists = true;

        if (haveSources) {
            // We want to hang on to the 'sources' even when its host FPA is blown away
            pmReadout *sourcesRO = pmFPAfileThisReadout(config->files, view, "PPSTACK.INPUT.SOURCES");
            globalSources = psMetadataLookupPtr(NULL, sourcesRO->analysis, "PSPHOT.SOURCES");
            psMemIncrRefCounter(globalSources);
            if (!globalSources) {
                psError(PS_ERR_UNKNOWN, true, "Unable to find sources.");
                psFree(view);
                return false;
            }
        } else {
            globalSources = ppStackSourceListCombine(sourceLists, config);
            psFree(sourceLists);
            if (!globalSources) {
                psError(PS_ERR_UNKNOWN, false, "Unable to add sources to list.");
                psFree(view);
                psFree(psfs);
                return false;
            }
#ifdef TESTING
            ppStackSourcesPrint(globalSources);
#endif
        }

        psFree(view);
        if (!filesIterateUp(config)) {
            return false;
        }
    }

    memDump("psf");

    psArray *imageNames = psArrayAlloc(num);
    psArray *maskNames = psArrayAlloc(num);
    psArray *weightNames = psArrayAlloc(num);
    for (int i = 0; i < num; i++) {
        psString imageName = NULL, maskName = NULL, weightName = NULL; // Names for convolved images
        psStringAppend(&imageName, "%s/%s.%d.%s", tempDir, tempName, i, tempImage);
        psStringAppend(&maskName, "%s/%s.%d.%s", tempDir, tempName, i, tempMask);
        psStringAppend(&weightName, "%s/%s.%d.%s", tempDir, tempName, i, tempWeight);
        psTrace("ppStack", 5, "Temporary files: %s %s %s\n", imageName, maskName, weightName);
        imageNames->data[i] = imageName;
        maskNames->data[i] = maskName;
        weightNames->data[i] = weightName;
    }
    // Free the outputName that we grabbed above to set up tempName.
    psFree(outputName);

    // Generate convolutions and write them to disk
    psTrace("ppStack", 1, "Convolving inputs to target PSF....\n");
    psArray *cells = psArrayAlloc(num); // Cells for convolved images --- a handle for reading again
    psArray *subKernels = psArrayAlloc(num); // Subtraction kernels --- required in the stacking
    psArray *subRegions = psArrayAlloc(num); // Subtraction regions --- required in the stacking
    psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator
    int numGood = 0;                    // Number of good frames
    int numCols = 0, numRows = 0;       // Size of image
    psVector *inputMask = psVectorAlloc(num, PS_TYPE_U8); // Mask for inputs
    psVectorInit(inputMask, 0);
    psVector *matchChi2 = psVectorAlloc(num, PS_TYPE_F32); // chi^2 for stamps when matching
    psVectorInit(matchChi2, NAN);
    psVector *weightings = psVectorAlloc(num, PS_TYPE_F32); // Combination weightings for images (1/noise^2)
    psVectorInit(weightings, NAN);
    for (int i = 0; i < num; i++) {
        psTrace("ppStack", 2, "Convolving input %d of %d to target PSF....\n", i, num);
        pmFPAfileActivate(config->files, false, NULL);
        fileActivationSingle(config, convolveFiles, true, i);
        pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File of interest
        pmFPAview *view = filesIterateDown(config);
        if (!view) {
            psFree(globalSources);
            psFree(indSources);
            psFree(targetPSF);
            psFree(rng);
            psFree(inputMask);
            psFree(matchChi2);
            return false;
        }

        pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); // Input readout
        psFree(view);

        if (numCols == 0 && numRows == 0) {
            numCols = readout->image->numCols;
            numRows = readout->image->numRows;
        } else if (numCols != readout->image->numCols || numRows != readout->image->numRows) {
            psError(PS_ERR_UNKNOWN, true, "Sizes of input images don't match: %dx%d vs %dx%d",
                    readout->image->numCols, readout->image->numRows, numCols, numRows);
            psFree(globalSources);
            psFree(indSources);
            psFree(targetPSF);
            psFree(rng);
            psFree(inputMask);
            psFree(matchChi2);
            return false;
        }

        // Background subtraction, scaling and normalisation is performed automatically by the image matching
        psArray *regions = NULL, *kernels = NULL; // Regions and kernels used in subtraction
        psArray *sources = haveSources ? globalSources : indSources->data[i]; // Sources for matching
        psTimerStart("PPSTACK_MATCH");
        if (!ppStackMatch(readout, &regions, &kernels, &matchChi2->data.F32[i], &weightings->data.F32[i],
                          sources, targetPSF, rng, config)) {
            psErrorStackPrint(stderr, "Unable to match image %d --- ignoring.", i);
            inputMask->data.U8[i] = PPSTACK_MASK_MATCH;
            psErrorClear();
            continue;
        }

        if (stats) {
            pmSubtractionKernels *kernels = psMetadataLookupPtr(NULL, readout->analysis,
                                                                PM_SUBTRACTION_ANALYSIS_KERNEL);// Conv kernel
            psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_MATCH", PS_META_DUPLICATE_OK,
                             "Time to match PSF", psTimerMark("PPSTACK_MATCH"));
            psMetadataAddF32(stats, PS_LIST_TAIL, "STAMP.MEAN", PS_META_DUPLICATE_OK,
                             "Mean deviation for stamps", kernels->mean);
            psMetadataAddF32(stats, PS_LIST_TAIL, "STAMP.RMS", PS_META_DUPLICATE_OK,
                             "RMS deviation for stamps", kernels->rms);
            psMetadataAddF32(stats, PS_LIST_TAIL, "STAMP.NUM", PS_META_DUPLICATE_OK,
                             "Number of stamps", kernels->numStamps);

        }
        psLogMsg("ppStack", PS_LOG_INFO, "Time to match image %d: %f sec", i, psTimerClear("PPSTACK_MATCH"));

        subRegions->data[i] = regions;
        subKernels->data[i] = kernels;

        // Write the temporary convolved files
        pmHDU *hdu = readout->parent->parent->parent->hdu; // HDU for convolved image
        assert(hdu);
        writeImage(imageNames->data[i],  hdu->header, readout->image, config);
        writeImage(maskNames->data[i],   hdu->header, readout->mask, config);
        writeImage(weightNames->data[i], hdu->header, readout->weight, config);

        cells->data[i] = psMemIncrRefCounter(readout->parent);
        if (!filesIterateUp(config)) {
            return false;
        }
        numGood++;

        memDump("match");
    }
    psFree(globalSources);
    psFree(indSources);
    psFree(targetPSF);
    psFree(rng);

    if (numGood == 0) {
        psError(PS_ERR_UNKNOWN, false, "No good images to combine.");
        psFree(subKernels);
        psFree(subRegions);
        psFree(cells);
        psFree(inputMask);
        psFree(matchChi2);
        return false;
    }

    // Reject images out-of-hand on the basis of their match chi^2
    {
        psVector *values = psVectorAllocEmpty(num, PS_TYPE_F32); // Values to sort
        for (int i = 0; i < num; i++) {
            if (inputMask->data.PS_TYPE_MASK_DATA[i] & PPSTACK_MASK_ALL) {
                continue;
            }
            values->data.F32[values->n++] = matchChi2->data.F32[i];
        }
        assert(values->n == numGood);
        if (!psVectorSortInPlace(values)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to sort vector.");
            psFree(subKernels);
            psFree(subRegions);
            psFree(cells);
            psFree(inputMask);
            psFree(matchChi2);
            psFree(values);
            return false;
        }
        float median = numGood % 2 ? values->data.F32[numGood / 2] :
            0.5 * (values->data.F32[numGood / 2 - 1] + values->data.F32[numGood / 2]);

        float rms = 0.74 * (values->data.F32[numGood * 3 / 4] -
                            values->data.F32[numGood / 4]); // Estimated RMS from interquartile range

        psFree(values);

        float rej = psMetadataLookupF32(NULL, recipe, "MATCH.REJ"); // Rejection threshold (stdevs)
        if (isfinite(rej)) {
            float thresh = median + rej * rms; // Threshold for rejection
            psLogMsg("ppStack", PS_LOG_INFO, "chi^2 rejection threshold = %f + %f * %f = %f",
                     median, rej, rms, thresh);

            int numRej = 0;                 // Number rejected
            numGood = 0;                    // Number of good images
            for (int i = 0; i < num; i++) {
                if (inputMask->data.PS_TYPE_MASK_DATA[i] & PPSTACK_MASK_ALL) {
                    continue;
                }
                if (matchChi2->data.F32[i] > thresh) {
                    numRej++;
                    inputMask->data.PS_TYPE_MASK_DATA[i] |= PPSTACK_MASK_CHI2;
                    psLogMsg("ppStack", PS_LOG_INFO, "Rejecting image %d because of large matching chi^2: %f",
                             i, matchChi2->data.F32[i]);
                } else {
                    numGood++;
                }
            }
        }
    }

    if (numGood == 0) {
        psError(PS_ERR_UNKNOWN, false, "No good images to combine.");
        psFree(subKernels);
        psFree(subRegions);
        psFree(cells);
        psFree(inputMask);
        psFree(matchChi2);
        return false;
    }

#ifdef TESTING
    psTraceSetLevel("psModules.imcombine", 7);
#endif


    // Start threading
    ppStackThreadInit();
    ppStackThreadData *stack = ppStackThreadDataSetup(cells, imageNames, maskNames, weightNames, config);

    psTimerStart("PPSTACK_INITIAL");

    memDump("preinitial");

    // Stack the convolved files
    psTrace("ppStack", 1, "Initial stack of convolved images....\n");
    pmReadout *outRO = NULL;            // Output readout
    pmFPAview *view = NULL;             // View to readout
    psArray *inspect = NULL;            // Array of arrays of pixels to inspect
    psVector *exptimes = psVectorAlloc(num, PS_TYPE_F32); // Exposure times for each input
    {
        int row0, col0;                 // Offset for readout
        int numCols, numRows;           // Size of readout
        ppStackThread *thread = stack->threads->data[0]; // Representative thread
        if (!pmReadoutStackSetOutputSize(&col0, &row0, &numCols, &numRows, thread->readouts)) {
            psError(PS_ERR_UNKNOWN, false, "problem setting output readout size.");
            psFree(subKernels);
            psFree(subRegions);
            psFree(stack);
            psFree(inputMask);
            psFree(matchChi2);
            psFree(exptimes);
            psFree(cells);
            return false;
        }

        pmFPAfileActivate(config->files, false, NULL);
        fileActivation(config, combineFiles, true);
        view = filesIterateDown(config);
        if (!view) {
            psFree(subKernels);
            psFree(subRegions);
            psFree(stack);
            psFree(inputMask);
            psFree(matchChi2);
            psFree(exptimes);
            psFree(cells);
            return false;
        }

        pmCell *outCell = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT"); // Output cell
        outRO = pmReadoutAlloc(outCell); // Output readout

        psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits to mask for bad
        psMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels
        if (!pmReadoutStackDefineOutput(outRO, col0, row0, numCols, numRows, true, true, maskBad)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to prepare output.");
            psFree(subKernels);
            psFree(subRegions);
            psFree(stack);
            psFree(inputMask);
            psFree(matchChi2);
            psFree(view);
            psFree(outRO);
            psFree(exptimes);
            psFree(cells);
            return false;
        }

        psList *fpaList = psListAlloc(NULL); // List of input FPAs, for concept averaging
        psList *cellList = psListAlloc(NULL); // List of input cells, for concept averaging
        for (int i = 0; i < num; i++) {
            pmCell *inCell = cells->data[i]; // Input cell
            if (!inCell) {
                exptimes->data.F32[i] = 0.0;
                continue;
            }
            exptimes->data.F32[i] = psMetadataLookupF32(NULL, inCell->concepts, "CELL.EXPOSURE");
            psListAdd(cellList, PS_LIST_TAIL, inCell);
            psListAdd(fpaList, PS_LIST_TAIL, inCell->parent->parent);
        }

        // Copy concepts
        pmChip *outChip = outCell->parent;  // Output chip
        pmFPA *outFPA = outChip->parent;    // Output FPA
        pmConceptsAverageFPAs(outFPA, fpaList);
        pmConceptsAverageCells(outCell, cellList, NULL, NULL, true);
        psFree(fpaList);
        psFree(cellList);

        psFree(cells);

        bool status;                    // Status of read
        int numChunk;                   // Number of chunks
        for (numChunk = 0; true; numChunk++) {
            ppStackThread *thread = ppStackThreadRead(&status, stack, config, numChunk, overlap);
            if (!status) {
                // Something went wrong
                psError(PS_ERR_UNKNOWN, false, "Unable to read chunk %d", numChunk);
                psFree(subKernels);
                psFree(subRegions);
                psFree(stack);
                psFree(inputMask);
                psFree(matchChi2);
                psFree(view);
                psFree(outRO);
                psFree(exptimes);
                return false;
            }
            if (!thread) {
                // Nothing more to read
                break;
            }

            // call: ppStackReadoutInitial(config, outRO, readouts, subRegions, subKernels)
            psThreadJob *job = psThreadJobAlloc("PPSTACK_INITIAL_COMBINE"); // Job to start
            psArrayAdd(job->args, 1, thread);
            psArrayAdd(job->args, 1, config);
            psArrayAdd(job->args, 1, outRO);
            psArrayAdd(job->args, 1, subRegions);
            psArrayAdd(job->args, 1, subKernels);
            psArrayAdd(job->args, 1, weightings);
            psArrayAdd(job->args, 1, matchChi2);
            if (!psThreadJobAddPending(job)) {
                psFree(job);
                psFree(subKernels);
                psFree(subRegions);
                psFree(stack);
                psFree(inputMask);
                psFree(weightings);
                psFree(matchChi2);
                psFree(view);
                psFree(outRO);
                psFree(exptimes);
                return false;
            }
            psFree(job);
        }

        if (!psThreadPoolWait(false)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to do initial combination.");
            psFree(subKernels);
            psFree(subRegions);
            psFree(stack);
            psFree(inputMask);
            psFree(matchChi2);
            psFree(view);
            psFree(outRO);
            psFree(exptimes);
            return false;
        }
        psFree(matchChi2);

        // Harvest the jobs, gathering the inspection lists
        inspect = psArrayAlloc(num);
        for (int i = 0; i < num; i++) {
            if (inputMask->data.U8[i]) {
                continue;
            }
            inspect->data[i] = psArrayAllocEmpty(numChunk);
        }
        psThreadJob *job;               // Completed job
        while ((job = psThreadJobGetDone())) {
            psAssert(strcmp(job->type, "PPSTACK_INITIAL_COMBINE") == 0,
                     "Job has incorrect type: %s", job->type);
            psArray *results = job->results; // Results of job
            for (int i = 0; i < num; i++) {
                if (inputMask->data.U8[i]) {
                    continue;
                }
                inspect->data[i] = psArrayAdd(inspect->data[i], 1, results->data[i]);
            }
            psFree(job);
        }

        memDump("initial");
    }

    if (stats) {
        psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_INITIAL", 0,
                         "Time to make initial stack", psTimerMark("PPSTACK_INITIAL"));
    }
    psLogMsg("ppStack", PS_LOG_INFO, "Time to make initial stack: %f sec", psTimerClear("PPSTACK_INITIAL"));

    psTimerStart("PPSTACK_REJECT");

    // Pixel rejection
    psArray *rejected = psArrayAlloc(num);
    {
        int numRejected = 0;        // Number of inputs rejected completely

        // Count images rejected out of hand
        for (int i = 0; i < num; i++) {
            if (inputMask->data.U8[i]) {
                numRejected++;
            }
        }

        for (int i = 0; i < num; i++) {
            if (inputMask->data.U8[i]) {
                continue;
            }

            psThreadJob *job = psThreadJobAlloc("PPSTACK_INSPECT"); // Job to start
            psArrayAdd(job->args, 1, inspect);
            PS_ARRAY_ADD_SCALAR(job->args, i, PS_TYPE_S32);
            if (!psThreadJobAddPending(job)) {
                psFree(job);
                psFree(subKernels);
                psFree(subRegions);
                psFree(stack);
                psFree(inputMask);
                psFree(view);
                psFree(outRO);
                psFree(inspect);
                psFree(rejected);
                psFree(exptimes);
                return false;
            }
            psFree(job);
        }

        if (!psThreadPoolWait(true)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to concatenate inspection lists.");
            psFree(subKernels);
            psFree(subRegions);
            psFree(stack);
            psFree(inputMask);
            psFree(view);
            psFree(outRO);
            psFree(inspect);
            psFree(rejected);
            psFree(exptimes);
            return false;
        }

        if (psMetadataLookupS32(NULL, config->arguments, "-threads") > 0) {
            pmStackRejectThreadsInit();
        }

        int stride = psMetadataLookupS32(NULL, ppsub, "STRIDE"); // Size of convolution patches

        // Reject bad pixels
        for (int i = 0; i < num; i++) {
            if (inputMask->data.U8[i]) {
                continue;
            }
            psTimerStart("PPSTACK_REJECT");

#ifdef TESTING
            {
                psImage *mask = psPixelsToMask(NULL, inspect->data[i],
                                               psRegionSet(0, numCols - 1, 0, numRows - 1),
                                               0xff); // Mask image
                psString name = NULL;           // Name of image
                psStringAppend(&name, "inspect_%03d.fits", i);
                psFits *fits = psFitsOpen(name, "w");
                psFree(name);
                psFitsWriteImage(fits, NULL, mask, 0, NULL);
                psFree(mask);
                psFitsClose(fits);
            }
#endif

            psPixels *reject = pmStackReject(inspect->data[i], numCols, numRows, threshold, poorFrac, stride,
                                             subRegions->data[i], subKernels->data[i]); // Rejected pixels

#ifdef TESTING
            {
                psImage *mask = psPixelsToMask(NULL, reject, psRegionSet(0, numCols - 1, 0, numRows - 1),
                                               0xff); // Mask image
                psString name = NULL;           // Name of image
                psStringAppend(&name, "reject_%03d.fits", i);
                psFits *fits = psFitsOpen(name, "w");
                psFree(name);
                psFitsWriteImage(fits, NULL, mask, 0, NULL);
                psFree(mask);
                psFitsClose(fits);
            }
#endif

            psFree(inspect->data[i]);
            inspect->data[i] = NULL;

            if (!reject) {
                psWarning("Rejection on image %d didn't work --- reject entire image.", i);
                numRejected++;
                inputMask->data.U8[i] = PPSTACK_MASK_REJECT;
            } else {
                float frac = reject->n / (float)(numCols * numRows); // Pixel fraction
                psLogMsg("ppStack", PS_LOG_INFO, "%ld pixels rejected from image %d (%.1f%%)",
                         reject->n, i, frac * 100.0);
                if (frac > imageRej) {
                    psWarning("Image %d rejected completely because rejection fraction (%.3f) "
                              "exceeds limit (%.3f)", i, frac, imageRej);
                    psFree(reject);
                    // reject == NULL means reject image completely
                    reject = NULL;
                    inputMask->data.U8[i] = PPSTACK_MASK_BAD;
                    numRejected++;
                }
            }

            // Images without a list of rejected pixels (the list may be empty) are rejected completely
            rejected->data[i] = reject;

            if (stats) {
                psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_REJECT", PS_META_DUPLICATE_OK,
                                 "Time to perform rejection", psTimerMark("PPSTACK_REJECT"));
                psMetadataAddS32(stats, PS_LIST_TAIL, "REJECT_PIXELS", PS_META_DUPLICATE_OK,
                                 "Number of pixels rejected", reject ? reject->n : 0);
            }
            psLogMsg("ppStack", PS_LOG_INFO, "Time to perform rejection on image %d: %f sec", i,
                     psTimerClear("PPSTACK_REJECT"));
        }

        psFree(inspect);
        psFree(subKernels);
        psFree(subRegions);

        if (numRejected >= num - 1) {
            psError(PS_ERR_UNKNOWN, true, "All inputs completely rejected; unable to proceed.");
            psFree(stack);
            psFree(rejected);
            psFree(inputMask);
            psFree(view);
            psFree(outRO);
            psFree(exptimes);
            return false;
        }

        if (stats) {
            psMetadataAddS32(stats, PS_LIST_TAIL, "REJECT_IMAGES", 0,
                             "Number of images rejected completely", numRejected);
        }
   }

    psTimerStart("PPSTACK_FINAL");

    memDump("reject");

    // Final combination
    psTrace("ppStack", 2, "Final stack of convolved images....\n");
    {
        stack->lastScan = 0;            // Reset read
        bool status;                    // Status of read
        for (int numChunk = 0; true; numChunk++) {
            ppStackThread *thread = ppStackThreadRead(&status, stack, config, numChunk, 0);
            if (!status) {
                // Something went wrong
                psError(PS_ERR_UNKNOWN, false, "Unable to read chunk %d", numChunk);
                psFree(stack);
                psFree(rejected);
                psFree(inputMask);
                psFree(view);
                psFree(outRO);
                psFree(exptimes);
                return false;
            }
            if (!thread) {
                // Nothing more to read
                break;
            }

            // call: ppStackReadoutFinal(config, outRO, readouts, rejected)
            psThreadJob *job = psThreadJobAlloc("PPSTACK_FINAL_COMBINE"); // Job to start
            psArrayAdd(job->args, 1, thread);
            psArrayAdd(job->args, 1, config);
            psArrayAdd(job->args, 1, outRO);
            psArrayAdd(job->args, 1, rejected);
            psArrayAdd(job->args, 1, weightings);
            if (!psThreadJobAddPending(job)) {
                psFree(job);
                psFree(stack);
                psFree(rejected);
                psFree(inputMask);
                psFree(view);
                psFree(outRO);
                psFree(exptimes);
                return false;
            }
            psFree(job);
        }

        if (!psThreadPoolWait(true)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to do final combination.");
            psFree(stack);
            psFree(inputMask);
            psFree(rejected);
            psFree(view);
            psFree(outRO);
            psFree(exptimes);
            return false;
        }
    }

    memDump("final");

    if (stats) {
        psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_FINAL", 0,
                         "Time to make final stack", psTimerMark("PPSTACK_FINAL"));
    }
    psLogMsg("ppStack", PS_LOG_INFO, "Time to make final stack: %f sec", psTimerClear("PPSTACK_FINAL"));

    psTrace("ppStack", 2, "Cleaning up after combination....\n");

#if 0
    // Ensure masked regions really look masked
    {
        psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits for bad
        psMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels
        if (!pmReadoutMaskApply(outRO, maskBad)) {
            psWarning("Unable to apply mask");
        }
    }
#endif

    // Close up
    bool wcsDone = false;           // Have we done the WCS?
    float totExposure = 0.0;            // Total exposure time
    for (int i = 0; i < num; i++) {
        if (inputMask->data.U8[i]) {
            continue;
        }
        totExposure += exptimes->data.F32[i];

        ppStackThread *thread = stack->threads->data[0]; // Representative stack
        pmReadout *inRO = thread->readouts->data[i]; // Template readout
        if (inRO && !wcsDone) {
            // Copy astrometry over
            wcsDone = true;
            pmHDU *inHDU = pmHDUFromCell(inRO->parent); // Template HDU
            pmHDU *outHDU = pmHDUFromCell(outRO->parent); // Output HDU
            pmChip *outChip = outRO->parent->parent; // Output chip
            pmFPA *outFPA = outChip->parent; // Output FPA
            if (!outHDU || !inHDU) {
                psWarning("Unable to find HDU at FPA level to copy astrometry.");
            } else {
                if (!pmAstromReadWCS(outFPA, outChip, inHDU->header, 1.0)) {
                    psErrorClear();
                    psWarning("Unable to read WCS astrometry from input FPA.");
                    wcsDone = false;
                } else {
                    if (!outHDU->header) {
                        outHDU->header = psMetadataAlloc();
                    }
                    if (!pmAstromWriteWCS(outHDU->header, outFPA, outChip, WCS_TOLERANCE)) {
                        psErrorClear();
                        psWarning("Unable to write WCS astrometry to output FPA.");
                        wcsDone = false;
                    }
                }
            }
        }

        if (tempDelete) {
            psString imageResolved = pmConfigConvertFilename(imageNames->data[i], config, false, false);
            psString maskResolved = pmConfigConvertFilename(maskNames->data[i], config, false, false);
            psString weightResolved = pmConfigConvertFilename(weightNames->data[i], config, false, false);
            if (unlink(imageResolved) == -1 || unlink(maskResolved) == -1 ||
                unlink(weightResolved) == -1) {
                psWarning("Unable to delete temporary files for image %d", i);
            }
            psFree(imageResolved);
            psFree(maskResolved);
            psFree(weightResolved);
        }
    }
    psFree(imageNames);
    psFree(maskNames);
    psFree(weightNames);

    psFree(exptimes);
    psFree(inputMask);
    psFree(stack);

    memDump("cleanup");

    // Generate binned JPEGs
    {
        int bin1 = psMetadataLookupS32(NULL, recipe, "BIN1"); // First binning level
        int bin2 = psMetadataLookupS32(NULL, recipe, "BIN2"); // Second binning level

        // Target cells
        pmCell *cell1 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG1");
        pmCell *cell2 = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT.JPEG2");
        psMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask

        pmReadout *ro1 = pmReadoutAlloc(cell1), *ro2 = pmReadoutAlloc(cell2); // Binned readouts
        if (!pmReadoutRebin(ro1, outRO, maskValue, bin1, bin1) || !pmReadoutRebin(ro2, ro1, 0, bin2, bin2)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to bin output.");
            psFree(ro1);
            psFree(ro2);
            psFree(outRO);
            return false;
        }
        psFree(ro1);
        psFree(ro2);
    }

    if (psMetadataLookupBool(&mdok, recipe, "RENORM")) {
        // Statistics for renormalisation
        psStatsOptions renormMean = psStatsOptionFromString(psMetadataLookupStr(&mdok, recipe,
                                                                                "RENORM.MEAN"));
        psStatsOptions renormStdev = psStatsOptionFromString(psMetadataLookupStr(&mdok, recipe,
                                                                                 "RENORM.STDEV"));
        int renormNum = psMetadataLookupS32(&mdok, recipe, "RENORM.NUM"); // Number of samples
        float renormWidth = psMetadataLookupS32(&mdok, recipe, "RENORM.WIDTH"); // Width of Gaussian phot
        psMaskType maskValue = pmConfigMaskGet("BLANK", config); // Bits to mask
        if (!pmReadoutWeightRenormPhot(outRO, maskValue, renormNum, renormWidth,
                                       renormMean, renormStdev, NULL)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to renormalise variances.");
            psFree(outRO);
            return false;
        }
    }

    if (psMetadataLookupBool(&mdok, recipe, "PHOTOMETRY")) {
        psTrace("ppStack", 1, "Photometering stacked image....\n");

        psTimerStart("PPSTACK_PHOT");

        fileActivation(config, combineFiles, false);
        fileActivation(config, photFiles, true);
        pmFPAview *photView = filesIterateDown(config);
        if (!ppStackPhotometry(config, outRO, photView)) {
            psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry on output.");
            psFree(outRO);
            psFree(photView);
            return false;
        }
        psFree(photView);

        fileActivation(config, combineFiles, true);

        if (stats) {
            pmFPAfile *photFile = psMetadataLookupPtr(NULL, config->files, "PSPHOT.INPUT"); // File
            pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout with the sources
            psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources
            psMetadataAddS32(stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected", sources->n);
            psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_PHOT", 0,
                             "Time to do photometry", psTimerMark("PPSTACK_PHOT"));
        }
        psLogMsg("ppStack", PS_LOG_INFO, "Time to do photometry: %f sec", psTimerClear("PPSTACK_PHOT"));
    }

    psThreadPoolFinalize();

    memDump("phot");

    // Statistics on output
    if (stats) {
        psTrace("ppStack", 1, "Gathering statistics on stacked image....\n");
        psString maskBadStr = psMetadataLookupStr(NULL, recipe, "MASK.BAD"); // Name of bits for bad
        psMaskType maskBad = pmConfigMaskGet(maskBadStr, config); // Bits to mask for bad pixels

        ppStatsFPA(stats, outRO->parent->parent->parent, view, maskBad, config);
    }

    memDump("stats");

    // Put version information into the header
    pmHDU *hdu = pmHDUFromCell(outRO->parent);
    if (!hdu) {
        psError(PS_ERR_UNKNOWN, false, "Unable to find HDU for output.");
        psFree(outRO);
        psFree(view);
        return false;
    }
    if (!hdu->header) {
        hdu->header = psMetadataAlloc();
    }
    ppStackVersionMetadata(hdu->header);

    psFree(outRO);
    psFree(view);

    // Write out summary statistics
    if (stats) {
        psMetadataAddF32(stats, PS_LIST_TAIL, "TIME_STACK", 0,
                         "Time to do photometry", psTimerClear("PPSTACK_TOTAL"));

        const char *statsMDC = psMetadataConfigFormat(stats);
        if (!statsMDC || strlen(statsMDC) == 0) {
            psError(PS_ERR_IO, false, "Unable to get statistics MDC file.\n");
        } else {
            fprintf(statsFile, "%s", statsMDC);
        }
        psFree((void *)statsMDC);
        fclose(statsFile);

        psFree(stats);
    }

    // Write out the output files
    if (!filesIterateUp(config)) {
        return false;
    }

    memDump("finish");

    return true;
}
