#include "streaksremove.h"

static pmConfig *parseArguments(int argc, char **argv);
static bool readAndCopyToOutput(streakFiles *sf, bool exciseAll);
static StreakPixels * getStreakPixels(streakFiles *sfiles, Streaks *streaks);
static void exciseNonWarpedPixels(streakFiles *sfiles, double newMaskValue);
static bool warpedPixel(streakFiles *sfiles, PixelPos *cellCoord);
static void excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, double newMaskValue);
static void writeImages(streakFiles *sf, bool exciseImageCube);
static bool replicateOutputs(streakFiles *sfiles);

int
main(int argc, char *argv[])
{
    bool status;

    psLibInit(NULL);
    psTimerStart("STREAKSREMOVE");

    pmConfig *config = parseArguments(argc, argv);
    if (!config) {
        psError(PS_ERR_UNKNOWN, false, "failed to parse arguments\n");
        return PS_EXIT_CONFIG_ERROR;
    }

    psMetadata *masks = psMetadataLookupMetadata(&status, config->recipes, "MASKS");
    if (!status) {
        psError(PM_ERR_CONFIG, false, "failed to lookup MASKS in recipes\n");
        return PS_EXIT_CONFIG_ERROR;
    }
    double maskStreak = (double) psMetadataLookupU8(&status, masks, "STREAK");
    if (!status) {
        psError(PM_ERR_CONFIG, false, "failed to lookup mask value for STREAK in recipes\n");
        return PS_EXIT_CONFIG_ERROR;
    }

    psString streaksFileName = psMetadataLookupStr(NULL, config->arguments, "STREAKS");

    Streaks *streaks = readStreaksFile(streaksFileName);
    if (!streaks) {
        psErrorStackPrint(stderr, "failed to read streaks file: %s", streaksFileName);
        exit (PS_EXIT_PROG_ERROR);
    }

    streakFiles *sfiles = openFiles(config, true);
    setupAstrometry(sfiles);

    bool exciseAll = false;
    // --keepnonwarped is a test and debug mode
    bool keepNonWarpedPixels = psMetadataLookupBool(&status, config->arguments, "KEEP_NON_WARPED");

    // we need to check for non warped pixels unless we've been asked not to or the stage is diff
    // (By definition pixels in diff images were included in a diff)
    bool checkNonWarpedPixels = ! (keepNonWarpedPixels || (sfiles->stage == IPP_STAGE_DIFF) );

    if (checkNonWarpedPixels ) {
        // From ICD:
        // In the raw and detrended images, the pixels which were not
        // included in any of the streak-processed warps must also be masked.
        // Note that the warp and difference skycells are only generated if more
        // than a small fraction of the pixels are lit by the input image.
        // if no skycells are provided sfiles->exciseAll is set to true

        psTimerStart("COMPUTE_WARPED_PIXELS");
        if (! computeWarpedPixels(sfiles) ) {
            // we have no choice to excise all pixels
            exciseAll = true;
        }
        psF64 cwp_t = psTimerClear("COMPUTE_WARPED_PIXELS");
        printf("time to compute warped pixels: %f\n", cwp_t);
    }
    
    if (sfiles->stage == IPP_STAGE_RAW) {
        // copy PHU to output files
        copyPHU(sfiles, true);

        // advance to the first image extension
        if (!streakFilesNextExtension(sfiles)) {
            psErrorStackPrint(stderr, "failed to advance to first extension of input images");
            exit (PS_EXIT_PROG_ERROR);
        }
    }

    int totalPixels = 0;
    int totalStreakPixels = 0;

    // Iterate through each component of the input (only one except for raw stage)
    do {
        bool exciseImageCube = false;

        psTimerStart("PROCESS_COMPONENT");

        // read the images and copy the data from the inputs to the outputs
        // This also sets up the astrometry and initializes the recovery image(s)

        if (!readAndCopyToOutput(sfiles, exciseAll)) {
            // false return value indicates that this extension is not an image and
            // the contents has already been copied to the output file.
            // we don't need to process this extesion for streaks.
            continue;
        }

        totalPixels += sfiles->inImage->numRows * sfiles->inImage->numCols;

        if (!exciseAll) {
            // Identify pixels to mask because of streaks

            psTimerStart("GET_STREAK_PIXELS");

            StreakPixels *pixels = getStreakPixels(sfiles, streaks);


            printf("time to get streak pixels: %f\n", psTimerClear("GET_STREAK_PIXELS"));

            if (checkNonWarpedPixels) {
                psTimerStart("EXCISE_NON_WARPED");

                exciseNonWarpedPixels(sfiles, maskStreak);

                printf("time to excise non warped pixels: %f\n", psTimerClear("EXCISE_NON_WARPED"));
            }
            
            if (sfiles->inImage->image) {
                totalStreakPixels +=  psArrayLength(pixels);
                psTimerStart("REMOVE_STREAKS");
                for (int i = 0; i < psArrayLength (pixels); ++i) {
                    PixelPos *pixelPos = psArrayGet (pixels, i);

                    if (!checkNonWarpedPixels || warpedPixel(sfiles, pixelPos)) {

                        excisePixel(sfiles, pixelPos->x, pixelPos->y, true, maskStreak);

                    } else {
                        // This pixel was not included in any warp and has thus already excised
                        // by exciseNonWarpedPixels
                    }
                }
                printf("time to remove streak pixels: %f\n", psTimerClear("REMOVE_STREAKS"));
            } else { 
                // this component contains an image cube, excise it completely
                exciseImageCube = true;
            }
            psArrayElementsFree (pixels);
        }

        // write out the destreaked temporary images and the recovery images
        writeImages(sfiles, exciseImageCube);

        printf("time to process component %d: %f\n", sfiles->extnum, psTimerClear("PROCESS_COMPONENT"));
    } while (streakFilesNextExtension(sfiles));

    printf("pixels: %ld streak pixels: %ld %4.2f%%\n", totalPixels, totalStreakPixels, 100. * totalStreakPixels / totalPixels);

    psTimerStart("CLOSE_IMAGES");
    // close all files
    closeImages(sfiles);
    printf("time to close images: %f\n", psTimerClear("CLOSE_IMAGES"));

    // NOTE: from here on we can't just quit if something goes wrong.
    // especially if we're working at the raw stage

    if (!replicateOutputs(sfiles)) {
        psError(PS_ERR_UNKNOWN, false, "failed to replicate output files");
        psErrorStackPrint(stderr, "");
        exit(PS_EXIT_UNKNOWN_ERROR);
    }

#ifdef NOTYET
    if (psMetadataLookupBool(&status, config->arguments, "REPLACE")) {
        //     swap the instances for the input and output
        //     Note this is a database operation. No file I/O is performed
        if (!swapOutputsToInputs(sfiles)) {
            psError(PS_ERR_UNKNOWN, false, "failed to swap files");

            // XXX: Now what? I guess swapOutputsToInputs will need to undo anything that
            // it has done and give a detailed report of what happened

            psErrorStackPrint(stderr, "");
            exit(PS_EXIT_UNKNOWN_ERROR);
        }

        if (psMetadataLookupBool(&status, config->arguments, "REMOVE")) {
            //      delete the temporary storage objects (which now points to the original image(s)
            if (!deleteTemps(sfiles)) {
                psError(PS_ERR_UNKNOWN, false, "failed to delete temporary files");
                // XXX: Now what? At this point the output files have been swapped, so we can't
                // repeat the operation.

                // Returning error status here is problematic. The inputs have been streak removed
                // but they're still lying around
                // Maybe just print an error message and
                // let other system tools clean up
                psErrorStackPrint(stderr, "");
                exit(PS_EXIT_UNKNOWN_ERROR);
            }
        }
    }
#endif  // REPLACE, REMOVE
    printf("time to run streaksremove: %f\n", psTimerClear("STREAKSREMOVE"));

    // nebServerFree(ourNebServer);
    psFree(config);
    pmConceptsDone();
    pmConfigDone();
    psLibFinalize();

    //  PAU
    fprintf(stderr, "Found %d leaks at %s\n", psMemCheckLeaks (0, NULL, NULL, false), "streaksremove");


    return 0;
}

static void usage()
{
    fprintf(stderr, "USAGE: streaksremove -stage raw|chip|warp|diff -image IMAGE.fits -tmproot directory -streaks streaks_file\n\n");
    fprintf(stderr, "Optional arguments:\n");
    fprintf(stderr, "\t-class_id class_id: class_id (required for raw and chip stages)\n");
    fprintf(stderr, "\t-recovery: directory to save excised pixel images\n");
    fprintf(stderr, "\t-mask MASK.fits mask file to de-streak\n");
    fprintf(stderr, "\t-weight WEIGHT.fits: weight file to de-streak\n");
    fprintf(stderr, "\t-replace: replace the input images with the output\n");
    fprintf(stderr, "\t-remove: remove the original image after processing (requires -replace)\n");
    fprintf(stderr, "\t-keepnonwarped: do not exise pixels that were not part of difference processing\n");
    fprintf(stderr, "\t-transparent val: instead of setting exicsed pixel to NAN add val\n");

    exit(2);


}
static pmConfig *parseArguments(int argc, char **argv)
{

    pmConfig *config = pmConfigRead(&argc, argv, NULL);
    if (!config) {
        streaksExit("failed to parse arguments", PS_EXIT_CONFIG_ERROR);
    }

    int argnum;
    ippStage stage = IPP_STAGE_NONE;
    if ((argnum = psArgumentGet(argc, argv, "-stage"))) {
        psArgumentRemove(argnum, &argc, argv);
        stage = parseStage(argv[argnum]);
        psMetadataAddS32(config->arguments, PS_LIST_TAIL, "STAGE", 0,
                "pipeline stage for streak removal", stage);
        psArgumentRemove(argnum, &argc, argv);
    } else {
        psError(PS_ERR_UNKNOWN, true, "-stage is required\n");
        usage();
    }

    if ((argnum = psArgumentGet(argc, argv, "-class_id"))) {
        psArgumentRemove(argnum, &argc, argv);
        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "CLASS_ID", 0,
                "class_id", argv[argnum]);
        psArgumentRemove(argnum, &argc, argv);
    } else if ((stage == IPP_STAGE_RAW) || (stage == IPP_STAGE_CHIP)) {
        psError(PS_ERR_UNKNOWN, true, "-class_id is required\n");
        usage();
    }

    if ((argnum = psArgumentGet(argc, argv, "-streaks"))) {
        psArgumentRemove(argnum, &argc, argv);
        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "STREAKS", 0,
                "STREAKS", argv[argnum]);
        psArgumentRemove(argnum, &argc, argv);
    } else {
        psError(PS_ERR_UNKNOWN, true, "-streaks is required\n");
        usage();
    }


    // If input image is in nebulous we require mask, weight, and tmproot to all be
    // nebulous paths
    bool nebulousImage = false;
    if ((argnum = psArgumentGet(argc, argv, "-image"))) {
        // for raw and chip level images we use psFits for I/O and ...
        nebulousImage = IN_NEBULOUS(argv[argnum+1]);
        if (CHIP_LEVEL_INPUT(stage)) {
            psArgumentRemove(argnum, &argc, argv);
            psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT", 0,
                    "name of input image", argv[argnum]);
            psArgumentRemove(argnum, &argc, argv);
        } else  {
            // ... for warped images we use pmFPAfiles
            if (!pmConfigFileSetsMD(config->arguments, &argc, argv, "INPUT", "-image", NULL)) { ;
                psError(PS_ERR_UNKNOWN, false, "failed to process -image");
                usage();
            }
        }
    } else {
        psError(PS_ERR_UNKNOWN, true, "-image is required\n");
        usage();
    }

    bool gotReplace = false;
    if ((argnum = psArgumentGet(argc, argv, "-replace"))) {
        gotReplace = true;
        psArgumentRemove(argnum, &argc, argv);
        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REPLACE", 0, "replace input files",
            true);
    }
    
    if ((argnum = psArgumentGet(argc, argv, "-keepnonwarped"))) {
        psArgumentRemove(argnum, &argc, argv);
        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "KEEP_NON_WARPED", 0,
            "skip excising of non warped pixels", true);
    }

    if ((argnum = psArgumentGet(argc, argv, "-transparent"))) {
        if (gotReplace) {
            psError(PS_ERR_UNKNOWN, true, "cannot make transparent streaks if -replace");
            usage();
        }
        psArgumentRemove(argnum, &argc, argv);
        double transparentStreaks = atof(argv[argnum]);
        psMetadataAddF64(config->arguments, PS_LIST_TAIL, "TRANSPARENT_STREAKS", 0,
            "value to adjust excised pixels", transparentStreaks);
        psArgumentRemove(argnum, &argc, argv);
    }
        
    // if skycells are not provided then we have to execise all pixels  unless -keepnonwarped
    pmConfigFileSetsMD(config->arguments, &argc, argv, "SKYCELLS", "-skycell", "-skycelllist");

    if (!pmConfigFileSetsMD(config->arguments, &argc, argv, "ASTROM", "-astrom", NULL)) { ;
        if (CHIP_LEVEL_INPUT(stage)) {
            psError(PS_ERR_UNKNOWN, true, "-astrom is required for raw and chip stages\n");
            usage();
        }
    }

    if ((argnum = psArgumentGet(argc, argv, "-mask"))) {
        psArgumentRemove(argnum, &argc, argv);
        bool nebulousMask = IN_NEBULOUS(argv[argnum]);
        if (nebulousMask != nebulousImage) {
            psError(PS_ERR_UNKNOWN, true, "mask image must be %snebulous path with %s image path\n",
                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
            usage();
        }
        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.MASK", 0,
                "name of input mask image", argv[argnum]);
        psArgumentRemove(argnum, &argc, argv);
    }

    if ((argnum = psArgumentGet(argc, argv, "-weight"))) {
        psArgumentRemove(argnum, &argc, argv);
        bool nebulousWeight = IN_NEBULOUS(argv[argnum]);
        if (nebulousWeight != nebulousImage) {
            psError(PS_ERR_UNKNOWN, true, "weight image must have %snebulous path with %s image path\n",
                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
            usage();
        }
        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "INPUT.WEIGHT", 0,
                "name of input weight image", argv[argnum]);
        psArgumentRemove(argnum, &argc, argv);
    }

    if ((argnum = psArgumentGet(argc, argv, "-tmproot"))) {
        psArgumentRemove(argnum, &argc, argv);
        bool nebulousTmp = IN_NEBULOUS(argv[argnum]);
        if (gotReplace && (nebulousTmp != nebulousImage)) {
            psError(PS_ERR_UNKNOWN, true, "tmproot must have %snebulous path with %s image path with -replace\n",
                nebulousImage ? "" : "non ", nebulousImage ? "nebulous" : "regular");
            usage();
        }
        psString dir = pathToDirectory(argv[argnum]);
        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "directory for temporary files",
            dir);
        psArgumentRemove(argnum, &argc, argv);
    } else {
        psError(PS_ERR_UNKNOWN, true, "-tmproot is required\n");
        usage();
    }

    if ((argnum = psArgumentGet(argc, argv, "-recovery"))) {
        psArgumentRemove(argnum, &argc, argv);
        psString dir = pathToDirectory(argv[argnum]);
        psMetadataAddStr(config->arguments, PS_LIST_TAIL, "RECOVERY", 0, "directory for recovery files",
            dir);
        psArgumentRemove(argnum, &argc, argv);
    } else if ((stage == IPP_STAGE_RAW) && gotReplace) {
        psError(PS_ERR_UNKNOWN, true, "-recovery is required for -stage raw with -replace\n");
        usage();
    }

    if ((argnum = psArgumentGet(argc, argv, "-remove"))) {
        if (!gotReplace) {
            psError(PS_ERR_UNKNOWN, true, "-replace is required with -remove\n");
            usage();
        }
        psArgumentRemove(argnum, &argc, argv);
        psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REMOVE", 0, "remove original files",
            true);
    }

    if (argc != 1) {
        psString unexpectedArguments = NULL;
        for (int i=1; i<argc; i++) {
            psStringAppend(&unexpectedArguments, "%s ", argv[i]);
        }
        psError(PS_ERR_UNKNOWN, true, "unexpected arguments: %s", unexpectedArguments);
        psFree(unexpectedArguments);
        usage();
    }

    return config;
}

// set the image for the output files to the contents of the corresponding input file.
static bool
readAndCopyToOutput(streakFiles *sf, bool exciseAll)
{
    bool    updateAstrometry = false;
    if (sf->inImage->pmfile) {
        // image data from pmFPAfile (diff or warp file)
        readImageFrom_pmFile(sf);
        sf->astrom = streakSetAstrometry(sf->astrom, sf->stage, sf->inAstrom->fpa,
                sf->chip, true, sf->cell->concepts, sf->inImage->numCols, sf->inImage->numRows);
    } else {
        // image data directly from psFits
        readImage(sf->inImage, sf->extnum, sf->stage);
        if (SFILE_IS_IMAGE(sf->inImage)) {
            sf->astrom = streakSetAstrometry(sf->astrom, sf->stage, sf->inAstrom->fpa, sf->chip, false,
                sf->inImage->header, sf->inImage->numCols, sf->inImage->numRows);
        }
    }
    if (!sf->astrom) {
        psError(PS_ERR_UNKNOWN, false, "failed to set up astrometry for extension %d", sf->extnum);
        streaksExit("", PS_EXIT_UNKNOWN_ERROR);
    }
    if (sf->stage == IPP_STAGE_CHIP) {
        // For the chip level files, copy the WCS from the astrometry file to the header
        if (!sf->bilevelAstrometry) {
            updateAstrometry = true;
            if (!pmAstromWriteWCS(sf->inImage->header, sf->inAstrom->fpa, sf->chip, 0.001)) {
                psError(PS_ERR_UNKNOWN, false, "failed to update astrometry for extension %d", sf->extnum);
                streaksExit("", PS_EXIT_UNKNOWN_ERROR);
            }
        }
    }
    sf->outImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header);
    if (sf->recImage) {
        sf->recImage->header = (psMetadata*) psMemIncrRefCounter(sf->inImage->header);
    }

    if (!SFILE_IS_IMAGE(sf->inImage)) {
        // fprintf(stderr, "skipping extnum: %d not an image\n", sf->extnum);
        // XXX: This extension is probably a video table.
        copyTable(sf->outImage, sf->inImage, sf->extnum);
        return false;
    }

    if (sf->inImage->image) {
        setupImageRefs(sf->outImage, sf->recImage, sf->inImage, sf->extnum, exciseAll);
    }  else {
        // Image cubes are handeled specially
    }

    // set up the compression parameters
#ifdef STREAKS_COMPRESS_OUTPUT
    // compression of the image pixels is disabled for now. Some consortium members
    // have problems reading them
    copyFitsOptions(sf->outImage, sf->recImage, sf->inImage);

    // XXX: TODO: can we derive these values from the input header?
    // psFitsCompressionGet(sf->inImage->image) gives compression none
    // perhaps we should just use the definition of COMP_IMG in the configuration 
    psFitsSetCompression(sf->outImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
    if (sf->recImage) {
        psFitsSetCompression(sf->recImage->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
    }
#endif

    if (sf->inMask) {
        readImage(sf->inMask, sf->extnum, sf->stage);
        sf->outMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
        if (sf->recMask) {
            sf->recMask->header = (psMetadata*) psMemIncrRefCounter(sf->inMask->header);
        }
        if (updateAstrometry) {
            pmAstromWriteWCS(sf->outMask->header, sf->inAstrom->fpa, sf->chip, 0.001);
        }

        setupImageRefs(sf->outMask, sf->recMask, sf->inMask, sf->extnum, exciseAll);

#ifdef STREAKS_COMPRESS_OUTPUT
        // XXX: see note above
        copyFitsOptions(sf->outMask, sf->recMask, sf->inMask);
        psFitsSetCompression(sf->outMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
        if (sf->recMask) {
            psFitsSetCompression(sf->recMask->fits, PS_FITS_COMPRESS_PLIO, sf->tiles, 8, 0, 0);
        }
#endif
    }

    if (sf->inWeight) {
        readImage(sf->inWeight, sf->extnum, sf->stage);
        sf->outWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header);
        if (sf->recWeight) {
            sf->recWeight->header = (psMetadata*) psMemIncrRefCounter(sf->inWeight->header);
        }
        if (updateAstrometry) {
            pmAstromWriteWCS(sf->inWeight->header, sf->inAstrom->fpa, sf->chip, 0.001);
        }
        setupImageRefs(sf->outWeight, sf->recWeight, sf->inWeight, sf->extnum, exciseAll);

#ifdef STREAKS_COMPRESS_OUTPUT
        copyFitsOptions(sf->outWeight, sf->recWeight, sf->inWeight);
        // XXX: see note above
        psFitsSetCompression(sf->outWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
        if (sf->recWeight) {
            psFitsSetCompression(sf->recWeight->fits, PS_FITS_COMPRESS_RICE, sf->tiles, 8, 0, 0);
        }
#endif
    }
    // and for raw images, create sub images that represent the actual image
    // area (no overscan)

    return true;
}



static void
writeImages(streakFiles *sf, bool exciseImageCube)
{
    psString extname = NULL;
    if (sf->nHDU > 1) {
        bool mdok;
        extname = psMetadataLookupStr(&mdok, sf->inImage->header, "EXTNAME");
    }
    if (sf->inImage->numZPlanes == 0)  {
        // note exciseing complete images is handled in readAndCopyToOutput
        writeImage(sf->outImage, extname, sf->extnum);
        writeImage(sf->recImage, extname, sf->extnum);
    } else {
        // we have an image cube
        double initValue;
        if (exciseImageCube) {
            // copy the entire input image to the recovery image
            writeImageCube(sf->recImage, sf->inImage->imagecube, extname, sf->extnum);
            initValue = NAN;
        } else {
            // otherwise write it to the output 
            writeImageCube(sf->outImage, sf->inImage->imagecube, extname, sf->extnum);
            initValue = 0;
        }

        // borrow one of the images from the imagecube and set it to init value
        psImage *image = psArrayGet (sf->inImage->imagecube, 0);
        psMemIncrRefCounter(image);
        psImageInit(image, initValue);
        if (exciseImageCube) {
            sf->outImage->image = image;
            writeImage(sf->outImage, extname, sf->extnum);
        } else {
            // write zero valued image to reccovery
            if (sf->recImage) {
                sf->recImage->image = image;
                writeImage(sf->recImage, extname, sf->extnum);
            }
        }
        // Assumption: there are no mask and weight images for video cells
        return;
    }
    if (sf->outMask) {
        writeImage(sf->outMask, extname, sf->extnum);
        writeImage(sf->recMask, extname, sf->extnum);
    }
    if (sf->outWeight) {
        writeImage(sf->outWeight, extname, sf->extnum);
        writeImage(sf->recWeight, extname, sf->extnum);
    }
}

// for any of the outputs that are stored in Nebulous set their extended attributes
// and replicate the files

static bool
replicateOutputs(streakFiles *sfiles)
{
    bool status = false;

    // XXX: TODO: need a nebGetXatrr function, but there isn't one
    // another option would be to take the number of copies to be
    // created as an option. That way the system could decide
    // whether to replicate anything other than raw Image files
    void *xattr = NULL;

    if (!replicate(sfiles->outImage, xattr)) {
        psError(PM_ERR_SYS, false, "failed to replicate outImage.");
        return false;
    }

#ifdef notyet
    // XXX: don't replicate mask and weight images until we can look up
    // the input's xattr. There may be a perl program that can getXattr
    if (sfiles->outMask) {
        // get xattr from input to see if we need to replicate
        if (!replicate(sfiles->outMask, xattr)) {
            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
            return false;
        }
    }
    if (sfiles->outWeight) {
        // get xattr from input to see if we need to replicate
        if (!replicate(sfiles->outWeight, xattr)) {
            psError(PM_ERR_SYS, false, "failed to replicate outImage.");
            return false;
        }
    }
#endif

//      replicate the recovery images (if in nebulous)
//      perhaps whether we do that or not should be configurable.
//      Sounds like we need a recipe

    return true;
}


static void
excisePixel(streakFiles *sfiles, unsigned int x, unsigned int y, bool streak, double newMaskValue)
{
    // we clip so that the streak calculation code doesn't have to
    if ((x < 0) || (x >= sfiles->inImage->numCols) || (y < 0) || (y >= sfiles->inImage->numRows)) {
        return;
    }

    double imageValue  = psImageGet (sfiles->inImage->image,  x, y);
    if (sfiles->recImage && !isnan(imageValue) ) {
        psImageSet (sfiles->recImage->image,  x, y, imageValue);
    }

    if (sfiles->transparentStreaks == 0) {
        psImageSet (sfiles->outImage->image,  x, y, NAN);
    } else {
        if (streak) {
            // as a visualization aid don't mask the pixel, just change the intensity
            psImageSet (sfiles->outImage->image,  x, y, imageValue + sfiles->transparentStreaks);
        } else {
            psImageSet (sfiles->outImage->image,  x, y, NAN);
        }
    }

    if (sfiles->inWeight) {
        if (sfiles->recWeight) {
            double weightValue = psImageGet (sfiles->inWeight->image, x, y);
            psImageSet (sfiles->recWeight->image, x, y, weightValue);
        }
        psImageSet (sfiles->outWeight->image, x, y, NAN);
    }
    if (sfiles->inMask) {
        if (sfiles->recMask) {
            double maskValue   = psImageGet (sfiles->inMask->image,   x, y);
            psImageSet (sfiles->recMask->image,   x, y, maskValue);
        }
        psImageSet (sfiles->outMask->image,   x, y, newMaskValue);
    }
}

static void
exciseNonWarpedPixels(streakFiles *sfiles, double newMaskValue)
{
    int cell_x0 = sfiles->astrom->cell_x0;
    int cell_y0 = sfiles->astrom->cell_y0;
    int xParity = sfiles->astrom->xParity;
    int yParity = sfiles->astrom->yParity;
    int numCols = sfiles->inImage->numCols;
    int numRows = sfiles->inImage->numRows;

//    printf("%2d x0: %4d y0: %4d xpar: %d ypar: %d\n", sfiles->extnum, cell_x0, cell_y0, xParity, yParity);

    for (int yCell = 0; yCell < numRows; yCell++) {
        int yChip;
        if (yParity == 1) {
            yChip = cell_y0 + yCell;
        } else {
            yChip = cell_y0 - yCell;
        }

        psU8 *pixels = sfiles->warpedPixels->data.U8[yChip];

        if (xParity == 1) {
            pixels += cell_x0;
            for (int xCell = 0; xCell < numCols; xCell++, pixels++) {
                if (! *pixels ) {
                    excisePixel(sfiles, xCell, yCell, false, newMaskValue);
                }
            }
        } else {
            // negative parity
            // point to the right most pixel in chip coordinates (lowest in cell coords)
            pixels += cell_x0 - numCols;
            for (int xCell = numCols - 1; xCell >= 0 ; xCell--, pixels++) {
                if (!*pixels) {
                    excisePixel(sfiles, xCell, yCell, false, newMaskValue);
                }
            }
        }
    }
}

static bool
warpedPixel(streakFiles *sfiles, PixelPos *cellCoord)
{
    PixelPos chipCoord;

    if (!CHIP_LEVEL_INPUT(sfiles->stage)) {
        // if we're here on a skycell image by definition this pixel was warped
        return true;
    }

    // we clip so that the streak calculation code doesn't have to
    if ((cellCoord->x < 0) || (cellCoord->x >= sfiles->inImage->numCols) ||
        (cellCoord->y < 0) || (cellCoord->y >= sfiles->inImage->numRows)) {
        return false;
    }

    cellToChip(&chipCoord, sfiles->astrom, cellCoord);

#ifdef notdef
    // Shouldn't need this but for raw images I'm getting 
    // x = warpedPixels->numCols for cell xy70 when cell.x = 0
    // Perhaps cell_x0 should be IMNPIX-1 since IMNPIX is in the 1 based system of the fits
    // headers
    if (chipCoord.x < 0 || chipCoord.x >= sfiles->warpedPixels->numCols) {
        return false;
    }
    if (chipCoord.y < 0 || chipCoord.y >= sfiles->warpedPixels->numRows) {
        return false;
    }
#endif

    return psImageGet(sfiles->warpedPixels, chipCoord.x, chipCoord.y) ? true : false;
}

static bool
streakEndOnComponent(streak *streak, double r_min, double r_max, double d_min, double d_max)
{
    if ((streak->ra1  >= r_min) && (streak->ra1  <= r_max) &&
        (streak->dec1 >= d_min) && (streak->dec1 <= d_max)) {
        return true;
    }
    if ((streak->ra2  >= r_min) && (streak->ra2  <= r_max) &&
        (streak->dec2 >= d_min) && (streak->dec2 <= d_max)) {
        return true;
    }
    return false;
}


static Streaks *
restrictStreaksToComponent(streakFiles *sfiles, Streaks *streaks)
{
    Streaks *out = psAlloc(sizeof(Streaks));

    strkPt ll, ur;
    cellToSky(&ll, sfiles->astrom, 0, 0);
    cellToSky(&ur, sfiles->astrom, sfiles->inImage->numCols-1, sfiles->inImage->numRows-1);

    double r_min, r_max, d_min, d_max;
    if (ll.x < ur.x) {
        r_min = ll.x; r_max = ur.x;
    } else {
        r_min = ur.x; r_max = ll.x;
    }
    if (ll.y < ur.y) {
        d_min = ll.y; d_max = ur.y;
    } else {
        d_min = ur.y; d_max = ll.y;
    }

    out->list = psAlloc(streaks->size * sizeof(streak));
    int j=0;
    for (int i=0 ; i<streaks->size; i++) {
        if (streakEndOnComponent(&streaks->list[i], r_min, r_max, d_min, d_max)) {
            out->list[j++] = streaks->list[i];
        }
            
    }
    out->size = j;
    return out;
}

static StreakPixels * 
getStreakPixels(streakFiles *sfiles, Streaks *streaks)
{
    StreakPixels *pixels;
#ifdef notyet
    Streaks *componentStreaks = NULL;
    Streaks *streaksToProcess;

    bool only_component_streaks = false; // XXX flesh this out and make it an option
    if (only_component_streaks) {
        componentStreaks = restrictStreaksToComponent(sfiles, streaks);
        streaksToProcess = componentStreaks;
    } else { 
        streaksToProcess = streaks;
    }

    pixels = streak_on_component (streaksToProcess, sfiles->astrom,
                                  sfiles->inImage->numCols, sfiles->inImage->numRows);
    if (componentStreaks) {
        psFree(componentStreaks);
    }
#else
        pixels = streak_on_component (streaks, sfiles->astrom,
                                      sfiles->inImage->numCols, sfiles->inImage->numRows);
#endif
    return pixels;
}

