Index: /branches/pap_branch_080617/ppStack/.cvsignore =================================================================== --- /branches/pap_branch_080617/ppStack/.cvsignore (revision 18170) +++ /branches/pap_branch_080617/ppStack/.cvsignore (revision 18170) @@ -0,0 +1,17 @@ +aclocal.m4 +compile +config.guess +config.log +config.status +config.sub +configure +depcomp +install-sh +libtool +ltmain.sh +Makefile +Makefile.in +missing +ppStac.pc +test +autom4te.cache Index: /branches/pap_branch_080617/ppStack/Makefile.am =================================================================== --- /branches/pap_branch_080617/ppStack/Makefile.am (revision 18170) +++ /branches/pap_branch_080617/ppStack/Makefile.am (revision 18170) @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = *~ core core.* Index: /branches/pap_branch_080617/ppStack/README.txt =================================================================== --- /branches/pap_branch_080617/ppStack/README.txt (revision 18170) +++ /branches/pap_branch_080617/ppStack/README.txt (revision 18170) @@ -0,0 +1,25 @@ +ppStack needs to be optimised, principally to reduce the memory +footprint (10 or so images will get it to over 4 GB). The proposed +strategy follows, more or less, the same as for ppMerge. + +* Convolution steps: + - For each file: + + read entire image + + convolve to reference PSF + + write convolved image + +* Combination step: + - For each chunk: + + For each file, read chunk + + combine chunk + - Probably have to repeat, to do the multi-pass rejection + + +To do: +* Need file PPSTACK.OUTPUT.CONV (image, mask, weight versions) +* Need file PPSTACK.INPUT.CONV (image, mask, weight versions) +* Check: does pmFPAfileChecks do piece-by-piece read? +* Can we reset a file, so that we can read it twice through? +* pmStack functions probably need work so that they operate on a piece + of the image at a time. Will there be issues with the masks in the + second pass? Index: /branches/pap_branch_080617/ppStack/autogen.sh =================================================================== --- /branches/pap_branch_080617/ppStack/autogen.sh (revision 18170) +++ /branches/pap_branch_080617/ppStack/autogen.sh (revision 18170) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ppStack +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $LIBTOOlIZE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" + DIE=1 +} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOlIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/pap_branch_080617/ppStack/configure.ac =================================================================== --- /branches/pap_branch_080617/ppStack/configure.ac (revision 18170) +++ /branches/pap_branch_080617/ppStack/configure.ac (revision 18170) @@ -0,0 +1,34 @@ +AC_PREREQ(2.61) + +AC_INIT([ppStack], [0.1.1], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDCFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0]) +PKG_CHECK_MODULES([PSPHOT], [psphot >= 0.8.0]) +PKG_CHECK_MODULES([PPSTATS], [ppStats >= 1.0.0]) + +IPP_STDOPTS +CFLAGS="${CFLAGS=} -Wall -Werror" + +AC_SUBST([PPSTACK_CFLAGS]) +AC_SUBST([PPSTACK_LIBS]) + +AC_CONFIG_FILES([ + Makefile + src/Makefile +]) +AC_OUTPUT Index: /branches/pap_branch_080617/ppStack/src/.cvsignore =================================================================== --- /branches/pap_branch_080617/ppStack/src/.cvsignore (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/.cvsignore (revision 18170) @@ -0,0 +1,10 @@ +.deps +Makefile +Makefile.in +.libs +*.lo +*.la +ppStack +config.h +config.h.in +stamp-h1 Index: /branches/pap_branch_080617/ppStack/src/Makefile.am =================================================================== --- /branches/pap_branch_080617/ppStack/src/Makefile.am (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/Makefile.am (revision 18170) @@ -0,0 +1,26 @@ +bin_PROGRAMS = ppStack + +ppStack_CFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PSPHOT_CFLAGS) $(PPSTATS_CFLAGS) $(PPSTACK_CFLAGS) +ppStack_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PSPHOT_LIBS) $(PPSTATS_LIBS) $(PPSTACK_LIBS) + +ppStack_SOURCES = \ + ppStack.c \ + ppStackArguments.c \ + ppStackCamera.c \ + ppStackLoop.c \ + ppStackPSF.c \ + ppStackReadout.c \ + ppStackPhotometry.c \ + ppStackVersion.c \ + ppStackMatch.c + +noinst_HEADERS = \ + ppStack.h + +CLEANFILES = *~ + +clean-local: + -rm -f TAGS +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` Index: /branches/pap_branch_080617/ppStack/src/ppStack.c =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStack.c (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStack.c (revision 18170) @@ -0,0 +1,65 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppStack.h" + +#define TIMER_NAME "PPSTACK" // Name of timer + +int main(int argc, char *argv[]) +{ + psExit exitValue = PS_EXIT_SUCCESS; // Exit value + psTimerStart(TIMER_NAME); + psLibInit(NULL); + + pmConfig *config = pmConfigRead(&argc, argv, PPSTACK_RECIPE); // Configuration + if (!config) { + psErrorStackPrint(stderr, "Error reading configuration."); + exitValue = PS_EXIT_CONFIG_ERROR; + goto die; + } + + (void) psTraceSetLevel("ppStack", 5); + + if (!ppStackArguments(argc, argv, config)) { + psErrorStackPrint(stderr, "Error reading arguments.\n"); + exitValue = PS_EXIT_CONFIG_ERROR; + goto die; + } + + if (!pmModelClassInit()) { + psErrorStackPrint(stderr, "Error initialising model classes.\n"); + exitValue = PS_EXIT_PROG_ERROR; + goto die; + } + + if (!ppStackCamera(config)) { + psErrorStackPrint(stderr, "Error setting up input files.\n"); + exitValue = PS_EXIT_CONFIG_ERROR; + goto die; + } + + if (!ppStackLoop(config)) { + psErrorStackPrint(stderr, "Error performing combination.\n"); + exitValue = PS_EXIT_DATA_ERROR; + goto die; + } + + + // Common code for the death. +die: + psTrace("ppStack", 1, "Finished at %f sec\n", psTimerMark(TIMER_NAME)); + psTimerStop(); + + psFree(config); + pmModelClassCleanup(); + pmConfigDone(); + psLibFinalize(); + + exit(exitValue); +} + Index: /branches/pap_branch_080617/ppStack/src/ppStack.h =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStack.h (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStack.h (revision 18170) @@ -0,0 +1,69 @@ +#ifndef PP_STACK_H +#define PP_STACK_H + +#define PPSTACK_RECIPE "PPSTACK" // Name of the recipe +#define PPSTACK_INSPECT_PIXELS "PPSTACK.PIXELS" // Name of rejected pixels metadata items + +#include + +// Parse command-line arguments +bool ppStackArguments(int argc, char *argv[], // Command-line arguments + pmConfig *config // Configuration + ); + +// Parse cameras +bool ppStackCamera(pmConfig *config // Configuration + ); + +// Loop over the inputs, doing the combination +bool ppStackLoop(pmConfig *config // Configuration + ); + +// Determine target PSF for input images +pmPSF *ppStackPSF(const pmConfig *config, // Configuration + int numCols, int numRows, // Size of image + const psArray *psfs // List of input PSFs + ); + +// Perform stacking on a readout +bool ppStackReadoutInitial(const pmConfig *config, // Configuration + pmReadout *outRO, // Output readout + const psArray *readouts, // Input readouts + const psArray *regions, // Array with array of regions used in each PSF matching + const psArray *kernels // Array with array of kernels used in each PSF matching + ); + +// Perform stacking on a readout +bool ppStackReadoutFinal(const pmConfig *config, // Configuration + pmReadout *outRO, // Output readout + const psArray *readouts, // Input readouts + const psArray *rejected // Array with pixels rejected in each image + ); + +// Perform photometry on stack +bool ppStackPhotometry(pmConfig *config, // Configuration + const pmReadout *readout, // Readout to be photometered + const pmFPAview *view // View to readout + ); + +// Return software version +psString ppStackVersion(void); + +// Return long description of software version +psString ppStackVersionLong(void); + +// Supplement metadata with software version +void ppStackVersionMetadata(psMetadata *metadata // Metadata to supplement + ); + +/// Convolve image to match specified seeing +bool ppStackMatch(pmReadout *readout, ///< Readout to be convolved; replaced with output + psArray **regions, // Array of regions used in each PSF matching, returned + psArray **kernels, // Array of kernels used in each PSF matching, returned + const pmReadout *sourcesRO, ///< Readout with sources + const pmPSF *psf, ///< Target PSF + psRandom *rng, ///< Random number generator + const pmConfig *config ///< Configuration + ); + +#endif Index: /branches/pap_branch_080617/ppStack/src/ppStackArguments.c =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStackArguments.c (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStackArguments.c (revision 18170) @@ -0,0 +1,253 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppStack.h" + +// XXX add in the version info as in ppImage + +// Print usage information and die +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + pmConfig *config // Configuration + ) +{ + fprintf(stderr, "\nPan-STARRS Image combination\n\n"); + fprintf(stderr, + "Usage: %s INPUTS.mdc OUTPUT_ROOT [-sources STAMPS.cmf | -stamps STAMPS.dat]\n" + "where INPUTS.mdc contains various METADATAs, each with:\n" + "\tIMAGE(STR): Image filename\n" + "\tMASK(STR): Mask filename\n" + "\tWEIGHT(STR) Weight map filename\n" + "\tPSF(STR) PSF filename\n" + "\tWEIGHTING(F32): Relative weighting to be applied\n", + program); + fprintf(stderr, "\n"); + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(PS_EXIT_CONFIG_ERROR); +} + +// Get a float-point value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_FLOAT(ARGNAME, RECIPENAME, TYPE) { \ + ps##TYPE value = psMetadataLookup##TYPE(NULL, arguments, ARGNAME); \ + if (isnan(value)) { \ + bool mdok; \ + value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \ + if (!mdok) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPSTACK_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAdd##TYPE(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, value); \ +} + +// Get an integer value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_INT(ARGNAME, RECIPENAME, TYPE, UNSET) { \ + ps##TYPE value = psMetadataLookup##TYPE(NULL, arguments, ARGNAME); \ + if (value == UNSET) { \ + bool mdok; \ + value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \ + if (!mdok) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPSTACK_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAdd##TYPE(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, value); \ +} + +// Get a mask value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_MASK(ARGNAME, RECIPENAME) { \ + bool mdok; \ + const char *name = psMetadataLookupStr(&mdok, arguments, ARGNAME); \ + if (!mdok || !name || strlen(name) == 0) { \ + name = psMetadataLookupStr(NULL, recipe, RECIPENAME); \ + if (!name) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPSTACK_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMaskType value = pmConfigMask(name, config); \ + psMetadataAddU8(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, value); \ +} + +// Get a statistic name from the command-line or recipe, and add the enum to the arguments +#define VALUE_ARG_RECIPE_STAT(ARGNAME, RECIPENAME) { \ + const char *stat = psMetadataLookupStr(NULL, arguments, ARGNAME); \ + if (!stat) { \ + stat = psMetadataLookupStr(NULL, recipe, RECIPENAME); \ + if (!stat) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find %s in recipe %s", \ + RECIPENAME, PPSTACK_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAddS32(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, psStatsOptionFromString(stat)); \ +} + +// Get a string value from the command-line and add it to the target +static bool valueArgStr(psMetadata *arguments, // Command-line arguments + const char *argName, // Argument name in the command-line arguments + const char *mdName, // Name for value in the metadata + psMetadata *target // Target metadata to which to add value + ) +{ + psString value = psMetadataLookupStr(NULL, arguments, argName); // Value of interest + if (value && strlen(value) > 0) { + return psMetadataAddStr(target, PS_LIST_TAIL, mdName, 0, NULL, value); + } + return false; +} + +// Get a string value from the command-line or recipe and add it to the target +static bool valueArgRecipeStr(psMetadata *arguments, // Command-line arguments + psMetadata *recipe, // Recipe + const char *argName, // Argument name in the command-line arguments + const char *mdName, // Name for value in the metadata and recipe + psMetadata *target // Target metadata to which to add value + ) +{ + psString value = psMetadataLookupStr(NULL, arguments, argName); // Value of interest + if (!value) { + value = psMetadataLookupStr(NULL, recipe, mdName); + if (!value) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", + mdName, PPSTACK_RECIPE); + return false; + } + } + return psMetadataAddStr(target, PS_LIST_TAIL, mdName, 0, NULL, value); +} + +bool ppStackArguments(int argc, char *argv[], pmConfig *config) +{ + assert(config); + // bool mdok; // Status of MD lookup + + pmConfigFileSetsMD(config->arguments, &argc, argv, "PPSTACK.SOURCES", "-sources", NULL); + + psMetadata *arguments = psMetadataAlloc(); // Command-line arguments + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stamps", 0, "Stamps file with x,y,flux per line", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "Statistics file", NULL); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-iter", 0, "Number of rejection iterations", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-combine-rej", 0, "Combination rejection thresold (sigma)", NAN); + psMetadataAddU8(arguments, PS_LIST_TAIL, "-mask-bad", 0, "Mask value for bad pixels", 0); + psMetadataAddU8(arguments, PS_LIST_TAIL, "-mask-blank", 0, "Mask value for blank region", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-threshold-mask", 0, "Threshold for mask deconvolution", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-image-rej", 0, "Pixel rejection fraction threshold for rejecting entire image", NAN); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-rows", 0, "Rows to read at once", 128); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-photometry", 0, "Do photometry on stacked image?", false); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-psf-instances", 0, "Number of instances for PSF generation", 5); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-psf-radius", 0, "Radius for PSF generation", 20.0); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-psf-model", 0, "Model name for PSF generation", "PS_MODEL_RGAUSS"); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-psf-order", 0, "Spatial order for PSF generation", 3); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-variance", 0, "Use variance for rejection?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-safe", 0, "Play safe with small numbers of pixels to combine?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-renorm", 0, "Renormalise variance maps?", false); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-renorm-mean", 0, "Statistic for mean in renormalisation", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-renorm-stdev", 0, "Statistic for stdev in renormalisation", NULL); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-renorm-width", 0, "Width of renormalisation boxes", 0); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-image", 0, "Suffix for temporary images", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-mask", 0, "Suffix for temporary masks", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-temp-weight", 0, "Suffix for temporary weight maps", NULL); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-temp-delete", 0, "Delete temporary files on completion?", false); + + if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 3) { + usage(argv[0], arguments, config); + } + + const char *stampsName = psMetadataLookupStr(NULL, arguments, "-stamps"); // Name of stamps file + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "STAMPS", 0, "Stamps file", stampsName); + // if (!stampsName && !psMetadataLookupPtr(&mdok, config->arguments, "PPSTACK.SOURCES")) { + // psError(PS_ERR_BAD_PARAMETER_VALUE, true, "One of -sources or -stamps must be specified."); + //goto ERROR; + // } + + unsigned int numBad = 0; // Number of bad lines + psMetadata *inputs = psMetadataConfigRead(NULL, &numBad, argv[1], false); // Information about inputs + if (!inputs || numBad > 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to cleanly read MDC file with inputs."); + goto ERROR; + } + psMetadataAddMetadata(config->arguments, PS_LIST_TAIL, "INPUTS", 0, + "Metadata with input details", inputs); + psFree(inputs); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, + "Root name of the output image list", argv[2]); + + valueArgStr(arguments, "-stats", "STATS", config->arguments); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSTACK_RECIPE); // Recipe for ppSim + if (!recipe) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSTACK_RECIPE); + goto ERROR; + } + + VALUE_ARG_RECIPE_INT("-iter", "ITER", S32, 0); + VALUE_ARG_RECIPE_FLOAT("-combine-rej", "COMBINE.REJ", F32); + VALUE_ARG_RECIPE_MASK("-mask-bad", "MASK.BAD"); + VALUE_ARG_RECIPE_MASK("-mask-blank", "MASK.BLANK"); + VALUE_ARG_RECIPE_FLOAT("-threshold-mask", "THRESHOLD.MASK", F32); + VALUE_ARG_RECIPE_FLOAT("-image-rej", "IMAGE.REJ", F32); + VALUE_ARG_RECIPE_INT("-rows", "ROWS", S32, 0); + + VALUE_ARG_RECIPE_INT("-psf-instances", "PSF.INSTANCES", S32, 0); + VALUE_ARG_RECIPE_FLOAT("-psf-radius", "PSF.RADIUS", F32); + VALUE_ARG_RECIPE_INT("-psf-order", "PSF.ORDER", S32, 0); + valueArgStr(arguments, "-psf-model", "PSF.MODEL", config->arguments); + + if (psMetadataLookupBool(NULL, arguments, "-photometry") || + psMetadataLookupBool(NULL, recipe, "PHOTOMETRY")) { + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "PHOTOMETRY", 0, + "Do photometry on stacked image?", true); + } + + if (psMetadataLookupBool(NULL, arguments, "-variance") || + psMetadataLookupBool(NULL, recipe, "VARIANCE")) { + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "VARIANCE", 0, + "Use variance for rejection?", true); + } + + if (psMetadataLookupBool(NULL, arguments, "-safe") || + psMetadataLookupBool(NULL, recipe, "SAFE")) { + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "SAFE", 0, + "Play safe with small number of pixels to combine?", true); + } + + if (psMetadataLookupBool(NULL, arguments, "-renorm") || + psMetadataLookupBool(NULL, recipe, "RENORM")) { + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "RENORM", 0, "Renormalise variance maps?", true); + } + VALUE_ARG_RECIPE_INT("-renorm-width", "RENORM.WIDTH", S32, 0); + VALUE_ARG_RECIPE_STAT("-renorm-mean", "RENORM.MEAN"); + VALUE_ARG_RECIPE_STAT("-renorm-stdev", "RENORM.STDEV"); + + valueArgRecipeStr(arguments, recipe, "-temp-image", "TEMP.IMAGE", config->arguments); + valueArgRecipeStr(arguments, recipe, "-temp-mask", "TEMP.MASK", config->arguments); + valueArgRecipeStr(arguments, recipe, "-temp-weight", "TEMP.WEIGHT", config->arguments); + + if (psMetadataLookupBool(NULL, arguments, "-temp-delete") || + psMetadataLookupBool(NULL, recipe, "TEMP.DELETE")) { + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "TEMP.DELETE", 0, + "Delete temporary files on completion?", true); + } + + psTrace("ppStack", 1, "Done reading command-line arguments\n"); + psFree(arguments); + return true; + +ERROR: + psFree(arguments); + return false; +} Index: /branches/pap_branch_080617/ppStack/src/ppStackCamera.c =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStackCamera.c (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStackCamera.c (revision 18170) @@ -0,0 +1,343 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppStack.h" + + +#if 0 +// Define an output convolved image file +static pmFPAfile *defineOutputConvolved(const char *name, // FPA file name + pmFPA *fpa, // FPA to bind + const pmConfig *config, // Configuration + pmFPAfileType type // Expected type + ) +{ + pmFPAfile *file = pmFPAfileDefineOutput(config, fpa, name); + if (!file) { + psError(PS_ERR_UNKNOWN, false, "Unable to define output convolved file %s", name); + return NULL; + } + if (file->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPSTACK.OUTCONV is not of type %s", pmFPAfileStringFromType(type)); + return NULL; + } + + return file; +} + +// Define an input convolved image file, using the output as a basis +static pmFPAfile *defineInputConvolved(const char *inputName, // Input FPA file name + pmFPAfile *outFile, // Corresponding output FPA file + pmConfig *config, // Configuration + pmFPAfileType type // Expected type + ) +{ + pmFPAview *view = pmFPAviewAlloc(0); // View into sky cells + view->chip = view->cell = view->readout = 0; + + psString imageName = pmFPANameFromRule(outFile->filerule, outFile->fpa, view); + psArray *imageNames = psArrayAlloc(1); + imageNames->data[0] = imageName; + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "INCONV.FILENAMES", PS_META_REPLACE, + "Filenames of input convolved image files", imageNames); + psFree(imageNames); + bool found = false; // Found the file? + pmFPAfile *imageFile = pmFPAfileDefineFromArgs(&found, config, "PPSTACK.INCONV", + "INCONV.FILENAMES"); + psMetadataRemoveKey(config->arguments, "INCONV.FILENAMES"); + if (!imageFile || !found) { + psError(PS_ERR_UNKNOWN, false, "Unable to define %s file", inputName); + return NULL; + } + if (imageFile->type != type) { + psError(PS_ERR_IO, true, "PPSTACK.INCONV is not of type %s", + pmFPAfileStringFromType(type)); + return NULL; + } + + return imageFile; +} +#endif + + + +bool ppStackCamera(pmConfig *config) +{ + bool haveWeights = false; // Do we have weight maps? + bool havePSFs = false; // Do we have PSFs? + + psMetadata *inputs = psMetadataLookupMetadata(NULL, config->arguments, "INPUTS"); // The inputs info + psMetadataIterator *iter = psMetadataIteratorAlloc(inputs, PS_LIST_HEAD, NULL); // Iterator + psMetadataItem *item; // Item from iteration + int i = 0; // Counter + while ((item = psMetadataGetAndIncrement(iter))) { + if (item->type != PS_DATA_METADATA) { + psError(PS_ERR_BAD_PARAMETER_TYPE, true, + "Component %s of the input metadata is not of type METADATA", item->name); + psFree(iter); + return false; + } + + psMetadata *input = item->data.md; // The input metadata of interest + + psString image = psMetadataLookupStr(NULL, input, "IMAGE"); // Name of image + if (!image || strlen(image) == 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Component %s lacks IMAGE of type STR", item->name); + psFree(iter); + return false; + } + + bool mdok; + psString mask = psMetadataLookupStr(&mdok, input, "MASK"); // Name of mask + psString weight = psMetadataLookupStr(&mdok, input, "WEIGHT"); // Name of weight map + psString psf = psMetadataLookupStr(&mdok, input, "PSF"); // Name of PSF + + float weighting = psMetadataLookupF32(&mdok, input, "WEIGHTING"); // Relative weighting + if (!mdok || isnan(weighting) || weighting == 0.0) { + psWarning("Component %s lacks WEIGHTING of type F32 --- assuming 1.0", item->name); + weighting = 1.0; + } + + // Add the image file + psArray *imageFiles = psArrayAlloc(1); // Array of filenames for this FPA + imageFiles->data[0] = psMemIncrRefCounter(image); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "IMAGE.FILENAMES", PS_META_REPLACE, + "Filenames of image files", imageFiles); + psFree(imageFiles); + + bool found = false; // Found the file? + pmFPAfile *imageFile = pmFPAfileDefineFromArgs(&found, config, "PPSTACK.INPUT", "IMAGE.FILENAMES"); + if (!imageFile || !found) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from image %d (%s)", i, image); + return false; + } + if (imageFile->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPSTACK.INPUT is not of type IMAGE"); + return false; + } + + // Optionally add the mask file + if (mask && strlen(mask) > 0) { + psArray *maskFiles = psArrayAlloc(1); // Array of filenames for this FPA + maskFiles->data[0] = psMemIncrRefCounter(mask); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "MASK.FILENAMES", PS_META_REPLACE, + "Filenames of mask files", maskFiles); + psFree(maskFiles); + + bool status; + pmFPAfile *maskFile = pmFPAfileBindFromArgs(&status, imageFile, config, "PPSTACK.INPUT.MASK", + "MASK.FILENAMES"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from mask %d (%s)", i, mask); + return false; + } + if (maskFile->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PPSTACK.INPUT.MASK is not of type MASK"); + return false; + } + } + + // Optionally add the weight file + if (weight && strlen(weight) > 0) { + haveWeights = true; + psArray *weightFiles = psArrayAlloc(1); // Array of filenames for this FPA + weightFiles->data[0] = psMemIncrRefCounter(weight); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "WEIGHT.FILENAMES", PS_META_REPLACE, + "Filenames of weight files", weightFiles); + psFree(weightFiles); + + bool status; + pmFPAfile *weightFile = pmFPAfileBindFromArgs(&status, imageFile, config, "PPSTACK.INPUT.WEIGHT", + "WEIGHT.FILENAMES"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from weight %d (%s)", i, weight); + return false; + } + if (weightFile->type != PM_FPA_FILE_WEIGHT) { + psError(PS_ERR_IO, true, "PPSTACK.INPUT.WEIGHT is not of type WEIGHT"); + return false; + } + } + + // Add the psf file + if (!psf || strlen(psf) == 0) { + if (havePSFs) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find PSF %d", i); + return false; + } + } else { + if (!havePSFs && i != 0) { + psWarning("PSF not provided for all inputs --- ignoring."); + } else { + psArray *psfFiles = psArrayAlloc(1); // Array of filenames for this FPA + psfFiles->data[0] = psMemIncrRefCounter(psf); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, "PSF.FILENAMES", PS_META_REPLACE, + "Filenames of PSF files", psfFiles); + psFree(psfFiles); + + bool status; + pmFPAfile *psfFile = pmFPAfileBindFromArgs(&status, imageFile, config, "PPSTACK.INPUT.PSF", + "PSF.FILENAMES"); + if (!status) { + psError(PS_ERR_UNKNOWN, false, "Unable to define file from psf %d (%s)", i, psf); + return false; + } + if (psfFile->type != PM_FPA_FILE_PSF) { + psError(PS_ERR_IO, true, "PPSTACK.INPUT.PSF is not of type PSF"); + return false; + } + havePSFs = true; + } + } + +#if 0 + // Output convolved files + pmFPAfile *outconvImage = defineOutputConvolved("PPSTACK.OUTCONV", imageFile->fpa, config, + PM_FPA_FILE_IMAGE); + pmFPAfile *outconvMask = defineOutputConvolved("PPSTACK.OUTCONV.MASK", imageFile->fpa, config, + PM_FPA_FILE_MASK); + pmFPAfile *outconvWeight = defineOutputConvolved("PPSTACK.OUTCONV.WEIGHT", imageFile->fpa, config, + PM_FPA_FILE_WEIGHT); + if (!outconvImage || !outconvMask || !outconvWeight) { + return false; + } + + // Input convolved files + pmFPAfile *inconvImage = defineInputConvolved("PPSTACK.INCONV", outconvImage, config, + PM_FPA_FILE_IMAGE); + pmFPAfile *inconvMask = defineInputConvolved("PPSTACK.INCONV.MASK", outconvMask, config, + PM_FPA_FILE_MASK); + pmFPAfile *inconvWeight = defineInputConvolved("PPSTACK.INCONV.WEIGHT", outconvWeight, config, + PM_FPA_FILE_WEIGHT); + if (!inconvImage || !inconvMask || !inconvWeight) { + return false; + } +#endif + + psMetadataAddF32(imageFile->fpa->analysis, PS_LIST_TAIL, "PPSTACK.WEIGHTING", 0, + "Relative weighting for image", weighting); + + i++; + } + psFree(iter); + psMetadataRemoveKey(config->arguments, "IMAGE.FILENAMES"); + if (psMetadataLookup(config->arguments, "MASK.FILENAMES")) { + psMetadataRemoveKey(config->arguments, "MASK.FILENAMES"); + } + if (psMetadataLookup(config->arguments, "WEIGHT.FILENAMES")) { + psMetadataRemoveKey(config->arguments, "WEIGHT.FILENAMES"); + } + if (psMetadataLookup(config->arguments, "PSF.FILENAMES")) { + psMetadataRemoveKey(config->arguments, "PSF.FILENAMES"); + } + + psMetadataAddS32(config->arguments, PS_LIST_TAIL, "INPUTS.NUM", 0, "Number of input files", i); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "HAVE.PSF", 0, "Have PSFs available?", havePSFs); + + // Output image + pmFPA *fpa = pmFPAConstruct(config->camera, config->cameraName); // FPA to contain the output + if (!fpa) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to construct an FPA from camera configuration."); + return false; + } + pmFPAfile *output = pmFPAfileDefineOutput(config, fpa, "PPSTACK.OUTPUT"); + psFree(fpa); // Drop reference + if (!output) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPSTACK.OUTPUT")); + return false; + } + if (output->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPSTACK.OUTPUT is not of type IMAGE"); + return false; + } + output->save = true; + + if (!pmFPAAddSourceFromFormat(fpa, "Stack", output->format)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate output FPA."); + psFree(fpa); + return false; + } + + // Output mask + pmFPAfile *outMask = pmFPAfileDefineOutput(config, output->fpa, "PPSTACK.OUTPUT.MASK"); + if (!outMask) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPSTACK.OUTPUT.MASK")); + return false; + } + if (outMask->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PPSTACK.OUTPUT.MASK is not of type MASK"); + return false; + } + outMask->save = true; + + // Output weight + if (haveWeights) { + pmFPAfile *outWeight = pmFPAfileDefineOutput(config, output->fpa, "PPSTACK.OUTPUT.WEIGHT"); + if (!outWeight) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPSTACK.OUTPUT.WEIGHT")); + return false; + } + if (outWeight->type != PM_FPA_FILE_WEIGHT) { + psError(PS_ERR_IO, true, "PPSTACK.OUTPUT.WEIGHT is not of type WEIGHT"); + return false; + } + outWeight->save = true; + } + + + // For photometry, we operate on the chip-mosaicked image + // we create a copy of the mosaicked image for psphot so we can write out a clean image + bool mdok = false; + bool doPhotom = psMetadataLookupBool(&mdok, config->arguments, "PHOTOMETRY"); // perform photometry + if (doPhotom) { + // This file, PSPHOT.INPUT, is just used as a carrier; output files (eg, PSPHOT.RESID) are defined by + // psphotDefineFiles + pmFPAfile *psphotInput = pmFPAfileDefineFromFPA(config, output->fpa, 1, 1, "PSPHOT.INPUT"); + if (!psphotInput) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PSPHOT.INPUT")); + return false; + } + + // Define associated psphot input/output files + if (!psphotDefineFiles(config, psphotInput)) { + psError(PSPHOT_ERR_CONFIG, false, + "Trouble defining the additional input/output files for psphot"); + return false; + } + } else { + // Output PSF --- only required if photometry is not being performed + pmFPAfile *outPSF = pmFPAfileDefineOutputFromFile(config, output, "PSPHOT.PSF.SAVE"); + if (!outPSF) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PSPHOT.PSF.SAVE")); + return false; + } + if (outPSF->type != PM_FPA_FILE_PSF) { + psError(PS_ERR_IO, true, "PSPHOT.PSF.SAVE is not of type PSF"); + return false; + } + outPSF->save = true; + } + + // Sources for use as stamps + bool status = false; // Found the file? + if (havePSFs) { + pmFPAfile *sources = pmFPAfileDefineFromArgs(&status, config, "PPSTACK.SOURCES", "PPSTACK.SOURCES"); + if (!status) { + psError(PS_ERR_IO, false, "Failed to load file definition PPSTACK.SOURCES"); + return false; + } + if (sources && sources->type != PM_FPA_FILE_CMF) { + psError(PS_ERR_IO, true, "PPSTACK.SOURCES is not of type CMF"); + return false; + } + } + + return true; +} Index: /branches/pap_branch_080617/ppStack/src/ppStackLoop.c =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStackLoop.c (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStackLoop.c (revision 18170) @@ -0,0 +1,704 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include "ppStack.h" + +//#define TESTING + +// Here follows lists of files for activation/deactivation at various stages. Each must be NULL-terminated. + +// Files required in preparation for convolution +static char *prepareFiles[] = { "PPSTACK.INPUT.PSF", "PPSTACK.SOURCES", NULL }; + +// Files required for the convolution +static char *convolveFiles[] = { "PPSTACK.INPUT", "PPSTACK.INPUT.MASK", "PPSTACK.INPUT.WEIGHT", NULL }; + +// Output files for the combination +static char *combineFiles[] = { "PPSTACK.OUTPUT", "PPSTACK.OUTPUT.MASK", "PPSTACK.OUTPUT.WEIGHT", NULL }; + +// Files for photometry +static char *photFiles[] = { "PSPHOT.INPUT", "PSPHOT.OUTPUT", "PSPHOT.RESID", "PSPHOT.BACKMDL", + "PSPHOT.BACKMDL.STDEV", "PSPHOT.BACKGND", "PSPHOT.BACKSUB", + "SOURCE.PLOT.MOMENTS", "SOURCE.PLOT.PSFMODEL", "SOURCE.PLOT.APRESID", + "PSPHOT.INPUT.CMF", NULL }; + + +// Activate/deactivate a list of files +static void fileActivation(pmConfig *config, // Configuration + char **files, // Files to turn on/off + bool state // Activation state + ) +{ + assert(config); + assert(files); + + for (int i = 0; files[i] != NULL; i++) { + pmFPAfileActivate(config->files, state, files[i]); + } + return; +} + +// Activate/deactivate a single element for a list +static void fileActivationSingle(pmConfig *config, // Configuration + char **files, // Files to turn on/off + bool state, // Activation state + int num // Number of file in sequence + ) +{ + assert(config); + assert(files); + + for (int i = 0; files[i] != NULL; i++) { + pmFPAfileActivateSingle(config->files, state, files[i], num); // Activated file + } + return; +} + +#if 0 +// Set the data level for a list of files +static void fileSetDataLevel(pmConfig *config, // Configuration + char **files, // Files for which to set level + pmFPALevel level // Level to set + ) +{ + assert(config); + assert(files); + + for (int i = 0; files[i] != NULL; i++) { + psArray *selected = pmFPAfileSelect(config->files, files[i]); // Selected files of interest + for (int j = 0; j < selected->n; j++) { + pmFPAfile *file = selected->data[j]; + assert(file); + file->dataLevel = level; + } + psFree(selected); + } + return; +} +#endif + +// Iterate down the hierarchy, loading files; we can get away with this because we're working on skycells +static pmFPAview *filesIterateDown(pmConfig *config // Configuration + ) +{ + assert(config); + + pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return NULL; + } + view->chip = 0; + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return NULL; + } + view->cell = 0; + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return NULL; + } + view->readout = 0; + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return NULL; + } + return view; +} + +// Iterate up the hierarchy, writing files; we can get away with this because we're working on skycells +static bool filesIterateUp(pmConfig *config // Configuration + ) +{ + assert(config); + + pmFPAview *view = pmFPAviewAlloc(0);// Pointer into FPA hierarchy + view->chip = view->cell = view->readout = 0; + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + return false; + } + view->readout = -1; + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + return false; + } + view->cell = -1; + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + return false; + } + view->chip = -1; + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + return false; + } + psFree(view); + return true; +} + +// Write an image to a FITS file +static bool writeImage(const char *name, // Name of image + psMetadata *header, // Header + const psImage *image, // Image + pmConfig *config // Configuration + ) +{ + assert(name); + assert(image); + + psString resolved = pmConfigConvertFilename(name, config, true, true); // Resolved file name + psFits *fits = psFitsOpen(resolved, "w"); + if (!fits) { + psError(PS_ERR_IO, false, "Unable to open FITS file %s to write image.", resolved); + psFree(resolved); + return false; + } + if (!psFitsWriteImage(fits, header, image, 0, NULL)) { + psError(PS_ERR_IO, false, "Unable to write FITS image %s.", resolved); + psFitsClose(fits); + psFree(resolved); + return false; + } + psFitsClose(fits); + psFree(resolved); + return true; +} + + +bool ppStackLoop(pmConfig *config) +{ + assert(config); + + bool mdok; // Status of MD lookup + psMaskType maskBlank = psMetadataLookupU8(NULL, config->arguments, "MASK.BLANK"); // Mask for blank reg. + bool tempDelete = psMetadataLookupBool(&mdok, config->arguments, "TEMP.DELETE"); // Delete temporary files? + const char *tempImage = psMetadataLookupStr(NULL, config->arguments, "TEMP.IMAGE"); // Suffix for temporary images + const char *tempMask = psMetadataLookupStr(NULL, config->arguments, "TEMP.MASK"); // Suffix for temporary masks + const char *tempWeight = psMetadataLookupStr(NULL, config->arguments, "TEMP.WEIGHT"); // Suffix for temporary weight (variance) maps + if (!tempImage || !tempMask || !tempWeight) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find TEMP.IMAGE, TEMP.MASK and TEMP.WEIGHT"); + return false; + } + + float threshold = psMetadataLookupF32(NULL, config->arguments, "THRESHOLD.MASK"); // Threshold for mask deconvolution + float imageRej = psMetadataLookupF32(NULL, config->arguments, "IMAGE.REJ"); // Maximum fraction of image to reject before rejecting entire image + + const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics + psMetadata *stats = NULL; // Container for statistics + FILE *statsFile = NULL; // File stream for statistics + if (statsName && strlen(statsName) > 0) { + psString resolved = pmConfigConvertFilename(statsName, config, true, true); // Resolved filename + statsFile = fopen(resolved, "w"); + if (!statsFile) { + psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved); + psFree(resolved); + return false; + } else { + stats = psMetadataAlloc(); + } + psFree(resolved); + } + + pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSTACK.OUTPUT"); // Output file + if (!output) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find output data!"); + return false; + } + int num = psMetadataLookupS32(NULL, config->arguments, "INPUTS.NUM"); // Number of inputs + int numScans = psMetadataLookupS32(NULL, config->arguments, "ROWS"); // Number of scans to read at once + psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe + int overlap = 2 * psMetadataLookupS32(NULL, ppsub, "KERNEL.SIZE"); // Overlap by kernel size between consecutive scans + + // Preparation iteration: Load the sources, and get a target PSF model + psTrace("ppStack", 1, "Determining target PSF....\n"); + pmReadout *sources = NULL; // Readout with sources to use for PSF matching + pmPSF *targetPSF = NULL; // Target PSF + if (psMetadataLookupBool(NULL, config->arguments, "HAVE.PSF")) { + pmFPAfileActivate(config->files, false, NULL); + fileActivation(config, prepareFiles, true); + pmFPAview *view = filesIterateDown(config); + if (!view) { + return false; + } + + // We want to hang on to the 'sources' even when its host FPA is blown away + sources = psMemIncrRefCounter(pmFPAfileThisReadout(config->files, view, "PPSTACK.SOURCES")); + if (!sources) { + psError(PS_ERR_UNKNOWN, true, "Unable to find sources."); + psFree(view); + return false; + } + + // Generate list of PSFs + psMetadataIterator *fileIter = psMetadataIteratorAlloc(config->files, PS_LIST_HEAD, + "^PPSTACK.INPUT$"); // Iterator + psMetadataItem *fileItem; // Item from iteration + psArray *psfs = psArrayAlloc(num); // PSFs for PSF envelope + int numCols = 0, numRows = 0; // Size of image + int index = 0; // Index for file + while ((fileItem = psMetadataGetAndIncrement(fileIter))) { + assert(fileItem->type == PS_DATA_UNKNOWN); + pmFPAfile *inputFile = fileItem->data.V; // An input file + pmChip *chip = pmFPAviewThisChip(view, inputFile->fpa); // The chip: holds the PSF + pmPSF *psf = psMetadataLookupPtr(NULL, chip->analysis, "PSPHOT.PSF"); // PSF + if (!psf) { + psError(PS_ERR_UNKNOWN, false, "Unable to find PSF."); + psFree(view); + psFree(sources); + psFree(fileIter); + psFree(psfs); + return false; + } + psfs->data[index++] = psMemIncrRefCounter(psf); + psMetadataRemoveKey(chip->analysis, "PSPHOT.PSF"); + + pmCell *cell = pmFPAviewThisCell(view, inputFile->fpa); // Cell of interest + pmHDU *hdu = pmHDUFromCell(cell); + assert(hdu && hdu->header); + int naxis1 = psMetadataLookupS32(NULL, hdu->header, "NAXIS1"); // Number of columns + int naxis2 = psMetadataLookupS32(NULL, hdu->header, "NAXIS2"); // Number of rows + if (naxis1 <= 0 || naxis2 <= 0) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine size of image from PSF."); + psFree(view); + psFree(sources); + psFree(fileIter); + psFree(psfs); + return false; + } + if (numCols == 0 && numRows == 0) { + numCols = naxis1; + numRows = naxis2; + } + } + psFree(fileIter); + psFree(view); + + targetPSF = ppStackPSF(config, numCols, numRows, psfs); + psFree(psfs); + if (!targetPSF) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine output PSF."); + psFree(sources); + return false; + } + + filesIterateUp(config); + } + + const char *outName = psMetadataLookupStr(NULL, config->arguments, "OUTPUT"); // Output root + assert(outName); + psArray *imageNames = psArrayAlloc(num); + psArray *maskNames = psArrayAlloc(num); + psArray *weightNames = psArrayAlloc(num); + for (int i = 0; i < num; i++) { + psString imageName = NULL, maskName = NULL, weightName = NULL; // Names for convolved images + psStringAppend(&imageName, "%s.%d.%s", outName, i, tempImage); + psStringAppend(&maskName, "%s.%d.%s", outName, i, tempMask); + psStringAppend(&weightName, "%s.%d.%s", outName, i, tempWeight); + imageNames->data[i] = imageName; + maskNames->data[i] = maskName; + weightNames->data[i] = weightName; + } + + // Generate convolutions and write them to disk + psTrace("ppStack", 1, "Convolving inputs to target PSF....\n"); + psArray *cells = psArrayAlloc(num); // Cells for convolved images --- a handle for reading again + psArray *subKernels = psArrayAlloc(num); // Subtraction kernels --- required in the stacking + psArray *subRegions = psArrayAlloc(num); // Subtraction regions --- required in the stacking + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator + int numGood = 0; // Number of good frames + for (int i = 0; i < num; i++) { + psTrace("ppStack", 2, "Convolving input %d of %d to target PSF....\n", i, num); + pmFPAfileActivate(config->files, false, NULL); + fileActivationSingle(config, convolveFiles, true, i); + pmFPAfile *file = pmFPAfileSelectSingle(config->files, "PPSTACK.INPUT", i); // File of interest + pmFPAview *view = filesIterateDown(config); + if (!view) { + psFree(sources); + psFree(targetPSF); + psFree(rng); + return false; + } + + pmReadout *readout = pmFPAviewThisReadout(view, file->fpa); // Input readout + psFree(view); + + // Background subtraction, scaling and normalisation is performed automatically by the image matching + psArray *regions = NULL, *kernels = NULL; // Regions and kernels used in subtraction + if (!ppStackMatch(readout, ®ions, &kernels, sources, targetPSF, rng, config)) { + psErrorStackPrint(stderr, "Unable to match image %d --- ignoring.", i); + psErrorClear(); + continue; + } + + subRegions->data[i] = regions; + subKernels->data[i] = kernels; + + // Write the temporary convolved files + pmHDU *hdu = readout->parent->parent->parent->hdu; // HDU for convolved image + assert(hdu); + writeImage(imageNames->data[i], hdu->header, readout->image, config); + writeImage(maskNames->data[i], hdu->header, readout->mask, config); + writeImage(weightNames->data[i], hdu->header, readout->weight, config); + + cells->data[i] = psMemIncrRefCounter(readout->parent); + filesIterateUp(config); + numGood++; + } + psFree(sources); + psFree(targetPSF); + psFree(rng); + + if (numGood == 0) { + psError(PS_ERR_UNKNOWN, false, "No good images to combine."); + return false; + } + + +#ifdef TESTING + psTraceSetLevel("psModules.imcombine", 7); +#endif + + + // Stack the convolved files + psTrace("ppStack", 1, "Initial stack of convolved images....\n"); + { + pmFPAfileActivate(config->files, false, NULL); + fileActivation(config, combineFiles, true); + pmFPAview *view = filesIterateDown(config); + if (!view) { + psFree(cells); + psFree(subKernels); + psFree(subRegions); + return false; + } + pmCell *outCell = pmFPAfileThisCell(config->files, view, "PPSTACK.OUTPUT"); // Output cell + pmReadout *outRO = pmReadoutAlloc(outCell); // Output readout + + psArray *readouts = psArrayAlloc(num); // Readouts for convolved images + for (int i = 0; i < num; i++) { + pmCell *cell = cells->data[i]; // Cell of interest + if (!cell) { + // Bad image + continue; + } + pmReadout *ro = cell->readouts->data[0]; // Readout of interest + if (!ro) { + ro = pmReadoutAlloc(cell); + } + readouts->data[i] = ro; // Readout into which to read + } + psFree(cells); + + // FITS files + psArray *imageFits = psArrayAlloc(num); + psArray *maskFits = psArrayAlloc(num); + psArray *weightFits = psArrayAlloc(num); + for (int i = 0; i < num; i++) { + if (!readouts->data[i]) { + // Bad image + continue; + } + // Resolved names + psString imageResolved = pmConfigConvertFilename(imageNames->data[i], config, false, false); + psString maskResolved = pmConfigConvertFilename(maskNames->data[i], config, false, false); + psString weightResolved = pmConfigConvertFilename(weightNames->data[i], config, false, false); + imageFits->data[i] = psFitsOpen(imageResolved, "r"); + maskFits->data[i] = psFitsOpen(maskResolved, "r"); + weightFits->data[i] = psFitsOpen(weightResolved, "r"); + psFree(imageResolved); + psFree(maskResolved); + psFree(weightResolved); + if (!imageFits->data[i] || !maskFits->data[i] || !weightFits->data[i]) { + psError(PS_ERR_UNKNOWN, false, "Unable to open convolved files %s, %s, %s", + (char*)imageNames->data[i], (char*)maskNames->data[i], (char*)weightNames->data[i]); + psFree(subKernels); + psFree(subRegions); + return false; + } + } + + // Read convolutions by chunks + bool more = true; // More to read? + for (int numChunk = 0; more; numChunk++) { + psTrace("ppStack", 2, "Initial stack of chunk %d....\n", numChunk); + for (int i = 0; i < num; i++) { + pmReadout *readout = readouts->data[i]; + if (!readout) { + // Bad image + continue; + } + + if (!pmReadoutReadChunk(readout, imageFits->data[i], 0, numScans, overlap, + config) || + !pmReadoutReadChunkMask(readout, maskFits->data[i], 0, numScans, overlap, + config) || + !pmReadoutReadChunkWeight(readout, weightFits->data[i], 0, numScans, overlap, + config)) { + psError(PS_ERR_IO, false, "Unable to read chunk %d for file %d", numChunk, i); + psFree(readouts); + psFree(subKernels); + psFree(subRegions); + psFree(outRO); + psFree(view); + return false; + } + } + +#ifndef PS_NO_TRACE + { + pmReadout *ro = NULL; // Representative readout + for (int i = 0; i < num && !ro; i++) { + ro = readouts->data[i]; + } + psAssert(ro, "There should be a readout here."); + psTrace("ppStack", 4, "Stack: [%d:%d,%d:%d]\n", ro->col0, ro->col0 + ro->image->numCols, + ro->row0, ro->row0 + ro->image->numRows); + } +#endif + + if (!ppStackReadoutInitial(config, outRO, readouts, subRegions, subKernels)) { + psError(PS_ERR_UNKNOWN, false, "Unable to stack images.\n"); + psFree(readouts); + psFree(subKernels); + psFree(subRegions); + psFree(outRO); + psFree(view); + return false; + } + + for (int i = 0; i < num && more; i++) { + pmReadout *readout = readouts->data[i]; + if (!readout) { + // Bad image + continue; + } + more &= pmReadoutMore(readout, imageFits->data[i], 0, numScans, config); + more &= pmReadoutMoreMask(readout, maskFits->data[i], 0, numScans, config); + more &= pmReadoutMoreWeight(readout, weightFits->data[i], 0, numScans, config); + } + } + + // Reset for the second read + // Extract the rejection lists + psArray *rejected = psArrayAlloc(num); // Pixels to inspect + int numRejected = 0; // Number of inputs rejected completely + for (int i = 0; i < num; i++) { + pmReadout *ro = readouts->data[i]; // Readout of interest + if (!ro) { + // Bad image + numRejected++; + continue; + } + psPixels *inspect = NULL; // Inspection list for this readout + psMetadataIterator *iter = psMetadataIteratorAlloc(ro->analysis, PS_LIST_HEAD, + "^" PPSTACK_INSPECT_PIXELS "$"); // Iterator + psMetadataItem *item; + while ((item = psMetadataGetAndIncrement(iter))) { + psPixels *pixels = item->data.V; // Rejected pixels + if (!pixels) { + continue; + } + psTrace("ppStack", 5, "Adding %ld pixels to inspect to image %d", pixels->n, i); + inspect = psPixelsConcatenate(inspect, pixels); + } + psFree(iter); + psMetadataRemoveKey(ro->analysis, PPSTACK_INSPECT_PIXELS); + pmReadoutFreeData(ro); + + psLogMsg("ppStack", PS_LOG_INFO, "%ld total pixels to inspect from image %d", inspect->n, i); + + psPixels *reject = pmStackReject(inspect, NULL, threshold, subRegions->data[i], + subKernels->data[i]); // Pixels to reject + psFree(inspect); + if (!reject) { + psWarning("Rejection on image %d didn't work --- reject entire image.", i); + numRejected++; + } else { + float frac = reject->n / (float)(outRO->image->numCols * outRO->image->numRows); // Pixel frac + psLogMsg("ppStack", PS_LOG_INFO, "%ld pixels rejected from image %d (%.1f%%)", + reject->n, i, frac * 100.0); + if (frac > imageRej) { + psWarning("Image %d rejected completely because rejection fraction (%.3f) " + "exceeds limit (%.3f)", i, frac, imageRej); + psFree(reject); + // reject == NULL means reject image completely + reject = NULL; + numRejected++; + } + } + rejected->data[i] = reject; + } + psFree(subKernels); + psFree(subRegions); + + if (numRejected == num) { + psError(PS_ERR_UNKNOWN, true, "All inputs completely rejected; unable to proceed."); + psFree(readouts); + psFree(rejected); + psFree(outRO); + psFree(view); + return false; + } + + // Read convolutions by chunks + psTrace("ppStack", 2, "Final stack of convolved images....\n"); + more = true; + for (int numChunk = 0; more; numChunk++) { + psTrace("ppStack", 2, "Final stack of chunk %d....\n", numChunk); + for (int i = 0; i < num; i++) { + if (!rejected->data[i]) { + continue; + } + pmReadout *readout = readouts->data[i]; + assert(readout); + + if (!pmReadoutReadChunk(readout, imageFits->data[i], 0, numScans, 0, config) || + !pmReadoutReadChunkMask(readout, maskFits->data[i], 0, numScans, 0, config) || + !pmReadoutReadChunkWeight(readout, weightFits->data[i], 0, numScans, 0, + config)) { + psError(PS_ERR_IO, false, "Unable to read chunk %d for file %d", numChunk, i); + psFree(readouts); + psFree(rejected); + psFree(outRO); + psFree(view); + return false; + } + } + +#ifndef PS_NO_TRACE + { + pmReadout *ro = NULL; + for (int i = 0; i < num && !ro; i++) { + if (!rejected->data[i]) { + continue; + } + ro = readouts->data[i]; + } + if (ro && ro->image) { + psTrace("ppStack", 4, "Stack: [%d:%d,%d:%d]\n", ro->col0, ro->col0 + ro->image->numCols, + ro->row0, ro->row0 + ro->image->numRows); + } + } +#endif + + if (!ppStackReadoutFinal(config, outRO, readouts, rejected)) { + psError(PS_ERR_UNKNOWN, false, "Unable to stack images.\n"); + psFree(readouts); + psFree(rejected); + psFree(outRO); + psFree(view); + return false; + } + + for (int i = 0; i < num && more; i++) { + pmReadout *readout = readouts->data[i]; + if (!readout) { + continue; + } + more &= pmReadoutMore(readout, imageFits->data[i], 0, numScans, config); + more &= pmReadoutMoreMask(readout, maskFits->data[i], 0, numScans, config); + more &= pmReadoutMoreWeight(readout, weightFits->data[i], 0, numScans, config); + } + } + + psFree(rejected); + for (int i = 0; i < readouts->n; i++) { + pmReadout *ro = readouts->data[i]; // Readout of interest + if (!ro) { + continue; + } + pmReadoutFreeData(ro); + } + psFree(readouts); + + // Close up + for (int i = 0; i < num; i++) { + if (!readouts->data[i]) { + continue; + } + psFitsClose(imageFits->data[i]); + psFitsClose(maskFits->data[i]); + psFitsClose(weightFits->data[i]); + imageFits->data[i] = NULL; + maskFits->data[i] = NULL; + weightFits->data[i] = NULL; + if (tempDelete) { + psString imageResolved = pmConfigConvertFilename(imageNames->data[i], config, false, false); + psString maskResolved = pmConfigConvertFilename(maskNames->data[i], config, false, false); + psString weightResolved = pmConfigConvertFilename(weightNames->data[i], config, false, false); + if (unlink(imageResolved) == -1 || unlink(maskResolved) == -1 || + unlink(weightResolved) == -1) { + psWarning("Unable to delete temporary files for image %d", i); + } + psFree(imageResolved); + psFree(maskResolved); + psFree(weightResolved); + } + } + psFree(imageNames); + psFree(maskNames); + psFree(weightNames); + psFree(imageFits); + psFree(maskFits); + psFree(weightFits); + + if (psMetadataLookupBool(&mdok, config->arguments, "PHOTOMETRY")) { + + fileActivation(config, combineFiles, false); + fileActivation(config, photFiles, true); + pmFPAview *photView = filesIterateDown(config); + + psTrace("ppStack", 1, "Photometering stacked image....\n"); + if (!ppStackPhotometry(config, outRO, view)) { + psError(PS_ERR_UNKNOWN, false, "Unable to perform photometry on output."); + psFree(outRO); + psFree(photView); + psFree(view); + return false; + } + psFree(photView); + + fileActivation(config, combineFiles, true); + } + + // Statistics on output + if (stats) { + psTrace("ppStack", 1, "Gathering statistics on stacked image....\n"); + ppStatsFPA(stats, outCell->parent->parent, view, maskBlank, config); + } + + // Put version information into the header + pmHDU *hdu = pmHDUFromCell(outRO->parent); + if (!hdu) { + psError(PS_ERR_UNKNOWN, false, "Unable to find HDU for output."); + return false; + } + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + ppStackVersionMetadata(hdu->header); + + // Write out the output files + filesIterateUp(config); + + psFree(outRO); + psFree(view); + } + + // Write out summary statistics + if (stats) { + const char *statsMDC = psMetadataConfigFormat(stats); + if (!statsMDC || strlen(statsMDC) == 0) { + psError(PS_ERR_IO, false, "Unable to get statistics MDC file.\n"); + } else { + fprintf(statsFile, "%s", statsMDC); + } + psFree((void *)statsMDC); + fclose(statsFile); + + psFree(stats); + } + + return true; +} Index: /branches/pap_branch_080617/ppStack/src/ppStackMatch.c =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStackMatch.c (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStackMatch.c (revision 18170) @@ -0,0 +1,214 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppStack.h" + +#define ARRAY_BUFFER 16 // Number to add to array at a time +#define MAG_IGNORE 99.0 // Ignore these magnitudes --- they're not real! +#define FAKE_SIZE 1 // Size of fake convolution kernel + +//#define TESTING + +bool ppStackMatch(pmReadout *readout, psArray **regions, psArray **kernels, + const pmReadout *sourcesRO, const pmPSF *psf, psRandom *rng, const pmConfig *config) +{ + assert(readout); + assert(regions && !*regions); + assert(kernels && !*kernels); + assert(config); + + pmReadout *output = pmReadoutAlloc(NULL); // Output readout, for holding results temporarily + + // Look up appropriate values from the ppSub recipe + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe + int size = psMetadataLookupS32(NULL, recipe, "KERNEL.SIZE"); // Kernel half-size + psMaskType maskBad = pmConfigMask(psMetadataLookupStr(NULL, recipe, "MASK.BAD"), config); // Value to mask + bool mdok; // Status of MD lookup + bool renorm = psMetadataLookupBool(&mdok, config->arguments, "RENORM"); // Renormalise variances? + psStatsOptions renormMean = psMetadataLookupS32(&mdok, config->arguments, + "RENORM.MEAN"); // Statistic for mean + psStatsOptions renormStdev = psMetadataLookupS32(&mdok, config->arguments, + "RENORM.STDEV"); // Statistic for stdev + int renormWidth = psMetadataLookupS32(&mdok, config->arguments, + "RENORM.WIDTH"); // Width for renormalisation box + + if (!pmReadoutMaskNonfinite(readout, maskBad)) { + psError(PS_ERR_UNKNOWN, false, "Unable to mask non-finite pixels in readout."); + return false; + } + + if (psMetadataLookupBool(&mdok, config->arguments, "HAVE.PSF")) { + assert(psf); + assert(sourcesRO); + + int order = psMetadataLookupS32(NULL, recipe, "SPATIAL.ORDER"); // Spatial polynomial order + float regionSize = psMetadataLookupF32(NULL, recipe, "REGION.SIZE"); // Size of iso-kernel regs + float spacing = psMetadataLookupF32(NULL, recipe, "STAMP.SPACING"); // Typical stamp spacing + int footprint = psMetadataLookupS32(NULL, recipe, "STAMP.FOOTPRINT"); // Stamp half-size + float threshold = psMetadataLookupF32(NULL, recipe, "STAMP.THRESHOLD"); // Threshold for stmps + int iter = psMetadataLookupS32(NULL, recipe, "ITER"); // Rejection iterations + float rej = psMetadataLookupF32(NULL, recipe, "REJ"); // Rejection threshold + pmSubtractionKernelsType type = pmSubtractionKernelsTypeFromString( + psMetadataLookupStr(NULL, recipe, "KERNEL.TYPE")); // Kernel type + psVector *widths = psMetadataLookupPtr(NULL, recipe, "ISIS.WIDTHS"); // ISIS Gaussian widths + psVector *orders = psMetadataLookupPtr(NULL, recipe, "ISIS.ORDERS"); // ISIS Polynomial orders + int inner = psMetadataLookupS32(NULL, recipe, "INNER"); // Inner radius + int ringsOrder = psMetadataLookupS32(NULL, recipe, "RINGS.ORDER"); // RINGS polynomial order + int binning = psMetadataLookupS32(NULL, recipe, "SPAM.BINNING"); // Binning for SPAM kernel + psMaskType maskBlank = pmConfigMask(psMetadataLookupStr(NULL, recipe, "MASK.BLANK"), + config); // Mask for blank reg. + float badFrac = psMetadataLookupF32(NULL, recipe, "BADFRAC"); // Maximum bad fraction + bool optimum = psMetadataLookupBool(&mdok, recipe, "OPTIMUM"); // Derive optimum parameters? + float optMin = psMetadataLookupF32(&mdok, recipe, "OPTIMUM.MIN"); // Minimum width for search + float optMax = psMetadataLookupF32(&mdok, recipe, "OPTIMUM.MAX"); // Maximum width for search + float optStep = psMetadataLookupF32(&mdok, recipe, "OPTIMUM.STEP"); // Step for search + float optThresh = psMetadataLookupF32(&mdok, recipe, "OPTIMUM.TOL"); // Tolerance for search + int optOrder = psMetadataLookupS32(&mdok, recipe, "OPTIMUM.ORDER"); // Order for search + + // These values are specified specifically for stacking + const char *stampsName = psMetadataLookupStr(NULL, config->arguments, "STAMPS"); // Stamps filename + + psVector *optWidths = NULL; // Vector with FWHMs for optimum search + if (optimum) { + optWidths = psVectorCreate(optWidths, optMin, optMax, optStep, PS_TYPE_F32); + } + + psArray *sources = NULL; // Sources in image + if (sourcesRO) { + sources = psMetadataLookupPtr(&mdok, sourcesRO->analysis, "PSPHOT.SOURCES"); + } + + // Generate a fake image to match to + float maxMag = -INFINITY; // Maximum magnitude of sources + for (int i = 0; i < sources->n; i++) { + pmSource *source = sources->data[i]; // Source of interest + if (source->psfMag > maxMag && source->psfMag != MAG_IGNORE) { + maxMag = source->psfMag; + } + } + + pmReadout *fake = pmReadoutAlloc(NULL); // Fake readout with target PSF + + if (!pmReadoutFakeFromSources(fake, readout->image->numCols, readout->image->numRows, sources, NULL, + NULL, psf, powf(10.0, -0.4 * maxMag), 0, false)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate fake image with target PSF."); + psFree(fake); + psFree(optWidths); + psFree(output); + return false; + } + +#ifdef TESTING + { + psFits *fits = psFitsOpen("fake.fits", "w"); + psFitsWriteImage(fits, NULL, fake->image, 0, NULL); + psFitsClose(fits); + } +#endif + + // Do the image matching + if (!pmSubtractionMatch(output, NULL, readout, fake, footprint, regionSize, spacing, threshold, + sources, stampsName, type, size, order, widths, orders, inner, ringsOrder, + binning, optimum, optWidths, optOrder, optThresh, iter, rej, maskBad, + maskBlank, badFrac, PM_SUBTRACTION_MODE_1)) { + psError(PS_ERR_UNKNOWN, false, "Unable to match images."); + psFree(fake); + psFree(optWidths); + psFree(output); + return false; + } + psFree(fake); + psFree(optWidths); + + // Replace original images with convolved + psFree(readout->image); + psFree(readout->mask); + psFree(readout->weight); + readout->image = psMemIncrRefCounter(output->image); + readout->mask = psMemIncrRefCounter(output->mask); + readout->weight = psMemIncrRefCounter(output->weight); + } else { + // Fake the convolution + psRegion *region = psRegionAlloc(0, readout->image->numCols - 1, 0, readout->image->numRows - 1); + psMetadataAddPtr(output->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_REGION, + PS_DATA_REGION | PS_META_DUPLICATE_OK, "Fake subtraction region", region); + psFree(region); + pmSubtractionKernels *kernels = pmSubtractionKernelsPOIS(FAKE_SIZE, 0, PM_SUBTRACTION_MODE_1); + // Set solution to delta function + kernels->solution1 = psVectorAlloc(kernels->num + 2, PS_TYPE_F64); + psVectorInit(kernels->solution1, 0.0); + int normIndex = PM_SUBTRACTION_INDEX_NORM(kernels); // Index for normalisation + kernels->solution1->data.F64[normIndex] = 1.0; + psMetadataAddPtr(output->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_KERNEL, + PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK, "Fake subtraction kernel", kernels); + psFree(kernels); + } + + // Extract the regions and solutions used in the image matching + // This stops them from being freed when we iterate back up the FPA + *regions = psArrayAllocEmpty(ARRAY_BUFFER); // Array of regions + { + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_REGION); + psMetadataIterator *iter = psMetadataIteratorAlloc(output->analysis, PS_LIST_HEAD, regex); // Iterator + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_REGION); + *regions = psArrayAdd(*regions, ARRAY_BUFFER, item->data.V); + } + psFree(iter); + } + *kernels = psArrayAllocEmpty(ARRAY_BUFFER); // Array of kernels + { + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL); + psMetadataIterator *iter = psMetadataIteratorAlloc(output->analysis, PS_LIST_HEAD, regex); // Iterator + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_UNKNOWN); + // Set the normalisation dimensions, since these will be otherwise unavailable when reading the + // images by scans. + pmSubtractionKernels *kernel = item->data.V; // Kernel used in subtraction + kernel->numCols = readout->image->numCols; + kernel->numRows = readout->image->numRows; + + *kernels = psArrayAdd(*kernels, ARRAY_BUFFER, kernel); + } + psFree(iter); + } + assert((*regions)->n == (*kernels)->n); + + // Renormalise the variances if desired + if (renorm) { + psLogMsg("ppStack", PS_LOG_INFO, "Renormalising variance map."); + if (!pmReadoutWeightRenorm(readout, maskBad, renormMean, renormStdev, renormWidth, rng)) { + psError(PS_ERR_UNKNOWN, false, "Unable to renormalise variances."); + psFree(output); + return false; + } + } + + // Ensure the background value is zero + psStats *bg = psStatsAlloc(PS_STAT_ROBUST_MEDIAN | PS_STAT_ROBUST_STDEV); // Statistics for background + if (!psImageBackground(bg, NULL, readout->image, readout->mask, maskBad, rng)) { + psWarning("Can't measure background for image."); + psErrorClear(); + } else { + psLogMsg("ppStack", PS_LOG_INFO, "Correcting convolved image background by %lf (+/- %lf)", + psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), psStatsGetValue(bg, PS_STAT_ROBUST_STDEV)); + (void)psBinaryOp(readout->image, readout->image, "+", + psScalarAlloc(- psStatsGetValue(bg, PS_STAT_ROBUST_MEDIAN), PS_TYPE_F32)); + } + psFree(bg); + + psFree(output); + + return true; +} Index: /branches/pap_branch_080617/ppStack/src/ppStackPSF.c =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStackPSF.c (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStackPSF.c (revision 18170) @@ -0,0 +1,28 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppStack.h" + +pmPSF *ppStackPSF(const pmConfig *config, int numCols, int numRows, const psArray *psfs) +{ + // Get the recipe values + int psfInstances = psMetadataLookupS32(NULL, config->arguments, "PSF.INSTANCES"); // Number of instances for PSF + float psfRadius = psMetadataLookupF32(NULL, config->arguments, "PSF.RADIUS"); // Radius for PSF + const char *psfModel = psMetadataLookupStr(NULL, config->arguments, "PSF.MODEL"); // Model for PSF + int psfOrder = psMetadataLookupS32(NULL, config->arguments, "PSF.ORDER"); // Spatial order for PSF + + // Solve for the target PSF + pmPSF *psf = pmPSFEnvelope(numCols, numRows, psfs, psfInstances, psfRadius, psfModel, + psfOrder, psfOrder); + if (!psf) { + psError(PS_ERR_UNKNOWN, false, "Unable to determine output PSF."); + return NULL; + } + + return psf; +} Index: /branches/pap_branch_080617/ppStack/src/ppStackPhotometry.c =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStackPhotometry.c (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStackPhotometry.c (revision 18170) @@ -0,0 +1,39 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppStack.h" + +bool ppStackPhotometry(pmConfig *config, const pmReadout *readout, const pmFPAview *view) +{ + pmFPAfile *photFile = psMetadataLookupPtr(NULL, config->files, "PSPHOT.INPUT"); + pmFPACopy(photFile->fpa, readout->parent->parent->parent); + + { + // Need to ensure aperture residual is not calculated + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PSPHOT_RECIPE); // Recipe + psMetadataItem *item = psMetadataLookup(recipe, "MEASURE.APTREND"); // Item determining aptrend + if (!item) { + psWarning("Unable to find MEASURE.APTREND in psphot recipe"); + psErrorClear(); + } else { + item->data.B = false; + } + } + + if (!psphotReadout(config, view)) { + // Clear the error, so that the output files are written. + psWarning("Unable to perform photometry on stacked image."); + psErrorStackPrint(stderr, "Error stack from photometry:"); + psErrorClear(); + } + + pmFPAfileActivate(config->files, false, "PSPHOT.INPUT"); + + return true; +} Index: /branches/pap_branch_080617/ppStack/src/ppStackReadout.c =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStackReadout.c (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStackReadout.c (revision 18170) @@ -0,0 +1,287 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppStack.h" + +#define WCS_TOLERANCE 0.001 // Tolerance for WCS + +//#define REJECTION_FILES // Write rejection mask? +//#define INSPECTION_FILES // Write inspection mask? +//#define COMBINED_FILES // Write combined images? + + +bool ppStackReadoutInitial(const pmConfig *config, pmReadout *outRO, const psArray *readouts, + const psArray *regions, const psArray *kernels) +{ + assert(config); + assert(outRO); + assert(readouts); + assert(regions); + assert(kernels); + assert(readouts->n == regions->n); + assert(regions->n == kernels->n); + static int sectionNum = 0; // Section number; for debugging outputs + + + // Get the recipe values + bool mdok; // Status of MD lookup + int iter = psMetadataLookupS32(NULL, config->arguments, "ITER"); // Rejection iterations + float combineRej = psMetadataLookupF32(NULL, config->arguments, "COMBINE.REJ"); // Combination threshold + psMaskType maskBad = psMetadataLookupU8(NULL, config->arguments, "MASK.BAD"); // Value to mask + psMaskType maskBlank = psMetadataLookupU8(NULL, config->arguments, "MASK.BLANK"); // Mask for blank reg. +// float threshold = psMetadataLookupF32(NULL, config->arguments, "THRESHOLD.MASK"); // Threshold for mask deconvolution + bool useVariance = psMetadataLookupBool(&mdok, config->arguments, "VARIANCE"); // Use variance for rejection? + bool safe = psMetadataLookupBool(&mdok, config->arguments, "SAFE"); // Play safe when combining small numbers of pixels? + psMetadata *ppsub = psMetadataLookupMetadata(NULL, config->recipes, "PPSUB"); // PPSUB recipe + int kernelSize = psMetadataLookupS32(NULL, ppsub, "KERNEL.SIZE"); // Kernel half-size + + + int num = readouts->n; // Number of inputs + psArray *stack = psArrayAlloc(num); // Array for stacking + + for (int i = 0; i < num; i++) { + pmReadout *ro = readouts->data[i]; + if (!ro) { + // Bad image + continue; + } + pmFPA *fpa = ro->parent->parent->parent; // Parent FPA + + float weighting = psMetadataLookupF32(&mdok, fpa->analysis, "PPSTACK.WEIGHTING"); // Relative weight + if (!mdok || !isfinite(weighting)) { + psWarning("No WEIGHTING supplied for image %d --- set to unity.", i); + weighting = 1.0; + } + + // Ensure there is a mask, or pmStackCombine will complain + if (!ro->mask) { + ro->mask = psImageAlloc(ro->image->numCols, ro->image->numRows, PS_TYPE_MASK); + psImageInit(ro->mask, 0); + } + + stack->data[i] = pmStackDataAlloc(ro, weighting); + } + + if (!pmStackCombine(outRO, stack, maskBad, maskBlank, kernelSize, iter, combineRej, true, + useVariance, safe)) { + psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts with rejection."); + psFree(stack); + return false; + } + +#ifdef COMBINED_FILES + { + psString name = NULL; // Name of image + psStringAppend(&name, "combined_initial_%03d.fits", sectionNum); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, NULL, outRO->image, 0, NULL); + psFitsClose(fits); + } +#endif +#ifdef INSPECTION_FILES + for (int i = 0; i < stack->n; i++) { + pmStackData *data = stack->data[i]; // Data for this image + if (!data) { + continue; + } + psImage *inspected = psPixelsToMask(NULL, data->inspect, + psRegionSet(0, outRO->image->numCols - 1, + 0, outRO->image->numRows - 1), + maskBlank); + psString name = NULL; // Name of image + psStringAppend(&name, "inspect_%03d_%03d.fits", sectionNum, i); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, NULL, inspected, 0, NULL); + psFree(inspected); + psFitsClose(fits); + } +#endif + + // Save list of pixels to inspect + for (int i = 0; i < num; i++) { + pmStackData *data = stack->data[i]; // Data for this image + if (!data) { + continue; + } + pmReadout *readout = data->readout; // Readout of interest + if (!readout) { + continue; + } + psMetadataAddPtr(readout->analysis, PS_LIST_TAIL, PPSTACK_INSPECT_PIXELS, + PS_DATA_PIXELS | PS_META_DUPLICATE_OK, "Pixels to inspect from initial combination", + data->inspect); + } + psFree(stack); + + sectionNum++; + + return true; +} + + + +bool ppStackReadoutFinal(const pmConfig *config, pmReadout *outRO, const psArray *readouts, + const psArray *rejected) +{ + assert(config); + assert(outRO); + assert(readouts); + assert(rejected); + assert(readouts->n == rejected->n); + + static int sectionNum = 0; + + // Get the recipe values + bool mdok; // Status of MD lookup + psMaskType maskBad = psMetadataLookupU8(NULL, config->arguments, "MASK.BAD"); // Value to mask + psMaskType maskBlank = psMetadataLookupU8(NULL, config->arguments, "MASK.BLANK"); // Mask for blank reg. + bool useVariance = psMetadataLookupBool(&mdok, config->arguments, "VARIANCE"); // Use variance for rejection? + + int num = readouts->n; // Number of inputs + psArray *stack = psArrayAlloc(num); // Array for stacking + + pmCell *outCell = outRO->parent; // Output cell + pmChip *outChip = outCell->parent; // Output chip + pmFPA *outFPA = outChip->parent; // Output FPA + + float totExposure = 0.0; // Total exposure time + psList *fpaList = psListAlloc(NULL); // List of input FPAs, for concept averaging + psList *cellList = psListAlloc(NULL); // List of input cells, for concept averaging + int numGood = num; // Number of good inputs: images that haven't been completely rejected + for (int i = 0; i < num; i++) { + if (!rejected->data[i]) { + // Image completely rejected + numGood--; + continue; + } + + pmReadout *ro = readouts->data[i]; + assert(ro); + pmFPA *fpa = ro->parent->parent->parent; // Parent FPA + + bool mdok; // Status of MD lookup + float weighting = psMetadataLookupF32(&mdok, fpa->analysis, "PPSTACK.WEIGHTING"); // Relative weight + if (!mdok || !isfinite(weighting)) { + psWarning("No WEIGHTING supplied for image %d --- set to unity.", i); + weighting = 1.0; + } + + float exposure = psMetadataLookupF32(NULL, ro->parent->concepts, "CELL.EXPOSURE"); // Exposure time + if (!isfinite(exposure)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "CELL.EXPOSURE is not set for input file %ld", stack->n); + psFree(fpaList); + psFree(cellList); + psFree(stack); + return false; + } + totExposure += exposure; // Total exposure time + +#if 0 + if (i == 0) { + // Copy astrometry over + pmHDU *hdu = fpa->hdu; // Template HDU + pmHDU *outHDU = outFPA->hdu; // Output HDU + if (!outHDU || !hdu) { + psWarning("Unable to find HDU at FPA level to copy astrometry."); + } else { + if (!pmAstromReadWCS(outFPA, outChip, hdu->header, 1.0)) { + psErrorClear(); + psWarning("Unable to read WCS astrometry from input FPA."); + } else { + if (!outHDU->header) { + outHDU->header = psMetadataAlloc(); + } + if (!pmAstromWriteWCS(outHDU->header, outFPA, outChip, WCS_TOLERANCE)) { + psErrorClear(); + psWarning("Unable to write WCS astrometry to output FPA."); + } + } + } + } +#endif + + // Ensure there is a mask, or pmStackCombine will complain + if (!ro->mask) { + ro->mask = psImageAlloc(ro->image->numCols, ro->image->numRows, PS_TYPE_MASK); + psImageInit(ro->mask, 0); + } + + psListAdd(fpaList, PS_LIST_TAIL, fpa); + psListAdd(cellList, PS_LIST_TAIL, ro->parent); + + pmStackData *data = pmStackDataAlloc(ro, weighting); + data->reject = psMemIncrRefCounter(rejected->data[i]); + stack->data[i] = data; + } + +#ifdef REJECTION_FILES + if (sectionNum == 0) { + for (int i = 0; i < stack->n; i++) { + pmStackData *data = stack->data[i]; // Data for this image + if (!data) { + continue; + } + psImage *reject = psPixelsToMask(NULL, data->reject, + psRegionSet(0, outRO->image->numCols - 1, + 0, outRO->image->numRows - 1), + maskBlank); + psString name = NULL; // Name of image + psStringAppend(&name, "reject_%03d.fits", i); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, NULL, reject, 0, NULL); + psFree(reject); + psFitsClose(fits); + } + } +#endif + + if (!pmStackCombine(outRO, stack, maskBad, maskBlank, 0, 0, NAN, numGood != num, useVariance, false)) { + psError(PS_ERR_UNKNOWN, false, "Unable to combine input readouts."); + psFree(fpaList); + psFree(cellList); + psFree(stack); + return false; + } + +#ifdef COMBINED_FILES + { + psString name = NULL; // Name of image + psStringAppend(&name, "combined_final_%03d.fits", sectionNum); + psFits *fits = psFitsOpen(name, "w"); + psFree(name); + psFitsWriteImage(fits, NULL, outRO->image, 0, NULL); + psFitsClose(fits); + } +#endif + + psMetadataAddF32(outCell->concepts, PS_LIST_TAIL, "CELL.EXPOSURE", PS_META_REPLACE, + "Summed exposure time (sec)", totExposure); + psMetadataAddF32(outCell->parent->parent->concepts, PS_LIST_TAIL, "FPA.EXPOSURE", PS_META_REPLACE, + "Summed exposure time (sec)", totExposure); + + outRO->data_exists = true; + outCell->data_exists = true; + outCell->parent->data_exists = true; + + // Copy other concepts + pmConceptsAverageFPAs(outFPA, fpaList); + pmConceptsAverageCells(outCell, cellList, NULL, NULL, true); + psFree(fpaList); + psFree(cellList); + + psFree(stack); + + sectionNum++; + + return true; +} Index: /branches/pap_branch_080617/ppStack/src/ppStackVersion.c =================================================================== --- /branches/pap_branch_080617/ppStack/src/ppStackVersion.c (revision 18170) +++ /branches/pap_branch_080617/ppStack/src/ppStackVersion.c (revision 18170) @@ -0,0 +1,60 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppStack.h" + +static const char *cvsTag = "$Name: not supported by cvs2svn $";// CVS tag name + +psString ppStackVersion(void) +{ + psString version = NULL; // Version, to return + psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION); + return version; +} + +psString ppStackVersionLong(void) +{ + psString version = ppStackVersion(); // Version, to return + psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag + psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__); + psFree(tag); + return version; +} + + +void ppStackVersionMetadata(psMetadata *metadata) +{ + PS_ASSERT_METADATA_NON_NULL(metadata,); + + psString pslib = psLibVersionLong();// psLib version + psString psmodules = psModulesVersionLong(); // psModules version + psString ppStats = ppStatsVersionLong(); // ppStats version + psString ppStack = ppStackVersionLong(); // ppStack version + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString head = NULL; // Head string + psStringAppend(&head, "ppStack processing at %s. Component information:", timeString); + psFree(timeString); + + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, head, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, pslib, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psmodules, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppStats, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppStack, ""); + + psFree(head); + psFree(pslib); + psFree(psmodules); + psFree(ppStats); + psFree(ppStack); + + return; +} Index: /branches/pap_branch_080617/ppSub/.cvsignore =================================================================== --- /branches/pap_branch_080617/ppSub/.cvsignore (revision 18170) +++ /branches/pap_branch_080617/ppSub/.cvsignore (revision 18170) @@ -0,0 +1,17 @@ +Makefile +Makefile.in +aclocal.m4 +autom4te.cache +compile +config.log +config.status +configure +depcomp +install-sh +missing +config.guess +libtool +ltmain.sh +stamp-h1 +config.sub +test Index: /branches/pap_branch_080617/ppSub/Makefile.am =================================================================== --- /branches/pap_branch_080617/ppSub/Makefile.am (revision 18170) +++ /branches/pap_branch_080617/ppSub/Makefile.am (revision 18170) @@ -0,0 +1,3 @@ +SUBDIRS = src + +CLEANFILES = *.pyc *~ core core.* Index: /branches/pap_branch_080617/ppSub/TO_DO =================================================================== --- /branches/pap_branch_080617/ppSub/TO_DO (revision 18170) +++ /branches/pap_branch_080617/ppSub/TO_DO (revision 18170) @@ -0,0 +1,177 @@ +Some ideas for improving the image subtraction +============================================== + +Paul Price +15 August 2007 + + +1. Dual convolution + +The Alard & Lupton (1998) and Alard (2000) algorithms are unable to +produce useful subtractions when the PSFs of the input and reference +images have misaligned position angles (e.g., / vs \), since this +would involve a deconvolution along one of the axes, which does not +work well. One way to solve this problem is through a dual +convolution. Instead of solving + + I(x,y) = k(u,v) * R(x,y) + +for k(u,v), we can solve + + k1(u,v) * I(x,y) = k2(u,v) * R(x,y) + +for k1(u,v) and k2(u,v). If we write + + k1(u,v) = sum_i a_i f_i(u,v) + +and + + k2(u,v) = sum_i b_i g_i(u,v) + +where f_i(u,v) and g_i(u,v) are sets of basis functions, then the +solution boils down to two equations: + + sum_j a_j A_j A_i = sum_j b_j B_j A_i + +and + + sum_j a_j A_j B_i = sum_j b_j B_j B_i + +where + + A_i = sum_x,y f_i(u,v) * I(x,y) / sigma(x,y) + +and + + B_i = sum_x,y g_i(u,v) * R(x,y) / sigma(x,y) + +Noting that the matrix terms of the RHS of the first equation and the +LHS of the second equation (A_i B_j) are the same leads us to attempt +to solve this system by iteration: + + 1. Set a_i = 1 if i = 0, otherwise a_i = 0; f_0 = delta(u,v); and + solve for b in the first equation. This corresponds to doing the + usual Alard & Lupton solution. + + 2. Use b in the second equation, and solve for a. + + 3. Use a in the first equation, and solve for b. + + 4. Proceed in this manner until the change in b between iterations + is small. + +This is basically doing A&L to get the reference image to match the +input image in the usual manner, then doing A&L on the input image to +match the convolved reference image, then doing A&L on the reference +image to match the convolved input image, etc., until the system +settles down to the solution. A more refined method of solving the +equations may exist, but this should work. + +There is a need to normalise one of the kernels: the solution is +currently non-unique because a and b can be scaled by some arbitrary +value. If we write + + k1(u,v) = delta(u,v) + sum_i a_i [k_i - delta(u,v)] + +and the k_i are normalised to a sum of unity, then + + sum_u,v k1(u,v) = 1. + +So k1 has a sum of unity, always. This means that k2 supplies the +scaling to match the photometry (as happens in Alard & Lupton), and k1 +is merely used to broaden the input image as required for the best +subtraction. + +Note that the expense for this method over the usual Alard & Lupton +roughly amounts to the extra image convolution, since that is +typically the dominant factor. Accumulation of the sums is not much +more than Alard & Lupton, and the iteration should be fairly fast. + + +2. Data-based kernel selection + +The quality of the subtraction is highly sensitive to the choice of +basis functions. In the case of ISIS kernels (which seem to be the +most useful of those experimented with so far, because it takes a +small number of parameters to generate a large kernel), the choice of +the Gaussian widths is very important. Some recipes exist for +choosing these widths, but these are generally motivated by experience +(through much trial and error) rather than directly from the data. It +would be nice to be able to throw down a large number of widths and +allow the least-squares solution to choose the best for the data at +hand (i.e., the most dominant contributors), but this is prohibitively +expensive --- at least for a full solution. + +What might be possible is to do a quick and dirty solution by reducing +the dimensionality. Instead of working with the full two-dimensional +kernel, k(u,v), let's work in one dimension, k(u). For each of our +stamps, let's take a cut through them in a consistent direction (e.g., +the x direction), and solve + + I(x) = k(u) * R(x) + +This is not nearly as computationally expensive as the full solution, +so we can pack k(u) with multiple Gaussian widths, solve the +least-squares problem, and identify the most important kernel +contributions which we will use (in suitable two-dimensional versions) +in solving the full problem. This method could even be applied with +multiple direction cuts to ensure the full range of required Gaussian +widths is obtained. + +Fitting the normalisation of a single kernel component, k(u) and the +background, we obtain the least-squares matrix and vector: + +M = ( sum_x C(x)^2/sigma(x)^2 sum_x C(x)/sigma(x)^2 ) + ( sum_x C(x)/sigma(x)^2 sum_x 1/sigma(x)^2 ) + +v = ( sum_x I(x)C(x)/sigma(x)^2 sum_x I(x)/sigma(x)^2 ) + +where C(x) = R(x) * k(u) + +Inverting the matrix, multiplying by the vector and taking the +component corresponding to the kernel normalisation, we get: + +f = (ab - cd) / (ae - c^2) + +where + +a = sum_x 1/sigma(x)^2 +b = sum_x I(x)C(x)/sigma(x)^2 +c = sum_x C(x)/sigma(x)^2 +d = sum_x I(x)/simga(x)^2 +e = sum_x C(x)^2/sigma(x)^2 + +Note that a and d are independent of the kernel, and so may be +measured once only. Note that we are ignoring any spatial variation +of the kernel here --- we are only interested in which kernel +components should be in the final solution, not the details of that +solution. + +Assuming that the kernel components are normalised, the kernel +component normalisation, f, is a measure of how important that kernel +component is in the final solution of the full problem. The following +algorithm is suggested: + +(a) For each original kernel component, extract a vector at 0, 45 and + 90 degrees (x, y and x-y axes). Ignore one of these sub-kernels + in what follows if the standard deviation of the subkernel is + zero. We use these three extraction angles to cover the u, v and + uv terms. + +(b) For each stamp, extract a vector at 0, 45, and 90 degrees (x, y, + and x-y axes). These sub-stamps go with the corresponding + sub-kernel. + +(c) For each kernel component, measure b, c, e and therefore determine + the kernel normalisation, f. Use the sub-kernel appropriate for + the sub-stamp (matched by extraction angle), and accumulate the + values together (sub-kernels all contribute to the same original + kernel component). + +(d) Take the kernel component with the largest |f| as the 'winner' of + this iteration. Apply this kernel to the sub-stamps for R(x), and + mark this kernel component in the list. It is part of the end + solution, but should not be used any more in the iteration. + +(e) Repeat from step (c) until the largest |f| is small, say 10^-3 of + the first obtained value of |f|. Index: /branches/pap_branch_080617/ppSub/autogen.sh =================================================================== --- /branches/pap_branch_080617/ppSub/autogen.sh (revision 18170) +++ /branches/pap_branch_080617/ppSub/autogen.sh (revision 18170) @@ -0,0 +1,103 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +srcdir=`dirname $0` +test -z "$srcdir" && srcdir=. + +ORIGDIR=`pwd` +cd $srcdir + +PROJECT=ppSub +TEST_TYPE=-f +# change this to be a unique filename in the top level dir +FILE=autogen.sh + +DIE=0 + +LIBTOOLIZE=libtoolize +ACLOCAL="aclocal $ACLOCAL_FLAGS" +AUTOHEADER=autoheader +AUTOMAKE=automake +AUTOCONF=autoconf + +($LIBTOOLIZE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $LIBTOOlIZE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/libtool/" + DIE=1 +} + +($ACLOCAL --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $ACLOCAL installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOHEADER --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOHEADER installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +($AUTOMAKE --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOMAKE installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/automake/" + DIE=1 +} + +($AUTOCONF --version) < /dev/null > /dev/null 2>&1 || { + echo + echo "You must have $AUTOCONF installed to compile $PROJECT." + echo "Download the appropriate package for your distribution," + echo "or get the source tarball at http://ftp.gnu.org/gnu/autoconf/" + DIE=1 +} + +if test "$DIE" -eq 1; then + exit 1 +fi + +test $TEST_TYPE $FILE || { + echo "You must run this script in the top-level $PROJECT directory" + exit 1 +} + +if test -z "$*"; then + echo "I am going to run ./configure with no arguments - if you wish " + echo "to pass any to it, please specify them on the $0 command line." +fi + +$LIBTOOLIZE --copy --force || echo "$LIBTOOlIZE failed" +$ACLOCAL || echo "$ACLOCAL failed" +$AUTOHEADER || echo "$AUTOHEADER failed" +$AUTOMAKE --add-missing --force-missing --copy || echo "$AUTOMAKE failed" +$AUTOCONF || echo "$AUTOCONF failed" + +cd $ORIGDIR + +run_configure=true +for arg in $*; do + case $arg in + --no-configure) + run_configure=false + ;; + *) + ;; + esac +done + +if $run_configure; then + $srcdir/configure --enable-maintainer-mode "$@" + echo + echo "Now type 'make' to compile $PROJECT." +else + echo + echo "Now run 'configure' and 'make' to compile $PROJECT." +fi Index: /branches/pap_branch_080617/ppSub/configure.ac =================================================================== --- /branches/pap_branch_080617/ppSub/configure.ac (revision 18170) +++ /branches/pap_branch_080617/ppSub/configure.ac (revision 18170) @@ -0,0 +1,37 @@ +dnl Process this file with autoconf to produce a configure script. +AC_PREREQ(2.61) + +AC_INIT([ppSub], [0.1.1], [ipp-support@ifa.hawaii.edu]) +AC_CONFIG_SRCDIR([src]) + +AM_INIT_AUTOMAKE([1.6 foreign dist-bzip2]) +AM_CONFIG_HEADER([src/config.h]) +AM_MAINTAINER_MODE + +IPP_STDCFLAGS + +AC_LANG(C) +AC_GNU_SOURCE +AC_PROG_CC_C99 +AC_PROG_INSTALL +AC_PROG_LIBTOOL +AC_SYS_LARGEFILE + +PKG_CHECK_MODULES([PSLIB], [pslib >= 1.0.0]) +PKG_CHECK_MODULES([PSMODULE], [psmodules >= 1.0.0]) +PKG_CHECK_MODULES([PPSTATS], [ppStats >= 1.0.0]) +PKG_CHECK_MODULES([PSPHOT], [psphot >= 0.9.0]) + +dnl Set CFLAGS for build +IPP_STDOPTS +CFLAGS="${CFLAGS} -Wall -Werror" + +AC_SUBST([PPSUB_CFLAGS]) +AC_SUBST([PPSUB_LIBS]) + +AC_CONFIG_FILES([ + Makefile + src/Makefile +]) + +AC_OUTPUT Index: /branches/pap_branch_080617/ppSub/src/.cvsignore =================================================================== --- /branches/pap_branch_080617/ppSub/src/.cvsignore (revision 18170) +++ /branches/pap_branch_080617/ppSub/src/.cvsignore (revision 18170) @@ -0,0 +1,10 @@ +*.o +*.lo +.libs +.deps +Makefile +Makefile.in +ppSub +config.h +config.h.in +stamp-h1 Index: /branches/pap_branch_080617/ppSub/src/Makefile.am =================================================================== --- /branches/pap_branch_080617/ppSub/src/Makefile.am (revision 18170) +++ /branches/pap_branch_080617/ppSub/src/Makefile.am (revision 18170) @@ -0,0 +1,22 @@ +bin_PROGRAMS = ppSub +ppSub_CPPFLAGS = $(PSLIB_CFLAGS) $(PSMODULE_CFLAGS) $(PPSTATS_CFLAGS) $(PSPHOT_CFLAGS) $(PPSUB_CFLAGS) +ppSub_LDFLAGS = $(PSLIB_LIBS) $(PSMODULE_LIBS) $(PPSTATS_LIBS) $(PSPHOT_LIBS) $(PPSUB_LIBS) + +ppSub_SOURCES = \ + ppSub.c \ + ppSubArguments.c \ + ppSubCamera.c \ + ppSubLoop.c \ + ppSubReadout.c \ + ppSubVersion.c + +noinst_HEADERS = \ + ppSub.h + +clean-local: + -rm -f TAGS + +# Tags for emacs +tags: + etags `find . -name \*.[ch] -print` + Index: /branches/pap_branch_080617/ppSub/src/ppSub.c =================================================================== --- /branches/pap_branch_080617/ppSub/src/ppSub.c (revision 18170) +++ /branches/pap_branch_080617/ppSub/src/ppSub.c (revision 18170) @@ -0,0 +1,58 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "ppSub.h" + +int main(int argc, char *argv[]) +{ + psExit exitValue = PS_EXIT_SUCCESS; // Exit value + psTimerStart("ppSub"); + psLibInit(NULL); + + pmConfig *config = pmConfigRead(&argc, argv, PPSUB_RECIPE); // Configuration + if (!config) { + psErrorStackPrint(stderr, "Error reading configuration."); + exitValue = PS_EXIT_CONFIG_ERROR; + goto die; + } + + if (!ppSubArguments(argc, argv, config)) { + psErrorStackPrint(stderr, "Error reading arguments.\n"); + exitValue = PS_EXIT_CONFIG_ERROR; + goto die; + } + + if (!pmModelClassInit()) { + psErrorStackPrint(stderr, "Error initialising model classes.\n"); + exitValue = PS_EXIT_PROG_ERROR; + goto die; + } + + if (!ppSubCamera(config)) { + psErrorStackPrint(stderr, "Error setting up camera.\n"); + exitValue = PS_EXIT_CONFIG_ERROR; + goto die; + } + + if (!ppSubLoop(config)) { + psErrorStackPrint(stderr, "Error performing subtraction.\n"); + exitValue = PS_EXIT_PROG_ERROR; + goto die; + } + + die: + psTrace("ppSub", 1, "Finished at %f sec\n", psTimerMark("ppSub")); + psTimerStop(); + + psFree(config); + pmModelClassCleanup(); + pmConfigDone(); + psLibFinalize(); + + exit(exitValue); +} Index: /branches/pap_branch_080617/ppSub/src/ppSub.h =================================================================== --- /branches/pap_branch_080617/ppSub/src/ppSub.h (revision 18170) +++ /branches/pap_branch_080617/ppSub/src/ppSub.h (revision 18170) @@ -0,0 +1,30 @@ +#ifndef PP_SUB_H +#define PP_SUB_H + +#define PPSUB_RECIPE "PPSUB" /// Name of the recipe to use + +/// Parse the arguments +bool ppSubArguments(int argc, char *argv[], ///< Command-line arguments + pmConfig *config ///< Configuration + ); + +/// Parse the camera input +bool ppSubCamera(pmConfig *config ///< Configuration + ); + +/// Loop over the FPA hierarchy +bool ppSubLoop(pmConfig *config ///< Configuration + ); + +/// Perform PSF-matched image subtraction on the readout +bool ppSubReadout(pmConfig *config, ///< Configuration + psMetadata *stats, ///< Statistics, for output + const pmFPAview *view ///< View of readout to subtract + ); + +/// Put the program version information into a metadata +void ppSubVersionMetadata(psMetadata *metadata ///< Metadata to populate + ); + + +#endif Index: /branches/pap_branch_080617/ppSub/src/ppSubArguments.c =================================================================== --- /branches/pap_branch_080617/ppSub/src/ppSubArguments.c (revision 18170) +++ /branches/pap_branch_080617/ppSub/src/ppSubArguments.c (revision 18170) @@ -0,0 +1,328 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppSub.h" + +// Print usage information and die +static void usage(const char *program, // Name of the program + psMetadata *arguments, // Command-line arguments + pmConfig *config // Configuration + ) +{ + fprintf(stderr, "\nPan-STARRS PSF-matched image subtraction\n\n"); + fprintf(stderr, "Usage: %s INPUT.fits REFERENCE.fits OUTPUT_ROOT \n" + "\t[-psf REFERENCE.psf.fits] [-sources REFERENCE.cmf]\n\n" + "This subtracts the convolved REFERENCE from the INPUT, by default.\n", + program); + fprintf(stderr, "\n"); + psArgumentHelp(arguments); + psFree(arguments); + psFree(config); + pmConfigDone(); + psLibFinalize(); + exit(PS_EXIT_CONFIG_ERROR); +} + +// Get a float-point value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_FLOAT(ARGNAME, RECIPENAME, TYPE) { \ + ps##TYPE value = psMetadataLookup##TYPE(NULL, arguments, ARGNAME); \ + if (isnan(value)) { \ + bool mdok; \ + value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \ + if (!mdok) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPSUB_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAdd##TYPE(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, value); \ +} + +// Get an integer value from the command-line or recipe, and add it to the arguments +#define VALUE_ARG_RECIPE_INT(ARGNAME, RECIPENAME, TYPE, UNSET) { \ + ps##TYPE value = psMetadataLookup##TYPE(NULL, arguments, ARGNAME); \ + if (value == UNSET) { \ + bool mdok; \ + value = psMetadataLookup##TYPE(&mdok, recipe, RECIPENAME); \ + if (!mdok) { \ + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", \ + RECIPENAME, PPSUB_RECIPE); \ + goto ERROR; \ + } \ + } \ + psMetadataAdd##TYPE(config->arguments, PS_LIST_TAIL, RECIPENAME, 0, NULL, value); \ +} + +// Get a string value from the command-line and add it to the target +static bool valueArgStr(psMetadata *arguments, // Command-line arguments + const char *argName, // Argument name in the command-line arguments + const char *mdName, // Name for value in the metadata + psMetadata *target // Target metadata to which to add value + ) +{ + psString value = psMetadataLookupStr(NULL, arguments, argName); // Value of interest + if (value && strlen(value) > 0) { + return psMetadataAddStr(target, PS_LIST_TAIL, mdName, 0, NULL, value); + } + return false; +} + +// Get a string value from the command-line or recipe and add it to the target +static bool valueArgRecipeStr(psMetadata *arguments, // Command-line arguments + psMetadata *recipe, // Recipe + const char *argName, // Argument name in the command-line arguments + const char *mdName, // Name for value in the metadata and recipe + psMetadata *target // Target metadata to which to add value + ) +{ + bool mdok; // Status of MD lookup + psString value = psMetadataLookupStr(&mdok, arguments, argName); // Value of interest + if (!mdok) { + value = psMetadataLookupStr(&mdok, recipe, mdName); + if (!mdok) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to find %s in recipe %s", + mdName, PPSUB_RECIPE); + return false; + } + } + return psMetadataAddStr(target, PS_LIST_TAIL, mdName, 0, NULL, value); +} + +// Get a vector from the command-line or recipe, and add it to the target +static bool vectorArgRecipe(psMetadata *arguments, // Command-line arguments + const char *argName, // Argument name in the command-line arguments + const psMetadata *recipe, // Recipe + const char *recipeName, // Name for value in the recipe + psMetadata *target, // Target to which to add value + psElemType type // Type for vector + ) +{ + psVector *vector; // Vector + psString string = psMetadataLookupStr(NULL, arguments, argName); // String from arguments + if (string) { + psArray *array = psStringSplitArray(string, ", ", false); // Array of strings + vector = psVectorAlloc(array->n, type); + for (int i = 0; i < array->n; i++) { + const char *subString = array->data[i]; // String with a value + char *end; // Ptr to end of string parsed + + switch (type) { + case PS_TYPE_F32: + vector->data.F32[i] = strtof(subString, &end); + break; + case PS_TYPE_S32: { + long value = strtol(subString, &end, 10); + if (value > PS_MAX_S32 || value < PS_MIN_S32) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "%s list includes value beyond S32 representation: %s", + argName, string); + psFree(vector); + return false; + } + vector->data.S32[i] = value; + break; + } + default: + psAbort("Unsupported type: %x\n", type); + } + if (end == subString) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to decipher %s list: %s", + argName, string); + psFree(vector); + return false; + } + } + psFree(array); + } else { + vector = psMetadataLookupPtr(NULL, recipe, recipeName); + if (!psMemCheckVector(vector) || vector->type.type != type) { + psError(PS_ERR_BAD_PARAMETER_TYPE, false, "%s in recipe %s is not a vector of type F32.", + recipeName, PPSUB_RECIPE); + return false; + } + psMemIncrRefCounter(vector); + } + + psMetadataAddVector(target, PS_LIST_TAIL, recipeName, 0, NULL, vector); + psFree(vector); // Drop reference + + return true; +} + +// Add a single filename to the arguments as an array, so that it can be used with pmFPAfileBindFromArgs, etc +static void fileList(const char *file, // The symbolic name for the file + const char *name, // The name of the file + const char *comment, // Description of the file + pmConfig *config // Configuration + ) +{ + psArray *files = psArrayAlloc(1); // Array with file names + files->data[0] = psStringCopy(name); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, file, 0, comment, files); + psFree(files); + return; +} + +bool ppSubArguments(int argc, char *argv[], pmConfig *config) +{ + assert(config); + + pmConfigFileSetsMD(config->arguments, &argc, argv, "PPSUB.SOURCES", "-sources", NULL); + + psMetadata *arguments = psMetadataAlloc(); // Command-line arguments + psMetadataAddStr(arguments, PS_LIST_TAIL, "-inmask", 0, "Input mask image", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-inweight", 0, "Input weight image", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-refmask", 0, "Referencemask image", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-refweight", 0, "Referenceweight image", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stats", 0, "Statistics file", NULL); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-region", 0, "Size of iso-kernel region", NAN); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-size", 0, "Kernel half-size (pixels)", 0); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-order", 0, "Spatial polynomial order", -1); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-type", 0, "Kernel type (ISIS|POIS|SPAM|FRIES|GUNK|RINGS)", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-isis-widths", 0, "ISIS Gaussian FWHMs (comma-separated)", NULL); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-isis-orders", 0, "ISIS polynomial orders (comma-separated)", NULL); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-rings-order", 0, "RINGS polynomial order", -1); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-inner", 0, "SPAM and FRIES inner radius", -1); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-spam-binning", 0, "SPAM kernel binning", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-spacing", 0, "Typical stamp spacing (pixels)", NAN); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-footprint", 0, "Stamp footprint half-size (pixels)", -1); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-threshold", 0, "Minimum threshold for stamps (ADU)", NAN); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-iter", 0, "Number of rejection iterations", -1); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-rej", 0, "Rejection thresold (sigma)", NAN); + psMetadataAddU8(arguments, PS_LIST_TAIL, "-mask-bad", 0, "Mask value for bad pixels", 0); + psMetadataAddU8(arguments, PS_LIST_TAIL, "-mask-blank", 0, "Mask value for blank region", 0); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-badfrac", 0, "Maximum fraction of bad pixels to accept", 1.0); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-reverse", 0, "Reverse sense of subtraction?", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-generate-mask", 0, "Generate mask if not supplied?", false); + psMetadataAddStr(arguments, PS_LIST_TAIL, "-stamps", 0, "Stamps filename; file has x,y on each line", NULL); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-opt", 0, "Derive optimum parameters for ISIS kernels?", false); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-opt-min", 0, "Minimum value for optimum kernel search", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-opt-max", 0, "Minimum value for optimum kernel search", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-opt-step", 0, "Step value for optimum kernel search", NAN); + psMetadataAddF32(arguments, PS_LIST_TAIL, "-opt-tol", 0, "Tolerance for optimum kernel search", NAN); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-opt-order", 0, "Maximum order for optimum kernel search", -1); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-dual", 0, "Dual convolution", false); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-renorm", 0, "Renormalise weights?", false); + psMetadataAddS32(arguments, PS_LIST_TAIL, "-renorm-width", 0, "Renormalisation width", 0); + psMetadataAddBool(arguments, PS_LIST_TAIL, "-photometry", 0, "Perform photometry?", false); + + if (argc == 1 || !psArgumentParse(arguments, &argc, argv) || argc != 4) { + usage(argv[0], arguments, config); + } + + fileList("INPUT", argv[1], "Name of the input image", config); + fileList("REF", argv[2], "Name of the reference image", config); + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output image", argv[3]); + + const char *inMask = psMetadataLookupStr(NULL, arguments, "-inmask"); // Name of input mask + if (inMask && strlen(inMask) > 0) { + fileList("INPUT.MASK", inMask, "Name of the input mask image", config); + } + const char *inWeight = psMetadataLookupStr(NULL, arguments, "-inweight"); // Name of input weight + if (inWeight && strlen(inWeight) > 0) { + fileList("INPUT.WEIGHT", inWeight, "Name of the input weight image", config); + } + + const char *refMask = psMetadataLookupStr(NULL, arguments, "-refmask"); // Name of reference mask + if (refMask && strlen(refMask) > 0) { + fileList("REF.MASK", refMask, "Name of the reference mask image", config); + } + const char *refWeight = psMetadataLookupStr(NULL, arguments, "-refweight"); // Name of reference weight + if (refWeight && strlen(refWeight) > 0) { + fileList("REF.WEIGHT", refWeight, "Name of the reference weight image", config); + } + + valueArgStr(arguments, "-stats", "STATS", config->arguments); + valueArgStr(arguments, "-stamps", "STAMPS", config->arguments); + + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PPSUB_RECIPE); // Recipe for ppSim + if (!recipe) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find recipe %s", PPSUB_RECIPE); + goto ERROR; + } + + VALUE_ARG_RECIPE_FLOAT("-region", "REGION.SIZE", F32); + VALUE_ARG_RECIPE_INT("-size", "KERNEL.SIZE", S32, 0); + VALUE_ARG_RECIPE_INT("-order", "SPATIAL.ORDER", S32, -1); + VALUE_ARG_RECIPE_FLOAT("-spacing", "STAMP.SPACING", F32); + VALUE_ARG_RECIPE_INT("-rings-order", "RINGS.ORDER", S32, -1); + VALUE_ARG_RECIPE_INT("-inner", "INNER", S32, -1); + VALUE_ARG_RECIPE_INT("-spam-binning", "SPAM.BINNING", S32, 0); + VALUE_ARG_RECIPE_INT("-footprint", "STAMP.FOOTPRINT", S32, -1); + VALUE_ARG_RECIPE_FLOAT("-threshold", "STAMP.THRESHOLD", F32); + VALUE_ARG_RECIPE_INT("-iter", "ITER", S32, -1); + VALUE_ARG_RECIPE_FLOAT("-rej", "REJ", F32); + VALUE_ARG_RECIPE_FLOAT("-badfrac", "BADFRAC", F32); + + valueArgRecipeStr(arguments, recipe, "-mask-bad", "MASK.BAD", config->arguments); + valueArgRecipeStr(arguments, recipe, "-mask-blank", "MASK.BLANK", config->arguments); + + vectorArgRecipe(arguments, "-isis-widths", recipe, "ISIS.WIDTHS", config->arguments, PS_TYPE_F32); + vectorArgRecipe(arguments, "-isis-orders", recipe, "ISIS.ORDERS", config->arguments, PS_TYPE_S32); + + psVector *widths = psMetadataLookupPtr(NULL, config->arguments, "ISIS.WIDTHS"); // ISIS Gaussian widths + psVector *orders = psMetadataLookupPtr(NULL, config->arguments, "ISIS.ORDERS"); // ISIS Polynomial orders + if (widths->n != orders->n) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, "Size of vectors for ISIS widths and orders do not match."); + goto ERROR; + } + + if (psMetadataLookupBool(NULL, arguments, "-opt") || psMetadataLookupBool(NULL, recipe, "OPTIMUM")) { + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "OPTIMUM", 0, + "Derive optimum parameters for ISIS kernels?", true); + VALUE_ARG_RECIPE_FLOAT("-opt-min", "OPTIMUM.MIN", F32); + VALUE_ARG_RECIPE_FLOAT("-opt-max", "OPTIMUM.MAX", F32); + VALUE_ARG_RECIPE_FLOAT("-opt-step","OPTIMUM.STEP", F32); + VALUE_ARG_RECIPE_FLOAT("-opt-tol", "OPTIMUM.TOL", F32); + VALUE_ARG_RECIPE_INT("-opt-order", "OPTIMUM.ORDER", S32, -1); + } + + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "REVERSE", 0, "Reverse sense of subtraction", + psMetadataLookupBool(NULL, arguments, "-reverse")); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "MASK.GENERATE", 0, "Generate mask if not supplied", + psMetadataLookupBool(NULL, arguments, "-generate-mask")); + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "DUAL", 0, "Dual convolution?", + psMetadataLookupBool(NULL, arguments, "-dual")); + + if (psMetadataLookupBool(NULL, arguments, "-renorm") || psMetadataLookupBool(NULL, recipe, "RENORM")) { + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "RENORM", 0, "Renormalise weights?", true); + VALUE_ARG_RECIPE_INT("-renorm-width", "RENORM.WIDTH", S32, 0); + } + + if (psMetadataLookupBool(NULL, arguments, "-photometry") || + psMetadataLookupBool(NULL, recipe, "PHOTOMETRY")) { + psMetadataAddBool(config->arguments, PS_LIST_TAIL, "PHOTOMETRY", 0, "Perform photometry?", true); + } + + // Translate the kernel type + psString type = psMetadataLookupStr(NULL, arguments, "-type"); // Name of kernel type + if (!type || strlen(type) == 0) { + type = psMetadataLookupStr(NULL, recipe, "KERNEL.TYPE"); + if (!type || strlen(type) == 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, false, "Unable to find KERNEL.TYPE specified."); + goto ERROR; + } + } + pmSubtractionKernelsType kernelType = pmSubtractionKernelsTypeFromString(type); // Type of kernel + if (kernelType == PM_SUBTRACTION_KERNEL_NONE) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unrecognised kernel type: %s", type); + goto ERROR; + } + psMetadataAddS32(config->arguments, PS_LIST_TAIL, "KERNEL.TYPE", 0, "Type of kernel", kernelType); + + psTrace("ppSub", 1, "Done reading command-line arguments\n"); + psFree(arguments); + return true; + +ERROR: + psFree(arguments); + return false; +} + + Index: /branches/pap_branch_080617/ppSub/src/ppSubCamera.c =================================================================== --- /branches/pap_branch_080617/ppSub/src/ppSubCamera.c (revision 18170) +++ /branches/pap_branch_080617/ppSub/src/ppSubCamera.c (revision 18170) @@ -0,0 +1,169 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppSub.h" + +bool ppSubCamera(pmConfig *config) +{ + bool status = false; // Status of definition + + // Input image + pmFPAfile *input = pmFPAfileDefineFromArgs(&status, config, "PPSUB.INPUT", "INPUT"); + if (!status || !input) { + psError(PS_ERR_IO, false, "Failed to build FPA from PPSUB.INPUT"); + return false; + } + if (input->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPSUB.INPUT is not of type IMAGE"); + return false; + } + + // Input mask + pmFPAfile *inputMask = pmFPAfileBindFromArgs(&status, input, config, "PPSUB.INPUT.MASK", "INPUT.MASK"); + if (!status) { + psError (PS_ERR_UNKNOWN, false, "Failed to load file definition PPSUB.INPUT.MASK"); + return NULL; + } + if (inputMask && inputMask->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PPSUB.INPUT.MASK is not of type MASK"); + return false; + } + + // Input weight map + pmFPAfile *inputWeight = pmFPAfileBindFromArgs(&status, input, config, "PPSUB.INPUT.WEIGHT", "INPUT.WEIGHT"); + if (!status) { + psError (PS_ERR_UNKNOWN, false, "Failed to load file definition PPSUB.INPUT.WEIGHT"); + return NULL; + } + if (inputWeight && inputWeight->type != PM_FPA_FILE_WEIGHT) { + psError(PS_ERR_IO, true, "PPSUB.INPUT.WEIGHT is not of type WEIGHT"); + return false; + } + + // Reference image + status = false; + pmFPAfile *ref = pmFPAfileDefineFromArgs(&status, config, "PPSUB.REF", "REF"); + if (!ref) { + psError(PS_ERR_IO, false, "Failed to build FPA from PPSUB.REF"); + return false; + } + if (ref->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPSUB.REF is not of type IMAGE"); + return false; + } + + // Reference mask + pmFPAfile *refMask = pmFPAfileBindFromArgs(&status, ref, config, "PPSUB.REF.MASK", "REF.MASK"); + if (!status) { + psError (PS_ERR_UNKNOWN, false, "Failed to load file definition PPSUB.REF.MASK"); + return NULL; + } + if (refMask && refMask->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PPSUB.REF.MASK is not of type MASK"); + return false; + } + + // Reference weight map + pmFPAfile *refWeight = pmFPAfileBindFromArgs(&status, ref, config, "PPSUB.REF.WEIGHT", "REF.WEIGHT"); + if (!status) { + psError (PS_ERR_UNKNOWN, false, "Failed to load file definition PPSUB.REF.WEIGHT"); + return NULL; + } + if (refWeight && refWeight->type != PM_FPA_FILE_WEIGHT) { + psError(PS_ERR_IO, true, "PPSUB.REF.WEIGHT is not of type WEIGHT"); + return false; + } + + // Output image + pmFPAfile *output = pmFPAfileDefineSkycell(config, NULL, "PPSUB.OUTPUT"); + if (!output) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.OUTPUT")); + return false; + } + if (output->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PPSUB.OUTPUT is not of type IMAGE"); + return false; + } + output->save = true; + + // Output mask + pmFPAfile *outMask = pmFPAfileDefineSkycell(config, output->fpa, "PPSUB.OUTPUT.MASK"); + if (!outMask) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.OUTPUT.MASK")); + return false; + } + if (outMask->type != PM_FPA_FILE_MASK) { + psError(PS_ERR_IO, true, "PPSUB.OUTPUT.MASK is not of type MASK"); + return false; + } + outMask->save = true; + + // Output weight + if (inputWeight && refWeight) { + pmFPAfile *outWeight = pmFPAfileDefineSkycell(config, output->fpa, "PPSUB.OUTPUT.WEIGHT"); + if (!outWeight) { + psError(PS_ERR_IO, false, _("Unable to generate output file from PPSUB.OUTPUT.WEIGHT")); + return false; + } + if (outWeight->type != PM_FPA_FILE_WEIGHT) { + psError(PS_ERR_IO, true, "PPSUB.OUTPUT.WEIGHT is not of type WEIGHT"); + return false; + } + outWeight->save = true; + } + + if (!pmFPAAddSourceFromFormat(output->fpa, "Subtraction", output->format)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate output FPA."); + return false; + } + + pmFPAfile *sources = pmFPAfileDefineFromArgs(&status, config, "PPSUB.SOURCES", "PPSUB.SOURCES"); + if (!status) { + psError(PS_ERR_IO, false, "Failed to load file definition PPSUB.SOURCES"); + return false; + } + if (sources && sources->type != PM_FPA_FILE_CMF) { + psError(PS_ERR_IO, true, "PPSUB.SOURCES is not of type CMF"); + return false; + } + + // psPhot input + if (psMetadataLookupBool(NULL, config->arguments, "PHOTOMETRY")) { + psphotModelClassInit (); // load implementation-specific models + pmFPAfile *psphot = pmFPAfileDefineFromFPA(config, output->fpa, 1, 1, "PSPHOT.INPUT"); + if (!psphot) { + psError(PS_ERR_IO, false, "Failed to build FPA from PSPHOT.INPUT"); + return false; + } + if (psphot->type != PM_FPA_FILE_IMAGE) { + psError(PS_ERR_IO, true, "PSPHOT.INPUT is not of type IMAGE"); + return false; + } + + if (!psphotDefineFiles(config, psphot)) { + psError(PS_ERR_UNKNOWN, false, "Unable to set up psphot files."); + return false; + } + + // Internal-ish file for getting the PSF from the matched addition + pmFPAfile *psf = pmFPAfileDefineSkycell(config, psphot->fpa, "PSPHOT.PSF.LOAD"); + if (!psf) { + psError(PS_ERR_IO, false, "Failed to build FPA from PSPHOT.PSF.LOAD"); + return false; + } + if (psf->type != PM_FPA_FILE_PSF) { + psError(PS_ERR_IO, true, "PSPHOT.PSF.LOAD is not of type PSF"); + return false; + } + pmFPAfileActivate(config->files, false, "PSPHOT.PSF.LOAD"); + + } + + return true; +} Index: /branches/pap_branch_080617/ppSub/src/ppSubLoop.c =================================================================== --- /branches/pap_branch_080617/ppSub/src/ppSubLoop.c (revision 18170) +++ /branches/pap_branch_080617/ppSub/src/ppSubLoop.c (revision 18170) @@ -0,0 +1,186 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include "ppSub.h" + +bool ppSubLoop(pmConfig *config) +{ + // Value to mask + psMaskType maskBlank = pmConfigMask(psMetadataLookupStr(NULL, config->arguments, "MASK.BLANK"), config) | + pmConfigMask(psMetadataLookupStr(NULL, config->arguments, "MASK.BAD"), + config); // Mask for subtracted image + + bool mdok; // Status of MD lookup + const char *statsName = psMetadataLookupStr(&mdok, config->arguments, "STATS"); // Filename for statistics + psMetadata *stats = NULL; // Container for statistics + FILE *statsFile = NULL; // File stream for statistics + if (statsName && strlen(statsName) > 0) { + psString resolved = pmConfigConvertFilename(statsName, config, true, true); // Resolved filename + statsFile = fopen(resolved, "w"); + if (!statsFile) { + psError(PS_ERR_IO, true, "Unable to open statistics file %s for writing.\n", resolved); + psFree(resolved); + return false; + } else { + stats = psMetadataAlloc(); + } + psFree(resolved); + } + + pmFPAfile *input = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); + if (!input) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find input data!\n"); + return false; + } + + pmFPAfile *reference = psMetadataLookupPtr(NULL, config->files, "PPSUB.REF"); + if (!reference) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find reference data!\n"); + return false; + } + + pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSUB.OUTPUT"); + if (!output) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Can't find output data!\n"); + return false; + } + + bool doPhotom = false; + if (psMetadataLookup(config->arguments, "PSPHOT.PSF")) { + doPhotom = true; + } + + pmFPAview *view = pmFPAviewAlloc(0); // Pointer into FPA hierarchy + pmHDU *lastHDU = NULL; // Last HDU that was updated + + // Iterate over the FPA hierarchy + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return false; + } + + pmChip *inChip; // Input chip of interest + while ((inChip = pmFPAviewNextChip(view, input->fpa, 1)) != NULL) { + pmChip *refChip = pmFPAviewThisChip(view, reference->fpa); // Reference chip of interest + if ((!inChip->file_exists && refChip->file_exists) || + (inChip->file_exists && !refChip->file_exists)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "FPA format discrepency between input and reference"); + psFree(view); + return false; + } + + if (!inChip->file_exists) { + continue; + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return false; + } + + pmCell *inCell; // Cell of interest + while ((inCell = pmFPAviewNextCell(view, input->fpa, 1)) != NULL) { + pmCell *refCell = pmFPAviewThisCell(view, reference->fpa); // Reference cell of interest + if ((!inCell->file_exists && refCell->file_exists) || + (inCell->file_exists && !refCell->file_exists)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "FPA format discrepency between input and reference"); + psFree(view); + return false; + } + if (!inCell->file_exists) { + continue; + } + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return false; + } + + // Put version information into the header + pmHDU *hdu = pmHDUFromCell(inCell); + if (hdu && hdu != lastHDU) { + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + ppSubVersionMetadata(hdu->header); + lastHDU = hdu; + } + + pmReadout *inRO; // Readin of interest + while ((inRO = pmFPAviewNextReadout(view, input->fpa, 1))) { + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return false; + } + pmReadout *refRO = pmFPAviewThisReadout(view, reference->fpa);// Reference readout of interest + if (!refRO || (!inRO->data_exists && refRO->data_exists) || + (inRO->data_exists && !refRO->data_exists)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "FPA format discrepency between input and reference"); + psFree(view); + return false; + } + if (!inRO->data_exists) { + continue; + } + + // Perform the analysis + if (!ppSubReadout(config, stats, view)) { + psError(PS_ERR_UNKNOWN, false, "Unable to subtract images.\n"); + return false; + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + return false; + } + } + + // Perform statistics on the cell + if (stats) { + pmFPAfile *output = psMetadataLookupPtr(NULL, config->files, "PPSUB.OUTPUT"); // Output file + if (!output) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find file PPSUB.OUTPUT.\n"); + return false; + } + ppStatsFPA(stats, output->fpa, view, maskBlank, config); + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + return false; + } + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + return false; + } + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + return false; + } + + psFree(view); + + // Write out summary statistics + if (stats) { + psMetadataAddF32(stats, PS_LIST_TAIL, "DT_SUB", 0, "Time for subtraction completion", + psTimerMark("ppSub")); + + const char *statsMDC = psMetadataConfigFormat(stats); + if (!statsMDC || strlen(statsMDC) == 0) { + psWarning("Unable to generate statistics MDC file.\n"); + } else { + fprintf(statsFile, "%s", statsMDC); + } + psFree((void *)statsMDC); + fclose(statsFile); + + psFree(stats); + } + + return true; +} Index: /branches/pap_branch_080617/ppSub/src/ppSubReadout.c =================================================================== --- /branches/pap_branch_080617/ppSub/src/ppSubReadout.c (revision 18170) +++ /branches/pap_branch_080617/ppSub/src/ppSubReadout.c (revision 18170) @@ -0,0 +1,375 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include +#include + +#include "ppSub.h" + +#define WCS_TOLERANCE 0.001 // Tolerance for WCS +//#define TESTING // For test output + + +// Copy every instance of a single keyword from one metadata to another +static bool metadataCopySingle(psMetadata *target, psMetadata *source, const char *name) +{ + PS_ASSERT_METADATA_NON_NULL(target, false); + PS_ASSERT_METADATA_NON_NULL(source, false); + PS_ASSERT_STRING_NON_EMPTY(name, false); + + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", name); + psMetadataIterator *iter = psMetadataIteratorAlloc(source, PS_LIST_HEAD, regex); // Iterator + psFree(regex); + psMetadataItem *item; // Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + psMetadataAddItem(target, item, PS_LIST_TAIL, PS_META_DUPLICATE_OK); + } + psFree(iter); + + return true; +} + + +bool ppSubReadout(pmConfig *config, psMetadata *stats, const pmFPAview *view) +{ + pmReadout *inRO = pmFPAfileThisReadout(config->files, view, "PPSUB.INPUT"); // Input readout + pmReadout *refRO = pmFPAfileThisReadout(config->files, view, "PPSUB.REF"); // Reference readout + pmReadout *sourcesRO = pmFPAfileThisReadout(config->files, view, "PPSUB.SOURCES"); // Readout with sources + pmReadout *inConv = pmReadoutAlloc(NULL); // Convolved version of input + pmReadout *refConv = pmReadoutAlloc(NULL); // Convolved version of reference + pmCell *outCell = pmFPAfileThisCell(config->files, view, "PPSUB.OUTPUT"); // Output cell + pmReadout *outRO = pmReadoutAlloc(outCell); // Output readout: subtraction + pmFPA *outFPA = outCell->parent->parent; // Output FPA + pmHDU *outHDU = outFPA->hdu; // Output HDU + if (!outHDU->header) { + outHDU->header = psMetadataAlloc(); + } + + psImage *input = inRO->image; // Input image + psImage *reference = refRO->image; // Reference image + PS_ASSERT_IMAGES_SIZE_EQUAL(input, reference, false); + + // Look up appropriate values + bool mdok; // Status of MD lookup + int size = psMetadataLookupS32(NULL, config->arguments, "KERNEL.SIZE"); // Kernel half-size + int order = psMetadataLookupS32(NULL, config->arguments, "SPATIAL.ORDER"); // Spatial polynomial order + float regionSize = psMetadataLookupF32(NULL, config->arguments, "REGION.SIZE"); // Size of iso-kernel regs + float spacing = psMetadataLookupF32(NULL, config->arguments, "STAMP.SPACING"); // Typical stamp spacing + int footprint = psMetadataLookupS32(NULL, config->arguments, "STAMP.FOOTPRINT"); // Stamp half-size + float threshold = psMetadataLookupF32(NULL, config->arguments, "STAMP.THRESHOLD"); // Threshold for stmps + int iter = psMetadataLookupS32(NULL, config->arguments, "ITER"); // Rejection iterations + float rej = psMetadataLookupF32(NULL, config->arguments, "REJ"); // Rejection threshold + pmSubtractionKernelsType type = psMetadataLookupS32(NULL, config->arguments, + "KERNEL.TYPE"); // Kernel type + bool reverse = psMetadataLookupBool(NULL, config->arguments, "REVERSE"); // Reverse sense of subtraction? + psVector *widths = psMetadataLookupPtr(NULL, config->arguments, "ISIS.WIDTHS"); // ISIS Gaussian widths + psVector *orders = psMetadataLookupPtr(NULL, config->arguments, "ISIS.ORDERS"); // ISIS Polynomial orders + int inner = psMetadataLookupS32(NULL, config->arguments, "INNER"); // Inner radius + int ringsOrder = psMetadataLookupS32(NULL, config->arguments, "RINGS.ORDER"); // RINGS polynomial order + int binning = psMetadataLookupS32(NULL, config->arguments, "SPAM.BINNING"); // Binning for SPAM kernel + psMaskType maskBad = pmConfigMask(psMetadataLookupStr(NULL, config->arguments, "MASK.BAD"), + config); // Value to mask + psMaskType maskBlank = pmConfigMask(psMetadataLookupStr(NULL, config->arguments, "MASK.BLANK"), + config); // Mask for blank reg. + float badFrac = psMetadataLookupF32(NULL, config->arguments, "BADFRAC"); // Maximum bad fraction + const char *stampsName = psMetadataLookupStr(&mdok, config->arguments, "STAMPS"); // Filename for stamps + + bool optimum = psMetadataLookupBool(&mdok, config->arguments, "OPTIMUM"); // Derive optimum parameters? + float optMin = psMetadataLookupF32(&mdok, config->arguments, "OPTIMUM.MIN"); // Minimum width for search + float optMax = psMetadataLookupF32(&mdok, config->arguments, "OPTIMUM.MAX"); // Maximum width for search + float optStep = psMetadataLookupF32(&mdok, config->arguments, "OPTIMUM.STEP"); // Step for search + float optThresh = psMetadataLookupF32(&mdok, config->arguments, "OPTIMUM.TOL"); // Tolerance for search + int optOrder = psMetadataLookupS32(&mdok, config->arguments, "OPTIMUM.ORDER"); // Order for search + bool dual = psMetadataLookupBool(&mdok, config->arguments, "DUAL"); // Dual convolution? + bool renorm = psMetadataLookupBool(&mdok, config->arguments, "RENORM"); // Renormalise weights? + int renormWidth = psMetadataLookupS32(&mdok, config->arguments, "RENORM.WIDTH"); // Width for renormalise + + pmSubtractionMode mode = dual ? PM_SUBTRACTION_MODE_DUAL : PM_SUBTRACTION_MODE_UNSURE; // Subtraction mode + + // Generate masks if they don't exist + int numCols = input->numCols, numRows = input->numRows; // Image dimensions + if (!inRO->mask) { + if (psMetadataLookupBool(NULL, config->arguments, "MASK.GENERATE")) { + pmReadoutSetMask(inRO, pmConfigMask("SAT", config), pmConfigMask("BAD", config)); + } else { + inRO->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + psImageInit(inRO->mask, 0); + } + } + if (!refRO->mask) { + if (psMetadataLookupBool(NULL, config->arguments, "MASK.GENERATE")) { + pmReadoutSetMask(refRO, pmConfigMask("SAT", config), pmConfigMask("BAD", config)); + } else { + refRO->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + psImageInit(refRO->mask, 0); + } + } + + if (!pmReadoutMaskNonfinite(inRO, maskBad)) { + psError(PS_ERR_UNKNOWN, false, "Unable to mask non-finite pixels in input."); + return false; + } + if (!pmReadoutMaskNonfinite(refRO, maskBad)) { + psError(PS_ERR_UNKNOWN, false, "Unable to mask non-finite pixels in reference."); + return false; + } + + psVector *optWidths = NULL; // Vector with FWHMs for optimum search + if (optimum) { + optWidths = psVectorCreate(optWidths, optMin, optMax, optStep, PS_TYPE_F32); + } + + psArray *sources = NULL; // Sources in image + if (sourcesRO) { + sources = psMetadataLookupPtr(&mdok, sourcesRO->analysis, "PSPHOT.SOURCES"); + } + + if (!pmSubtractionMatch(inConv, refConv, inRO, refRO, footprint, regionSize, spacing, threshold, sources, + stampsName, type, size, order, widths, orders, inner, ringsOrder, + binning, optimum, optWidths, optOrder, optThresh, iter, rej, maskBad, + maskBlank, badFrac, mode)) { + psError(PS_ERR_UNKNOWN, false, "Unable to match images."); + psFree(inConv); + psFree(refConv); + psFree(outRO); + return false; + } + psFree(optWidths); + + // Add kernel descrption to header + pmSubtractionKernels *kernels = psMetadataLookupPtr(&mdok, inConv->analysis, + PM_SUBTRACTION_ANALYSIS_KERNEL); // Subtraction kernel + if (!kernels) { + kernels = psMetadataLookupPtr(&mdok, refConv->analysis, PM_SUBTRACTION_ANALYSIS_KERNEL); + } + if (!kernels) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find SUBTRACTION.KERNEL"); + psFree(inConv); + psFree(refConv); + psFree(outRO); + return false; + } + psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.KERNEL", 0, + "Subtraction kernel", kernels->description); + + + // Statistics on the matching + if (stats) { + if (psMetadataLookup(inConv->analysis, PM_SUBTRACTION_ANALYSIS_MODE)) { + metadataCopySingle(stats, inConv->analysis, PM_SUBTRACTION_ANALYSIS_MODE); + metadataCopySingle(stats, inConv->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS_NUM); + metadataCopySingle(stats, inConv->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS_RMS); + } + if (psMetadataLookup(refConv->analysis, PM_SUBTRACTION_ANALYSIS_MODE)) { + metadataCopySingle(stats, refConv->analysis, PM_SUBTRACTION_ANALYSIS_MODE); + metadataCopySingle(stats, refConv->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS_NUM); + metadataCopySingle(stats, refConv->analysis, PM_SUBTRACTION_ANALYSIS_STAMPS_RMS); + } + } + +#ifdef TESTING + psImage *kernelImage = psMetadataLookupPtr(&mdok, inConv->analysis, + "SUBTRACTION.KERNEL.IMAGE"); // Image of the kernels + if (!kernelImage) { + kernelImage = psMetadataLookupPtr(&mdok, refConv->analysis, "SUBTRACTION.KERNEL.IMAGE"); + } + psFits *fits = psFitsOpen("kernel.fits", "w"); + psFitsWriteImage(fits, NULL, kernelImage, 0, NULL); + psFitsClose(fits); +#endif + + // "Subtract" the mask and weight map + outRO->mask = (psImage*)psBinaryOp(outRO->mask, inConv->mask, "|", refConv->mask); + if (inConv->weight && refConv->weight) { + outRO->weight = (psImage*)psBinaryOp(outRO->weight, inConv->weight, "+", refConv->weight); + } + outRO->data_exists = outCell->data_exists = outCell->parent->data_exists = true; + + // Photometry is to be performed in two stages: + // 1. Measure the PSF using the PSF-matched images + // 2. Find and measure sources on the subtracted image + + // Photometry stage 1: measure the PSF + pmPSF *psf = NULL; // PSF for photometry + if (psMetadataLookupBool(NULL, config->arguments, "PHOTOMETRY")) { + // We use a summed image as the basis for the PSF: this will have the maximum S/N. + outRO->image = (psImage*)psBinaryOp(outRO->image, inConv->image, "+", refConv->image); + + pmFPAfile *photFile = psMetadataLookupPtr(NULL, config->files, "PSPHOT.INPUT"); + pmFPACopy(photFile->fpa, outRO->parent->parent->parent); + + // We have a list of sources, so we could use those to redetermine the PSF model. + // Until Gene makes the necessary adaptations to psphot, we will simply redetermine the PSF model from + // scratch. + + // Need to ensure aperture residual is not calculated + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PSPHOT_RECIPE); // Recipe + if (!recipe) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find %s recipe.", PSPHOT_RECIPE); + return false; + } + const char *breakpoint = psMetadataLookupStr(&mdok, recipe, "BREAK_POINT"); // Current break point + psMetadataAddStr(recipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, + "ALTERED break point for psphot operations", "PSFMODEL"); + + if (!psphotReadout(config, view)) { + psWarning("Unable to perform photometry on subtracted image."); + psErrorStackPrint(stderr, "Error stack from photometry:"); + psErrorClear(); + } + + if (breakpoint) { + psMetadataAddStr(recipe, PS_LIST_TAIL, "BREAK_POINT", PS_META_REPLACE, + "RESTORED break point for psphot operations", breakpoint); + } + + // Blow away the sources psphot found --- they're irrelevant for the subtraction + pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout with sources + psMetadataRemoveKey(photRO->analysis, "PSPHOT.SOURCES"); + psMetadataRemoveKey(photRO->analysis, "PSPHOT.HEADER"); + + psf = psMemIncrRefCounter(psMetadataLookupPtr(NULL, photRO->parent->parent->analysis, "PSPHOT.PSF")); + if (!psf) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find PSF from psphot"); + return false; + } + + pmCell *photCell = pmFPAfileThisCell(config->files, view, "PSPHOT.INPUT"); + pmCellFreeReadouts(photCell); + } + + // Do the subtraction + { + // Subtraction is: minuend - subtrahend + pmReadout *minuend = inConv; + pmReadout *subtrahend = refConv; + + if (reverse) { + pmReadout *temp = subtrahend; + subtrahend = minuend; + minuend = temp; + } + + outRO->image = (psImage*)psBinaryOp(outRO->image, minuend->image, "-", subtrahend->image); + +#ifdef TESTING + { + psFits *fits = psFitsOpen("minuend.fits", "w"); + psFitsWriteImage(fits, NULL, minuend->image, 0, NULL); + psFitsClose(fits); + } + { + psFits *fits = psFitsOpen("subtrahend.fits", "w"); + psFitsWriteImage(fits, NULL, subtrahend->image, 0, NULL); + psFitsClose(fits); + } +#endif + + pmReadoutMaskApply(outRO, maskBlank); + } + + psFree(inConv); + psFree(refConv); + +#ifdef TESTING + for (int y = 0; y < outRO->image->numRows; y++) { + for (int x = 0; x < outRO->image->numCols; x++) { + if (isnan(outRO->image->data.F32[y][x]) && !(outRO->mask->data.U8[y][x] & maskBlank)) { + printf("Unmasked NAN at %d %d --> %d\n", x, y, outRO->mask->data.U8[y][x]); + } + } + } +#endif + + if (!pmFPACopyConcepts(outCell->parent->parent, inRO->parent->parent->parent)) { + psError(PS_ERR_UNKNOWN, false, "Unable to copy concepts from input to output."); + psFree(outRO); + return false; + } + + if (renorm) { + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator + if (!pmReadoutWeightRenorm(outRO, maskBlank, PS_STAT_ROBUST_MEDIAN, PS_STAT_ROBUST_STDEV, + renormWidth, rng)) { + psError(PS_ERR_UNKNOWN, false, "Unable to renormalise weights for photometry."); + psFree(outRO); + return false; + } + psFree(rng); + } + + psTraceSetLevel("psphot", 6); + + // Photometry stage 2: find and measure sources on the subtracted image + if (psMetadataLookupBool(NULL, config->arguments, "PHOTOMETRY")) { + // The PSF should already be stored for the readout + pmFPAfile *photFile = psMetadataLookupPtr(NULL, config->files, "PSPHOT.INPUT"); + pmFPACopy(photFile->fpa, outRO->parent->parent->parent); + + pmReadout *psfRO = pmFPAfileThisReadout(config->files, view, "PSPHOT.PSF.LOAD"); + if (!psfRO) { + psError(PS_ERR_UNEXPECTED_NULL, true, "Unable to find file PSPHOT.PSF.LOAD"); + return false; + } + psMetadataAddPtr(psfRO->parent->parent->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN, + "PSF from matched addition", psf); + psFree(psf); + + // Need to ensure aperture residual is not calculated + psMetadata *recipe = psMetadataLookupMetadata(NULL, config->recipes, PSPHOT_RECIPE); // Recipe + psMetadataItem *item = psMetadataLookup(recipe, "MEASURE.APTREND"); // Item determining aptrend + if (!item) { + psWarning("Unable to find MEASURE.APTREND in psphot recipe"); + psErrorClear(); + } else { + item->data.B = false; + } + + if (!psphotReadout(config, view)) { + psWarning("Unable to perform photometry on subtracted image."); + psErrorStackPrint(stderr, "Error stack from photometry:"); + psErrorClear(); + } + + pmFPAfileActivate(config->files, false, "PSPHOT.INPUT"); + pmFPAfileActivate(config->files, false, "PSPHOT.LOAD.PSF"); + + if (stats) { + pmReadout *photRO = pmFPAviewThisReadout(view, photFile->fpa); // Readout with the sources + psArray *sources = psMetadataLookupPtr(NULL, photRO->analysis, "PSPHOT.SOURCES"); // Sources + psMetadataAddS32(stats, PS_LIST_TAIL, "NUM_SOURCES", 0, "Number of sources detected", sources->n); + } + } + + // Copy astrometry over + pmFPA *refFPA = refRO->parent->parent->parent; // Reference FPA + pmHDU *refHDU = refFPA->hdu; // Reference HDU + if (!outHDU || !refHDU) { + psWarning("Unable to find HDU at FPA level to copy astrometry."); + } else { + if (!pmAstromReadWCS(outFPA, outCell->parent, refHDU->header, 1.0)) { + psWarning("Unable to read WCS astrometry from reference FPA."); + psErrorClear(); + } else if (!pmAstromWriteWCS(outHDU->header, outFPA, outCell->parent, WCS_TOLERANCE)) { + psWarning("Unable to write WCS astrometry to output FPA."); + psErrorClear(); + } + } + + // Add additional data to the header + pmFPAfile *refFile = psMetadataLookupPtr(NULL, config->files, "PPSUB.REF"); // Reference file + pmFPAfile *inFile = psMetadataLookupPtr(NULL, config->files, "PPSUB.INPUT"); // Input file + psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.REFERENCE", 0, + "Subtraction reference", refFile->filename); + psMetadataAddStr(outHDU->header, PS_LIST_TAIL, "PPSUB.INPUT", 0, + "Subtraction input", inFile->filename); + + psFree(outRO); + + return true; +} Index: /branches/pap_branch_080617/ppSub/src/ppSubVersion.c =================================================================== --- /branches/pap_branch_080617/ppSub/src/ppSubVersion.c (revision 18170) +++ /branches/pap_branch_080617/ppSub/src/ppSubVersion.c (revision 18170) @@ -0,0 +1,60 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "ppSub.h" + +static const char *cvsTag = "$Name: not supported by cvs2svn $";// CVS tag name + +psString ppSubVersion(void) +{ + psString version = NULL; // Version, to return + psStringAppend(&version, "%s-%s",PACKAGE_NAME,PACKAGE_VERSION); + return version; +} + +psString ppSubVersionLong(void) +{ + psString version = ppSubVersion(); // Version, to return + psString tag = psStringStripCVS(cvsTag, "Name"); // CVS tag + psStringAppend(&version, " (cvs tag %s) %s, %s", tag, __DATE__, __TIME__); + psFree(tag); + return version; +} + + +void ppSubVersionMetadata(psMetadata *metadata) +{ + PS_ASSERT_METADATA_NON_NULL(metadata,); + + psString pslib = psLibVersionLong();// psLib version + psString psmodules = psModulesVersionLong(); // psModules version + psString ppStats = ppStatsVersionLong(); // ppStats version + psString ppSub = ppSubVersionLong(); // ppSub version + + psTime *time = psTimeGetNow(PS_TIME_TAI); // The time now + psString timeString = psTimeToISO(time); // The time in an ISO string + psFree(time); + psString head = NULL; // Head string + psStringAppend(&head, "ppSub processing at %s. Component information:", timeString); + psFree(timeString); + + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, head, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, pslib, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, psmodules, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppStats, ""); + psMetadataAddStr(metadata, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, ppSub, ""); + + psFree(head); + psFree(pslib); + psFree(psmodules); + psFree(ppStats); + psFree(ppSub); + + return; +} Index: /branches/pap_branch_080617/ppSub/test/.cvsignore =================================================================== --- /branches/pap_branch_080617/ppSub/test/.cvsignore (revision 18170) +++ /branches/pap_branch_080617/ppSub/test/.cvsignore (revision 18170) @@ -0,0 +1,1 @@ +* Index: /branches/pap_branch_080617/psModules/src/imcombine/.cvsignore =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/.cvsignore (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/.cvsignore (revision 18170) @@ -0,0 +1,6 @@ +.deps +.libs +Makefile +Makefile.in +*.la +*.lo Index: /branches/pap_branch_080617/psModules/src/imcombine/Makefile.am =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/Makefile.am (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/Makefile.am (revision 18170) @@ -0,0 +1,35 @@ +noinst_LTLIBRARIES = libpsmodulesimcombine.la + +libpsmodulesimcombine_la_CPPFLAGS = $(SRCINC) $(PSMODULES_CFLAGS) +libpsmodulesimcombine_la_LDFLAGS = -release $(PACKAGE_VERSION) + +libpsmodulesimcombine_la_SOURCES = \ + pmReadoutCombine.c \ + pmStack.c \ + pmStackReject.c \ + pmSubtraction.c \ + pmSubtractionEquation.c \ + pmSubtractionIO.c \ + pmSubtractionKernels.c \ + pmSubtractionMask.c \ + pmSubtractionMatch.c \ + pmSubtractionParams.c \ + pmSubtractionStamps.c \ + pmPSFEnvelope.c + +pkginclude_HEADERS = \ + pmImageCombine.h \ + pmReadoutCombine.h \ + pmStack.h \ + pmStackReject.h \ + pmSubtraction.h \ + pmSubtractionEquation.h \ + pmSubtractionIO.h \ + pmSubtractionKernels.h \ + pmSubtractionMask.h \ + pmSubtractionMatch.h \ + pmSubtractionParams.h \ + pmSubtractionStamps.h \ + pmPSFEnvelope.h + +CLEANFILES = *~ Index: /branches/pap_branch_080617/psModules/src/imcombine/demo_psftool.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/demo_psftool.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/demo_psftool.c (revision 18170) @@ -0,0 +1,194 @@ +// This is a very quick and dirty test program for pmPSFEnvelope +// +// gcc -g demo_psftool.c -o demo_psftool `psmodules-config --cflags --libs` --std=gnu99 -Wall +// +// ./demo_psftool test MOPS.skycell.0198767.wrp*.psf +// + +#include +#include +#include + +// Add a single filename to the arguments as an array, so that it can be used with pmFPAfileBindFromArgs, etc +static void fileList(const char *file, // The symbolic name for the file + const char *name, // The name of the file + const char *comment, // Description of the file + pmConfig *config // Configuration + ) +{ + psArray *files = psArrayAlloc(1); // Array with file names + files->data[0] = psStringCopy(name); + psMetadataAddArray(config->arguments, PS_LIST_TAIL, file, 0, comment, files); + psFree(files); + return; +} + + + +int main(int argc, char *argv[]) +{ + psLibInit(NULL); + pmConfig *config = pmConfigRead(&argc, argv, "PSF"); + if (!config) { + psErrorStackPrint(stderr, "Error reading configuration."); + exit(PS_EXIT_CONFIG_ERROR); + } + + psTraceSetLevel("psModules.imcombine", 5); + psTraceSetLevel("psModules.objects", 0); + psTraceSetLevel("psLib.math", 0); + + pmModelClassInit(); + + psMetadataAddStr(config->arguments, PS_LIST_TAIL, "OUTPUT", 0, "Name of the output", argv[1]); + + psArray *files = psArrayAlloc(argc - 2); + for (int i = 2; i < argc; i++) { + psString name = NULL; // Name of file list + psStringAppend(&name, "INPUT_%d", i); + + fileList(name, argv[i], "Input PSF", config); + + pmFPAfile *file = pmFPAfileDefineFromArgs(NULL, config, "PSPHOT.PSF.LOAD", name); + psFree(name); + if (!file) { + psErrorStackPrint(stderr, "Can't define PSF file from %s --> %s", name, argv[i]); + psFree(files); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + } + + files->data[i - 2] = psMemIncrRefCounter(file); + } + + pmFPAfile *outFile = pmFPAfileDefineOutput(config, NULL, "PSPHOT.PSF.SAVE"); + if (!outFile) { + psErrorStackPrint(stderr, "Can't define output PSF file"); + psFree(files); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + } + outFile->save = true; + + // XXX This is a bit dodgy; should be more rigorous for a real system + { + pmFPAview *phuView = pmFPAviewAlloc(0); + phuView->chip = 0; + if (!pmFPAAddSourceFromView(outFile->fpa, "Envelope PSF", phuView, outFile->format)) { + psError(PS_ERR_UNKNOWN, false, "Unable to add PHU to output."); + psFree(phuView); + return false; + } + psFree(phuView); + } + + pmFPAview *view = pmFPAviewAlloc(0); + + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psErrorStackPrint(stderr, "Problem in I/O"); + psFree(view); + psFree(files); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + } + + pmChip *chip; + while ((chip = pmFPAviewNextChip(view, outFile->fpa, 1)) != NULL) { + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psErrorStackPrint(stderr, "Problem in I/O"); + psFree(view); + psFree(files); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + } + +#if 0 + pmFPAfileActivate(config->files, "PSPHOT.PSF.SAVE", false); + pmCell *cell; + while ((cell = pmFPAviewNextCell(view, outFile->fpa, 1)) != NULL) { + if (!pmFPAfileIOChecks(config, view, PM_FPA_BEFORE)) { + psErrorStackPrint(stderr, "Problem in I/O"); + psFree(view); + psFree(files); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + } + } +#endif + + psArray *inputs = psArrayAlloc(files->n); + int numCols = 4501, numRows = 4751; + for (int i = 0; i < files->n; i++) { + pmFPAfile *file = files->data[i]; + pmChip *chip = pmFPAviewThisChip(view, file->fpa); + + pmPSF *psf = psMetadataLookupPtr(NULL, chip->analysis, "PSPHOT.PSF"); + if (!psf) { + psErrorStackPrint(stderr, "Can't find PSF in file %d", i); + psFree(inputs); + psFree(files); + psFree(config); + exit(PS_EXIT_PROG_ERROR); + } + +#if 0 + pmHDU *hdu = pmHDUGetLowest(file->fpa, chip, NULL); + int imaxis1 = psMetadataLookupS32(NULL, hdu->header, "IMAXIS1"); + int imaxis2 = psMetadataLookupS32(NULL, hdu->header, "IMAXIS2"); + if (imaxis1 == 0 || imaxis2 == 0) { + psErrorStackPrint(stderr, "Size of image %d can't be determined.", i); + psFree(inputs); + psFree(files); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + } + if (numCols == 0 && numRows == 0) { + numCols = imaxis1; + numRows = imaxis2; + } else if (imaxis1 != numCols || imaxis2 != numRows) { + psErrorStackPrint(stderr, "Image %d differs in size: %dx%d vs %dx%d", + i, imaxis1, imaxis2, numCols, numRows); + psFree(inputs); + psFree(files); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + } +#endif + + inputs->data[i] = psMemIncrRefCounter(psf); + } + + pmPSF *psf = pmPSFEnvelope(numCols, numRows, inputs, 5, 20, "PS_MODEL_RGAUSS", 3, 3); + psFree(inputs); + if (!psf) { + psErrorStackPrint(stderr, "Can't generate envelope PSF."); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + } + + psMetadataAddPtr(chip->analysis, PS_LIST_TAIL, "PSPHOT.PSF", PS_DATA_UNKNOWN, "Envelope PSF", psf); + psFree(psf); + chip->data_exists = true; + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psErrorStackPrint(stderr, "Problem in I/O"); + psFree(view); + psFree(files); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + return false; + } + } + + if (!pmFPAfileIOChecks(config, view, PM_FPA_AFTER)) { + psErrorStackPrint(stderr, "Problem in I/O"); + psFree(view); + psFree(files); + psFree(config); + exit(PS_EXIT_SYS_ERROR); + } + + psFree(config); + + exit(PS_EXIT_SUCCESS); +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmImageCombine.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmImageCombine.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmImageCombine.c (revision 18170) @@ -0,0 +1,683 @@ +/** @file pmImageCombine.c + * + * This file will perform image combination of several images of the + * same field, produce a list of questionable pixels, then tag some + * of those pixels as cosmic rays. + * + * @author Paul Price, IfA (original prototype) + * @author GLG, MHPCC + * + * @version $Revision: 1.12 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-04-04 22:42:48 $ + * + * XXX: pmRejectPixels() has a known bug with the pmImageTransform() call. + * + * Copyright 2004 Maui High Performance Computing Center, University of Hawaii + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include "pslib.h" + +#define PIXEL_LIST_BUFFER 100 // Size of the pixel list buffer + +// Data structure for use as a buffer in combining pixels +typedef struct +{ + psVector *pixels; // Pixel values + psVector *masks; // Pixel masks + psVector *errors; // Pixel errors + psStats *stats; // Statistics to use with combination +} +combineBuffer; + +void combineBufferFree(combineBuffer *buffer) +{ + psFree(buffer->pixels); + psFree(buffer->masks); + psFree(buffer->errors); + psFree(buffer->stats); +} + +combineBuffer *combineBufferAlloc(long numImages // Number of images that will be combined + ) +{ + combineBuffer *buffer = psAlloc(sizeof(combineBuffer)); + psMemSetDeallocator(buffer, (psFreeFunc)combineBufferFree); + + buffer->pixels = psVectorAlloc(numImages, PS_TYPE_F32); + buffer->masks = psVectorAlloc(numImages, PS_TYPE_MASK); + buffer->errors = psVectorAlloc(numImages, PS_TYPE_F32); + buffer->stats = psStatsAlloc(PS_STAT_SAMPLE_MEAN | PS_STAT_SAMPLE_STDEV); + + return buffer; +} + + +static bool combinePixels(psImage *combine, // Combined image, for output + psArray *questionablePixels, // Array of rejection masks + int x, int y, // Position in the images + const psArray *images, // Array of input images + const psArray *errors, // Array of input error images + const psArray *masks, // Array of input masks + psU32 maskVal, // Mask value + psS32 numIter, // Number of rejection iterations + psF32 sigmaClip, // Number of standard deviations at which to reject + combineBuffer *buffer // Buffer for combination; to avoid multiple allocations + ) +{ + assert(combine); + assert(x >= 0 && x < combine->numCols); + assert(y >= 0 && y < combine->numRows); + assert(images); + int numImages = images->n; // Number of images to combine + if (masks) { + assert(masks->n == numImages); + } + if (errors) { + assert(errors->n == numImages); + } + assert(numIter >= 0); + assert(sigmaClip > 0); + assert(!questionablePixels || (questionablePixels && questionablePixels->n == numImages)); + + if (buffer) { + psMemIncrRefCounter(buffer); + } else { + buffer = combineBufferAlloc(numImages); + } + + psVector *pixelData = buffer->pixels; // Values for the pixel of interest + psVector *pixelErrors = buffer->errors; // Errors for the pixel of interest + psVector *pixelMasks = buffer->masks; // Masks for the pixel of interest + psStats *stats = buffer->stats; // Statistics for combination + + // + // Loop through each image, extract the pixel/mask/error data into psVectors. + // + if (!masks) { + psVectorInit(pixelMasks, 0); + } + if (!errors) { + pixelErrors = NULL; + } + for (int i = 0; i < numImages; i++) { + // Set the pixel data + psImage *image = images->data[i]; // Image of interest + pixelData->data.F32[i] = image->data.F32[y][x]; + // Set the pixel mask data, if necessary + if (masks) { + psImage *mask = masks->data[i]; // Mask of interest + pixelMasks->data.U8[i] = mask->data.U8[y][x]; + } // Set the pixel error data, if necessary + if (errors) { + psImage *error = errors->data[i]; // Error image of interest + pixelErrors->data.F32[i] = error->data.F32[y][x]; + } + } + + // + // Iterate on the pixels, rejecting outliers + // + for (int iter = 0; iter < numIter; iter++) { + // Combine all the pixels, using the specified stat. + if (!psVectorStats(stats, pixelData, pixelErrors, pixelMasks, maskVal)) { + combine->data.F32[y][x] = NAN; + psFree(buffer); + return false; + } + float combinedPixel = stats->sampleMean; // Value of the combination + + if (iter == 0) { + // Save the value produced with no rejection, since it may be useful later + // (if the rejection turns out to be unnecessary) + combine->data.F32[y][x] = combinedPixel; + } + + // + // Reject all pixels that lie more that sigmaClip standard deviations from + // the combined pixel value. + // + int numRejects = 0; // Number of rejections + float stdev = stats->sampleStdev; + for (int i = 0; i < numImages; i++) { + if (!(pixelMasks->data.U8[i] & maskVal) && + fabs(pixelData->data.F32[i] - combinedPixel) > sigmaClip * stdev) { + // Reject pixel as questionable + numRejects++; + pixelMasks->data.U8[i] = maskVal; + if (questionablePixels) { + // Mark the pixel as questionable + psPixels *qp = questionablePixels->data[i]; // Questionable pixels for this image + int qpNum = qp->n; // Number of QPs in the image of interest + if (qpNum >= qp->nalloc) { + // Grow dynamically, if required + qp = psPixelsRealloc(qp, qp->nalloc + PIXEL_LIST_BUFFER); + questionablePixels->data[i] = qp; + } + qp->data[qpNum].x = x; + qp->data[qpNum].y = y; + qp->n++; + } + } + } + + // + // If the number of rejected pixels is zero, then there's no point continuing the loop. + // + if (numRejects == 0) { + break; + } + + // + // XXX: Is it possible to have all pixels rejected? If so, we should exit the loop. + // + } + + psFree(buffer); + return true; +} + +psImage *pmCombineImages( + psImage *combine, ///< Combined image (output) + psArray **questionablePixels, ///< Array of rejection masks + const psArray *images, ///< Array of input images + const psArray *errors, ///< Array of input error images + const psArray *masks, ///< Array of input masks + psU32 maskVal, ///< Mask value + const psPixels *pixels, ///< Pixels to combine + psS32 numIter, ///< Number of rejection iterations + psF32 sigmaClip ///< Number of standard deviations at which to reject +) +{ + psTrace("psModules.imcombine", 3, "Calling pmCombineImages(%ld)\n", images->n); + + PS_ASSERT_ARRAY_NON_NULL(images, NULL); + PS_ASSERT_INT_POSITIVE(images->n, NULL); + long numImages = images->n; // Number of images + int numCols = ((psImage*)images->data[0])->numCols; // Size in x + int numRows = ((psImage*)images->data[0])->numRows; // Size in y + + if (combine) { + PS_ASSERT_IMAGE_NON_NULL(combine, NULL); + PS_ASSERT_IMAGE_SIZE(combine, numCols, numRows, NULL); + } + if (questionablePixels && !*questionablePixels) { + PS_ASSERT_ARRAY_NON_NULL(*questionablePixels, NULL); + PS_ASSERT_ARRAY_SIZE(*questionablePixels, numImages, 0); + } + for (int i = 1; i < images->n; i++) { + psImage *image = images->data[i]; // Image of interest + PS_ASSERT_IMAGE_TYPE(image, PS_TYPE_F32, NULL); + PS_ASSERT_IMAGE_SIZE(image, numCols, numRows, NULL); + } + if (errors) { + PS_ASSERT_ARRAYS_SIZE_EQUAL(images, errors, NULL); + for (int i = 0; i < images->n; i++) { + psImage *error = errors->data[i]; + PS_ASSERT_IMAGE_SIZE(error, numCols, numRows, NULL); + PS_ASSERT_IMAGE_TYPE(error, PS_TYPE_F32, NULL); + } + } + if (masks) { + PS_ASSERT_ARRAYS_SIZE_EQUAL(images, masks, NULL); + for (int i = 0; i < images->n; i++) { + psImage *mask = masks->data[i]; + PS_ASSERT_IMAGE_SIZE(mask, numCols, numRows, NULL); + PS_ASSERT_IMAGE_TYPE(mask, PS_TYPE_MASK, NULL); + } + } + PS_ASSERT_INT_POSITIVE(numIter, NULL); + PS_ASSERT_FLOAT_LARGER_THAN(sigmaClip, 0.0, NULL); + + // Allocate and initialize the combined image, if necessary. + if (!combine) { + combine = psImageAlloc(numCols, numRows, PS_TYPE_F32); + if (pixels) { + // Set everything we're not working on to NAN + psImageInit(combine, NAN); + } + } + + // + // Allocate the questionablePixels psArray, if necesssary, then create a psPixels + // struct for each image. + // + if (questionablePixels) { + if (*questionablePixels == NULL) { + *questionablePixels = psArrayAlloc(numImages); + } else if ((*questionablePixels)->n != numImages) { + *questionablePixels = psArrayRealloc(*questionablePixels, numImages); + } + for (int i = 0; i < numImages; i++) { + psFree((*questionablePixels)->data[i]); + (*questionablePixels)->data[i] = psPixelsAlloc(PIXEL_LIST_BUFFER); + } + } + + combineBuffer *buffer = combineBufferAlloc(numImages); // Buffer for combination + + if (pixels) { + // Only those specified pixels should be combined. + + for (int p = 0; p < pixels->n; p++) { + int x = pixels->data[p].x; // Column of interest + int y = pixels->data[p].y; // Row of interest + + if (!combinePixels(combine, questionablePixels ? *questionablePixels : NULL, x, y, + images, errors, masks, maskVal, numIter, sigmaClip, buffer)) { + // Bad pixel --- no big deal + psErrorClear(); + } + } + } else { + // + // We get here if there is a NULL list of pixels to combine. + // Therefore, we combine all pixels in all images. + // + + // + // Loop over all pixels in all images, set the appropriate data, mask, + // error vectors, call psVectorStats(), and set the result in the + // combine image. + // + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if (!combinePixels(combine, questionablePixels ? *questionablePixels : NULL, x, y, + images, errors, masks, maskVal, numIter, sigmaClip, buffer)) { + // Bad pixel --- no big deal + psErrorClear(); + } + } + } + } + + psFree(buffer); + + psTrace("psModules.imcombine", 3, "Exiting pmCombineImages(%ld)\n", images->n); + return combine; +} + + +/****************************************************************************** +XXX: Directly from Paul Price + *****************************************************************************/ +static psF32 CalcGradient( + psImage *image, + psImage *imageMask, + psS32 x, + psS32 y +) +{ + psTrace("psModules.imcombine", 4, "Calling CalcGradient(%d, %d)\n", x, y); + int num = 0; + psVector *pixels = psVectorAlloc(8, PS_TYPE_F32); // Array of pixels + psVector *mask = psVectorAlloc(8, PS_TYPE_U8); // Corresponding mask + + // Get limits + int xMin = PS_MAX(x - 1, 0); + int xMax = PS_MIN(x + 1, image->numCols - 1); + int yMin = PS_MAX(y - 1, 0); + int yMax = PS_MIN(y + 1, image->numRows - 1); + if (imageMask != NULL) { + for (int j = yMin; j <= yMax; j++) { + for (int i = xMin; i <= xMax; i++) { + if ((i != x) && (j != y) && (0 == imageMask->data.U8[j][i])) { + pixels->data.F32[num] = image->data.F32[j][i]; + mask->data.U8[num] = 0; + num++; + } else { + mask->data.U8[num] = 1; + } + } + } + } else { + // + // This code is simply the previous loop without the imageMask. + // XXX: Consider restructuring this. + // + for (int j = yMin; j <= yMax; j++) { + for (int i = xMin; i <= xMax; i++) { + if ((i != x) && (j != y)) { + pixels->data.F32[num] = image->data.F32[j][i]; + mask->data.U8[num] = 0; + num++; + } else { + mask->data.U8[num] = 1; + } + } + } + } + + pixels->n = num; + mask->n = num; + + // Get the median + psStats *stats = psStatsAlloc(PS_STAT_SAMPLE_MEDIAN); + psVectorStats(stats, pixels, NULL, mask, 1); + float median = stats->sampleMedian; + psFree(stats); + psFree(pixels); + psFree(mask); + + psTrace("psModules.imcombine", 4, "Exiting CalcGradient(%d, %d)\n", x, y); + return(median / image->data.F32[y][x]); +} + +/****************************************************************************** +DetermineRegion(image, myOutToIn): for a psImage and a psPlaneTransform to that +image, this routine determines the size of the input image which maps to that +image, and returns the result in a psRegion struct. + +XXX: Basically, this routine is only guaranteed to work if the transform is +linear. + +XXX: Shouldn't this functionality be part of psImageTransform()? + *****************************************************************************/ +static psRegion DetermineRegion(psImage *image, + psPlaneTransform *myOutToIn) +{ + psTrace("psModules.imcombine", 4, "Calling DetermineRegion()\n"); + psRegion myRegion; + myRegion.x0 = PS_MAX_F32; + myRegion.x1 = PS_MIN_F32; + myRegion.y0 = PS_MAX_F32; + myRegion.y1 = PS_MIN_F32; + psPlane in; + psPlane out; + + in.x = 0.0; + in.y = 0.0; + + psPlaneTransformApply(&out, myOutToIn, &in); + if (out.x < myRegion.x0) { + myRegion.x0 = out.x; + } + if (out.x > myRegion.x1) { + myRegion.x1 = out.x; + } + if (out.y < myRegion.y0) { + myRegion.y0 = out.y; + } + if (out.y > myRegion.y1) { + myRegion.y1 = out.y; + } + + in.x = (psF32) (image->numCols); + in.y = 0.0; + psPlaneTransformApply(&out, myOutToIn, &in); + if (out.x < myRegion.x0) { + myRegion.x0 = out.x; + } + if (out.x > myRegion.x1) { + myRegion.x1 = out.x; + } + if (out.y < myRegion.y0) { + myRegion.y0 = out.y; + } + if (out.y > myRegion.y1) { + myRegion.y1 = out.y; + } + + in.x = (psF32) (image->numCols); + ; + in.y = 0.0; + psPlaneTransformApply(&out, myOutToIn, &in); + if (out.x < myRegion.x0) { + myRegion.x0 = out.x; + } + if (out.x > myRegion.x1) { + myRegion.x1 = out.x; + } + if (out.y < myRegion.y0) { + myRegion.y0 = out.y; + } + if (out.y > myRegion.y1) { + myRegion.y1 = out.y; + } + + in.x = (psF32) (image->numCols); + in.y = (psF32) (image->numRows); + psPlaneTransformApply(&out, myOutToIn, &in); + if (out.x < myRegion.x0) { + myRegion.x0 = out.x; + } + if (out.x > myRegion.x1) { + myRegion.x1 = out.x; + } + if (out.y < myRegion.y0) { + myRegion.y0 = out.y; + } + if (out.y > myRegion.y1) { + myRegion.y1 = out.y; + } + + psTrace("psModules.imcombine", 4, "Exiting DetermineRegion()\n"); + return(myRegion); +} + +/****************************************************************************** +XXX: Don't we have a psLib function for this? + *****************************************************************************/ +static psImage *ImageConvertF32(psImage *image) +{ + psTrace("psModules.imcombine", 4, "Calling ImageConvertF32()\n"); + psImage *imgF32 = psImageAlloc(image->numCols, image->numRows, PS_TYPE_F32); + + for (psS32 i = 0 ; i < image->numRows ; i++) { + for (psS32 j = 0 ; j < image->numCols ; j++) { + imgF32->data.F32[i][j] = (psF32) image->data.U8[i][j]; + } + } + + psTrace("psModules.imcombine", 4, "Exiting ImageConvertF32()\n"); + return(imgF32); +} + + +// +// The following macros define how big the initial pixel list will be, and +// how much it should be incremented when realloc'ed. +// +#define PS_REJECT_PIXEL_INITIAL_PIXEL_LIST_LENGTH 100 +#define PS_REJECT_PIXEL_INITIAL_PIXEL_LIST_LENGTH_INC 100 +/****************************************************************************** +pmRejectPixels(images, errors, inToOut, outToIn, rejThreshold, +gradLimit) + +XXX: Optimization: we don't need to transform the entire mask image. +XXX: The inToOut and outToIn transforms are confusing. Verify that what + I think they mean syncs with PWP. + *****************************************************************************/ +psArray *pmRejectPixels( + const psArray *images, ///< Array of input images + const psArray *masks, ///< Array of input image masks + const psArray *errors, ///< The pixels which were rejected in the combination + const psArray *inToOut, ///< Transformation from input to output system + const psArray *outToIn, ///< Transformation from output to input system + psF32 rejThreshold, ///< Rejection threshold + psF32 gradLimit ///< Gradient limit +) +{ + psTrace("psModules.imcombine", 3, "Calling pmRejectPixels()\n"); + PS_ASSERT_PTR_NON_NULL(images, NULL); + for (psS32 im = 0 ; im < images->n ; im++) { + psImage *tmpImage = (psImage *) images->data[im]; + PS_ASSERT_IMAGE_NON_NULL(tmpImage, NULL); + PS_ASSERT_IMAGE_NON_EMPTY(tmpImage, NULL); + PS_ASSERT_IMAGE_TYPE(tmpImage, PS_TYPE_F32, NULL); + if (masks != NULL) { + PS_ASSERT_INT_EQUAL(images->n, masks->n, NULL); + psImage *tmpMask = (psImage *) masks->data[im]; + PS_ASSERT_IMAGE_NON_NULL(tmpMask, NULL); + PS_ASSERT_IMAGE_NON_EMPTY(tmpMask, NULL); + PS_ASSERT_IMAGE_TYPE(tmpMask, PS_TYPE_F32, NULL); + PS_ASSERT_IMAGES_SIZE_EQUAL(tmpImage, tmpMask, NULL); + } + PS_ASSERT_IMAGES_SIZE_EQUAL(((psImage *) images->data[0]), tmpImage, NULL); + } + PS_ASSERT_PTR_NON_NULL(errors, NULL); + PS_ASSERT_PTR_NON_NULL(inToOut, NULL); + PS_ASSERT_PTR_NON_NULL(outToIn, NULL); + // Ensure that the psArray parameters have an element for each image. + psS32 numImages = images->n; + PS_ASSERT_INT_EQUAL(numImages, errors->n, NULL); + PS_ASSERT_INT_EQUAL(numImages, inToOut->n, NULL); + PS_ASSERT_INT_EQUAL(numImages, outToIn->n, NULL); + + // + // Create the psArray of psPixelLists, one for each image, for rejected pixels. + // + psArray *rejects = psArrayAlloc(numImages); + for (psS32 im = 0 ; im < numImages ; im++) { + rejects->data[im] = (psPtr *) psPixelsAlloc(PS_REJECT_PIXEL_INITIAL_PIXEL_LIST_LENGTH); + ((psPixels *)(rejects->data[im]))->n = ((psPixels *)(rejects->data[im]))->nalloc; + psPixels *pixels = (psPixels *) rejects->data[im]; + pixels->n = 0; + } + // + // rPtr is used to maintain a count of the questionable pixels for each image. + // + psVector *rPtr = psVectorAlloc(numImages, PS_TYPE_S32); + psVectorInit(rPtr, 0); + + psS32 numCols = ((psImage *) images->data[0])->numCols; + psS32 numRows = ((psImage *) images->data[0])->numRows; + psRegion myRegion = psRegionSet(0, numCols-1, 0, numRows-1); + psU32 maskVal = 1; // XXX: Is this appropriate? + + psPlane *inCoords = psAlloc(sizeof(psPlane)); + psPlane *outCoords = psAlloc(sizeof(psPlane)); + + for (psS32 im = 0 ; im < numImages ; im++) { + // + // Extract data from psArrays. + // + psPixels *pixelList = (psPixels *) errors->data[im]; + + psImage *currImage = (psImage *) images->data[im]; + myRegion.x0 = 0; + myRegion.x1 = currImage->numCols; + myRegion.y0 = 0; + myRegion.y1 = currImage->numRows; + psPlaneTransform *myInToOut = (psPlaneTransform *) inToOut->data[im]; + psPlaneTransform *myOutToIn = (psPlaneTransform *) outToIn->data[im]; + + // + // Create a psU8 mask image from the list of cosmic pixels. + // + psImage *maskImage = NULL; + maskImage = psPixelsToMask(maskImage, pixelList, myRegion, maskVal); + psImage *maskImageF32 = ImageConvertF32(maskImage); + + // + // Transform that mask image into detector coordinate space + // + psRegion myRegionXForm = DetermineRegion(maskImageF32, myOutToIn); + psImage *transformedImage = psImageTransform(NULL, NULL, maskImageF32, NULL, + 0, myOutToIn, myRegionXForm, NULL, + PS_INTERPOLATE_BILINEAR, 0); + + // + // Loop over all cosmic pixels. Transform their coords to detector space. + // If the value of the transformed mask is larger than rejThreshold, then + // this might be a cosmic ray pixel. We then calculate the mean gradient + // in other images. + // + + psImageInterpolateOptions *interp = psImageInterpolateOptionsAlloc(PS_INTERPOLATE_BILINEAR, + transformedImage, NULL, NULL, + 0, 0.0, 0.0, 0, 0, 0.0); + + for (psS32 p = 0 ; p < pixelList->n ; p++) { + inCoords->x = 0.5 + (psF32) (pixelList->data[p]).x; + inCoords->y = 0.5 + (psF32) (pixelList->data[p]).y; + psPlaneTransformApply(outCoords, myInToOut, inCoords); + double maskVal; + if (!psImageInterpolate(&maskVal, NULL, NULL, outCoords->x, outCoords->y, interp)) { + psError(PS_ERR_UNKNOWN, false, "Unable to interpolate image."); + psFree(interp); + psFree(maskImage); + psFree(maskImageF32); + psFree(transformedImage); + psFree(inCoords); + psFree(outCoords); + psFree(rejects); + return NULL; + } + if (maskVal > rejThreshold) { + + // This is a possible cosmic array pixel. We must calculate the gradient + // at this location in all input images. + psF32 meanGrads = 0.0; + psS32 numGrads = 0; + // + // Loop through all other images, calculate their mean gradient. + // + for (psS32 otherImg = 0 ; otherImg < numImages ; otherImg++) { + if (im != otherImg) { + // Map the outCoords to inCoords that for otherImg space. + psImage *tmpMask = NULL; + if (masks != NULL) { + tmpMask = masks->data[otherImg]; + } + psPlaneTransformApply(inCoords, + (psPlaneTransform * )outToIn->data[otherImg], + outCoords); + psS32 xPix = (int)(inCoords->x + 0.5); + psS32 yPix = (int)(inCoords->y + 0.5); + if ((xPix >= 0) && (xPix <= ((psImage*)(images->data[otherImg]))->numCols - 1) && + (yPix >= 0) && (yPix <= ((psImage*)(images->data[otherImg]))->numRows - 1)) { + meanGrads += CalcGradient(images->data[otherImg], tmpMask, xPix, yPix); + numGrads++; + } + } + } + if (numGrads > 0) { + meanGrads /= (psF32) numGrads; + } else { + // XXX: my idea. Verify with PWP: + meanGrads = 1.0 + gradLimit; + } + + // XXX: The SDRS and the prototype code differ significantly here: + // if (CalcGradient(inputs->data[i], pixelList->data.x, pixelList->data.y) < (gradLimit * meanGrads)) { + if (meanGrads < gradLimit) { + // + // Add this to the list of questionable pixels. We must ensure that the + // pixelList is large enough; if not, we realloc() + // + psS32 ptr = rPtr->data.S32[im]; + psPixels *pixelListPtr = (psPixels *) rejects->data[im]; + if (ptr >= pixelListPtr->nalloc) { + rejects->data[im] = (psPtr *) psPixelsRealloc(((psPixels *) rejects->data[im]), + ((((psPixels *) rejects->data[im])->nalloc) + PS_REJECT_PIXEL_INITIAL_PIXEL_LIST_LENGTH_INC)); + // XXX: Can the realloc() fail? Must we check for NULL? + } + + ((psPixels *) rejects->data[im])->data[ptr].x = (pixelList->data[p]).x; + ((psPixels *) rejects->data[im])->data[ptr].y = (pixelList->data[p]).y; + (rPtr->data.S32[im])++; + // XXX: this pixel ->n increment is wierd + (((psPixels *) rejects->data[im])->n)++; + } + } + } + + psFree(interp); + psFree(maskImage); + psFree(maskImageF32); + psFree(transformedImage); + } + + psFree(inCoords); + psFree(outCoords); + psTrace("psModules.imcombine", 3, "Exiting pmRejectPixels()\n"); + return(rejects); +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmImageCombine.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmImageCombine.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmImageCombine.h (revision 18170) @@ -0,0 +1,44 @@ +/* @file pmImageCombine.h + * + * This file will perform image combination of several images of the + * same field, produce a list of questionable pixels, then tag some + * of those pixels as cosmic rays. + * + * @author Paul Price, IfA (original prototype) + * @author GLG, MHPCC + * + * @version $Revision: 1.6 $ $Name: not supported by cvs2svn $ + * @date $Date: 2007-03-30 21:12:56 $ + * Copyright 2004 Maui High Performance Computing Center, University of Hawaii + */ + +#ifndef PM_IMAGE_COMBINE_H +#define PM_IMAGE_COMBINE_H + +/// @addtogroup imcombine Image Combinations +/// @{ + +psImage *pmCombineImages( + psImage *combine, ///< Combined image (output) + psArray **questionablePixels, ///< Array of rejection masks + const psArray *images, ///< Array of input images + const psArray *errors, ///< Array of input error images + const psArray *masks, ///< Array of input masks + psU32 maskVal, ///< Mask value + const psPixels *pixels, ///< Pixels to combine + psS32 numIter, ///< Number of rejection iterations + psF32 sigmaClip ///< Number of standard deviations at which to reject +); + +psArray *pmRejectPixels( + const psArray *images, ///< Array of input images + const psArray *masks, ///< Array of input image masks + const psArray *errors, ///< The pixels which were rejected in the combination + const psArray *inToOut, ///< Transformation from input to output system + const psArray *outToIn, ///< Transformation from output to input system + psF32 rejThreshold, ///< Rejection threshold + psF32 gradLimit ///< Gradient limit +); + +/// @} +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmPSFEnvelope.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmPSFEnvelope.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmPSFEnvelope.c (revision 18170) @@ -0,0 +1,307 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + + +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmReadoutFake.h" + +#include "pmMoments.h" +#include "pmResiduals.h" +#include "pmGrowthCurve.h" +#include "pmTrend2D.h" +#include "pmPSF.h" +#include "pmModel.h" +#include "pmModelClass.h" +#include "pmModelUtils.h" +#include "pmPeaks.h" +#include "pmSource.h" +#include "pmSourceUtils.h" +#include "pmSourceFitModel.h" +#include "pmPSFtry.h" + + +#include "pmPSFEnvelope.h" + + + +//#define TESTING // Enable test output +#define PEAK_FLUX 1.0e4 // Peak flux for each source +#define SKY_VALUE 0.0e0 // Sky value for fake image +#define WEIGHT_VAL 1.0 // Weighting for image +#define PSF_STATS PS_STAT_SAMPLE_MEDIAN | PS_STAT_SAMPLE_STDEV // Statistics options for measuring PSF + + + +// XXX To do: +// +// * PSF variation when only a portion of the image is present (e.g., the edge of an FPA overlapping a +// skycell) may mean a disastrously weird PSF in the missing regions. To counter this, get a region of +// validity for each PSF (perhaps from an associated mask, or have the user work it out), and taper the PSF +// when outside this region (perhaps multiply the peak flux by a Gaussian whose arguments are the distance +// from the valid region, and a width that the user supplies). + + +// We deliberately do not include the calculation of and storing of residuals (data - model) for the PSF +// model, because (1) there is no code in psModules to do this, and we're not going to implement it here; and +// (2) this is intended to generate "nice" or "ideal" PSFs to feed into pmSubtraction (PSF matching code), so +// any residuals will hopefully be dealt with by that. + + +pmPSF *pmPSFEnvelope(int numCols, int numRows, // Size of original image + const psArray *inputs, // Input PSF models + int instances, // Number of instances per dimension + int radius, // Radius of each PSF + const char *modelName,// Name of PSF model to use + int xOrder, int yOrder // Order for PSF variation fit + ) +{ + PS_ASSERT_INT_POSITIVE(numCols, NULL); + PS_ASSERT_INT_POSITIVE(numRows, NULL); + PS_ASSERT_ARRAY_NON_NULL(inputs, NULL); + PS_ASSERT_INT_POSITIVE(instances, NULL); + PS_ASSERT_INT_POSITIVE(radius, NULL); + PS_ASSERT_STRING_NON_EMPTY(modelName, NULL); + PS_ASSERT_INT_NONNEGATIVE(xOrder, NULL); + PS_ASSERT_INT_NONNEGATIVE(yOrder, NULL); + + float xOrigSpacing = (float)(numCols - 2 * radius) / (float)(instances - 1); // Spacing between instances + float yOrigSpacing = (float)(numRows - 2 * radius) / (float)(instances - 1); // Spacing between instances + int fakeSpacing = 2 * radius + 1; // Spacing between instances (x and y) in the fake image + int fakeSize = instances * fakeSpacing; // Size of fake image + + // Generate list of fake sources (instances of the PSF) + int numFakes = PS_SQR(instances); // Number of fake sources + psArray *fakes = psArrayAlloc(numFakes); // Fake sources + psVector *xOffset = psVectorAlloc(numFakes, PS_TYPE_S32); // X offset from fake position to image + psVector *yOffset = psVectorAlloc(numFakes, PS_TYPE_S32); // Y offset from fake position to image + for (int j = 0, index = 0; j < instances; j++) { + float yOrig = j * yOrigSpacing + radius; // Source position in original image + float yFake = j * fakeSpacing + radius; // Position in fake image + int dy = yFake - yOrig; // Difference between fake and original position + + for (int i = 0; i < instances; i++, index++) { + float xOrig = i * xOrigSpacing + radius; // Source position in original image + float xFake = i * fakeSpacing + radius; // Position in fake image + int dx = xFake - xOrig; // Difference between fake and original position + + pmSource *fake = pmSourceAlloc(); // Fake source + fake->peak = pmPeakAlloc(xFake - dx, yFake - dy, PEAK_FLUX, PM_PEAK_LONE); + fake->type = PM_SOURCE_TYPE_STAR; + fake->psfMag = -2.5 * log10(PEAK_FLUX); + + psTrace("psModules.imcombine", 5, "Source %d: %.2f,%.2f\n", + index, xOrig, yOrig); + + fakes->data[index] = fake; + xOffset->data.S32[index] = dx; + yOffset->data.S32[index] = dy; + } + } + + // Generate fake images with each PSF, and take the envelope + psImage *envelope = psImageAlloc(fakeSize, fakeSize, PS_TYPE_F32); // Image with envelope of PSFs + psImageInit(envelope, SKY_VALUE); + pmReadout *fakeRO = pmReadoutAlloc(NULL); // Fake readout + for (int i = 0; i < inputs->n; i++) { + pmPSF *psf = inputs->data[i]; // PSF of interest + pmResiduals *resid = psf->residuals;// PSF residuals + psf->residuals = NULL; + if (!pmReadoutFakeFromSources(fakeRO, fakeSize, fakeSize, fakes, xOffset, yOffset, psf, + NAN, radius, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate fake readout."); + psFree(envelope); + psFree(yOffset); + psFree(xOffset); + psFree(fakes); + psf->residuals = resid; + return NULL; + } + psf->residuals = resid; + + // Need to renormalise sources so they all have the same peak. You would think they do have the same + // peak already, but it seems that the residual map messes things up by adding extra flux + for (int j = 0; j < numFakes; j++) { + pmSource *source = fakes->data[j]; // Fake source + float x = source->peak->xf + xOffset->data.S32[j]; // x coordinate of source + float y = source->peak->yf + yOffset->data.S32[j]; // y coordinate of source + + double flux = fakeRO->image->data.F32[(int)y][(int)x]; + float norm = PEAK_FLUX / flux; // Normalisation for source + psRegion region = psRegionSet(x - radius, x + radius, y - radius, y + radius); // PSF region + psImage *subImage = psImageSubset(fakeRO->image, region); // Subimage of fake PSF + psImage *subEnv = psImageSubset(envelope, region); // Subimage of envelope + psBinaryOp(subImage, subImage, "*", psScalarAlloc(norm, PS_TYPE_F32)); + psBinaryOp(subEnv, subEnv, "MAX", subImage); + psFree(subImage); + psFree(subEnv); + } + +#ifdef TESTING + { + // Write out the PSF field + psString name = NULL; + psStringAppend(&name, "psf_field_%03d.fits", i); + psFits *fits = psFitsOpen(name, "w"); + psFitsWriteImage(fits, NULL, fakeRO->image, 0, NULL); + psFitsClose(fits); + psFree(name); + } +#endif + + } + psFree(fakeRO); + +#ifdef TESTING + { + // Write out the envelope + psFits *fits = psFitsOpen("psf_field_envelope.fits", "w"); + psFitsWriteImage(fits, NULL, envelope, 0, NULL); + psFitsClose(fits); + } +#endif + + // Put the fake sources onto a full-size image + pmReadout *readout = pmReadoutAlloc(NULL); // Readout to contain envelope pixels + readout->image = psImageAlloc(numCols, numRows, PS_TYPE_F32); + psImageInit(readout->image, 0.0); + for (int i = 0; i < numFakes; i++) { + pmSource *source = fakes->data[i]; // Fake source + // Position of source on fake image + int xFake = source->peak->x + xOffset->data.S32[i]; + int yFake = source->peak->y + yOffset->data.S32[i]; + psRegion region = psRegionSet(xFake - radius, xFake + radius, + yFake - radius, yFake + radius); // PSF region + psImage *subImage = psImageSubset(envelope, region); // Subimage of fake PSF + + // Position of source on "real" image + int x0 = source->peak->x - radius; + int y0 = source->peak->y - radius; + + if (!psImageOverlaySection(readout->image, subImage, x0, y0, "=")) { + psError(PS_ERR_UNKNOWN, false, "Unable to overlay PSF"); + psFree(subImage); + psFree(readout); + psFree(xOffset); + psFree(yOffset); + psFree(fakes); + return NULL; + } + psFree(subImage); + } + psFree(xOffset); + psFree(yOffset); + psFree(envelope); + + // XXX This seems the best way to set the weight image so that pixels aren't rejected as "insignificant" + readout->weight = (psImage*)psBinaryOp(NULL, readout->image, "*", readout->image); + readout->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + psImageInit(readout->mask, 0); + +#ifdef TESTING + { + // Write out the envelope + psFits *fits = psFitsOpen("psf_field_full.fits", "w"); + psFitsWriteImage(fits, NULL, readout->image, 0, NULL); + psFitsClose(fits); + } +#endif + + // Reset the sources to point to the new pixels, and measure the moments in preparation for PSF fitting + for (int i = 0; i < numFakes; i++) { + pmSource *source = fakes->data[i]; // Fake source + float x = source->peak->xf; // x coordinates of source + float y = source->peak->yf; // y coordinates of source + + psFree(source->pixels); + psFree(source->weight); + psFree(source->maskView); + psFree(source->maskObj); + source->pixels = NULL; + source->weight = NULL; + source->maskView = NULL; + source->maskObj = NULL; + + if (!pmSourceDefinePixels(source, readout, x, y, (float)radius)) { + psError(PS_ERR_UNKNOWN, false, "Unable to define pixels for source."); + psFree(readout); + psFree(fakes); + return NULL; + } + + if (!pmSourceMoments(source, radius)) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure moments for source."); + psFree(readout); + psFree(fakes); + return NULL; + } + } + + // Don't assume Poisson errors + pmPSFOptions *options = pmPSFOptionsAlloc(); // Options for fitting a PSF + options->poissonErrorsPhotLMM = true; + options->poissonErrorsPhotLin = false; + options->poissonErrorsParams = false; + options->stats = psStatsAlloc(PSF_STATS); + options->radius = radius; + options->psfTrendMode = PM_TREND_MAP; + options->psfTrendNx = xOrder; + options->psfTrendNy = yOrder; + options->psfFieldNx = numCols; + options->psfFieldNy = numRows; + options->psfFieldXo = 0; + options->psfFieldYo = 0; + + pmSourceFitModelInit (15, 0.01, WEIGHT_VAL, options->poissonErrorsPhotLMM); + + pmPSFtry *try = pmPSFtryModel(fakes, modelName, options, 0, 0xff); + psFree(options); + if (!try) { + psError(PS_ERR_UNKNOWN, false, "Unable to fit PSF model to PSF envelope."); + psFree(readout); + psFree(fakes); + return NULL; + } + + pmPSF *psf = psMemIncrRefCounter(try->psf); // Output PSF + psFree(try); + +#ifdef TESTING + { + // Need to translate peak flux --> integrated flux + pmModel *fakeModel = pmModelFromPSFforXY(psf, (float)numCols / 2.0, (float)numRows / 2.0, + 1.0); // Fake model, with central intensity of 1.0 + float flux0 = fakeModel->modelFlux(fakeModel->params); // Flux for central intensity of 1.0 + for (int i = 0; i < numFakes; i++) { + pmSource *source = fakes->data[i]; // Fake source + source->psfMag -= 2.5 * log10(flux0); + } + + pmReadout *generated = pmReadoutAlloc(NULL); // Generated image + pmReadoutFakeFromSources(generated, numCols, numRows, fakes, NULL, NULL, psf, NAN, radius, + false); + { + psFits *fits = psFitsOpen("psf_field_model.fits", "w"); + psFitsWriteImage(fits, NULL, generated->image, 0, NULL); + psFitsClose(fits); + } + psBinaryOp(generated->image, generated->image, "-", readout->image); + { + psFits *fits = psFitsOpen("psf_field_resid.fits", "w"); + psFitsWriteImage(fits, NULL, generated->image, 0, NULL); + psFitsClose(fits); + } + psFree(generated); + } +#endif + + psFree(fakes); + psFree(readout); + + return psf; +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmPSFEnvelope.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmPSFEnvelope.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmPSFEnvelope.h (revision 18170) @@ -0,0 +1,22 @@ +#ifndef PM_PSF_ENVELOPE_H +#define PM_PSF_ENVELOPE_H + +#include +#include +#include +#include +#include +#include + +/// Generate a PSF which is the envelope of an array of PSFs +/// +/// Generates multiple instances of the PSFs (distributed over an image) +pmPSF *pmPSFEnvelope(int numCols, int numRows, // Size of original image + const psArray *inputs, // Input PSF models + int instances, // Number of instances per dimension + int radius, // Radius of each PSF + const char *modelName, // Name of PSF model to use + int xOrder, int yOrder // Order for PSF variation + ); + +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmReadoutCombine.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmReadoutCombine.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmReadoutCombine.c (revision 18170) @@ -0,0 +1,375 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmHDUUtils.h" +#include "pmFPAMaskWeight.h" +#include "pmConceptsAverage.h" +#include "pmReadoutStack.h" + +#include "pmReadoutCombine.h" + +//#define SHOW_BUSY 1 // Show that the function is busy + + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Allocator for pmCombineParams +pmCombineParams *pmCombineParamsAlloc(psStatsOptions combine) +{ + pmCombineParams *params = psAlloc(sizeof(pmCombineParams)); + + params->combine = combine; + params->maskVal = 0; + params->blank = 0; + params->nKeep = 0; + params->fracHigh = 0.0; + params->fracHigh = 0.0; + params->iter = 1; + params->rej = INFINITY; + params->weights = false; + + return params; +} + + +// XXX: Maybe add support for S16 and S32 types. Currently, only F32 supported. +bool pmReadoutCombine(pmReadout *output, const psArray *inputs, const psVector *zero, const psVector *scale, + const pmCombineParams *params) +{ + // Check inputs + PS_ASSERT_PTR_NON_NULL(output, false); + PS_ASSERT_ARRAY_NON_NULL(inputs, false); + PS_ASSERT_PTR_NON_NULL(params, false); + if (zero) { + PS_ASSERT_VECTOR_TYPE(zero, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(zero, inputs->n, false); + } + if (scale) { + PS_ASSERT_VECTOR_TYPE(scale, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_SIZE(scale, inputs->n, false); + } + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracLow, 0.0, 1.0, false); + PS_ASSERT_FLOAT_WITHIN_RANGE(params->fracHigh, 0.0, 1.0, false); + if (params->combine != PS_STAT_SAMPLE_MEAN && params->combine != PS_STAT_SAMPLE_MEDIAN && + params->combine != PS_STAT_ROBUST_MEDIAN && params->combine != PS_STAT_FITTED_MEAN && + params->combine != PS_STAT_CLIPPED_MEAN) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Combination method is not SAMPLE_MEAN, SAMPLE_MEDIAN, " + "ROBUST_MEDIAN, FITTED_MEAN or CLIPPED_MEAN.\n"); + return false; + } + for (int i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + if (params->weights && !readout->weight) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "Rejection based on weights requested, but no weights supplied for image %d.\n", i); + return false; + } + } + + bool first = !output->image; // First pass through? + + pmHDU *hdu = pmHDUFromReadout(output); // Output HDU + if (!hdu) { + psError(PS_ERR_UNEXPECTED_NULL, false, "Unable to find HDU for readout.\n"); + return false; + } + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combining using statistic: %x", params->combine); + if (!hdu->header) { + hdu->header = psMetadataAlloc(); + } + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psStatsOptions combineStdev = 0; // Statistics option for weights + switch (params->combine) { + case PS_STAT_SAMPLE_MEAN: + case PS_STAT_SAMPLE_MEDIAN: + combineStdev = PS_STAT_SAMPLE_STDEV; + break; + case PS_STAT_ROBUST_MEDIAN: + combineStdev = PS_STAT_ROBUST_STDEV; + break; + case PS_STAT_FITTED_MEAN: + combineStdev = PS_STAT_FITTED_STDEV; + break; + case PS_STAT_CLIPPED_MEAN: + combineStdev = PS_STAT_CLIPPED_STDEV; + break; + default: + psAbort("Should never get here --- checked params->combine before.\n"); + } + + psStats *stats = psStatsAlloc(params->combine | combineStdev); // The statistics to use in the combination + if (params->combine == PS_STAT_CLIPPED_MEAN) { + stats->clipSigma = params->rej; + stats->clipIter = params->iter; + + if (first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Combination clipping: %d iterations, rejection at %f sigma", + params->iter, params->rej); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + } + if (params->weights && first) { + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, + "Using input weights to combine images", ""); + } + + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + inputs)) { + psError(PS_ERR_UNKNOWN, false, "No valid input readouts."); + return false; + } + + pmReadoutUpdateSize(output, minInputCols, minInputRows, xSize, ySize, true, params->weights, + params->blank); + psTrace("psModules.imcombine", 7, "Output minimum: %d,%d\n", output->col0, output->row0); + + psImage *counts = pmReadoutAnalysisImage(output, PM_READOUT_STACK_ANALYSIS_COUNT, xSize, ySize, + PS_TYPE_U16, 0); + if (!counts) { + return false; + } + psImage *sigma = pmReadoutAnalysisImage(output, PM_READOUT_STACK_ANALYSIS_SIGMA, xSize, ySize, + PS_TYPE_F32, NAN); + if (!sigma) { + psFree(counts); + return false; + } + + stats->options |= combineStdev; + + // We loop through each pixel in the output image. We loop through each input readout. We determine if + // that output pixel is contained in the image from that readout. If so, we save it in psVector pixels. + // If not, we set a mask for that element in pixels. Then, we mask off pixels not between fracLow and + // fracHigh. Then we call the vector stats routine on those pixels/mask. Then we set the output pixel + // value to the result of the stats call. + + psVector *pixels = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of pixels + psF32 *pixelsData = pixels->data.F32; // Dereference pixels + + psVector *mask = psVectorAlloc(inputs->n, PS_TYPE_U8); // Mask for stack + psU8 *maskData = mask->data.U8; // Dereference mask + + psVector *weights = NULL; // Stack of weights + psVector *errors = NULL; // Stack of errors (sqrt of variance/weights), for psVectorStats + psF32 *weightsData = NULL; // Dereference weights + if (params->weights) { + weights = psVectorAlloc(inputs->n, PS_TYPE_F32); // Stack of weights + weightsData = weights->data.F32; + } + psVector *index = NULL; // The indices to sort the pixels + + float keepFrac = 1.0 - params->fracLow - params->fracHigh; // Fraction of pixels to keep + if (keepFrac != 1.0 && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Min/max rejection: %f high, %f low, keep %d", + params->fracHigh, params->fracLow, params->nKeep); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + psMaskType maskVal = params->maskVal; // The mask value + if (maskVal && first) { + psString comment = NULL; // Comment to add to header + psStringAppend(&comment, "Mask for combination: %x", maskVal); + psMetadataAddStr(hdu->header, PS_LIST_TAIL, "HISTORY", PS_META_DUPLICATE_OK, comment, ""); + psFree(comment); + } + + #ifndef PS_NO_TRACE + psTrace("psModules.imcombine", 3, "Iterating output: %d --> %d, %d --> %d\n", + minInputCols - output->col0, maxInputCols - output->col0, + minInputRows - output->row0, maxInputRows - output->row0); + if (psTraceGetLevel("psModules.imcombine") >= 3) { + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + psTrace("psModules.imcombine", 3, "Iterating input %d: %d --> %d, %d --> %d\n", r, + minInputCols - readout->col0, maxInputCols - readout->col0, + minInputRows - readout->row0, maxInputRows - readout->row0); + } + } + #endif + + // Dereference output products + psF32 **outputImage = output->image->data.F32; // Output image + psU8 **outputMask = output->mask->data.U8; // Output mask + psF32 **outputWeight = NULL; // Output weight map + if (output->weight) { + outputWeight = output->weight->data.F32; + } + + psVector *invScale = NULL; // Inverse scale; pre-calculated for efficiency + if (scale) { + invScale = (psVector*)psBinaryOp(NULL, psScalarAlloc(1.0, PS_TYPE_F32), "/", (const psPtr)scale); + } + + for (int i = minInputRows; i < maxInputRows; i++) { + int yOut = i - output->row0; // y position on output readout + #ifdef SHOW_BUSY + + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("Processing row %d\r", i); + fflush(stdout); + } + #endif + for (int j = minInputCols; j < maxInputCols; j++) { + int xOut = j - output->col0; // x position on output readout + + int numValid = 0; // Number of valid pixels in the stack + memset(maskData, 0, mask->n * sizeof(psU8)); // Reset the mask + for (int r = 0; r < inputs->n; r++) { + pmReadout *readout = inputs->data[r]; // Input readout + int yIn = i - readout->row0; // y position on input readout + int xIn = j - readout->col0; // x position on input readout + psImage *image = readout->image; // The readout image + + #if 0 // This should have been taken care of already: + // Check bounds + if (xIn < 0 || xIn >= image->numCols || yIn < 0 || yIn >= image->numRows) { + continue; + } + #endif + + pixelsData[r] = image->data.F32[yIn][xIn]; + if (!isfinite(pixelsData[r])) { + maskData[r] = 1; + continue; + } + + // Check mask + psImage *roMask = readout->mask; // The mask image + if (roMask && roMask->data.U8[yIn][xIn] & maskVal) { + maskData[r] = 1; + continue; + } + + if (params->weights) { + weightsData[r] = readout->weight->data.F32[yIn][xIn]; + } + + if (zero) { + pixelsData[r] -= zero->data.F32[r]; + } + if (scale) { + pixelsData[r] *= invScale->data.F32[r]; + if (params->weights) { + weightsData[r] *= invScale->data.F32[r] * invScale->data.F32[r]; + } + } + + numValid++; + } + + if (numValid == 0) { + outputMask[yOut][xOut] = params->blank; + outputImage[yOut][xOut] = NAN; + counts->data.U16[yOut][xOut] = 0; + sigma->data.F32[yOut][xOut] = NAN; + continue; + } + + // Apply fracLow,fracHigh if there are enough pixels + if (numValid * keepFrac >= params->nKeep && keepFrac != 1.0) { + index = psVectorSortIndex(index, pixels); + int numLow = numValid * params->fracLow; // Number of low pixels to clip + int numHigh = numValid * params->fracHigh; // Number of high pixels to clip + // Low pixels + psS32 *indexData = index->data.S32; // Dereference index + for (int k = 0, numMasked = 0; numMasked < numLow && k < index->n; k++) { + // Don't count the ones that are already masked + if (!maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + numValid--; + } + } + // High pixels + for (int k = pixels->n - 1, numMasked = 0; numMasked < numHigh && k >= 0; k--) { + // Don't count the ones that are already masked + if (!maskData[indexData[k]]) { + maskData[indexData[k]] = 1; + numMasked++; + numValid--; + } + } + } + counts->data.U16[yOut][xOut] = numValid; + + // XXXXX this step probably is very expensive : convert errors to variance everywhere? + if (params->weights) { + errors = (psVector*)psUnaryOp(errors, weights, "sqrt"); + } + + // Combination + if (!psVectorStats(stats, pixels, errors, mask, 1)) { + // Can't do much about it, but it's not worth worrying about + psErrorClear(); + outputImage[yOut][xOut] = NAN; + outputMask[yOut][xOut] = params->blank; + if (params->weights) { + outputWeight[yOut][xOut] = NAN; + } + sigma->data.F32[yOut][xOut] = NAN; + } else { + outputImage[yOut][xOut] = psStatsGetValue(stats, params->combine); + outputMask[yOut][xOut] = isfinite(outputImage[yOut][xOut]) ? 0 : params->blank; + if (params->weights) { + float stdev = psStatsGetValue(stats, combineStdev); + outputWeight[yOut][xOut] = PS_SQR(stdev); // Variance + // XXXX this is not the correct formal error. + // also, the weighted mean is not obviously the correct thing here + } + sigma->data.F32[yOut][xOut] = psStatsGetValue(stats, combineStdev); + } + } + } + #ifdef SHOW_BUSY + if (psTraceGetLevel("psModules.imcombine") > 9) { + printf("\n"); + } + #endif + psFree(index); + psFree(pixels); + psFree(mask); + psFree(weights); + psFree(errors); + psFree(stats); + psFree(invScale); + psFree(counts); + psFree(sigma); + + // Update the "concepts" + psList *inputCells = psListAlloc(NULL); // List of cells + for (long i = 0; i < inputs->n; i++) { + pmReadout *readout = inputs->data[i]; // Readout of interest + psListAdd(inputCells, PS_LIST_TAIL, readout->parent); + } + bool success = pmConceptsAverageCells(output->parent, inputCells, NULL, NULL, true); + psFree(inputCells); + + output->data_exists = true; + output->parent->data_exists = true; + output->parent->parent->data_exists = true; + + return success; +} + Index: /branches/pap_branch_080617/psModules/src/imcombine/pmReadoutCombine.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmReadoutCombine.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmReadoutCombine.h (revision 18170) @@ -0,0 +1,47 @@ +/* @file pmReadoutCombine.h + * @brief Combine multiple readouts + * + * @author George Gusciora, MHPCC + * @author Paul Price, IfA + * + * @version $Revision: 1.13 $ $Name: not supported by cvs2svn $ + * @date $Date: 2008-03-29 03:10:17 $ + * Copyright 2004-2006 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_READOUT_COMBINE_H +#define PM_READOUT_COMBINE_H + +/// @addtogroup imcombine Image Combinations +/// @{ + +/// Combination parameters for pmReadoutCombine. +/// +/// These values define how the combination is performed, and should not vary by detector, so that it can be +/// re-used for multiple combinations. +typedef struct { + psStatsOptions combine; ///< Statistic to use when performing the combination + psMaskType maskVal; ///< Mask value + psMaskType blank; ///< Mask value to give blank (i.e., no data) pixels + int nKeep; ///< Mimimum number of pixels to keep + float fracHigh; ///< Fraction of high pixels to immediately throw + float fracLow; ///< Fraction of low pixels to immediately throw + int iter; ///< Number of iterations for clipping (for CLIPPED_MEAN only) + float rej; ///< Rejection threshould for clipping (for CLIPPED_MEAN only) + bool weights; ///< Use the supplied weights (instead of calculated stdev)? +} pmCombineParams; + +// Allocator for pmCombineParams +pmCombineParams *pmCombineParamsAlloc(psStatsOptions statsOptions ///< Statistic to use for combination + ); + +/// Combine multiple readouts, applying zero and scale, with optional minmax clipping +bool pmReadoutCombine(pmReadout *output,///< Output readout; altered and returned + const psArray *inputs, ///< Array of input readouts (F32 image and weight, U8 mask) + const psVector *zero, ///< Zero corrections to subtract from input, or NULL + const psVector *scale, ///< Scale corrections to divide into input, or NULL + const pmCombineParams *params ///< Combination parameters + ); + +/// @} +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmStack.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmStack.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmStack.c (revision 18170) @@ -0,0 +1,683 @@ +/** @file pmStack.c + * + * This file will perform image combination of several images of the + * same field, produce a list of questionable pixels, then tag some + * of those pixels as cosmic rays. + * + * @author Paul Price, IfA + * @author GLG, MHPCC + * + * @version $Revision: 1.31 $ $Name: not supported by cvs2svn $ + * @date $Date: 2008-05-14 22:22:28 $ + * Copyright 2004-2007 Institute for Astronomy, University of Hawaii + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmReadoutStack.h" +#include "pmConceptsAverage.h" + +#include "pmStack.h" + +#define PIXEL_LIST_BUFFER 100 // Number of entries to add to pixel list at a time +#define PIXEL_MAP_BUFFER 2 // Number of entries to add to pixel map at a time + +// Internal values for masks +#define MASK_BAD 0x01 // Mask value for pixels that have formerly been masked as bad +#define MASK_SUSPECT 0x02 // Mask value for pixels that are suspected bad (by pmStackCombine) + +#define NUM_DIRECT_STDEV 5 // For less than this number of values, measure stdev directly + + +// Data structure for use as a buffer when combining pixels +// Use of this structure means we don't have to do an allocation in the combination function for each pixel +typedef struct { + psVector *pixels; // Pixel values + psVector *masks; // Pixel masks + psVector *variances; // Pixel variances + psVector *weights; // Pixel weightings + psVector *sources; // Pixel sources (which image did they come from?) + psVector *sort; // Buffer for sorting (to get a robust estimator of the standard dev) +} combineBuffer; + +static void combineBufferFree(combineBuffer *buffer) +{ + psFree(buffer->pixels); + psFree(buffer->masks); + psFree(buffer->variances); + psFree(buffer->weights); + psFree(buffer->sources); + psFree(buffer->sort); + return; +} + +static combineBuffer *combineBufferAlloc(long numImages // Number of images that will be combined + ) +{ + combineBuffer *buffer = psAlloc(sizeof(combineBuffer)); + psMemSetDeallocator(buffer, (psFreeFunc)combineBufferFree); + + buffer->pixels = psVectorAlloc(numImages, PS_TYPE_F32); + buffer->masks = psVectorAlloc(numImages, PS_TYPE_MASK); + buffer->variances = psVectorAlloc(numImages, PS_TYPE_F32); + buffer->weights = psVectorAlloc(numImages, PS_TYPE_F32); + buffer->sources = psVectorAlloc(numImages, PS_TYPE_U16); + buffer->sort = psVectorAlloc(numImages, PS_TYPE_F32); + return buffer; +} + + +// Deallocator for the stack data +static void stackDataFree(pmStackData *data) +{ + psFree(data->readout); + psFree(data->reject); + psFree(data->inspect); + return; +} + + +// Determine a mean value and variance for the combination +// Not using psVectorStats because it assumes that the "weights" are errors, and weights by 1/error^2 +static bool combinationMeanVariance(float *mean, // Mean value, to return + float *var, // Variance value, to return + const psVector *values, // Values to combine + const psVector *variances, // Pixel variances to combine + const psVector *weights // Weights to apply + ) +{ + assert(mean); + assert(values && weights); + assert(values->n == weights->n); + assert((var && variances) || !var); + assert(!variances || variances->n == values->n); + assert(values->type.type == PS_TYPE_F32); + assert(!values || values->type.type == PS_TYPE_F32); + assert(weights->type.type == PS_TYPE_F32); + + // We're not using the input pixel variances to generate a weighted average for the pixel flux (because + // that introduces systematic biases), so the variance of the output pixel value should simply be: + // simga^2 = sum(weight_i^2 * sigma_i^2) / (sum(weight_i))^2 + // This reduces, when the weights are all identically unity, to: + // variance_combination = sum(variance_i) / N^2 + // and if the variances are all equal: + // variance_combination = variance_individual / N + // which makes sense --- the standard deviation of the combination is reduced by a factor of sqrt(N). + + float sumValueWeight = 0.0; // Sum of the value multiplied by the weight + float sumVarianceWeight = 0.0; // Sum of the pixel variances multiplied by the global weights + float sumWeight = 0.0; // Sum of the image weights + for (int i = 0; i < values->n; i++) { + sumValueWeight += values->data.F32[i] * weights->data.F32[i]; + sumWeight += weights->data.F32[i]; + if (variances) { + sumVarianceWeight += variances->data.F32[i] * PS_SQR(weights->data.F32[i]); + } + } + + if (sumWeight <= 0) { + return false; + } + + *mean = sumValueWeight / sumWeight; + if (var) { + *var = sumVarianceWeight / PS_SQR(sumWeight); + } + return true; +} + +// Return the median and standard deviation for the pixels +// Not using psVectorStats because it has additional allocations which slow things down +static bool combinationMedianStdev(float *median, // Median value, to return + float *stdev, // Standard deviation value, to return + const psVector *values, // Values to combine + const psVector *masks, // Mask to apply + psVector *sortBuffer // Buffer for sorting + ) +{ + assert(values); + assert(!masks || values->n == masks->n); + assert(values->type.type == PS_TYPE_F32); + assert(!masks || masks->type.type == PS_TYPE_MASK); + assert(sortBuffer && sortBuffer->nalloc >= values->n && sortBuffer->type.type == PS_TYPE_F32); + + // Need to filter out clipped values + int num = 0; // Number of valid values + for (int i = 0; i < values->n; i++) { + if (!masks || !masks->data.PS_TYPE_MASK_DATA[i]) { + sortBuffer->data.F32[num++] = values->data.F32[i]; + } + } + sortBuffer->n = num; + if (!psVectorSortInPlace(sortBuffer)) { + return false; + } + + if (num == 3) { + // Attempt to measure standard deviation with only three values (and one of those possibly corrupted) + *median = sortBuffer->data.F32[1]; + if (stdev) { + float diff1 = sortBuffer->data.F32[0] - *median; + float diff2 = sortBuffer->data.F32[2] - *median; + // This factor of sqrt(2) might not be exact, but it's about right + *stdev = M_SQRT2 * PS_MIN(fabsf(diff1), fabsf(diff2)); + } + } else { + *median = num % 2 ? sortBuffer->data.F32[num / 2] : + (sortBuffer->data.F32[num / 2 - 1] + sortBuffer->data.F32[num / 2]) / 2.0; + if (stdev) { + if (num <= NUM_DIRECT_STDEV) { + // If there are not many values, the direct standard deviation is better + double sum = 0.0; + for (int i = 0; i < num; i++) { + sum += PS_SQR(sortBuffer->data.F32[i] - *median); + } + *stdev = sqrt(sum / (double)(num - 1)); + } else { + // Standard deviation from the interquartile range + *stdev = 0.74 * (sortBuffer->data.F32[(int)(0.75 * num)] - + sortBuffer->data.F32[(int)(0.25 * num)]); + } + } + } + + return true; +} + + +// Mark a pixel for inspection +static inline void combineInspect(const psArray *inputs, // Stack data + int x, int y, // Pixel + int source // Source image index + ) +{ + pmStackData *data = inputs->data[source]; // Stack data of interest + if (!data) { + psWarning("Can't find input data for source %d", source); + return; + } + data->inspect = psPixelsAdd(data->inspect, PIXEL_LIST_BUFFER, x, y); + return; +} + +// Given a stack of images, combine with optional rejection. +// Pixels in the stack that are rejected are marked for subsequent inspection +static void combinePixels(psImage *image, // Combined image, for output + psImage *mask, // Combined mask, for output + psImage *variance, // Combined variance map, for output + const psArray *inputs, // Stack data + const psVector *weights, // Global (single value) weights for data, or NULL + const psVector *reject, // Indices of pixels to reject, or NULL + int x, int y, // Coordinates of interest; frame of output image + psMaskType maskVal, // Value to mask + psMaskType bad, // Value to give bad pixels + int numIter, // Number of rejection iterations + float rej, // Number of standard deviations at which to reject + bool useVariance, // Use variance for rejection when combining? + bool safe, // Combine safely? + combineBuffer *buffer // Buffer for combination; to avoid multiple allocations + ) +{ + // Rudimentary error checking + assert(image); + assert(mask); + assert(inputs); + assert(numIter >= 0); + assert(buffer); + assert((useVariance && variance) || !useVariance); + + psVector *pixelData = buffer->pixels; // Values for the pixel of interest + psVector *pixelMasks = buffer->masks; // Masks for the pixel of interest + psVector *pixelVariances = variance ? buffer->variances : NULL; // Variances for the pixel of interest + psVector *pixelWeights = buffer->weights; // Image weights for the pixel of interest + psVector *pixelSources = buffer->sources; // Sources for the pixel of interest + psVector *sort = buffer->sort; // Sort buffer + + // Extract the pixel and mask data + int num = 0; // Number of good images + for (int i = 0, j = 0; i < inputs->n; i++) { + // Check if this pixel has been rejected. Assumes that the rejection pixel list is sorted --- it + // should be because of how pixelMapGenerate works + if (reject && reject->data.U16[j] == i) { + j++; + continue; + } + + pmStackData *data = inputs->data[i]; // Stack data of interest + if (!data) { + continue; + } + + int xIn = x - data->readout->col0, yIn = y - data->readout->row0; // Coordinates on input readout + psImage *mask = data->readout->mask; // Mask of interest + if (mask->data.PS_TYPE_MASK_DATA[yIn][xIn] & maskVal) { + continue; + } + + psImage *image = data->readout->image; // Image of interest + psImage *variance = data->readout->weight; // Variance ("weight") map of interest + pixelData->data.F32[num] = image->data.F32[yIn][xIn]; + if (variance) { + pixelVariances->data.F32[num] = variance->data.F32[yIn][xIn]; + } + pixelWeights->data.F32[num] = data->weight; + pixelSources->data.U16[num] = i; + num++; + } + pixelData->n = num; + if (variance) { + pixelVariances->n = num; + } + pixelWeights->n = num; + pixelSources->n = num; + + + // The sensible thing to do varies according to how many good pixels there are. + // Default option is that the pixel is bad + float imageValue = NAN, varianceValue = NAN; // Value for combined image and variance map + psMaskType maskValue = bad; // Value for combined mask + switch (num) { + case 0: + // Nothing to combine: it's bad + break; + case 1: { + // Accept the single pixel unless we have to be safe + if (!safe) { + imageValue = pixelData->data.F32[0]; + if (variance) { + varianceValue = pixelVariances->data.F32[0]; + } + maskValue = 0; + } + break; + } + case 2: { + // Accept the mean of the pixels only if we're going to reject based on the variance, or we're not + // playing safe + if (useVariance || !safe) { + float mean, var; // Mean and variance from combination + if (combinationMeanVariance(&mean, &var, pixelData, pixelVariances, pixelWeights)) { + imageValue = mean; + varianceValue = var; + maskValue = 0; + } + } + if (useVariance && safe && numIter > 0) { + // Use variance to check that the two are consistent + float diff = pixelData->data.F32[0] - pixelData->data.F32[1]; + float sigma = sqrtf(pixelVariances->data.F32[0] + pixelVariances->data.F32[1]); + if (fabs(diff) > rej * sigma) { + // Not consistent: mark both for inspection + combineInspect(inputs, x, y, pixelSources->data.U16[0]); + combineInspect(inputs, x, y, pixelSources->data.U16[1]); + } + } + break; + } + default: { + // Record the value derived with no clipping, because pixels rejected using the harsh clipping + // applied in the first pass might later be accepted. + float mean, var; // Mean and variance of the combination + if (!combinationMeanVariance(&mean, &var, pixelData, pixelVariances, pixelWeights)) { + break; + } + imageValue = mean; + varianceValue = var; + maskValue = 0; + + // Prepare for clipping iteration + if (numIter > 0) { + pixelMasks->n = num; + psVectorInit(pixelMasks, 0); + if (useVariance) { + // Convert to rejection limits + for (int i = 0; i < num; i++) { + pixelVariances->data.F32[i] = rej * sqrtf(pixelVariances->data.F32[i]); + } + } + } + + // The clipping that follows is solely to identify suspect pixels. + // These suspect pixels will be inspected in more detail by other functions. + int numClipped = INT_MAX; // Number of pixels clipped per iteration + int totalClipped = 0; // Total number of pixels clipped + for (int i = 0; i < numIter && numClipped > 0 && num - totalClipped > 2; i++) { + numClipped = 0; + float median, stdev; // Median and stdev of the combination, for rejection + + if (!combinationMedianStdev(&median, useVariance ? NULL : &stdev, pixelData, pixelMasks, sort)) { + psWarning("Bad median/stdev at %d,%d", x, y); + break; + } + + float limit = NAN; // Rejection limit, when rejecting based on data properties + if (!useVariance) { + limit = rej * stdev; + } + + for (int j = 0; j < num; j++) { + if (pixelMasks->data.PS_TYPE_MASK_DATA[j]) { + continue; + } + float diff = fabsf(pixelData->data.F32[j] - median); // Difference from expected + if (diff > (useVariance ? pixelVariances->data.F32[i] : limit)) { + pixelMasks->data.PS_TYPE_MASK_DATA[j] = 0xff; + combineInspect(inputs, x, y, pixelSources->data.U16[j]); + numClipped++; + totalClipped++; + } + } + } + break; + } + } + + image->data.F32[y][x] = imageValue; + mask->data.PS_TYPE_MASK_DATA[y][x] = maskValue; + if (variance) { + variance->data.F32[y][x] = varianceValue; + } + + return; +} + + +// Ensure the input array of pmStackData is valid, and get some details out of it +static bool validateInputData(bool *haveVariances, // Do we have variance maps in the sky images? + bool *haveRejects, // Do we have lists of rejected pixels? + int *num, // Number of inputs + int *numCols, int *numRows, // Size of (sky) images + psArray *input // Input array of pmStackData to validate + ) +{ + PS_ASSERT_ARRAY_NON_NULL(input, false); + *num = input->n; + + pmStackData *data = NULL; // First image off the rank, used as a template + for (int i = 0; !data && i < input->n; i++) { + data = input->data[i]; + } + PS_ASSERT_PTR_NON_NULL(data, false); + assert(psMemGetDeallocator(data) == (psFreeFunc)stackDataFree); // Ensure it's the right type + *haveVariances = false; + PS_ASSERT_IMAGE_NON_NULL(data->readout->image, false); + PS_ASSERT_IMAGE_TYPE(data->readout->image, PS_TYPE_F32, false); + PS_ASSERT_IMAGE_NON_NULL(data->readout->mask, false); + PS_ASSERT_IMAGE_TYPE(data->readout->mask, PS_TYPE_MASK, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(data->readout->image, data->readout->mask, false); + *numCols = data->readout->image->numCols; + *numRows = data->readout->image->numRows; + if (data->readout->weight) { + *haveVariances = true; + PS_ASSERT_IMAGE_NON_NULL(data->readout->weight, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(data->readout->image, data->readout->weight, false); + PS_ASSERT_IMAGE_TYPE(data->readout->weight, PS_TYPE_F32, false); + } + *haveRejects = (data->reject != NULL); + + // Make sure the rest correspond with the first + for (int i = 1; i < *num; i++) { + pmStackData *data = input->data[i]; // Stack data for this input + if (!data) { + continue; + } + assert(psMemGetDeallocator(data) == (psFreeFunc)stackDataFree); // Ensure it's the right type + if (!data->readout) { + psError(PS_ERR_UNEXPECTED_NULL, true, "The readout is specified in some but not all inputs."); + return false; + } + if ((*haveRejects && !data->reject) || (data->reject && !*haveRejects)) { + psError(PS_ERR_UNEXPECTED_NULL, true, + "The rejected pixels are specified in some but not all inputs."); + return false; + } + PS_ASSERT_IMAGE_NON_NULL(data->readout->image, false); + PS_ASSERT_IMAGE_NON_NULL(data->readout->mask, false); + PS_ASSERT_IMAGE_TYPE(data->readout->image, PS_TYPE_F32, false); + PS_ASSERT_IMAGE_TYPE(data->readout->mask, PS_TYPE_MASK, false); + PS_ASSERT_IMAGE_SIZE(data->readout->image, *numCols, *numRows, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(data->readout->image, data->readout->mask, false); + if (*haveVariances) { + PS_ASSERT_IMAGE_NON_NULL(data->readout->weight, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(data->readout->image, data->readout->weight, false); + PS_ASSERT_IMAGE_TYPE(data->readout->weight, PS_TYPE_F32, false); + } + } + + return true; +} + + +// Generate a "pixel map". +// +// A "pixel map" is an image-like structure containing a vector that contains the indices of images. The idea +// is to provide a reverse lookup for an array of pixel lists, so that the image for which a pixel is flagged +// can be identified easily. +static psArray *pixelMapGenerate(const psArray *input, // Data to stack + int numCols, int numRows // Size of (sky) images + ) +{ + psArray *map = psArrayAlloc(numRows); // The pixel map + for (int y = 0; y < numRows; y++) { + map->data[y] = psArrayAlloc(numCols); + } + + for (int i = 0; i < input->n; i++) { + pmStackData *data = input->data[i]; + if (!data) { + continue; + } + assert(data->reject); + psPixels *pixels = data->reject; // The rejected pixels + for (int j = 0; j < pixels->n; j++) { + int x = pixels->data[j].x, y = pixels->data[j].y; // Coordinates of interest + if (x < 0 || x >= numCols || y < 0 || y >= numRows) { + // psWarning("Bad pixel coordinate: %d,%d --- ignored.", x, y); + continue; + } + psArray *columns = map->data[y]; // The columns for that row + psVector *images = columns->data[x]; // The images for that column + if (!images) { + images = columns->data[x] = psVectorAllocEmpty(PIXEL_MAP_BUFFER, PS_TYPE_U16); + } + int size = images->n; // Element number at which to add + columns->data[x] = psVectorExtend(images, PIXEL_MAP_BUFFER, 1); + images->data.U16[size] = i; + } + } + + return map; +} + +// Query a "pixel map", by returning the list of image indices for a particular pixel. +static psVector *pixelMapQuery(const psArray *map, // Pixel map + int x, int y // Coordinates of interest + ) +{ + assert(y >= 0 && y < map->n); + psArray *colMap = map->data[y]; // Columns for that row + assert(x >= 0 && x < colMap->n); + return colMap->data[x]; +} + + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +/// Constructor +pmStackData *pmStackDataAlloc(pmReadout *readout, float weight) +{ + pmStackData *data = psAlloc(sizeof(pmStackData)); // Stack data, to return + psMemSetDeallocator(data, (psFreeFunc)stackDataFree); + + data->readout = psMemIncrRefCounter(readout); + data->reject = NULL; + data->inspect = NULL; + data->weight = weight; + + return data; +} + +/// Stack input images +bool pmStackCombine(pmReadout *combined, psArray *input, psMaskType maskVal, psMaskType bad, + int kernelSize, int numIter, float rej, bool entire, bool useVariance, bool safe) +{ + PS_ASSERT_PTR_NON_NULL(combined, false); + bool haveVariances; // Do we have the variance maps? + bool haveRejects; // Do we have lists of rejected pixels? + int num; // Number of inputs + int numCols, numRows; // Size of (sky) images + if (!validateInputData(&haveVariances, &haveRejects, &num, &numCols, &numRows, input)) { + return false; + } + PS_ASSERT_INT_NONNEGATIVE(kernelSize, false); + PS_ASSERT_INT_POSITIVE(bad, false); + PS_ASSERT_INT_NONNEGATIVE(numIter, false); + if (isnan(rej)) { + PS_ASSERT_INT_EQUAL(numIter, 0, false); + } else { + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, false); + } + if (haveRejects) { + // This is a subsequent combination, so expect that the image and mask already exist + PS_ASSERT_IMAGE_NON_NULL(combined->image, false); + PS_ASSERT_IMAGE_TYPE(combined->image, PS_TYPE_F32, false); + PS_ASSERT_IMAGE_NON_NULL(combined->mask, false); + PS_ASSERT_IMAGE_TYPE(combined->mask, PS_TYPE_MASK, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(combined->image, combined->mask, false); + } + if (useVariance && !haveVariances) { + psWarning("Unable to use variance in rejection if no variance maps supplied --- option turned off"); + useVariance = false; + } + + psVector *weights = psVectorAlloc(num, PS_TYPE_F32); // Relative weighting for each image + psArray *stack = psArrayAlloc(num); // Stack of readouts + for (int i = 0; i < num; i++) { + pmStackData *data = input->data[i]; // Stack data for this input + if (!data) { + weights->data.F32[i] = 0.0; + continue; + } + weights->data.F32[i] = data->weight; + stack->data[i] = psMemIncrRefCounter(data->readout); + } + + int minInputCols, maxInputCols, minInputRows, maxInputRows; // Smallest and largest values to combine + int xSize, ySize; // Size of the output image + if (!pmReadoutStackValidate(&minInputCols, &maxInputCols, &minInputRows, &maxInputRows, &xSize, &ySize, + stack)) { + psError(PS_ERR_UNKNOWN, false, "Input stack is not valid."); + psFree(stack); + return false; + } + psFree(stack); + pmReadoutUpdateSize(combined, minInputCols, minInputRows, xSize, ySize, true, true, bad); + psTrace("psModules.imcombine", 1, "Have for combination [%d:%d,%d:%d] (%dx%d)\n", + minInputCols, maxInputCols, minInputRows, maxInputRows, xSize, ySize); + + // Reduce combination area by the size of the kernel + minInputCols += kernelSize; + maxInputCols -= kernelSize; + minInputRows += kernelSize; + maxInputRows -= kernelSize; + psTrace("psModules.imcombine", 1, "Combining on [%d:%d,%d:%d]\n", + minInputCols, maxInputCols, minInputRows, maxInputRows); + + + // Buffer for combination + combineBuffer *buffer = combineBufferAlloc(num); + + if (haveRejects) { + psImage *combinedImage = combined->image; // Combined image + psImage *combinedMask = combined->mask; // Combined mask + psImage *combinedVariance = combined->weight; // Combined variance map + + psArray *pixelMap = pixelMapGenerate(input, maxInputCols, maxInputRows); // Map of pixels to source + psPixels *pixels = NULL; // Total list of pixels, with no duplicates + for (int i = 0; i < num; i++) { + pmStackData *data = input->data[i]; // Stacking data; contains the list of pixels + if (!data) { + continue; + } + pixels = psPixelsConcatenate(pixels, data->reject); + } + + if (entire) { + // Combine entire image + for (int y = minInputRows; y < maxInputRows; y++) { + for (int x = minInputCols; x < maxInputCols; x++) { + psVector *reject = pixelMapQuery(pixelMap, x, y); // Inspect these images closely + combinePixels(combinedImage, combinedMask, combinedVariance, input, weights, reject, x, y, + maskVal, bad, numIter, rej, useVariance, safe, buffer); + } + } + } else { + // Only combine previously rejected pixels + for (int i = 0; i < pixels->n; i++) { + // Pixel coordinates are in the frame of the original image + int x = pixels->data[i].x, y = pixels->data[i].y; // Coordinates of interest + if (x < minInputCols || x >= maxInputCols || y < minInputRows || y >= maxInputRows) { + continue; + } + psVector *reject = pixelMapQuery(pixelMap, x, y); // Inspect these images closely + combinePixels(combinedImage, combinedMask, combinedVariance, input, weights, reject, x, y, + maskVal, bad, numIter, rej, useVariance, safe, buffer); + } + } + psFree(pixels); + psFree(pixelMap); + } else { + // Pull the products out, allocate if necessary + psImage *combinedImage = combined->image; // Combined image + if (!combinedImage) { + combined->image = psImageAlloc(numCols, numRows, PS_TYPE_F32); + combinedImage = combined->image; + } + psImage *combinedMask = combined->mask; // Combined mask + if (!combinedMask) { + combined->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + combinedMask = combined->mask; + } + + psImage *combinedVariance = combined->weight; // Combined variance map + if (haveVariances && !combinedVariance) { + combined->weight = psImageAlloc(numCols, numRows, PS_TYPE_F32); + combinedVariance = combined->weight; + } + + for (int y = minInputRows; y < maxInputRows; y++) { + for (int x = minInputCols; x < maxInputCols; x++) { + combinePixels(combinedImage, combinedMask, combinedVariance, input, weights, NULL, x, y, + maskVal, bad, numIter, rej, useVariance, safe, buffer); + } + } + +#ifndef PS_NO_TRACE + if (psTraceGetLevel("psModules.imcombine") >= 5) { + for (int i = 0; i < num; i++) { + pmStackData *data = input->data[i]; // Stack data for this input + if (!data || !data->inspect) { + continue; + } + psTrace("psModules.imcombine", 5, "Image %d: %ld pixels to inspect.\n", i, data->inspect->n); + } + } +#endif + } + + psFree(weights); + psFree(buffer); + + return true; +} + Index: /branches/pap_branch_080617/psModules/src/imcombine/pmStack.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmStack.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmStack.h (revision 18170) @@ -0,0 +1,54 @@ +/* @file pmStack.h + * + * This file will perform image combination of several images of the + * same field, produce a list of questionable pixels, then tag some + * of those pixels as defects. + * + * @author Paul Price, IfA + * @author GLG, MHPCC + * + * @version $Revision: 1.7 $ $Name: not supported by cvs2svn $ + * @date $Date: 2008-03-17 21:38:43 $ + * + * Copyright 2004-2007 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_STACK_H +#define PM_STACK_H + +#include +#include +#include + +/// @addtogroup imcombine Image Combinations +/// @{ + + +/// Container for input image +typedef struct { + pmReadout *readout; ///< Warped readout (sky cell) + psPixels *reject; ///< Pixels to reject + psPixels *inspect; ///< Pixels to inspect + float weight; ///< Relative weighting for image +} pmStackData; + +/// Constructor +pmStackData *pmStackDataAlloc(pmReadout *readout, ///< Warped readout (sky cell) + float weight ///< Weight to apply + ); + +/// Stack input images +bool pmStackCombine(pmReadout *combined,///< Combined readout (output) + psArray *input, ///< Input array of pmStackData + psMaskType maskVal, ///< Mask value of bad pixels + psMaskType bad, ///< Mask value to give rejected pixels + int kernelSize, ///< Half-size of the convolution kernel + int numIter, ///< Number of iterations + float rej, ///< Rejection limit (standard deviations) + bool entire, ///< Combine entire image even if rejection lists provided? + bool useVariance, ///< Use variance values for rejection? + bool safe ///< Play safe with small numbers of input pixels (mask if N <= 2)? + ); + +/// @} +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmStackReject.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmStackReject.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmStackReject.c (revision 18170) @@ -0,0 +1,190 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pmSubtraction.h" +#include "pmSubtractionKernels.h" + +#define PIXEL_LIST_BUFFER 100 // Number of pixels to add to list at a time + +//#define TESTING // Testing output + + +psPixels *pmStackReject(const psPixels *in, const psRegion *valid, float threshold, + const psArray *subRegions, const psArray *kernels) +{ + PS_ASSERT_PIXELS_NON_NULL(in, NULL); + PS_ASSERT_FLOAT_LARGER_THAN_OR_EQUAL(threshold, 0.0, NULL); + PS_ASSERT_FLOAT_LESS_THAN_OR_EQUAL(threshold, 1.0, NULL); + PS_ASSERT_ARRAY_NON_NULL(subRegions, NULL); + PS_ASSERT_ARRAY_NON_NULL(kernels, NULL); + PS_ASSERT_ARRAYS_SIZE_EQUAL(subRegions, kernels, NULL); + + // Get the original image size + int numRegions = subRegions->n; // Number of regions + int numCols = 0, numRows = 0; // Size of original image + int minCols = INT_MAX, minRows = INT_MAX; // Minimum coordinate for image + int size = 0; // Size of kernel + for (int i = 0; i < numRegions; i++) { + psRegion *subRegion = subRegions->data[i]; // Region of interest + if (subRegion->x0 < minCols) { + minCols = subRegion->x0; + } + if (subRegion->y0 < minRows) { + minRows = subRegion->y0; + } + if (subRegion->x1 > numCols) { + numCols = subRegion->x1; + } + if (subRegion->y1 > numRows) { + numRows = subRegion->y1; + } + + pmSubtractionKernels *kernel = kernels->data[i]; // Kernel of interest + if (size == 0) { + size = kernel->size; + } else if (kernel->size != size) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Kernel sizes are not identical: %d vs %d", + size, kernel->size); + return NULL; + } + } + + // Adjust the size for the size of the subimage + if (valid) { + minCols = PS_MAX(valid->x0, minCols); + minRows = PS_MAX(valid->y0, minRows); + numCols = PS_MIN(valid->x1, numCols); + numRows = PS_MIN(valid->y1, numRows); + } + psTrace("psModules.imcombine", 1, "Rejecting [%d:%d,%d:%d]\n", minCols, numCols, minRows, numRows); + + psImage *mask = psPixelsToMask(NULL, in, psRegionSet(minCols, numCols - 1, minRows, numRows - 1), + 0x01); // Mask + psImage *image = psImageCopy(NULL, mask, PS_TYPE_F32); // Floating-point version, so we can convolve + psFree(mask); + + // Convolve the image with the kernel --- we're basically applying a matched filter and then thresholding + pmReadout *convRO = pmReadoutAlloc(NULL); // Readout with convolved image + pmReadout *inRO = pmReadoutAlloc(NULL); // Readout with input image + inRO->image = image; + inRO->col0 = minCols; + inRO->row0 = minRows; + for (int i = 0; i < numRegions; i++) { + psRegion *region = subRegions->data[i]; // Region of interest + if (valid && (region->x0 > valid->x1 || region->x1 < valid->x0 || + region->y0 > valid->y1 || region->y1 < valid->y0)) { + // Region is outside of our sub-image + continue; + } + pmSubtractionKernels *kernel = kernels->data[i]; // Kernel of interest + if (!pmSubtractionConvolve(convRO, NULL, inRO, NULL, NULL, 0, region, kernel, false, true)) { + psError(PS_ERR_UNKNOWN, false, "Unable to convolve mask image in region %d.", i); + psFree(convRO); + psFree(inRO); + return NULL; + } + + // Need to adjust the thresholding level for the normalisation of the kernel --- the application of + // the kernel may scale the unit level that we've inserted. + + // Image of the kernel at the centre of the region + float xNorm = (region->x0 + 0.5 * (region->x1 - region->x0) - kernel->numCols/2.0) / + (float)kernel->numCols; + float yNorm = (region->y0 + 0.5 * (region->y1 - region->y0) - kernel->numRows/2.0) / + (float)kernel->numRows; + psImage *image = pmSubtractionKernelImage(kernel, xNorm, yNorm, false); + if (!image) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate kernel image."); + psFree(convRO); + psFree(inRO); + return NULL; + } + float sum = 0.0; + for (int y = 0; y < image->numRows; y++) { + for (int x = 0; x < image->numCols; x++) { + sum += image->data.F32[y][x]; + } + } + psFree(image); + + // Range for normalisation + int yMin = PS_MAX(minRows, region->y0) - inRO->row0; + int yMax = PS_MIN(numRows - 1, region->y1) - inRO->row0; + int xMin = PS_MAX(minCols, region->x0) - inRO->col0; + int xMax = PS_MIN(numCols - 1, region->x1) - inRO->col0; + psTrace("psModules.imcombine", 2, "Normalising convolved mask image by %f over %d:%d,%d:%d\n", + sum, xMin, xMax, yMin, yMax); + for (int y = yMin; y <= yMax; y++) { + for (int x = xMin; x <= xMax; x++) { + convRO->image->data.F32[y][x] /= sum; + } + } + } + psFree(inRO); + psImage *convolved = psMemIncrRefCounter(convRO->image); + psFree(convRO); + +#ifdef TESTING + { + static int seqNum = 0; // Sequence number + psString name = NULL; // Name of image + psStringAppend(&name, "inspect_conv_%02d.fits", seqNum); + seqNum++; + psFits *fits = psFitsOpen(name, "w"); // FITS file pointer + psFree(name); + psFitsWriteImage(fits, NULL, convolved, 0, NULL); + psFitsClose(fits); + } +#endif + + // Threshold the convolved image + psPixels *bad = psPixelsAllocEmpty(PIXEL_LIST_BUFFER); // List of pixels that should be masked + for (int y = size; y < convolved->numRows - size; y++) { + for (int x = size; x < convolved->numCols - size; x++) { + if (convolved->data.F32[y][x] > threshold) { + // Pixel coordinates in "bad" correspond to the full image + bad = psPixelsAdd(bad, PIXEL_LIST_BUFFER, x + minCols, y + minRows); + } + } + } + psFree(convolved); + + // Now, grow the mask to include everything that touches a bad pixel in the convolution + int x0 = minCols, y0 = minRows; // Offset for mask image + mask = psPixelsToMask(NULL, bad, psRegionSet(x0, numCols - 1, y0, numRows - 1), 0xff); + for (int i = 0; i < bad->n; i++) { + int xPix = bad->data[i].x - x0, yPix = bad->data[i].y - y0; // Coordinates in frame of mask image + // Convolution limits + int xMin = PS_MAX(xPix - size, 0); + int xMax = PS_MIN(xPix + size, mask->numCols - 1); + int yMin = PS_MAX(yPix - size, 0); + int yMax = PS_MIN(yPix + size, mask->numRows - 1); + for (int y = yMin; y <= yMax; y++) { + for (int x = xMin; x <= xMax; x++) { + assert(x < mask->numCols && y < mask->numRows); + mask->data.PS_TYPE_MASK_DATA[y][x] = 0xff; + } + } + } + bad = psPixelsFromMask(bad, mask, 0xff); + psFree(mask); + + // Convert coordinates to frame of original image + for (int i = 0; i < bad->n; i++) { + int x = bad->data[i].x + x0; + int y = bad->data[i].y + y0; + if (x < 0 || x >= numCols || y < 0 || y >= numRows) { + psWarning("Bad pixel coordinate %d: %d,%d --- ignored.", + i, x, y); + continue; + } + bad->data[i].x = x; + bad->data[i].y = y; + } + + return bad; +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmStackReject.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmStackReject.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmStackReject.h (revision 18170) @@ -0,0 +1,19 @@ +#ifndef PM_STACK_REJECT_H +#define PM_STACK_REJECT_H + +#include +#include + +/// Given a list of pixels from the convolved image, find the corresponding (smaller subset of) pixels in the +/// original image, and then convolve those to get the list of all pixels which should be rejected +/// +/// We apply a matched filter to the corresponding mask image, and threshold to find the original pixels +psPixels *pmStackReject(const psPixels *in, ///< List of pixels in the convolved image + const psRegion *valid, ///< Valid region to consider + float threshold, ///< Threshold on convolved image, 0..1 + const psArray *regions, ///< Array of image regions for image + const psArray *kernels ///< Array of kernel parameters for each region + ); + + +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtraction.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtraction.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtraction.c (revision 18170) @@ -0,0 +1,897 @@ +/** @file pmSubtraction.c + * + * @author Paul Price, IfA + * @author GLG, MHPCC + * + * @version $Revision: 1.94 $ $Name: not supported by cvs2svn $ + * @date $Date: 2008-06-17 22:16:38 $ + * + * Copyright 2004-2007 Institute for Astronomy, University of Hawaii + * + */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" // Required for pmFPA.h +#include "pmFPA.h" +#include "pmSubtractionStamps.h" +#include "pmSubtractionEquation.h" + +#include "pmSubtraction.h" + +//#define TESTING + +#define PIXEL_LIST_BUFFER 100 // Number of entries to add to pixel list at a time + + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Private (file-static) functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Generate the kernel to apply to the variance from the normal kernel +static psKernel *varianceKernel(psKernel *out, // Output kernel + psKernel *normalKernel // Normal kernel + ) +{ + // Kernel range + int xMin = normalKernel->xMin, xMax = normalKernel->xMax; + int yMin = normalKernel->yMin, yMax = normalKernel->yMax; + + if (!out) { + out = psKernelAlloc(xMin, xMax, yMin, yMax); + } + + // Take the square of the normal kernel + double sumNormal = 0.0, sumVariance = 0.0; // Sum of the normal and variance kernels + for (int v = yMin; v <= yMax; v++) { + for (int u = xMin; u <= xMax; u++) { + float value = normalKernel->kernel[v][u]; // Value of interest + float value2 = PS_SQR(value); // Value squared + sumNormal += value; + sumVariance += value2; + out->kernel[v][u] = value2; + } + } + + // Normalise so that the sum of the variance kernel is the square of the sum of the normal kernel + // This is required to keep the relative scaling between the image and the weight map + psBinaryOp(out->image, out->image, "*", psScalarAlloc(PS_SQR(sumNormal) / sumVariance, PS_TYPE_F32)); + + return out; +} + +// Generate an image of the solved kernel +static psKernel *solvedKernel(psKernel *kernel, // Kernel, to return + const pmSubtractionKernels *kernels, // Kernel basis functions + const psImage *polyValues, // Spatial polynomial values + bool wantDual // Want the dual (second) kernel? + ) +{ + assert(kernels); + assert(polyValues); + + int numKernels = kernels->num; // Number of kernel basis functions + int size = kernels->size; // Kernel half-size + if (!kernel) { + kernel = psKernelAlloc(-size, size, -size, size); + } + psImageInit(kernel->image, 0.0); + + for (int i = 0; i < numKernels; i++) { + double value = p_pmSubtractionSolutionCoeff(kernels, polyValues, i, wantDual); // Polynomial value + + switch (kernels->type) { + case PM_SUBTRACTION_KERNEL_POIS: { + int u = kernels->u->data.S32[i]; // Offset in x + int v = kernels->v->data.S32[i]; // Offset in y + kernel->kernel[v][u] += value; + kernel->kernel[0][0] -= value; + break; + } + /* SPAM and FRIES use the same method */ + case PM_SUBTRACTION_KERNEL_SPAM: + case PM_SUBTRACTION_KERNEL_FRIES: { + int uStart = kernels->u->data.S32[i]; + int uStop = kernels->uStop->data.S32[i]; + int vStart = kernels->v->data.S32[i]; + int vStop = kernels->vStop->data.S32[i]; + + // Normalising sum of kernel component to unity + float norm = 1.0 / (float)((uStop - uStart + 1) * (vStop - vStart + 1)); + + for (int v = vStart; v <= vStop; v++) { + for (int u = uStart; u <= uStop; u++) { + kernel->kernel[v][u] += norm * value; + kernel->kernel[0][0] -= value; + } + } + break; + } + case PM_SUBTRACTION_KERNEL_GUNK: { + if (i < kernels->inner) { + // Using pre-calculated function + psKernel *preCalc = kernels->preCalc->data[i]; // Precalculated values + // Iterating over the kernel + for (int v = -size; v <= size; v++) { + for (int u = -size; u <= size; u++) { + kernel->kernel[v][u] += preCalc->kernel[v][u] * value; + // Photometric scaling is built into the preCalc kernel --- no subtraction! + } + } + } else { + // Using delta function + int u = kernels->u->data.S32[i]; // Offset in x + int v = kernels->v->data.S32[i]; // Offset in y + kernel->kernel[v][u] += value; + kernel->kernel[0][0] -= value; + } + break; + } + case PM_SUBTRACTION_KERNEL_ISIS: { + psKernel *preCalc = kernels->preCalc->data[i]; // Precalculated values + // Iterating over the kernel + for (int v = -size; v <= size; v++) { + for (int u = -size; u <= size; u++) { + kernel->kernel[v][u] += preCalc->kernel[v][u] * value; + // Photometric scaling is built into the preCalc kernel --- no subtraction! + } + } + break; + } + case PM_SUBTRACTION_KERNEL_RINGS: { + psArray *preCalc = kernels->preCalc->data[i]; // Precalculated data + psVector *uCoords = preCalc->data[0]; // u coordinates + psVector *vCoords = preCalc->data[1]; // v coordinates + psVector *poly = preCalc->data[2]; // Polynomial values + int num = uCoords->n; // Number of pixels + + for (int j = 0; j < num; j++) { + int u = uCoords->data.S32[j], v = vCoords->data.S32[j]; // Kernel coordinates + kernel->kernel[v][u] += poly->data.F32[j] * value; + } + // Photometric scaling is built into the kernel --- no subtraction! + break; + } + default: + psAbort("Should never get here."); + } + } + + // Put in the normalisation component + kernel->kernel[0][0] += (wantDual ? 1.0 : p_pmSubtractionSolutionNorm(kernels)); + + return kernel; +} + +// Subtract the (0,0) element to preserve photometric scaling +static void convolveSub(psKernel *convolved, // Convolved image + const psKernel *image, // Image being convolved + int footprint // Size of region of interest + ) +{ + // Can't use psBinaryOp because the images are of different size + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + convolved->kernel[y][x] -= image->kernel[y][x]; + } + } + return; +} + +// Generate the convolution given some offset +static psKernel *convolveOffset(const psKernel *image, // Image to convolve (a kernel for convenience) + int u, int v, // Offset to apply + int footprint // Size of region of interest + ) +{ + psKernel *convolved = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Convolved image + int numBytes = (2 * footprint + 1) * PSELEMTYPE_SIZEOF(PS_TYPE_F32); // Number of bytes to copy + for (int y = -footprint; y <= footprint; y++) { + // Convolution with a delta function is just the value specified by the offset + memcpy(&convolved->kernel[y][-footprint], &image->kernel[y - v][-footprint - u], numBytes); + } + return convolved; +} + + +// Convolve an image using FFT +static void convolveFFT(psImage *target,// Place the result in here + const psImage *image, // Image to convolve + const psImage *mask, // Mask image + psMaskType maskVal, // Value to mask + const psKernel *kernel, // Kernel by which to convolve + psRegion region,// Region of interest + float background, // Background to add + int size // Size of (square) kernel + ) +{ + psRegion border = psRegionSet(region.x0 - size, region.x1 + size, + region.y0 - size, region.y1 + size); // Add a border + + // Casting away const so psImageSubset can add the child + psImage *subImage = psImageSubset((psImage*)image, border); // Subimage to convolve + psImage *subMask = mask ? psImageSubset((psImage*)mask, border) : NULL; // Subimage mask + psImage *convolved = psImageConvolveFFT(NULL, subImage, subMask, maskVal, kernel); // Convolution + psFree(subImage); + psFree(subMask); + + // Now, we have to chop off the borders, and stick it in where it belongs + psImage *subConv = psImageSubset(convolved, psRegionSet(size, -size, size, -size)); // Cut off the edges + psImage *subTarget = psImageSubset(target, region); // Target for this subregion + if (background != 0.0) { + psBinaryOp(subTarget, subConv, "+", psScalarAlloc(background, PS_TYPE_F32)); + } else { + int numBytes = subTarget->numCols * PSELEMTYPE_SIZEOF(PS_TYPE_F32); // Number of bytes to copy + for (int y = 0; y < subTarget->numRows; y++) { + memcpy(subTarget->data.F32[y], subConv->data.F32[y], numBytes); + } + } + psFree(subConv); + psFree(convolved); + psFree(subTarget); + return; +} + +// Convolve an image directly +static void convolveDirect(psImage *target, // Put the result here + const psImage *image, // Image to convolve + const psKernel *kernel, // Kernel by which to convolve + psRegion region,// Region of interest + float background, // Background to add + int size // Size of (square) kernel + ) +{ + for (int y = region.y0; y < region.y1; y++) { + for (int x = region.x0; x < region.x1; x++) { + target->data.F32[y][x] = background; + for (int v = -size; v <= size; v++) { + for (int u = -size; u <= size; u++) { + target->data.F32[y][x] += kernel->kernel[v][u] * + image->data.F32[y - v][x - u]; + } + } + } + } + return; +} + +// Convolve a region of an image +static inline void convolveRegion(psImage *convImage, // Convolved image (output) + psImage *convWeight, // Convolved weight map (output), or NULL + psKernel **kernelImage, // Convolution kernel for the image + psKernel **kernelWeight, // Convolution kernel for the weight map, or NULL + const psImage *image, // Image to convolve + const psImage *weight, // Weight map to convolve, or NULL + const psImage *subMask, // Subtraction mask + psMaskType maskVal, // Mask value to apply in convolution + const pmSubtractionKernels *kernels, // Kernels + psImage *polyValues, // Polynomial values + float background, // Background value to apply + psRegion region, // Region to convolve + bool useFFT, // Use FFT to convolve? + bool wantDual // Want the dual convolution? + ) +{ + *kernelImage = solvedKernel(*kernelImage, kernels, polyValues, wantDual); + if (weight) { + *kernelWeight = varianceKernel(*kernelWeight, *kernelImage); + } + + if (useFFT) { + // Use Fast Fourier Transform to do the convolution + // This provides a big speed-up for large kernels + convolveFFT(convImage, image, subMask, maskVal, *kernelImage, region, background, kernels->size); + if (weight) { + convolveFFT(convWeight, weight, subMask, maskVal, *kernelWeight, region, 0.0, kernels->size); + } + } else { + convolveDirect(convImage, image, *kernelImage, region, background, kernels->size); + if (weight) { + convolveDirect(convWeight, weight, *kernelWeight, region, 0.0, kernels->size); + } + } + + return; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Semi-public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +psKernel *p_pmSubtractionConvolveStampPrecalc(const psKernel *image, const psKernel *kernel) +{ + PS_ASSERT_KERNEL_NON_NULL(image, NULL); + PS_ASSERT_KERNEL_NON_NULL(kernel, NULL); + + psImage *conv = psImageConvolveFFT(NULL, image->image, NULL, 0, kernel); // Convolved image + int x0 = - image->xMin, y0 = - image->yMin; // Position of centre of convolved image + psKernel *convolved = psKernelAllocFromImage(conv, x0, y0); // Kernel version + psFree(conv); + return convolved; +} + +psImage *p_pmSubtractionPolynomial(psImage *output, int spatialOrder, float x, float y) +{ + assert(spatialOrder >= 0); + assert(x >= -1 && x <= 1); + assert(y >= -1 && y <= 1); + + output = psImageRecycle(output, spatialOrder + 1, spatialOrder + 1, PS_TYPE_F64); + output->data.F64[0][0] = 1.0; + + double value = 1.0; + for (int i = 1; i <= spatialOrder; i++) { + value *= x; + output->data.F64[0][i] = value; + } + + value = 1.0; + for (int j = 1; j <= spatialOrder; j++) { + value *= y; + output->data.F64[j][0] = value; + } + + for (int j = 1; j <= spatialOrder; j++) { + for (int i = 1; i <= spatialOrder - j; i++) { + output->data.F64[j][i] = output->data.F64[j][0] * output->data.F64[0][i]; + } + } + + return output; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + +// Convolve a stamp by a single kernel basis function +static psKernel *convolveStampSingle(const pmSubtractionKernels *kernels, // Kernel basis functions + int index, // Kernel basis function index + const psKernel *image, // Image to convolve (a kernel for convenience) + int footprint // Size of region of interest + ) +{ +#if 0 + if (index == kernels->num) { + // Normalisation component + return convolveOffset(image, 0, 0, footprint); + } +#endif + + switch (kernels->type) { + case PM_SUBTRACTION_KERNEL_POIS: { + int u = kernels->u->data.S32[index]; // Offset in x + int v = kernels->v->data.S32[index]; // Offset in y + psKernel *convolved = convolveOffset(image, u, v, footprint); // Convolved image + convolveSub(convolved, image, footprint); + return convolved; + } + // Method for SPAM and FRIES is the same + case PM_SUBTRACTION_KERNEL_SPAM: + case PM_SUBTRACTION_KERNEL_FRIES: { + psKernel *convolved = psKernelAlloc(-footprint, footprint, + -footprint, footprint); // Convolved image + int uStart = kernels->u->data.S32[index]; + int uStop = kernels->uStop->data.S32[index]; + int vStart = kernels->v->data.S32[index]; + int vStop = kernels->vStop->data.S32[index]; + float norm = 1.0 / (uStop - uStart + 1) * (vStop - vStart + 1); // Normalisation + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + double sum = 0.0; + for (int v = vStart; v <= vStop; v++) { + for (int u = uStart; u <= uStop; u++) { + sum += image->kernel[y - v][x - u]; + } + } + convolved->kernel[y][x] = norm * sum; + } + } + convolveSub(convolved, image, footprint); + return convolved; + } + case PM_SUBTRACTION_KERNEL_GUNK: { + if (index < kernels->inner) { + // Photometric scaling is already built in to the precalculated kernel + return p_pmSubtractionConvolveStampPrecalc(image, kernels->preCalc->data[index]); + } + // Using delta function + int u = kernels->u->data.S32[index]; // Offset in x + int v = kernels->v->data.S32[index]; // Offset in y + psKernel *convolved = convolveOffset(image, u, v, footprint); // Convolved image + convolveSub(convolved, image, footprint); + return convolved; + } + case PM_SUBTRACTION_KERNEL_ISIS: { + // Photometric scaling is already built in to the precalculated kernel + return p_pmSubtractionConvolveStampPrecalc(image, kernels->preCalc->data[index]); + } + case PM_SUBTRACTION_KERNEL_RINGS: { + psKernel *convolved = psKernelAlloc(-footprint, footprint, + -footprint, footprint); // Convolved image + psArray *preCalc = kernels->preCalc->data[index]; // Precalculated data + psVector *uCoords = preCalc->data[0]; // u coordinates + psVector *vCoords = preCalc->data[1]; // v coordinates + psVector *poly = preCalc->data[2]; // Polynomial values + int num = uCoords->n; // Number of pixels + for (int y = -footprint; y <= footprint; y++) { + for (int x = -footprint; x <= footprint; x++) { + double sum = 0.0; // Accumulated sum from convolution + for (int j = 0; j < num; j++) { + int u = uCoords->data.S32[j], v = vCoords->data.S32[j]; // Kernel coordinates + sum += image->kernel[y - v][x - u] * poly->data.F32[j]; + } + convolved->kernel[y][x] = sum; + // Photometric scaling is built into the kernel --- no subtraction! + } + } + return convolved; + } + default: + psAbort("Should never get here."); + } + return NULL; +} + +// Convolve the stamp by each of the kernel basis functions +static psArray *convolveStamp(psArray *convolutions, // The convolutions + const psKernel *image, // Image to convolve + const pmSubtractionKernels *kernels, // Kernel basis functions + int footprint // Stamp half-size + ) +{ + assert(image); + assert(kernels); + assert(footprint >= 0); + + if (convolutions) { + // Already done + return convolutions; + } + + int numKernels = kernels->num; // Number of kernels + convolutions = psArrayAlloc(numKernels); + + for (int i = 0; i < numKernels; i++) { + convolutions->data[i] = convolveStampSingle(kernels, i, image, footprint); + } + + return convolutions; +} + + +bool pmSubtractionConvolveStamp(pmSubtractionStamp *stamp, const pmSubtractionKernels *kernels, int footprint) +{ + PS_ASSERT_PTR_NON_NULL(stamp, false); + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, false); + PS_ASSERT_INT_NONNEGATIVE(footprint, false); + + if (stamp->status != PM_SUBTRACTION_STAMP_CALCULATE) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Stamp not marked for calculation."); + return false; + } + + switch (kernels->mode) { + case PM_SUBTRACTION_MODE_1: + stamp->convolutions1 = convolveStamp(stamp->convolutions1, stamp->image1, kernels, footprint); + break; + case PM_SUBTRACTION_MODE_2: + stamp->convolutions2 = convolveStamp(stamp->convolutions2, stamp->image2, kernels, footprint); + break; + case PM_SUBTRACTION_MODE_UNSURE: + case PM_SUBTRACTION_MODE_DUAL: + stamp->convolutions1 = convolveStamp(stamp->convolutions1, stamp->image1, kernels, footprint); + stamp->convolutions2 = convolveStamp(stamp->convolutions2, stamp->image2, kernels, footprint); + break; + default: + psAbort("Unsupported subtraction mode: %x", kernels->mode); + } + + return true; +} + + + + +int pmSubtractionRejectStamps(float *rmsPtr, int *numPtr, pmSubtractionStampList *stamps, + const psVector *deviations, psImage *subMask, float sigmaRej, + int footprint) +{ + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, -1); + PS_ASSERT_VECTOR_NON_NULL(deviations, -1); + PS_ASSERT_VECTOR_TYPE(deviations, PS_TYPE_F32, -1); + PS_ASSERT_IMAGE_NON_EMPTY(subMask, -1); + PS_ASSERT_IMAGE_TYPE(subMask, PS_TYPE_MASK, -1); + + double totalSquareDev = 0.0; // Total square deviation from zero + int numStamps = 0; // Number of used stamps + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (stamp->status != PM_SUBTRACTION_STAMP_USED) { + continue; + } + totalSquareDev += PS_SQR(deviations->data.F32[i]); + numStamps++; + } + + float rms = sqrt(totalSquareDev / (double)numStamps); // Convert to RMS + + if (rmsPtr) { + *rmsPtr = rms; + } + if (numPtr) { + *numPtr = numStamps; + } + + if (numStamps == 0) { + psError(PS_ERR_UNKNOWN, true, "No good stamps found."); + return -1; + } + + if (!isfinite(sigmaRej) || sigmaRej <= 0.0) { + // User just wanted to calculate and record the RMS for posterity + psLogMsg("psModules.imcombine", PS_LOG_INFO, "RMS deviation from %d stamps: %lf", + numStamps, rms); + return 0; + } + + float limit = sigmaRej * rms; // Limit on maximum deviation + psTrace("psModules.imcombine", 1, "Deviation limit: %f\n", limit); + + int numRejected = 0; // Number of stamps rejected + int numGood = 0; // Number of good stamps + double newSquareDev = 0.0; // New square deviation + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (stamp->status == PM_SUBTRACTION_STAMP_USED) { + if (deviations->data.F32[i] > limit) { + // Mask out the stamp in the image so you it's not found again + psTrace("psModules.imcombine", 3, "Rejecting stamp %d (%d,%d)\n", i, + (int)(stamp->x + 0.5), (int)(stamp->y + 0.5)); + numRejected++; + for (int y = stamp->y - footprint; y <= stamp->y + footprint; y++) { + for (int x = stamp->x - footprint; x <= stamp->x + footprint; x++) { + subMask->data.PS_TYPE_MASK_DATA[y][x] |= PM_SUBTRACTION_MASK_REJ; + } + } + + // Set stamp for replacement + stamp->x = 0; + stamp->y = 0; + stamp->xNorm = NAN; + stamp->yNorm = NAN; + stamp->status = PM_SUBTRACTION_STAMP_REJECTED; + // Recalculate convolutions + psFree(stamp->convolutions1); + psFree(stamp->convolutions2); + stamp->convolutions1 = stamp->convolutions2 = NULL; + psFree(stamp->image1); + psFree(stamp->image2); + psFree(stamp->weight); + stamp->image1 = stamp->image2 = stamp->weight = NULL; + psFree(stamp->matrix1); + psFree(stamp->matrix2); + psFree(stamp->matrixX); + stamp->matrix1 = stamp->matrix2 = stamp->matrixX = NULL; + psFree(stamp->vector1); + psFree(stamp->vector2); + stamp->vector1 = stamp->vector2 = NULL; + } else { + numGood++; + newSquareDev += PS_SQR(deviations->data.F32[i]); + } + } + } + + if (numRejected > 0) { + psLogMsg("psModules.imcombine", PS_LOG_INFO, + "%d good stamps; %d rejected.\nRMS deviation: %f --> %f\n", + numGood, numRejected, rms, + sqrt(newSquareDev / (double)numGood)); + } else { + psLogMsg("psModules.imcombine", PS_LOG_INFO, + "%d good stamps; 0 rejected.\nRMS deviation: %f\n", + numGood, rms); + } + + return numRejected; +} + +psImage *pmSubtractionKernelImage(const pmSubtractionKernels *kernels, float x, float y, bool wantDual) +{ + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, NULL); + PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, NULL); + PS_ASSERT_FLOAT_WITHIN_RANGE(x, -1.0, 1.0, NULL); + PS_ASSERT_FLOAT_WITHIN_RANGE(y, -1.0, 1.0, NULL); + + // Precalulate polynomial values + psImage *polyValues = p_pmSubtractionPolynomial(NULL, kernels->spatialOrder, x, y); + + // The appropriate kernel + psKernel *kernel = solvedKernel(NULL, kernels, polyValues, wantDual); + + psFree(polyValues); + + psImage *image = psMemIncrRefCounter(kernel->image); // Image of the kernel + psFree(kernel); + + return image; +} + + +float pmSubtractionVarianceFactor(const pmSubtractionKernels *kernels, float x, float y, bool wantDual) +{ + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, NAN); + PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, NAN); + PS_ASSERT_FLOAT_WITHIN_RANGE(x, -1.0, 1.0, NAN); + PS_ASSERT_FLOAT_WITHIN_RANGE(y, -1.0, 1.0, NAN); + + // Precalulate polynomial values + psImage *polyValues = p_pmSubtractionPolynomial(NULL, kernels->spatialOrder, x, y); + + psKernel *kernel = solvedKernel(NULL, kernels, polyValues, wantDual); // The appropriate kernel + psFree(polyValues); + + double sumKernel2 = 0.0; // Sum of the kernel squared + for (int y = kernel->yMin; y <= kernel->yMax; y++) { + for (int x = kernel->xMin; x <= kernel->xMax; x++) { + sumKernel2 += PS_SQR(kernel->kernel[y][x]); + } + } + + psFree(kernel); + + return sumKernel2; +} + +#if 0 +psArray *pmSubtractionKernelSolutions(const pmSubtractionKernels *kernels, float x, float y, bool wantDual) +{ + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, NULL); + PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, NULL); + PS_ASSERT_FLOAT_WITHIN_RANGE(x, -1.0, 1.0, NULL); + PS_ASSERT_FLOAT_WITHIN_RANGE(y, -1.0, 1.0, NULL); + + psArray *images = psArrayAlloc(solution->n - 1); // Images of each kernel to return + psVector *fakeSolution = psVectorAlloc(solution->n, PS_TYPE_F64); // Fake solution vector + psVectorInit(fakeSolution, 0.0); + + for (int i = 0; i < solution->n - 1; i++) { + fakeSolution->data.F64[i] = solution->data.F64[i]; + images->data[i] = pmSubtractionKernelImage(kernels, x, y, wantDual); + fakeSolution->data.F64[i] = 0.0; + } + + psFree(fakeSolution); + + return images; +} +#endif + + + +bool pmSubtractionConvolve(pmReadout *out1, pmReadout *out2, const pmReadout *ro1, const pmReadout *ro2, + const psImage *subMask, psMaskType blank, const psRegion *region, + const pmSubtractionKernels *kernels, bool doBG, bool useFFT) +{ + int numCols = 0, numRows = 0; // Image dimensions + int x0 = 0, y0 = 0; // Image offset + if (kernels->mode == PM_SUBTRACTION_MODE_1 || kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + PM_ASSERT_READOUT_NON_NULL(out1, false); + PM_ASSERT_READOUT_NON_NULL(ro1, false); + PM_ASSERT_READOUT_IMAGE(ro1, false); + numCols = ro1->image->numCols; + numRows = ro1->image->numRows; + x0 = ro1->col0; + y0 = ro1->row0; + } + if (kernels->mode == PM_SUBTRACTION_MODE_2 || kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + PM_ASSERT_READOUT_NON_NULL(out2, false); + PM_ASSERT_READOUT_NON_NULL(ro2, false); + PM_ASSERT_READOUT_IMAGE(ro2, false); + if (numCols == 0 && numRows == 0) { + numCols = ro2->image->numCols; + numRows = ro2->image->numRows; + x0 = ro2->col0; + y0 = ro2->row0; + } + } + if (kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + PS_ASSERT_IMAGES_SIZE_EQUAL(ro1->image, ro2->image, false); + } + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, false); + PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, false); + if (subMask) { + PS_ASSERT_IMAGE_NON_NULL(subMask, false); + PS_ASSERT_IMAGE_TYPE(subMask, PS_TYPE_MASK, false); + PS_ASSERT_IMAGE_SIZE(subMask, numCols, numRows, false); + } + if (region && psRegionIsNaN(*region)) { + psString string = psRegionToString(*region); + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Input region (%s) contains NAN values", string); + psFree(string); + return false; + } + + // Inputs + psImage *image1 = NULL, *weight1 = NULL; // Image and weight map from input 1 + if (ro1) { + image1 = ro1->image; + weight1 = ro1->weight; + } + psImage *image2 = NULL, *weight2 = NULL; // Image and weight map from input 2 + if (ro2) { + image2 = ro2->image; + weight2 = ro2->weight; + } + + // Outputs + psImage *convImage1 = NULL, *convWeight1 = NULL; // Convolved image and weight from input 1 + if (kernels->mode == PM_SUBTRACTION_MODE_1 || kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + convImage1 = out1->image; + if (!convImage1) { + convImage1 = out1->image = psImageAlloc(numCols, numRows, PS_TYPE_F32); + } + if (weight1) { + if (!out1->weight) { + out1->weight = psImageAlloc(numCols, numRows, PS_TYPE_F32); + } + convWeight1 = out1->weight; + psImageInit(convWeight1, 0.0); + } + } + psImage *convImage2 = NULL, *convWeight2 = NULL; // Convolved image and weight from input 2 + if (kernels->mode == PM_SUBTRACTION_MODE_2 || kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + convImage2 = out2->image; + if (!convImage2) { + convImage2 = out2->image = psImageAlloc(numCols, numRows, PS_TYPE_F32); + } + if (weight2) { + if (!out2->weight) { + out2->weight = psImageAlloc(numCols, numRows, PS_TYPE_F32); + } + convWeight2 = out2->weight; + psImageInit(convWeight2, 0.0); + } + } + psImage *convMask = NULL; // Convolved mask image (common to inputs 1 and 2) + if (subMask) { + if (kernels->mode == PM_SUBTRACTION_MODE_1 || kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + if (!out1->mask) { + out1->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + } + convMask = out1->mask; + } + if (kernels->mode == PM_SUBTRACTION_MODE_2 || kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + if (convMask) { + if (out2->mask) { + psFree(out2->mask); + } + out2->mask = psMemIncrRefCounter(convMask); + } else { + if (!out2->mask) { + out2->mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); + } + convMask = out2->mask; + } + } + psImageInit(convMask, 0); + } + + + int size = kernels->size; // Half-size of kernel + int fullSize = 2 * size + 1; // Full size of kernel + + // Get region for convolution: [xMin:xMax,yMin:yMax] + int xMin = size, xMax = numCols - size; + int yMin = size, yMax = numRows - size; + if (region) { + xMin = PS_MAX(region->x0, xMin); + xMax = PS_MIN(region->x1, xMax); + yMin = PS_MAX(region->y0, yMin); + yMax = PS_MIN(region->y1, yMax); + } + + // Size to use when calculating normalised coordinates (different from actual size when convolving + // subimage) + int xNormSize = (kernels->numCols > 0 ? kernels->numCols : numCols); + int yNormSize = (kernels->numRows > 0 ? kernels->numRows : numRows); + + psMaskType maskSource; // Mask these pixels when convolving + psMaskType maskTarget; // Mark these pixels as bad when propagating the subtractionMask + switch (kernels->mode) { + case PM_SUBTRACTION_MODE_1: + maskSource = PM_SUBTRACTION_MASK_BAD_1; + maskTarget = PM_SUBTRACTION_MASK_BAD_2 | PM_SUBTRACTION_MASK_CONVOLVE_1; + break; + case PM_SUBTRACTION_MODE_2: + maskSource = PM_SUBTRACTION_MASK_BAD_2; + maskTarget = PM_SUBTRACTION_MASK_BAD_1 | PM_SUBTRACTION_MASK_CONVOLVE_2; + break; + case PM_SUBTRACTION_MODE_DUAL: + maskSource = PM_SUBTRACTION_MASK_BAD_1 | PM_SUBTRACTION_MASK_BAD_2; + maskTarget = PM_SUBTRACTION_MASK_CONVOLVE_1 | PM_SUBTRACTION_MASK_CONVOLVE_2; + break; + default: + psAbort("Unsupported subtraction mode: %x", kernels->mode); + } + + psImage *polyValues = NULL; // Pre-calculated polynomial values + psKernel *kernelImage = NULL; // Kernel for the images + psKernel *kernelWeight = NULL; // Kernel for the weight maps + + for (int j = yMin; j < yMax; j += fullSize) { + int ySubMax = PS_MIN(j + fullSize, yMax); // Range for subregion of interest + float yNorm = 2.0 * (float)(j + y0 + size + 1 - yNormSize/2.0) / + (float)yNormSize; // Normalised coordinate + for (int i = xMin; i < xMax; i += fullSize) { + int xSubMax = PS_MIN(i + fullSize, xMax); // Range for subregion of interest + float xNorm = 2.0 * (float)(i + x0 + size + 1 - xNormSize/2.0) / + (float)xNormSize; // Normalised coordinate + + // Only generate polynomial values every kernel footprint, since we have already assumed + // (with the stamps) that it does not vary rapidly on this scale. + polyValues = p_pmSubtractionPolynomial(polyValues, kernels->spatialOrder, xNorm, yNorm); + float background = doBG ? p_pmSubtractionSolutionBackground(kernels, polyValues) : + 0.0; // Background term + psRegion subRegion = psRegionSet(i, xSubMax, j, ySubMax); // Sub-region to convolve + + if (kernels->mode == PM_SUBTRACTION_MODE_1 || kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + convolveRegion(convImage1, convWeight1, &kernelImage, &kernelWeight, image1, weight1, + subMask, maskSource, kernels, polyValues, background, subRegion, useFFT, + false); + } + if (kernels->mode == PM_SUBTRACTION_MODE_2 || kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + convolveRegion(convImage2, convWeight2, &kernelImage, &kernelWeight, image2, weight2, + subMask, maskSource, kernels, polyValues, background, subRegion, useFFT, + kernels->mode == PM_SUBTRACTION_MODE_DUAL); + } + + // Propagate the mask + if (subMask) { + for (int y = j; y < ySubMax; y++) { + for (int x = i; x < xSubMax; x++) { + if (subMask->data.PS_TYPE_MASK_DATA[y][x] & maskTarget) { + convMask->data.PS_TYPE_MASK_DATA[y][x] |= blank; + } + } + } + } + } + } + psFree(kernelImage); + psFree(kernelWeight); + psFree(polyValues); + + // Copy anything that wasn't convolved + switch (kernels->mode) { + case PM_SUBTRACTION_MODE_1: + if (out2) { + out2->image = psMemIncrRefCounter(ro2->image); + out2->weight = psMemIncrRefCounter(ro2->weight); + out2->mask = psMemIncrRefCounter(ro2->mask); + } + break; + case PM_SUBTRACTION_MODE_2: + if (out1) { + out1->image = psMemIncrRefCounter(ro1->image); + out1->weight = psMemIncrRefCounter(ro1->weight); + out1->mask = psMemIncrRefCounter(ro1->mask); + } + break; + case PM_SUBTRACTION_MODE_DUAL: + break; + default: + psAbort("Should never get here."); + } + + return true; +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtraction.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtraction.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtraction.h (revision 18170) @@ -0,0 +1,114 @@ +/* @file pmSubtraction.h + * + * PSF-matched image subtraction, based on the Alard & Lupton (1998) and Alard (2000) methods. + * + * @author Paul Price, IfA + * @author GLG, MHPCC + * + * @version $Revision: 1.25 $ $Name: not supported by cvs2svn $ + * @date $Date: 2008-05-24 00:40:16 $ + * Copyright 2004-207 Institute for Astronomy, University of Hawaii + */ + +#ifndef PM_SUBTRACTION_H +#define PM_SUBTRACTION_H + +#include + +#include +#include +#include +#include + +/// @addtogroup imcombine Image Combinations +/// @{ + +/// Mask values for the subtraction mask +typedef enum { + PM_SUBTRACTION_MASK_CLEAR = 0x00, // No masking + PM_SUBTRACTION_MASK_BAD_1 = 0x01, // Image 1 is bad + PM_SUBTRACTION_MASK_BAD_2 = 0x02, // Image 2 is bad + PM_SUBTRACTION_MASK_CONVOLVE_1 = 0x04, // If image 1 is convolved, would be bad + PM_SUBTRACTION_MASK_CONVOLVE_2 = 0x08, // If image 2 is convolved, would be bad + PM_SUBTRACTION_MASK_FOOTPRINT_1 = 0x10, // Bad pixel within the stamp footprint of image 1 + PM_SUBTRACTION_MASK_FOOTPRINT_2 = 0x20, // Bad pixel within the stamp footprint of image 2 + PM_SUBTRACTION_MASK_BORDER = 0x40, // Image border + PM_SUBTRACTION_MASK_REJ = 0x80, // Previously tried as a stamp, and rejected +} pmSubtractionMasks; + + +/// Number of terms in a polynomial +#define PM_SUBTRACTION_POLYTERMS(ORDER) (((ORDER) + 1) * ((ORDER) + 2) / 2) + +/// Set the indices for the normalisation and background terms +#define PM_SUBTRACTION_INDICES(NORM,BG,KERNELS) { \ + int numSpatial = PM_SUBTRACTION_POLYTERMS((KERNELS)->spatialOrder); /* Number of spatial terms */ \ + NORM = (KERNELS)->num * numSpatial; \ + BG = NORM + 1; \ +} + +/// Return the index for the start of the normalisation terms +#define PM_SUBTRACTION_INDEX_NORM(KERNELS) \ + ((KERNELS)->num * PM_SUBTRACTION_POLYTERMS((KERNELS)->spatialOrder)) + +/// Return the index for the start of the background terms +#define PM_SUBTRACTION_INDEX_BG(KERNELS) \ + (((KERNELS)->num * PM_SUBTRACTION_POLYTERMS((KERNELS)->spatialOrder)) + 1) + + +/// Convolve the reference stamp with the kernel components +bool pmSubtractionConvolveStamp(pmSubtractionStamp *stamp, ///< Stamp to convolve + const pmSubtractionKernels *kernels, ///< Kernel parameters + int footprint ///< Half-size of region over which to calculate equation + ); + +/// Reject stamps +int pmSubtractionRejectStamps(float *rms, ///< RMS deviation, to return + int *num, ///< Number of good stamps, to return + pmSubtractionStampList *stamps, ///< Stamps + const psVector *deviations, ///< Deviations for each stamp + psImage *subMask, ///< Subtraction mask + float sigmaRej, ///< Number of RMS deviations above zero at which to reject + int footprint ///< Half-size of stamp + ); + +/// Generate an image of the convolution kernel +psImage *pmSubtractionKernelImage(const pmSubtractionKernels *kernels, ///< Kernel parameters + float x, float y,///< Normalised position [-1,1] for which to generate image + bool wantDual ///< Calculate for the dual kernel? + ); + +/// Generate images of the convolution kernel elements +psArray *pmSubtractionKernelSolutions(const psVector *solution, ///< Solution vector + const pmSubtractionKernels *kernels, ///< Kernel parameters + float x, float y ///< Normalised position [-1,1] for images + ); + +/// Convolve image in preparation for subtraction +bool pmSubtractionConvolve(pmReadout *out1, ///< Output image 1 + pmReadout *out2, ///< Output image 2 (DUAL mode only) + const pmReadout *ro1, // Input image 1 + const pmReadout *ro2, // Input image 2 + const psImage *subMask, ///< Subtraction mask (or NULL) + psMaskType blank, ///< Mask value for blank regions + const psRegion *region, ///< Region to convolve (or NULL) + const pmSubtractionKernels *kernels, ///< Kernel parameters + bool doBG, ///< Apply background term? + bool useFFT ///< Use Fast Fourier Transform for the convolution? + ); + +/// Generate the convolution of an image, given a precalculated kernel +/// +/// The 'image' is a kernel for convenience --- intended to be a stamp +psKernel *p_pmSubtractionConvolveStampPrecalc(const psKernel *image, ///< Image to convolve + const psKernel *kernel ///< Kernel by which to convolve + ); + +/// Given (normalised) coordinates (x,y), generate a matrix where the elements (i,j) are x^i * y^j +psImage *p_pmSubtractionPolynomial(psImage *output, ///< Output matrix, or NULL + int spatialOrder, ///< Maximum spatial polynomial order + float x, float y ///< Normalised position of interest, [-1,1] + ); + +/// @} +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionEquation.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionEquation.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionEquation.c (revision 18170) @@ -0,0 +1,1051 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +#include "pmSubtraction.h" +#include "pmSubtractionKernels.h" +#include "pmSubtractionStamps.h" + +#include "pmSubtractionEquation.h" + + +//#define TESTING + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Private (file-static) functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Calculate the sum over a stamp product +static inline double calculateSumProduct(const psKernel *image1, // First image in multiplication + const psKernel *image2, // Second image in multiplication + const psKernel *weight, // Weight image + int footprint // (Half-)Size of stamp + ) +{ + double sum = 0.0; // Sum of the image products + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + sum += image1->kernel[y][x] * image2->kernel[y][x] / weight->kernel[y][x]; + } + } + return sum; +} + +// Calculate a single element of the least-squares matrix, with the polynomial expansions in one direction +static inline bool calculateMatrixElement1(psImage *matrix, // Matrix to calculate + int i, int j, // Coordinates of element + const psKernel *image1, // First image in multiplication + const psKernel *image2, // Second image in multiplication + const psKernel *weight, // Weight image + const psImage *polyValues, // Spatial polynomial values + int numKernels, // Number of kernel basis functions + int footprint, // (Half-)Size of stamp + int spatialOrder, // Maximum order of spatial variation + bool symmetric // Is the matrix symmetric? + ) +{ + double sum = calculateSumProduct(image1, image2, weight, footprint); // Sum of the image products + if (!isfinite(sum)) { + return false; + } + + // Generate the pseudo-convolutions from the spatial polynomial terms + for (int iyOrder = 0, iIndex = i; iyOrder <= spatialOrder; iyOrder++) { + for (int ixOrder = 0; ixOrder <= spatialOrder - iyOrder; ixOrder++, iIndex += numKernels) { + double convPoly = sum * polyValues->data.F64[iyOrder][ixOrder]; + + assert(iIndex < matrix->numRows && j < matrix->numCols); + + matrix->data.F64[iIndex][j] = convPoly; + if (symmetric) { + + assert(iIndex < matrix->numCols && j < matrix->numRows); + + matrix->data.F64[j][iIndex] = convPoly; + } + } + } + return true; +} + +// Calculate a single element of the least-squares matrix, with the polynomial expansions in both directions +static inline bool calculateMatrixElement2(psImage *matrix, // Matrix to calculate + int i, int j, // Coordinates of element + const psKernel *image1, // First image in multiplication + const psKernel *image2, // Second image in multiplication + const psKernel *weight, // Weight image + const psImage *polyValues, // Spatial polynomial values + int numKernels, // Number of kernel basis functions + int footprint, // (Half-)Size of stamp + int spatialOrder, // Maximum order of spatial variation + bool symmetric // Is the matrix symmetric? + ) +{ + double sum = calculateSumProduct(image1, image2, weight, footprint); // Sum of the image products + if (!isfinite(sum)) { + return false; + } + + // Generate the pseudo-convolutions from the spatial polynomial terms + for (int iyOrder = 0, iIndex = i; iyOrder <= spatialOrder; iyOrder++) { + for (int ixOrder = 0; ixOrder <= spatialOrder - iyOrder; ixOrder++, iIndex += numKernels) { + double iPoly = polyValues->data.F64[iyOrder][ixOrder]; // Value of polynomial + for (int jyOrder = 0, jIndex = j; jyOrder <= spatialOrder; jyOrder++) { + for (int jxOrder = 0; jxOrder <= spatialOrder - jyOrder; jxOrder++, jIndex += numKernels) { + double convPoly = sum * iPoly * polyValues->data.F64[jyOrder][jxOrder]; + + assert(iIndex < matrix->numRows && jIndex < matrix->numCols); + + matrix->data.F64[iIndex][jIndex] = convPoly; + if (symmetric) { + + assert(iIndex < matrix->numCols && jIndex < matrix->numRows); + + matrix->data.F64[jIndex][iIndex] = convPoly; + } + } + } + } + } + return true; +} + +// Calculate the square part of the matrix derived from multiplying convolutions +static bool calculateMatrixSquare(psImage *matrix, // Matrix to calculate + const psArray *convolutions1, // Convolutions for element 1 + const psArray *convolutions2, // Convolutions for element 2 + const psKernel *weight, // Weight image + const psImage *polyValues, // Polynomial values + int numKernels, // Number of kernel basis functions + int spatialOrder, // Order of spatial variation + int footprint // Half-size of stamp + ) +{ + bool symmetric = (convolutions1 == convolutions2 ? true : false); // Is matrix symmetric? + + for (int i = 0; i < numKernels; i++) { + psKernel *iConv = convolutions1->data[i]; // Convolution for i-th element + + for (int j = (symmetric ? i : 0); j < numKernels; j++) { + psKernel *jConv = convolutions2->data[j]; // Convolution for j-th element + + if (!calculateMatrixElement2(matrix, i, j, iConv, jConv, weight, polyValues, numKernels, + footprint, spatialOrder, symmetric)) { + psTrace("psModules.imcombine", 2, "Bad sumCC at %d, %d", i, j); + return false; + } + } + } + + return true; +} + +// Calculate least-squares matrix and vector +static bool calculateMatrix(psImage *matrix, // Matrix to calculate + const pmSubtractionKernels *kernels, // Kernel components + const psArray *convolutions, // Convolutions of source with kernels + const psKernel *input, // Input stamp, or NULL + const psKernel *weight, // Weight stamp + const psImage *polyValues, // Spatial polynomial values + int footprint, // (Half-)Size of stamp + bool normAndBG // Calculate normalisation and background terms? + ) +{ + int numKernels = kernels->num; // Number of kernel components + int spatialOrder = kernels->spatialOrder; // Maximum order of spatial variation + int numSpatial = PM_SUBTRACTION_POLYTERMS(spatialOrder); // Number of spatial variation terms + int bgOrder = kernels->bgOrder; // Maximum order of background fit + int numBackground = normAndBG ? PM_SUBTRACTION_POLYTERMS(bgOrder) : 0; // Number of background terms + int numTerms = numKernels * numSpatial + (normAndBG ? 1 + numBackground : 0); // Total number of terms + assert(matrix); + assert(matrix->numCols == matrix->numRows); + assert(matrix->numCols == numTerms); + assert(convolutions && convolutions->n == numKernels); + assert(polyValues); + assert(!normAndBG || input); // If we want the normalisation and BG, then we need the input image + + // Square part of the matrix (convolution-convolution products) + if (!calculateMatrixSquare(matrix, convolutions, convolutions, weight, polyValues, numKernels, + spatialOrder, footprint)) { + return false; + } + + // XXX To support higher-order background model than simply constant, the below code needs to be updated. + if (normAndBG) { + int normIndex = PM_SUBTRACTION_INDEX_NORM(kernels); // Index for normalisation + int bgIndex = PM_SUBTRACTION_INDEX_BG(kernels); // Index in matrix for background + + for (int i = 0; i < numKernels; i++) { + psKernel *conv = convolutions->data[i]; // Convolution for i-th element + + // Normalisation-convolution terms + if (!calculateMatrixElement1(matrix, i, normIndex, conv, input, weight, polyValues, numKernels, + footprint, spatialOrder, true)) { + psTrace("psModules.imcombine", 2, "Bad sumIC at %d", i); + return false; + } + + // Background-convolution terms + double sumC = 0.0; // Sum of the convolution + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + sumC += conv->kernel[y][x] / weight->kernel[y][x]; + } + } + if (!isfinite(sumC)) { + psTrace("psModules.imcombine", 2, "Bad sumC at %d", i); + return false; + } + + for (int yOrder = 0, index = i; yOrder <= spatialOrder; yOrder++) { + for (int xOrder = 0; xOrder <= spatialOrder - yOrder; xOrder++, index += numKernels) { + double value = sumC * polyValues->data.F64[yOrder][xOrder]; + matrix->data.F64[index][bgIndex] = value; + matrix->data.F64[bgIndex][index] = value; + } + } + } + + // Background only, normalisation only, and background-normalisation terms + double sum1 = 0.0; // Sum of the weighting + double sumI = 0.0; // Sum of the input + double sumII = 0.0; // Sum of the input squared + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + double invNoise2 = 1.0 / weight->kernel[y][x]; + double value = input->kernel[y][x] * invNoise2; + sumI += value; + sumII += value * input->kernel[y][x]; + sum1 += invNoise2; + } + } + if (!isfinite(sumI)) { + psTrace("psModules.imcombine", 2, "Bad sumI detected"); + return false; + } + if (!isfinite(sumII)) { + psTrace("psModules.imcombine", 2, "Bad sumII detected"); + return false; + } + if (!isfinite(sum1)) { + psTrace("psModules.imcombine", 2, "Bad sum1 detected"); + return false; + } + matrix->data.F64[normIndex][normIndex] = sumII; + matrix->data.F64[bgIndex][bgIndex] = sum1; + matrix->data.F64[normIndex][bgIndex] = sumI; + matrix->data.F64[bgIndex][normIndex] = sumI; + } + + return true; +} + + +// Calculate least-squares matrix and vector +static bool calculateVector(psVector *vector, // Vector to calculate, or NULL + const pmSubtractionKernels *kernels, // Kernel components + const psArray *convolutions, // Convolutions of source with kernels + const psKernel *input, // Input stamp, or NULL if !normAndBG + const psKernel *target, // Target stamp + const psKernel *weight, // Weight stamp + const psImage *polyValues, // Spatial polynomial values + int footprint, // (Half-)Size of stamp + bool normAndBG // Calculate normalisation and background terms? + ) +{ + int numKernels = kernels->num; // Number of kernel components + int spatialOrder = kernels->spatialOrder; // Maximum order of spatial variation + int numSpatial = PM_SUBTRACTION_POLYTERMS(spatialOrder); // Number of spatial variation terms + int bgOrder = kernels->bgOrder; // Maximum order of background fit + int numBackground = normAndBG ? PM_SUBTRACTION_POLYTERMS(bgOrder) : 0; // Number of background terms + int numTerms = numKernels * numSpatial + (normAndBG ? 1 + numBackground : 0); // Total number of terms + assert(vector && vector->n == numTerms); + assert(convolutions && convolutions->n == numKernels); + assert(target); + assert(polyValues); + assert(!normAndBG || input); // If we want the normalisation and BG, then we need the input image + + // Convolution terms + for (int i = 0; i < numKernels; i++) { + psKernel *conv = convolutions->data[i]; // Convolution for i-th element + double sumTC = 0.0; // Sum of the target and convolution + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + sumTC += target->kernel[y][x] * conv->kernel[y][x] / weight->kernel[y][x]; + } + } + if (!isfinite(sumTC)) { + psTrace("psModules.imcombine", 2, "Bad sumTC at %d", i); + return false; + } + for (int yOrder = 0, index = i; yOrder <= spatialOrder; yOrder++) { + for (int xOrder = 0; xOrder <= spatialOrder - yOrder; xOrder++, index += numKernels) { + vector->data.F64[index] = sumTC * polyValues->data.F64[yOrder][xOrder]; + } + } + } + + if (normAndBG) { + // Background terms + double sumT = 0.0; // Sum of the target + double sumIT = 0.0; // Sum of the input-target product + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + float value = target->kernel[y][x] / weight->kernel[y][x]; + sumIT += value * input->kernel[y][x]; + sumT += value; + } + } + if (!isfinite(sumT)) { + psTrace("psModules.imcombine", 2, "Bad sumI detected"); + return false; + } + if (!isfinite(sumIT)) { + psTrace("psModules.imcombine", 2, "Bad sumIT detected"); + return false; + } + + int normIndex = PM_SUBTRACTION_INDEX_NORM(kernels); // Index for normalisation term + vector->data.F64[normIndex] = sumIT; + int bgIndex = PM_SUBTRACTION_INDEX_BG(kernels); // Index for background term + vector->data.F64[bgIndex] = sumT; + } + + return true; +} + + + +// Calculate the cross-matrix, composed of convolutions of each image +// Note that the cross-matrix is NOT square +static bool calculateMatrixCross(psImage *matrix, // Matrix to calculate + const pmSubtractionKernels *kernels, // Kernel components + const psArray *convolutions1, // Convolutions of image 1 + const psArray *convolutions2, // Convolutions of image 2 + const psKernel *image1, // Image 1 stamp + const psKernel *weight, // Weight stamp + const psImage *polyValues, // Spatial polynomial values + int footprint // (Half-)Size of stamp + ) +{ + assert(matrix); + int numKernels = kernels->num; // Number of kernel components + int spatialOrder = kernels->spatialOrder; // Maximum order of spatial variation + int numSpatial = PM_SUBTRACTION_POLYTERMS(spatialOrder); // Number of spatial polynomial terms + int numBackground = PM_SUBTRACTION_POLYTERMS(kernels->bgOrder); // Number of background terms + int numCols = numKernels * numSpatial + 1 + numBackground; // Number of columns + int numRows = numKernels * numSpatial; // Number of rows + assert(matrix->numCols == numCols && matrix->numRows == numRows); + assert(convolutions1 && convolutions1->n == numKernels); + assert(convolutions2 && convolutions2->n == numKernels); + + int normIndex, bgIndex; // Indices in matrix for normalisation and background terms + PM_SUBTRACTION_INDICES(normIndex, bgIndex, kernels); + + if (!calculateMatrixSquare(matrix, convolutions1, convolutions2, weight, polyValues, numKernels, + spatialOrder, footprint)) { + return false; + } + + for (int i = 0; i < numKernels; i++) { + // Normalisation + psKernel *conv = convolutions2->data[i]; // Convolution + if (!calculateMatrixElement1(matrix, i, normIndex, conv, image1, weight, polyValues, numKernels, + footprint, spatialOrder, false)) { + psTrace("psModules.imcombine", 2, "Bad sumIC at %d", i); + return false; + } + + // Background + double sumC = 0.0; // Sum of the weighting + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + sumC += conv->kernel[y][x] / weight->kernel[y][x]; + } + } + if (!isfinite(sumC)) { + psTrace("psModules.imcombine", 2, "Bad sumC detected at %d", i); + return false; + } + for (int yOrder = 0, index = i; yOrder <= spatialOrder; yOrder++) { + for (int xOrder = 0; xOrder <= spatialOrder - yOrder; xOrder++, index += numKernels) { + matrix->data.F64[index][bgIndex] = sumC * polyValues->data.F64[yOrder][xOrder]; + } + } + } + + return true; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Semi-public functions +// XXX EAM these cannot be inline :: talk to Josh +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Calculate the value of a polynomial, specified by coefficients and polynomial values +double p_pmSubtractionCalculatePolynomial(const psVector *coeff, // Coefficients + const psImage *polyValues, // Polynomial values + int order, // Order of polynomials + int index, // Index at which to begin + int step // Step between subsequent indices + ) +{ + double sum = 0.0; // Value of the polynomial sum + for (int yOrder = 0; yOrder <= order; yOrder++) { + for (int xOrder = 0; xOrder <= order - yOrder; xOrder++, index += step) { + + assert(index < coeff->n); + + sum += coeff->data.F64[index] * polyValues->data.F64[yOrder][xOrder]; + } + } + return sum; +} + +double p_pmSubtractionSolutionCoeff(const pmSubtractionKernels *kernels, const psImage *polyValues, + int index, bool wantDual) +{ +#if 0 + // This is probably in a tight loop, so don't check inputs + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, NAN); + PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, NAN); + PS_ASSERT_IMAGE_NON_NULL(polyValues, NAN); + PS_ASSERT_INT_POSITIVE(index, NAN); +#endif + + psVector *solution = wantDual ? kernels->solution2 : kernels->solution1; // Solution vector + return p_pmSubtractionCalculatePolynomial(solution, polyValues, kernels->spatialOrder, index, + kernels->num); +} + +double p_pmSubtractionSolutionNorm(const pmSubtractionKernels *kernels) +{ +#if 0 + // This is probably in a tight loop, so don't check inputs + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, NAN); + PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, NAN); + PS_ASSERT_IMAGE_NON_NULL(polyValues, NAN); +#endif + + int normIndex = PM_SUBTRACTION_INDEX_NORM(kernels); // Index for normalisation + return kernels->solution1->data.F64[normIndex]; +} + +double p_pmSubtractionSolutionBackground(const pmSubtractionKernels *kernels, + const psImage *polyValues) +{ +#if 0 + // This is probably in a tight loop, so don't check inputs + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, NAN); + PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, NAN); + PS_ASSERT_IMAGE_NON_NULL(polyValues, NAN); +#endif + + int bgIndex = PM_SUBTRACTION_INDEX_BG(kernels); // Index for background + return p_pmSubtractionCalculatePolynomial(kernels->solution1, polyValues, kernels->bgOrder, bgIndex, 1); +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +bool pmSubtractionCalculateEquation(pmSubtractionStampList *stamps, const pmSubtractionKernels *kernels) +{ + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false); + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, false); + + int footprint = stamps->footprint; // Half-size of stamps + int spatialOrder = kernels->spatialOrder; // Maximum order of spatial variation + int numKernels = kernels->num; // Number of kernel basis functions + int numSpatial = PM_SUBTRACTION_POLYTERMS(spatialOrder); // Number of spatial variations + int numBackground = PM_SUBTRACTION_POLYTERMS(kernels->bgOrder); // Number of background terms + + // Total number of parameters to solve for: coefficient of each kernel basis function, multipled by the + // number of coefficients for the spatial polynomial, normalisation and a constant background offset. + int numParams = numKernels * numSpatial + 1 + numBackground; + + psImage *polyValues = NULL; // Polynomial terms + + // We iterate over each stamp, allocate the matrix and vectors if + // necessary, and then calculate those matrix/vectors. + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (stamp->status != PM_SUBTRACTION_STAMP_CALCULATE) { + continue; + } + + // Generate convolutions + if (!pmSubtractionConvolveStamp(stamp, kernels, footprint)) { + psError(PS_ERR_UNKNOWN, false, "Unable to convolve stamp %d.", i); + psFree(polyValues); + return NULL; + } + +#ifdef TESTING + for (int j = 0; j < numKernels; j++) { + if (stamp->convolutions1) { + psString convName = NULL; + psStringAppend(&convName, "conv1_%03d_%03d.fits", i, j); + psFits *fits = psFitsOpen(convName, "w"); + psFree(convName); + psKernel *conv = stamp->convolutions1->data[j]; + psFitsWriteImage(fits, NULL, conv->image, 0, NULL); + psFitsClose(fits); + } + + if (stamp->convolutions2) { + psString convName = NULL; + psStringAppend(&convName, "conv2_%03d_%03d.fits", i, j); + psFits *fits = psFitsOpen(convName, "w"); + psFree(convName); + psKernel *conv = stamp->convolutions2->data[j]; + psFitsWriteImage(fits, NULL, conv->image, 0, NULL); + psFitsClose(fits); + } + } +#endif + + polyValues = p_pmSubtractionPolynomial(polyValues, spatialOrder, stamp->xNorm, stamp->yNorm); + + + stamp->matrix1 = psImageAlloc(numParams, numParams, PS_TYPE_F64); + stamp->vector1 = psVectorAlloc(numParams, PS_TYPE_F64); +#ifdef TESTING + psImageInit(stamp->matrix1, NAN); + psVectorInit(stamp->vector1, NAN); +#endif + + bool status; // Status of least-squares matrix/vector calculation + switch (kernels->mode) { + case PM_SUBTRACTION_MODE_1: + status = calculateMatrix(stamp->matrix1, kernels, stamp->convolutions1, stamp->image1, + stamp->weight, polyValues, footprint, true); + status &= calculateVector(stamp->vector1, kernels, stamp->convolutions1, stamp->image1, + stamp->image2, stamp->weight, polyValues, footprint, true); + break; + case PM_SUBTRACTION_MODE_2: + status = calculateMatrix(stamp->matrix1, kernels, stamp->convolutions2, stamp->image2, + stamp->weight, polyValues, footprint, true); + status &= calculateVector(stamp->vector1, kernels, stamp->convolutions2, stamp->image2, + stamp->image1, stamp->weight, polyValues, footprint, true); + break; + case PM_SUBTRACTION_MODE_DUAL: + stamp->matrix2 = psImageAlloc(numKernels * numSpatial, numKernels * numSpatial, PS_TYPE_F64); + stamp->matrixX = psImageAlloc(numParams, numKernels * numSpatial, PS_TYPE_F64); + stamp->vector2 = psVectorAlloc(numKernels * numSpatial, PS_TYPE_F64); +#ifdef TESTING + psImageInit(stamp->matrix2, NAN); + psImageInit(stamp->matrixX, NAN); + psVectorInit(stamp->vector2, NAN); +#endif + status = calculateMatrix(stamp->matrix1, kernels, stamp->convolutions1, stamp->image1, + stamp->weight, polyValues, footprint, true); + status &= calculateMatrix(stamp->matrix2, kernels, stamp->convolutions2, NULL, + stamp->weight, polyValues, footprint, false); + status &= calculateMatrixCross(stamp->matrixX, kernels, stamp->convolutions1, + stamp->convolutions2, stamp->image1, stamp->weight, polyValues, + footprint); + status &= calculateVector(stamp->vector1, kernels, stamp->convolutions1, stamp->image1, + stamp->image2, stamp->weight, polyValues, footprint, true); + status &= calculateVector(stamp->vector2, kernels, stamp->convolutions2, NULL, + stamp->image2, stamp->weight, polyValues, footprint, false); + break; + default: + psAbort("Unsupported subtraction mode: %x", kernels->mode); + } + + if (!status) { + stamp->status = PM_SUBTRACTION_STAMP_REJECTED; + psWarning("Rejecting stamp %d (%d,%d) because of bad equation", + i, (int)(stamp->x + 0.5), (int)(stamp->y + 0.5)); + } else { + stamp->status = PM_SUBTRACTION_STAMP_USED; + } + +#ifdef TESTING + if (psTraceGetLevel("psModules.imcombine.equation") >= 10) { + psString matrixName = NULL; + psStringAppend(&matrixName, "matrix1_%d.fits", i); + psFits *matrixFile = psFitsOpen(matrixName, "w"); + psFree(matrixName); + psFitsWriteImage(matrixFile, NULL, stamp->matrix1, 0, NULL); + psFitsClose(matrixFile); + + matrixName = NULL; + psStringAppend(&matrixName, "vector1_%d.fits", i); + psImage *dummy = psImageAlloc(stamp->vector1->n, 1, PS_TYPE_F64); + memcpy(dummy->data.F64[0], stamp->vector1->data.F64, + PSELEMTYPE_SIZEOF(PS_TYPE_F64) * stamp->vector1->n); + matrixFile = psFitsOpen(matrixName, "w"); + psFree(matrixName); + psFitsWriteImage(matrixFile, NULL, dummy, 0, NULL); + psFree(dummy); + psFitsClose(matrixFile); + + if (stamp->vector2) { + matrixName = NULL; + psStringAppend(&matrixName, "vector2_%d.fits", i); + dummy = psImageAlloc(stamp->vector2->n, 1, PS_TYPE_F64); + memcpy(dummy->data.F64[0], stamp->vector2->data.F64, + PSELEMTYPE_SIZEOF(PS_TYPE_F64) * stamp->vector2->n); + matrixFile = psFitsOpen(matrixName, "w"); + psFree(matrixName); + psFitsWriteImage(matrixFile, NULL, dummy, 0, NULL); + psFree(dummy); + psFitsClose(matrixFile); + } + + if (kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + matrixName = NULL; + psStringAppend(&matrixName, "matrix2_%d.fits", i); + matrixFile = psFitsOpen(matrixName, "w"); + psFree(matrixName); + psFitsWriteImage(matrixFile, NULL, stamp->matrix2, 0, NULL); + psFitsClose(matrixFile); + + matrixName = NULL; + psStringAppend(&matrixName, "matrixX_%d.fits", i); + matrixFile = psFitsOpen(matrixName, "w"); + psFree(matrixName); + psFitsWriteImage(matrixFile, NULL, stamp->matrixX, 0, NULL); + psFitsClose(matrixFile); + } + } +#endif + + } + psFree(polyValues); + + return true; +} + +bool pmSubtractionSolveEquation(pmSubtractionKernels *kernels, const pmSubtractionStampList *stamps) +{ + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, false); + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false); + + // Check inputs + int numParams = -1; // Number of parameters + int numParams2 = 0; // Number of parameters for part solution (DUAL mode) + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + PS_ASSERT_PTR_NON_NULL(stamp, false); + if (stamp->status != PM_SUBTRACTION_STAMP_USED) { + continue; + } + + PS_ASSERT_VECTOR_NON_NULL(stamp->vector1, false); + if (numParams == -1) { + numParams = stamp->vector1->n; + } + PS_ASSERT_VECTOR_SIZE(stamp->vector1, (long)numParams, false); + PS_ASSERT_VECTOR_TYPE(stamp->vector1, PS_TYPE_F64, false); + PS_ASSERT_IMAGE_NON_NULL(stamp->matrix1, false); + PS_ASSERT_IMAGE_SIZE(stamp->matrix1, numParams, numParams, false); + PS_ASSERT_IMAGE_TYPE(stamp->matrix1, PS_TYPE_F64, false); + + if (kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + PS_ASSERT_IMAGE_NON_NULL(stamp->matrix2, false); + PS_ASSERT_IMAGE_NON_NULL(stamp->matrixX, false); + if (numParams2 == 0) { + numParams2 = stamp->matrix2->numCols; + } + PS_ASSERT_IMAGE_SIZE(stamp->matrix2, numParams2, numParams2, false); + PS_ASSERT_IMAGE_SIZE(stamp->matrixX, numParams, numParams2, false); + PS_ASSERT_IMAGE_TYPE(stamp->matrix2, PS_TYPE_F64, false); + PS_ASSERT_IMAGE_TYPE(stamp->matrixX, PS_TYPE_F64, false); + PS_ASSERT_VECTOR_NON_NULL(stamp->vector2, false); + PS_ASSERT_VECTOR_SIZE(stamp->vector2, (long)numParams2, false); + PS_ASSERT_VECTOR_TYPE(stamp->vector2, PS_TYPE_F64, false); + } + } + if (numParams == -1) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No suitable stamps found."); + return NULL; + } + + if (kernels->mode != PM_SUBTRACTION_MODE_DUAL) { + // Accumulate the least-squares matricies and vectors + psImage *sumMatrix = psImageAlloc(numParams, numParams, PS_TYPE_F64); // Combined matrix + psVector *sumVector = psVectorAlloc(numParams, PS_TYPE_F64); // Combined vector + psVectorInit(sumVector, 0.0); + psImageInit(sumMatrix, 0.0); + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (stamp->status == PM_SUBTRACTION_STAMP_USED) { + (void)psBinaryOp(sumMatrix, sumMatrix, "+", stamp->matrix1); + (void)psBinaryOp(sumVector, sumVector, "+", stamp->vector1); + } + } + + psVector *permutation = NULL; // Permutation vector, required for LU decomposition + psImage *luMatrix = psMatrixLUD(NULL, &permutation, sumMatrix); + psFree(sumMatrix); + if (!luMatrix) { + psError(PS_ERR_UNKNOWN, true, "LU Decomposition of least-squares matrix failed.\n"); + psFree(sumVector); + psFree(luMatrix); + psFree(permutation); + return NULL; + } + kernels->solution1 = psMatrixLUSolve(kernels->solution1, luMatrix, sumVector, permutation); + psFree(sumVector); + psFree(luMatrix); + psFree(permutation); + if (!kernels->solution1) { + psError(PS_ERR_UNKNOWN, true, "Failed to solve the least-squares system.\n"); + return NULL; + } + } else { + // Dual convolution solution + + // Accumulation of stamp matrices/vectors + psImage *sumMatrix1 = psImageAlloc(numParams, numParams, PS_TYPE_F64); + psImage *sumMatrix2 = psImageAlloc(numParams2, numParams2, PS_TYPE_F64); + psImage *sumMatrixX = psImageAlloc(numParams, numParams2, PS_TYPE_F64); + psVector *sumVector1 = psVectorAlloc(numParams, PS_TYPE_F64); + psVector *sumVector2 = psVectorAlloc(numParams, PS_TYPE_F64); + psImageInit(sumMatrix1, 0.0); + psImageInit(sumMatrix2, 0.0); + psImageInit(sumMatrixX, 0.0); + psVectorInit(sumVector1, 0.0); + psVectorInit(sumVector2, 0.0); + + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (stamp->status == PM_SUBTRACTION_STAMP_USED) { + (void)psBinaryOp(sumMatrix1, sumMatrix1, "+", stamp->matrix1); + (void)psBinaryOp(sumMatrix2, sumMatrix2, "+", stamp->matrix2); + (void)psBinaryOp(sumMatrixX, sumMatrixX, "+", stamp->matrixX); + (void)psBinaryOp(sumVector1, sumVector1, "+", stamp->vector1); + (void)psBinaryOp(sumVector2, sumVector2, "+", stamp->vector2); + } + } + +#if 0 + // Apply weighting to maximise the difference between solution coefficients for the same functions + double fudge = PS_SQR(2 * stamps->footprint + 1); // Fudge factor + for (int i = 0; i < kernels->num; i++) { + sumMatrix1->data.F64[i][i] -= fudge; + sumMatrix2->data.F64[i][i] += fudge; + } +#endif + + // Pure matrix operations + + // A * a = Ct * b + d + // C * a = B * b + e + // + // a = (Ct * Bi * C - A)i (Ct * Bi * e - d) + // b = Bi * (C * a - e) + psVector *a = psVectorRecycle(kernels->solution1, numParams, PS_TYPE_F64); + psVector *b = psVectorRecycle(kernels->solution2, numParams2, PS_TYPE_F64); +#ifdef TESTING + psVectorInit(a, NAN); + psVectorInit(b, NAN); +#endif + psImage *A = sumMatrix1; + psImage *B = sumMatrix2; + psImage *C = sumMatrixX; + psVector *d = sumVector1; + psVector *e = sumVector2; + + assert(a->n == numParams); + assert(b->n == numParams2); + assert(A->numRows == numParams && A->numCols == numParams); + assert(B->numRows == numParams2 && B->numCols == numParams2); + assert(C->numRows == numParams2 && C->numCols == numParams); + assert(d->n == numParams); + assert(e->n == numParams2); + + psImage *Bi = psMatrixInvert(NULL, B, NULL); + assert(Bi->numRows == numParams2 && Bi->numCols == numParams2); + psImage *Ct = psMatrixTranspose(NULL, C); + assert(Ct->numRows == numParams && Ct->numCols == numParams2); + + psImage *BiC = psMatrixMultiply(NULL, Bi, C); + assert(BiC->numRows == numParams2 && BiC->numCols == numParams); + psImage *CtBi = psMatrixMultiply(NULL, Ct, Bi); + assert(CtBi->numRows == numParams && CtBi->numCols == numParams2); + + psImage *CtBiC = psMatrixMultiply(NULL, Ct, BiC); + assert(CtBiC->numRows == numParams && CtBiC->numCols == numParams); + + psImage *F = (psImage*)psBinaryOp(NULL, CtBiC, "-", A); + assert(F->numRows == numParams && F->numCols == numParams); + float det = NAN; + psImage *Fi = psMatrixInvert(NULL, F, &det); + assert(Fi->numRows == numParams && Fi->numCols == numParams); + psTrace("psModules.imcombine", 4, "Determinant of F: %f\n", det); + + psVector *g = psVectorAlloc(numParams, PS_TYPE_F64); +#ifdef TESTING + psVectorInit(g, NAN); +#endif + assert(CtBi->numRows == numParams && CtBi->numCols == numParams2); + assert(e->n == numParams2); + assert(d->n == numParams); + for (int i = 0; i < numParams; i++) { + double value = 0.0; + for (int j = 0; j < numParams2; j++) { + value += CtBi->data.F64[i][j] * e->data.F64[j]; + } + g->data.F64[i] = value - d->data.F64[i]; + } + + assert(Fi->numRows == numParams && Fi->numCols == numParams); + assert(g->n == numParams); + for (int i = 0; i < numParams; i++) { + double value = 0.0; + for (int j = 0; j < numParams; j++) { + value += Fi->data.F64[i][j] * g->data.F64[j]; + } + a->data.F64[i] = value; + } + + psVector *h = psVectorAlloc(numParams2, PS_TYPE_F64); +#ifdef TESTING + psVectorInit(h, NAN); +#endif + assert(C->numRows == numParams2 && C->numCols == numParams); + assert(a->n == numParams); + assert(e->n == numParams2); + for (int i = 0; i < numParams2; i++) { + double value = 0.0; + for (int j = 0; j < numParams; j++) { + value += C->data.F64[i][j] * a->data.F64[j]; + } + h->data.F64[i] = value - e->data.F64[i]; + } + + assert(Bi->numRows == numParams2 && Bi->numCols == numParams2); + assert(h->n == numParams2); + for (int i = 0; i < numParams2; i++) { + double value = 0.0; + for (int j = 0; j < numParams2; j++) { + value += Bi->data.F64[i][j] * h->data.F64[j]; + } + b->data.F64[i] = value; + } + + +#if 0 + for (int i = 0; i < numParams; i++) { + double aVal1 = 0.0, bVal1 = 0.0; + for (int j = 0; j < numParams2; j++) { + aVal1 += A->data.F64[i][j] * a->data.F64[j]; + bVal1 += Ct->data.F64[i][j] * b->data.F64[j]; + } + bVal1 += d->data.F64[i]; + for (int j = numParams2; j < numParams; j++) { + aVal1 += A->data.F64[i][j] * a->data.F64[j]; + } + printf("%d: %lf\n", i, aVal1 - bVal1); + } + + for (int i = 0; i < numParams2; i++) { + double aVal2 = 0.0, bVal2 = 0.0; + for (int j = 0; j < numParams2; j++) { + aVal2 += C->data.F64[i][j] * a->data.F64[j]; + bVal2 += B->data.F64[i][j] * b->data.F64[j]; + } + bVal2 += e->data.F64[i]; + for (int j = numParams2; j < numParams; j++) { + aVal2 += C->data.F64[i][j] * a->data.F64[j]; + } + printf("%d: %lf\n", i, aVal2 - bVal2); + } +#endif + + { + psFits *fits = psFitsOpen("sumMatrix1.fits", "w"); + psFitsWriteImage(fits, NULL, sumMatrix1, 0, NULL); + psFitsClose(fits); + } + { + psFits *fits = psFitsOpen("sumMatrix2.fits", "w"); + psFitsWriteImage(fits, NULL, sumMatrix2, 0, NULL); + psFitsClose(fits); + } + { + psFits *fits = psFitsOpen("sumMatrixX.fits", "w"); + psFitsWriteImage(fits, NULL, sumMatrixX, 0, NULL); + psFitsClose(fits); + } + { + psFits *fits = psFitsOpen("sumFinverse.fits", "w"); + psFitsWriteImage(fits, NULL, Fi, 0, NULL); + psFitsClose(fits); + } + + + kernels->solution1 = a; + kernels->solution2 = b; + + // XXXXX Free temporary matrices and vectors + + } + + if (psTraceGetLevel("psModules.imcombine") >= 7) { + for (int i = 0; i < kernels->solution1->n; i++) { + psTrace("psModules.imcombine", 7, "Solution 1 %d: %f\n", i, kernels->solution1->data.F64[i]); + } + if (kernels->mode == PM_SUBTRACTION_MODE_DUAL) { + for (int i = 0; i < kernels->solution2->n; i++) { + psTrace("psModules.imcombine", 7, "Solution 2 %d: %f\n", i, kernels->solution2->data.F64[i]); + } + } + } + + return true; +} + +psVector *pmSubtractionCalculateDeviations(pmSubtractionStampList *stamps, + const pmSubtractionKernels *kernels) +{ + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, NULL); + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, NULL); + PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(kernels, NULL); + + psVector *deviations = psVectorAlloc(stamps->num, PS_TYPE_F32); // Mean deviation for stamps + int footprint = stamps->footprint; // Half-size of stamps + long numPixels = PS_SQR(2 * footprint + 1); // Number of pixels in footprint + double devNorm = 1.0 / (double)numPixels; // Normalisation for deviations + int numKernels = kernels->num; // Number of kernels + + psImage *polyValues = NULL; // Polynomial values + psKernel *residual = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Residual image + + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // The stamp of interest + if (stamp->status != PM_SUBTRACTION_STAMP_USED) { + deviations->data.F32[i] = NAN; + continue; + } + + // Calculate coefficients of the kernel basis functions + polyValues = p_pmSubtractionPolynomial(polyValues, kernels->spatialOrder, stamp->xNorm, stamp->yNorm); + double norm = p_pmSubtractionSolutionNorm(kernels); // Normalisation + double background = p_pmSubtractionSolutionBackground(kernels, polyValues);// Difference in background + + // Calculate residuals + psKernel *weight = stamp->weight; // Weight postage stamp + psImageInit(residual->image, 0.0); + if (kernels->mode != PM_SUBTRACTION_MODE_DUAL) { + psKernel *target; // Target postage stamp + psKernel *source; // Source postage stamp + psArray *convolutions; // Convolution postage stamps for each kernel basis function + switch (kernels->mode) { + case PM_SUBTRACTION_MODE_1: + target = stamp->image2; + source = stamp->image1; + convolutions = stamp->convolutions1; + break; + case PM_SUBTRACTION_MODE_2: + target = stamp->image1; + source = stamp->image2; + convolutions = stamp->convolutions2; + break; + default: + psAbort("Unsupported subtraction mode: %x", kernels->mode); + } + + for (int j = 0; j < numKernels; j++) { + psKernel *convolution = convolutions->data[j]; // Convolution + double coefficient = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, + false); // Coefficient + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + residual->kernel[y][x] -= convolution->kernel[y][x] * coefficient; + } + } + } + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + residual->kernel[y][x] += target->kernel[y][x] - background - source->kernel[y][x] * norm; + } + } + } else { + // Dual convolution + psArray *convolutions1 = stamp->convolutions1; // Convolutions of the first image + psArray *convolutions2 = stamp->convolutions2; // Convolutions of the second image + psKernel *image1 = stamp->image1; // The first image + psKernel *image2 = stamp->image2; // The second image + + for (int j = 0; j < numKernels; j++) { + psKernel *conv1 = convolutions1->data[j]; // Convolution of first image + psKernel *conv2 = convolutions2->data[j]; // Convolution of second image + double coeff1 = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, false); // Coefficient 1 + double coeff2 = p_pmSubtractionSolutionCoeff(kernels, polyValues, j, true); // Coefficient 2 + + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + residual->kernel[y][x] += conv2->kernel[y][x] * coeff2 - conv1->kernel[y][x] * coeff1; + } + } + } + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + residual->kernel[y][x] += image2->kernel[y][x] - background - image1->kernel[y][x] * norm; + } + } + } + + double deviation = 0.0; // Sum of differences + for (int y = - footprint; y <= footprint; y++) { + for (int x = - footprint; x <= footprint; x++) { + double dev = PS_SQR(residual->kernel[y][x]) / weight->kernel[y][x]; + deviation += dev; +#ifdef TESTING + residual->kernel[y][x] = dev; +#endif + } + } + deviations->data.F32[i] = sqrtf(devNorm * deviation); + psTrace("psModules.imcombine", 5, "Deviation for stamp %d (%d,%d): %f\n", + i, (int)(stamp->x + 0.5), (int)(stamp->y + 0.5), deviations->data.F32[i]); + if (!isfinite(deviations->data.F32[i])) { + stamp->status = PM_SUBTRACTION_STAMP_REJECTED; + psTrace("psModules.imcombine", 5, + "Rejecting stamp %d (%d,%d) because of non-finite deviation\n", + i, (int)(stamp->x + 0.5), (int)(stamp->y + 0.5)); + continue; + } + +#ifdef TESTING + { + psString filename = NULL; + psStringAppend(&filename, "resid_%03d.fits", i); + psFits *fits = psFitsOpen(filename, "w"); + psFree(filename); + psFitsWriteImage(fits, NULL, residual->image, 0, NULL); + psFitsClose(fits); + } + { + psString filename = NULL; + psStringAppend(&filename, "stamp_image_%03d.fits", i); + psFits *fits = psFitsOpen(filename, "w"); + psFree(filename); + psFitsWriteImage(fits, NULL, stamp->image1->image, 0, NULL); + psFitsClose(fits); + } + { + psString filename = NULL; + psStringAppend(&filename, "stamp_weight_%03d.fits", i); + psFits *fits = psFitsOpen(filename, "w"); + psFree(filename); + psFitsWriteImage(fits, NULL, stamp->weight->image, 0, NULL); + psFitsClose(fits); + } +#endif + + } + psFree(residual); + psFree(polyValues); + + return deviations; +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionEquation.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionEquation.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionEquation.h (revision 18170) @@ -0,0 +1,46 @@ +#ifndef PM_SUBTRACTION_EQUATION_H +#define PM_SUBTRACTION_EQUATION_H + +#include "pmSubtractionStamps.h" +#include "pmSubtractionKernels.h" + +/// Calculate the least-squares equation to match the image quality +bool pmSubtractionCalculateEquation(pmSubtractionStampList *stamps, ///< Stamps + const pmSubtractionKernels *kernels ///< Kernel parameters + ); + +/// Solve the least-squares equation to match the image quality +bool pmSubtractionSolveEquation(pmSubtractionKernels *kernels, ///< Kernel parameters + const pmSubtractionStampList *stamps ///< Stamps + ); + +/// Calculate deviations +psVector *pmSubtractionCalculateDeviations(pmSubtractionStampList *stamps, ///< Stamps + const pmSubtractionKernels *kernels ///< Kernel parameters + ); + +/// Calculate the value of a polynomial, specified by coefficients and polynomial values +inline double p_pmSubtractionCalculatePolynomial(const psVector *coeff, ///< Coefficients + const psImage *polyValues, ///< Polynomial values + int order, ///< Order of polynomials + int index, ///< Index at which to begin + int step ///< Step between subsequent indices + ); + +/// Return the specified coefficient in the solution +inline double p_pmSubtractionSolutionCoeff(const pmSubtractionKernels *kernels, ///< Kernel parameters + const psImage *polyValues, ///< Polynomial values + int index, ///< Coefficient index to calculate + bool wantDual ///< Calculate the coefficient for the dual solution? + ); + +/// Return the normalisation in the solution +inline double p_pmSubtractionSolutionNorm(const pmSubtractionKernels *kernels ///< Kernel parameters + ); + +/// Return the background (difference) in the solution +inline double p_pmSubtractionSolutionBackground(const pmSubtractionKernels *kernels, ///< Kernel parameters + const psImage *polyValues ///< Polynomial values + ); + +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionIO.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionIO.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionIO.c (revision 18170) @@ -0,0 +1,284 @@ +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" + +#include "pmSubtraction.h" +#include "pmSubtractionKernels.h" +#include "pmSubtractionMatch.h" + +#define ARRAY_BUFFER 16 // Number to add to array at a time + +// Names of FITS table columns +#define NAME_XMIN "XMIN" // Region of applicability: minimum x value +#define NAME_XMAX "XMAX" // Region of applicability: maximum x value +#define NAME_YMIN "YMIN" // Region of applicability: minimum y value +#define NAME_YMAX "YMAX" // Region of applicability: maximum y value +#define NAME_KERNEL "KERNEL" // Kernel description +#define NAME_TYPE "TYPE" // Kernel type +#define NAME_SIZE "SIZE" // Kernel half-size +#define NAME_INNER "INNER" // Size of inner region (only applicable for some kernel types) +#define NAME_SPATIAL "SPATIAL_ORDER" // Order of spatial polynomial +#define NAME_BG "BG_ORDER" // Order of background polynomial +#define NAME_MODE "MODE" // Matching mode +#define NAME_COLS "COLUMNS" // Number of columns +#define NAME_ROWS "ROWS" // Number of rows +#define NAME_SOL1 "SOLUTION_1" // Solution for convolving image 1 +#define NAME_SOL2 "SOLUTION_2" // Solution for convolving image 2 + + +// Generate an extension name for the subtraction kernel +static psString subtractionKernelExtname(const pmReadout *ro // Readout containing kernel + ) +{ + psAssert(ro, "Required parameter"); + + psString extname = NULL; // Name of FITS extension + pmCell *cell = ro->parent; // Parent cell + if (cell && cell->parent) { + pmChip *chip = cell->parent; // Parent chip + const char *chipName = psMetadataLookupStr(NULL, chip->concepts, "CHIP.NAME"); // Name of chip + const char *cellName = psMetadataLookupStr(NULL, cell->concepts, "CELL.NAME"); // Name of cell + psAssert(chipName && cellName, "Chip and cell names should be defined in concepts."); + int roNum = -1; // Index of readout + for (int i = 0; i < cell->readouts->n && roNum == -1; i++) { + if (cell->readouts->data[i] == ro) { + roNum = i; + } + } + if (roNum != -1) { + psStringAppend(&extname, "%s_%s_%d", chipName, cellName, roNum); + } else { + psWarning("Unable to find readout number"); + } + } + + if (!extname) { + extname = psStringCopy("SUBTRACTION_KERNEL"); + } + + return extname; +} + +bool pmReadoutWriteSubtractionKernels(pmReadout *ro, psFits *fits) +{ + PM_ASSERT_READOUT_NON_NULL(ro, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + + // Extract the regions and solutions used in the image matching + psArray *regions = psArrayAllocEmpty(ARRAY_BUFFER); // Array of regions + { + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_REGION); + psMetadataIterator *iter = psMetadataIteratorAlloc(ro->analysis, PS_LIST_HEAD, regex); // Iterator + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_REGION); + regions = psArrayAdd(regions, ARRAY_BUFFER, item->data.V); + } + psFree(iter); + } + if (regions->n == 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "No subtraction regions found."); + psFree(regions); + return false; + } + + psArray *kernels = psArrayAllocEmpty(ARRAY_BUFFER); // Array of kernels + { + psString regex = NULL; // Regular expression + psStringAppend(®ex, "^%s$", PM_SUBTRACTION_ANALYSIS_KERNEL); + psMetadataIterator *iter = psMetadataIteratorAlloc(ro->analysis, PS_LIST_HEAD, regex); // Iterator + psFree(regex); + psMetadataItem *item = NULL;// Item from iteration + while ((item = psMetadataGetAndIncrement(iter))) { + assert(item->type == PS_DATA_UNKNOWN); + // Set the normalisation dimensions, since these will be otherwise unavailable when reading the + // images by scans. + pmSubtractionKernels *kernel = item->data.V; // Kernel used in subtraction + kernel->numCols = ro->image->numCols; + kernel->numRows = ro->image->numRows; + + kernels = psArrayAdd(kernels, ARRAY_BUFFER, kernel); + } + psFree(iter); + } + + if (regions->n != kernels->n) { + psError(PS_ERR_BAD_PARAMETER_SIZE, true, "Number of regions (%ld) and kernels (%ld) don't match.\n", + regions->n, kernels->n); + psFree(regions); + psFree(kernels); + return false; + } + + // Format for writing a table + int num = regions->n; // Number of regions and kernels + psArray *rows = psArrayAlloc(num); // Array of FITS table rows + for (int i = 0; i < num; i++) { + psMetadata *row = psMetadataAlloc(); // Row of interest + rows->data[i] = psMemIncrRefCounter(row); + + psRegion *region = regions->data[i]; // Region of interest + psMetadataAddS32(row, PS_LIST_TAIL, NAME_XMIN, 0, "Applicability minimum x", region->x0); + psMetadataAddS32(row, PS_LIST_TAIL, NAME_XMAX, 0, "Applicability maximum x", region->x1); + psMetadataAddS32(row, PS_LIST_TAIL, NAME_YMIN, 0, "Applicability minimum y", region->y0); + psMetadataAddS32(row, PS_LIST_TAIL, NAME_YMAX, 0, "Applicability maximum y", region->y1); + + pmSubtractionKernels *kernel = kernels->data[i]; // Kernel + psMetadataAddStr(row, PS_LIST_TAIL, NAME_KERNEL, 0, "Kernel description", kernel->description); + +#if 0 + psMetadataAddS32(row, PS_LIST_TAIL, NAME_TYPE, 0, "Kernel type (enum)", kernel->type); + psMetadataAddS32(row, PS_LIST_TAIL, NAME_SIZE, 0, "Kernel half-size", kernel->size); + psMetadataAddS32(row, PS_LIST_TAIL, NAME_INNER, 0, "Size of inner region", kernel->inner); + psMetadataAddS32(row, PS_LIST_TAIL, NAME_SPATIAL, 0, "Polynomial order for spatial variations", + kernel->spatialOrder); +#endif + + psMetadataAddS32(row, PS_LIST_TAIL, NAME_BG, 0, "Polynomial order for background fitting", + kernel->bgOrder); + psMetadataAddS32(row, PS_LIST_TAIL, NAME_MODE, 0, "Matching mode (enum)", kernel->mode); + psMetadataAddS32(row, PS_LIST_TAIL, NAME_COLS, 0, "Number of columns", kernel->numCols); + psMetadataAddS32(row, PS_LIST_TAIL, NAME_ROWS, 0, "Number of rows", kernel->numRows); + if (kernel->mode == PM_SUBTRACTION_MODE_1 || kernel->mode == PM_SUBTRACTION_MODE_DUAL) { + psMetadataAddVector(row, PS_LIST_TAIL, NAME_SOL1, 0, "Solution vector 1", kernel->solution1); + } + if (kernel->mode == PM_SUBTRACTION_MODE_2 || kernel->mode == PM_SUBTRACTION_MODE_DUAL) { + psMetadataAddVector(row, PS_LIST_TAIL, NAME_SOL2, 0, "Solution vector 2", kernel->solution2); + } + } + psFree(regions); + psFree(kernels); + + psMetadata *header = psMetadataAlloc(); // Header for FITS file + + // CVS tags, used to identify the version of this file (in case incompatibilities are introduced) + psString cvsFile = psStringCopy("$RCSfile: pmSubtractionIO.c,v $"); + psString cvsRev = psStringCopy("$Revision: 1.1 $"); + psString cvsDate = psStringCopy("$Date: 2008-06-16 21:59:43 $"); + psStringSubstitute(&cvsFile, NULL, "RCSfile: "); + psStringSubstitute(&cvsRev, NULL, "Revision: "); + psStringSubstitute(&cvsDate, NULL, "Date: "); + + psString version = NULL; // Version information, for header + psStringAppend(&version, "%s %s %s", cvsFile, cvsRev, cvsDate); + psFree(cvsFile); + psFree(cvsRev); + psFree(cvsDate); + psStringSubstitute(&version, NULL, "$"); + psMetadataAddStr(header, PS_LIST_TAIL, "PSVERSION", 0, "S/W version", version); + psFree(version); + + psString extname = subtractionKernelExtname(ro); // Extension name + + if (!psFitsWriteTable(fits, header, rows, extname)) { + psError(PS_ERR_IO, false, "Unable to write subtraction kernel to FITS table."); + psFree(header); + psFree(rows); + psFree(extname); + return false; + } + + psFree(header); + psFree(rows); + psFree(extname); + + return true; +} + + +bool pmReadoutReadSubtractionKernels(pmReadout *ro, psFits *fits) +{ + PM_ASSERT_READOUT_NON_NULL(ro, false); + PS_ASSERT_FITS_NON_NULL(fits, false); + + psString extname = subtractionKernelExtname(ro); // Extension name + if (!psFitsMoveExtName(fits, extname)) { + psError(PS_ERR_IO, false, "Unable to move to subtraction kernel table."); + psFree(extname); + return false; + } + psFree(extname); + + psArray *table = psFitsReadTable(fits); // Table of interest + if (!table) { + psError(PS_ERR_IO, false, "Unable to read FITS table"); + return false; + } + + // Look up a column value for a row +#define TABLE_LOOKUP(TYPE, SUFFIX, TARGET, NAME) \ + TYPE TARGET; \ + { \ + bool mdok; \ + TARGET = psMetadataLookup##SUFFIX(&mdok, row, NAME); \ + if (!mdok) { \ + psError(PS_ERR_UNKNOWN, false, "Unable to find column %s in subtraction kernel table.", NAME); \ + psFree(table); \ + return false; \ + } \ + } + + for (int i = 0; i < table->n; i++) { + psMetadata *row = table->data[i]; // Table row + + TABLE_LOOKUP(int, S32, xMin, NAME_XMIN); + TABLE_LOOKUP(int, S32, xMax, NAME_XMAX); + TABLE_LOOKUP(int, S32, yMin, NAME_YMIN); + TABLE_LOOKUP(int, S32, yMax, NAME_YMAX); + + psRegion *region = psRegionAlloc(xMin, xMax, yMin, yMax); // Region of applicability + psMetadataAddPtr(ro->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_REGION, PS_DATA_REGION, + "Subtraction region", region); + psFree(region); + + TABLE_LOOKUP(const char *, Str, description, NAME_KERNEL); + TABLE_LOOKUP(pmSubtractionMode, S32, mode, NAME_MODE); + +#if 0 + TABLE_LOOKUP(int, S32, size, NAME_SIZE); + TABLE_LOOKUP(int, S32, inner, NAME_INNER); + TABLE_LOOKUP(int, S32, spatial, NAME_SPATIAL); +#endif + + TABLE_LOOKUP(int, S32, bg, NAME_BG); + TABLE_LOOKUP(int, S32, numCols, NAME_COLS); + TABLE_LOOKUP(int, S32, numRows, NAME_ROWS); + + pmSubtractionKernels *kernels = pmSubtractionKernelsFromDescription(description, bg, mode); + kernels->numCols = numCols; + kernels->numRows = numRows; + + bool mdok; // Status of MD lookup + if (mode == PM_SUBTRACTION_MODE_1 || mode == PM_SUBTRACTION_MODE_DUAL) { + kernels->solution1 = psMemIncrRefCounter(psMetadataLookupPtr(&mdok, row, NAME_SOL1)); + if (!mdok) { + psError(PS_ERR_UNKNOWN, false, "Unable to find column %s in subtraction kernel table.", + NAME_SOL1); + psFree(kernels); + psFree(table); + return false; + } + } + if (mode == PM_SUBTRACTION_MODE_2 || mode == PM_SUBTRACTION_MODE_DUAL) { + kernels->solution2 = psMemIncrRefCounter(psMetadataLookupPtr(&mdok, row, NAME_SOL2)); + if (!mdok) { + psError(PS_ERR_UNKNOWN, false, "Unable to find column %s in subtraction kernel table.", + NAME_SOL2); + psFree(kernels); + psFree(table); + return false; + } + } + + psMetadataAddPtr(ro->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_KERNEL, PS_DATA_UNKNOWN, + "Subtraction kernels", kernels); + psFree(kernels); + } + psFree(table); + return true; +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionIO.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionIO.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionIO.h (revision 18170) @@ -0,0 +1,22 @@ +#ifndef PM_SUBTRACTION_IO_H +#define PM_SUBTRACTION_IO_H + +#include + +#include +#include + +/// Write subtraction kernels within a readout to a FITS file +bool pmReadoutWriteSubtractionKernels( + pmReadout *readout, ///< Readout for which to write subtraction kernels (in analysis MD) + psFits *fits ///< FITS file to which to write + ); + + +bool pmReadoutReadSubtractionKernels( + pmReadout *readout, ///< Readout for which to read subtraction kernels (into analysis MD) + psFits *fits ///< FITS file to which to write + ); + + +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionKernels.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionKernels.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionKernels.c (revision 18170) @@ -0,0 +1,728 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmSubtraction.h" +#include "pmSubtractionKernels.h" + +#define RINGS_BUFFER 10 // Buffer size for RINGS data + + +// Free function for pmSubtractionKernels +static void subtractionKernelsFree(pmSubtractionKernels *kernels) +{ + psFree(kernels->description); + psFree(kernels->u); + psFree(kernels->v); + psFree(kernels->widths); + psFree(kernels->uStop); + psFree(kernels->vStop); + psFree(kernels->preCalc); + psFree(kernels->solution1); + psFree(kernels->solution2); +} + +// Raise an integer to an integer power +static inline long power(int value, // Value + int exp // Exponent + ) +{ + if (exp == 0) { + return 1.0; + } + long result = value; // Result to return + for (int i = 2; i <= exp; i++) { + result *= value; + } + return result; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Semi-public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +bool p_pmSubtractionKernelsAddGrid(pmSubtractionKernels *kernels, int start, int size) +{ + PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(kernels, false); + PS_ASSERT_INT_NONNEGATIVE(start, false); + PS_ASSERT_INT_NONNEGATIVE(size, false); + + int numNew = PS_SQR(2 * size + 1) - 1; // Number of new kernel parameters to add + + // Ensure the sizes match + kernels->widths = psVectorRealloc(kernels->widths, start + numNew); + kernels->u = psVectorRealloc(kernels->u, start + numNew); + kernels->v = psVectorRealloc(kernels->v, start + numNew); + kernels->preCalc = psArrayRealloc(kernels->preCalc, start + numNew); + kernels->inner = start; + + // Generate a set of kernels for each (u,v) + for (int v = - size, index = start; v <= size; v++) { + for (int u = - size; u <= size; u++, index++) { + if (v == 0 && u == 0) { + // Skip normalisation component: added explicitly + index--; + continue; + } + kernels->widths->data.F32[index] = NAN; + kernels->u->data.S32[index] = u; + kernels->v->data.S32[index] = v; + kernels->preCalc->data[index] = NULL; + + psTrace("psModules.imcombine", 7, "Kernel %d: %d %d\n", index, u, v); + } + } + + return true; +} + +pmSubtractionKernels *p_pmSubtractionKernelsRawISIS(int size, int spatialOrder, + const psVector *fwhms, const psVector *orders, + pmSubtractionMode mode) +{ + PS_ASSERT_VECTOR_NON_NULL(fwhms, NULL); + PS_ASSERT_VECTOR_TYPE(fwhms, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_NON_NULL(orders, NULL); + PS_ASSERT_VECTOR_TYPE(orders, PS_TYPE_S32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(fwhms, orders, NULL); + PS_ASSERT_INT_POSITIVE(size, NULL); + PS_ASSERT_INT_NONNEGATIVE(spatialOrder, NULL); + + int numGaussians = fwhms->n; // Number of Gaussians + + int num = 0; // Number of basis functions + psString params = NULL; // List of parameters + for (int i = 0; i < numGaussians; i++) { + int gaussOrder = orders->data.S32[i]; // Polynomial order to apply to Gaussian + psStringAppend(¶ms, "(%.1f,%d)", fwhms->data.F32[i], orders->data.S32[i]); + num += (gaussOrder + 1) * (gaussOrder + 2) / 2; + } + + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_ISIS, + size, spatialOrder, mode); // The kernels + psStringAppend(&kernels->description, "ISIS(%d,%s,%d)", size, params, spatialOrder); + + psLogMsg("psModules.imcombine", PS_LOG_INFO, "ISIS kernel: %s,%d --> %d elements", + params, spatialOrder, num); + psFree(params); + + // Set the kernel parameters + for (int i = 0, index = 0; i < numGaussians; i++) { + float sigma = fwhms->data.F32[i] / (2.0 * sqrtf(2.0 * logf(2.0))); // Gaussian sigma + float norm = 1.0 / (M_2_PI * sqrtf(sigma)); // Normalisation for Gaussian + // Iterate over (u,v) order + for (int uOrder = 0; uOrder <= orders->data.S32[i]; uOrder++) { + for (int vOrder = 0; vOrder <= orders->data.S32[i] - uOrder; vOrder++, index++) { + // Set the pre-calculated kernel + psKernel *preCalc = psKernelAlloc(-size, size, -size, size); + double sum = 0.0; // Normalisation + for (int v = -size; v <= size; v++) { + for (int u = -size; u <= size; u++) { + sum += preCalc->kernel[v][u] = norm * power(u, uOrder) * power(v, vOrder) * + expf(-0.5 * (PS_SQR(u) + PS_SQR(v)) / PS_SQR(sigma)); + } + } + + // Normalise sum of kernel component to unity for even functions + if (uOrder % 2 == 0 && vOrder % 2 == 0) { + for (int v = -size; v <= size; v++) { + for (int u = -size; u <= size; u++) { + preCalc->kernel[v][u] = preCalc->kernel[v][u] / sum; + } + } + preCalc->kernel[0][0] -= 1.0; + } + + kernels->widths->data.F32[index] = fwhms->data.F32[i]; + kernels->u->data.S32[index] = uOrder; + kernels->v->data.S32[index] = vOrder; + if (kernels->preCalc->data[index]) { + psFree(kernels->preCalc->data[index]); + } + kernels->preCalc->data[index] = preCalc; + + psTrace("psModules.imcombine", 7, "Kernel %d: %f %d %d\n", index, + fwhms->data.F32[i], uOrder, vOrder); + } + } + } + + return kernels; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pmSubtractionKernels *pmSubtractionKernelsAlloc(int numBasisFunctions, pmSubtractionKernelsType type, + int size, int spatialOrder, pmSubtractionMode mode) +{ + pmSubtractionKernels *kernels = psAlloc(sizeof(pmSubtractionKernels)); // Kernels, to return + psMemSetDeallocator(kernels, (psFreeFunc)subtractionKernelsFree); + + kernels->type = type; + kernels->description = NULL; + kernels->num = numBasisFunctions; + kernels->u = psVectorAlloc(numBasisFunctions, PS_TYPE_S32); + kernels->v = psVectorAlloc(numBasisFunctions, PS_TYPE_S32); + kernels->widths = psVectorAlloc(numBasisFunctions, PS_TYPE_F32); + kernels->preCalc = psArrayAlloc(numBasisFunctions); + kernels->uStop = NULL; + kernels->vStop = NULL; + kernels->size = size; + kernels->inner = 0; + kernels->spatialOrder = spatialOrder; + kernels->bgOrder = 0; + kernels->mode = mode; + kernels->numCols = 0; + kernels->numRows = 0; + kernels->solution1 = NULL; + kernels->solution2 = NULL; + + return kernels; +} + +pmSubtractionKernels *pmSubtractionKernelsPOIS(int size, int spatialOrder, pmSubtractionMode mode) +{ + PS_ASSERT_INT_POSITIVE(size, NULL); + PS_ASSERT_INT_NONNEGATIVE(spatialOrder, NULL); + + int num = PS_SQR(2 * size + 1) - 1; // Number of basis functions + + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_POIS, + size, spatialOrder, mode); // The kernels + psStringAppend(&kernels->description, "POIS(%d,%d)", size, spatialOrder); + psLogMsg("psModules.imcombine", PS_LOG_INFO, "POIS kernel: %d,%d --> %d elements", + size, spatialOrder, num); + + if (!p_pmSubtractionKernelsAddGrid(kernels, 0, size)) { + psAbort("Should never get here."); + } + + return kernels; +} + + +pmSubtractionKernels *pmSubtractionKernelsISIS(int size, int spatialOrder, + const psVector *fwhms, const psVector *orders, + pmSubtractionMode mode) +{ + pmSubtractionKernels *kernels = p_pmSubtractionKernelsRawISIS(size, spatialOrder, + fwhms, orders, mode); // Kernels + if (!kernels) { + return NULL; + } + + if (psTraceGetLevel("psModules.imcombine.kernel") >= 10) { + for (int i = 0; i < kernels->num; i++) { + psKernel *kernel = kernels->preCalc->data[i]; // Kernel of interest + psString kernelName = NULL; + psStringAppend(&kernelName, "kernel%03d.fits", i); + psFits *kernelFile = psFitsOpen(kernelName, "w"); + psFree(kernelName); + psFitsWriteImage(kernelFile, NULL, kernel->image, 0, NULL); + psFitsClose(kernelFile); + double sum = 0.0; + for (int y = 0; y < kernel->image->numRows; y++) { + for (int x = 0; x < kernel->image->numCols; x++) { + sum += kernel->image->data.F32[y][x]; + } + } + psTrace("psModules.imcombine.kernel", 10, "Kernel %d sum: %le\n", i, sum); + } + } + + return kernels; +} + +pmSubtractionKernels *pmSubtractionKernelsSPAM(int size, int spatialOrder, int inner, int binning, + pmSubtractionMode mode) +{ + PS_ASSERT_INT_POSITIVE(size, NULL); + PS_ASSERT_INT_NONNEGATIVE(spatialOrder, NULL); + PS_ASSERT_INT_NONNEGATIVE(inner, NULL); + PS_ASSERT_INT_LARGER_THAN(size, inner, NULL); + PS_ASSERT_INT_POSITIVE(binning, NULL); + + // The outer region should be divisible by the "binning"; otherwise allocate remainder to the inner region + int numOuter = (size - inner) / binning; // Number of summed pixels in the outer region + int numInner = inner + (size - inner) % binning; // Number of pixels in the inner region + assert(numOuter * binning + numInner == size); + int numTotal = numOuter + numInner; // Total number of summed pixels + + psTrace("psModules.imcombine", 3, "Inner: %d Outer: %d\n", numInner, numOuter); + + int num = PS_SQR(2 * numTotal + 1) - 1; // Number of basis functions + + psTrace("psModules.imcombine", 3, "Number of basis functions: %d\n", num); + + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_SPAM, + size, spatialOrder, mode); // The kernels + psStringAppend(&kernels->description, "SPAM(%d,%d,%d,%d)", size, inner, binning, spatialOrder); + + psLogMsg("psModules.imcombine", PS_LOG_INFO, "SPAM kernel: %d,%d,%d,%d --> %d elements", + size, inner, binning, spatialOrder, num); + + kernels->uStop = psVectorAlloc(num, PS_TYPE_S32); + kernels->vStop = psVectorAlloc(num, PS_TYPE_S32); + + psVector *locations = psVectorAlloc(2 * numTotal + 1, PS_TYPE_S32); // Locations for each kernel element + psVector *widths = psVectorAlloc(2 * numTotal + 1, PS_TYPE_S32); // Widths for each kernel element + locations->data.S32[numTotal] = 0; + widths->data.S32[numTotal] = 0; + for (int i = 1; i <= numInner; i++) { + locations->data.S32[numTotal + i] = i; + widths->data.S32[numTotal + i] = 0; + locations->data.S32[numTotal - i] = - i; + widths->data.S32[numTotal - i] = 0; + } + for (int i = numInner + 1; i <= numTotal; i++) { + locations->data.S32[numTotal + i] = locations->data.S32[numTotal + i - 1] + + widths->data.S32[numTotal + i - 1] + 1; + widths->data.S32[numTotal + i] = binning - 1; + locations->data.S32[numTotal - i] = locations->data.S32[numTotal - i + 1] - binning; + widths->data.S32[numTotal - i] = binning - 1; + } + + if (psTraceGetLevel("psModules.imcombine") >= 10) { + for (int i = 0; i < 2 * numTotal + 1; i++) { + psTrace("psModules.imcombine", 10, "%d: %d -> %d\n", i, locations->data.S32[i], + locations->data.S32[i] + widths->data.S32[i]); + } + } + + // Set the kernel parameters + for (int i = - numTotal, index = 0; i <= numTotal; i++) { + int u = locations->data.S32[numTotal + i]; // Location of pixel + int uStop = u + widths->data.S32[numTotal + i]; // Width of pixel + + for (int j = - numTotal; j <= numTotal; j++, index++) { + if (i == 0 && j == 0) { + // Skip normalisation component: added explicitly + index--; + continue; + } + int v = locations->data.S32[numTotal + j]; // Location of pixel + int vStop = v + widths->data.S32[numTotal + j]; // Width of pixel + + kernels->u->data.S32[index] = u; + kernels->v->data.S32[index] = v; + kernels->uStop->data.S32[index] = uStop; + kernels->vStop->data.S32[index] = vStop; + + psTrace("psModules.imcombine", 7, "Kernel %d: %d %d %d %d\n", index, + u, uStop, v, vStop); + } + } + + psFree(locations); + psFree(widths); + + return kernels; +} + + +pmSubtractionKernels *pmSubtractionKernelsFRIES(int size, int spatialOrder, int inner, pmSubtractionMode mode) +{ + PS_ASSERT_INT_POSITIVE(size, NULL); + PS_ASSERT_INT_NONNEGATIVE(spatialOrder, NULL); + PS_ASSERT_INT_NONNEGATIVE(inner, NULL); + PS_ASSERT_INT_LARGER_THAN(size, inner, NULL); + + int fibNum = 0; // Number of Fibonacci values + int fibLast = 1, fibTotal = 2; // Fibonacci sequence + while (fibTotal < size - inner) { + int temp = fibTotal; + fibTotal += fibLast; + fibLast = temp; + fibNum++; + } + + int numInner = inner; // Number of pixels in the inner region + int numOuter = fibNum; // Number of summed pixels in the outer region + int numTotal = numOuter + numInner; // Total number of summed pixels + + psTrace("psModules.imcombine", 3, "Inner: %d Outer: %d\n", numInner, numOuter); + + int num = PS_SQR(2 * numTotal + 1) - 1; // Number of basis functions + + psTrace("psModules.imcombine", 3, "Number of basis functions: %d\n", num); + + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_FRIES, + size, spatialOrder, mode); // The kernels + psStringAppend(&kernels->description, "FRIES(%d,%d,%d)", size, inner, spatialOrder); + + psLogMsg("psModules.imcombine", PS_LOG_INFO, "FRIES kernel: %d,%d,%d --> %d elements", + size, inner, spatialOrder, num); + + kernels->uStop = psVectorAlloc(num, PS_TYPE_S32); + kernels->vStop = psVectorAlloc(num, PS_TYPE_S32); + + psVector *start = psVectorAlloc(2 * numTotal + 1, PS_TYPE_S32); + psVector *stop = psVectorAlloc(2 * numTotal + 1, PS_TYPE_S32); + start->data.S32[numTotal] = 0; + stop->data.S32[numTotal] = 0; + for (int i = 1; i <= numInner; i++) { + start->data.S32[numTotal + i] = i; + stop->data.S32[numTotal + i] = i; + start->data.S32[numTotal - i] = -i; + stop->data.S32[numTotal - i] = -i; + } + for (int i = numInner + 1, fibLast = 1, fib = 2, temp; i <= numTotal; + i++, fib = (temp = fib) + fibLast, fibLast = temp) { + start->data.S32[numTotal + i] = stop->data.S32[numTotal + i - 1] + 1; + stop->data.S32[numTotal + i] = PS_MIN(start->data.S32[numTotal + i] + fib - 1, size); + start->data.S32[numTotal - i] = - stop->data.S32[numTotal + i]; + stop->data.S32[numTotal - i] = - start->data.S32[numTotal + i]; + } + + if (psTraceGetLevel("psModules.imcombine") >= 10) { + for (int i = 0; i < 2 * numTotal + 1; i++) { + psTrace("psModules.imcombine", 10, "%d: %d -> %d\n", i, start->data.S32[i], stop->data.S32[i]); + } + } + + // Set the kernel parameters + for (int i = - numTotal, index = 0; i <= numTotal; i++) { + int u = start->data.S32[numTotal + i]; // Location of pixel + int uStop = stop->data.S32[numTotal + i]; // Width of pixel + for (int j = - numTotal; j <= numTotal; j++, index++) { + if (i == 0 && j == 0) { + // Skip normalisation component: added explicitly + index--; + continue; + } + int v = start->data.S32[numTotal + j]; // Location of pixel + int vStop = stop->data.S32[numTotal + j]; // Width of pixel + + kernels->u->data.S32[index] = u; + kernels->v->data.S32[index] = v; + kernels->uStop->data.S32[index] = uStop; + kernels->vStop->data.S32[index] = vStop; + + psTrace("psModules.imcombine", 7, "Kernel %d: %d %d %d %d\n", index, + u, uStop, v, vStop); + } + } + + psFree(start); + psFree(stop); + + return kernels; +} + +// Grid United with Normal Kernel +pmSubtractionKernels *pmSubtractionKernelsGUNK(int size, int spatialOrder, const psVector *fwhms, + const psVector *orders, int inner, pmSubtractionMode mode) +{ + PS_ASSERT_INT_POSITIVE(size, NULL); + PS_ASSERT_INT_NONNEGATIVE(spatialOrder, NULL); + PS_ASSERT_VECTOR_NON_NULL(fwhms, NULL); + PS_ASSERT_VECTOR_TYPE(fwhms, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_NON_NULL(orders, NULL); + PS_ASSERT_VECTOR_TYPE(orders, PS_TYPE_S32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(fwhms, orders, NULL); + PS_ASSERT_INT_NONNEGATIVE(inner, NULL); + PS_ASSERT_INT_LESS_THAN(inner, size, NULL); + + pmSubtractionKernels *kernels = p_pmSubtractionKernelsRawISIS(size, spatialOrder, + fwhms, orders, mode); // Kernels + psStringPrepend(&kernels->description, "GUNK="); + psStringAppend(&kernels->description, "+POIS(%d,%d)", inner, spatialOrder); + + int numISIS = kernels->num; // Number of ISIS kernels + + if (!p_pmSubtractionKernelsAddGrid(kernels, numISIS, inner)) { + psAbort("Should never get here."); + } + + return kernels; +} + +// RINGS --- just what it says +pmSubtractionKernels *pmSubtractionKernelsRINGS(int size, int spatialOrder, int inner, int ringsOrder, + pmSubtractionMode mode) +{ + PS_ASSERT_INT_POSITIVE(size, NULL); + PS_ASSERT_INT_NONNEGATIVE(spatialOrder, NULL); + PS_ASSERT_INT_NONNEGATIVE(inner, NULL); + PS_ASSERT_INT_LESS_THAN_OR_EQUAL(inner, size, NULL); + PS_ASSERT_INT_NONNEGATIVE(ringsOrder, NULL); + + int fibNum = 0; // Number of Fibonacci values + { + int fibIndex = 1, fibIndexMinus1 = 0; // Fibonnacci parameters + int radius = inner; + while (radius + fibIndex < size) { + radius++; + int fibNew = fibIndex + fibIndexMinus1; + fibIndexMinus1 = fibIndex; + fibIndex = fibNew; + radius += fibIndex; + fibNum++; + } + } + + int numInner = inner - 1; // Number of pixels in the inner region + int numOuter = fibNum; // Number of summed pixels in the outer region + + int numRings = numOuter + numInner; // Number of rings (not including the central pixel) + int numPoly = PM_SUBTRACTION_POLYTERMS(ringsOrder); // Number of polynomial variants of each ring + + int num = numRings * numPoly; // Total number of basis functions + + pmSubtractionKernels *kernels = pmSubtractionKernelsAlloc(num, PM_SUBTRACTION_KERNEL_RINGS, + size, spatialOrder, mode); // The kernels + psStringAppend(&kernels->description, "RINGS(%d,%d,%d,%d)", size, inner, ringsOrder, spatialOrder); + + psLogMsg("psModules.imcombine", PS_LOG_INFO, "RINGS kernel: %d,%d,%d,%d --> %d elements", + size, inner, ringsOrder, spatialOrder, num); + + // Set the Gaussian kernel parameters + int fibIndex = 1, fibIndexMinus1 = 0; // Fibonnacci parameters + int radiusLast = 1; // Last radius + for (int i = 1, index = 0; i < numRings + 1; i++) { + float lower2; // Lower limit of radius^2 + float upper2; // Upper limit of radius^2 + if (i <= inner) { + // A ring every pixel width + float radius = i; + lower2 = PS_SQR(radius - 0.5); + upper2 = PS_SQR(radius + 0.5); + radiusLast = i; + } else { + // Rings Fibonacci distributed (2, 3, 5...) + int fibNew = fibIndex + fibIndexMinus1; + fibIndexMinus1 = fibIndex; + fibIndex = fibNew; + + float radiusLower = radiusLast + 1; + radiusLast = radiusLower + fibIndex; + float radiusUpper = radiusLast; + + lower2 = PS_SQR(radiusLower - 0.5); + upper2 = PS_SQR(radiusUpper + 0.5); + } + + psTrace("psModules.imcombine", 8, "Radius limits: %f --> %f\n", sqrtf(lower2), sqrtf(upper2)); + + // Iterate over (u,v) order + for (int uOrder = 0; uOrder <= (i == 0 ? 0 : ringsOrder); uOrder++) { + for (int vOrder = 0; vOrder <= (i == 0 ? 0 : ringsOrder - uOrder); vOrder++, index++) { + + psArray *data = psArrayAlloc(3); // Container for data + psVector *uCoords = data->data[0] = psVectorAllocEmpty(RINGS_BUFFER, PS_TYPE_S32); // u coords + psVector *vCoords = data->data[1] = psVectorAllocEmpty(RINGS_BUFFER, PS_TYPE_S32); // v coords + psVector *poly = data->data[2] = psVectorAllocEmpty(RINGS_BUFFER, PS_TYPE_F32); // Polynomial + + if (i == 0) { + // Central pixel is easy + uCoords->data.S32[0] = vCoords->data.S32[0] = 0; + poly->data.F32[0] = 1.0; + uCoords->n = vCoords->n = poly->n = 1; + radiusLast = 0; + } else { + int j = 0; // Index for data + double norm = 0.0; // Normalisation + for (int v = -size; v <= size; v++) { + int v2 = PS_SQR(v); // Square of v + float vPoly = powf(v/(float)size, vOrder); // Value of v^vOrder + + for (int u = -size; u <= size; u++) { + int u2 = PS_SQR(u); // Square of u + int distance2 = u2 + v2; // Distance from the centre + if (distance2 > lower2 && distance2 < upper2) { + float uPoly = powf(u/(float)size, uOrder); // Value of u^uOrder + + float polyVal = uPoly * vPoly; // Value of polynomial + if (polyVal != 0) { // No point adding it otherwise + uCoords->data.S32[j] = u; + vCoords->data.S32[j] = v; + poly->data.F32[j] = polyVal; + norm += polyVal; + + psVectorExtend(uCoords, RINGS_BUFFER, 1); + psVectorExtend(vCoords, RINGS_BUFFER, 1); + psVectorExtend(poly, RINGS_BUFFER, 1); + psTrace("psModules.imcombine", 9, "u = %d, v = %d, poly = %f\n", + u, v, poly->data.F32[j]); + j++; + } + } + } + } + // Normalise kernel component to unit sum + if (uOrder % 2 == 0 && vOrder % 2 == 0) { + psBinaryOp(poly, poly, "*", psScalarAlloc(1.0 / norm, PS_TYPE_F32)); + // Add subtraction of 0,0 component to preserve photometric scaling + uCoords->data.S32[j] = 0; + vCoords->data.S32[j] = 0; + poly->data.F32[j] = -1.0; + psVectorExtend(uCoords, RINGS_BUFFER, 1); + psVectorExtend(vCoords, RINGS_BUFFER, 1); + psVectorExtend(poly, RINGS_BUFFER, 1); + } else { + norm = powf(size, uOrder) * powf(size, vOrder); + psBinaryOp(poly, poly, "*", psScalarAlloc(1.0 / norm, PS_TYPE_F32)); + } + } + + psTrace("psModules.imcombine", 8, "%ld pixels in kernel\n", uCoords->n); + + kernels->preCalc->data[index] = data; + kernels->u->data.S32[index] = uOrder; + kernels->v->data.S32[index] = vOrder; + + psTrace("psModules.imcombine", 7, "Kernel %d: %d %d %d\n", index, + i, uOrder, vOrder); + } + } + } + + return kernels; +} + +pmSubtractionKernels *pmSubtractionKernelsGenerate(pmSubtractionKernelsType type, int size, int spatialOrder, + const psVector *fwhms, const psVector *orders, int inner, + int binning, int ringsOrder, pmSubtractionMode mode) +{ + switch (type) { + case PM_SUBTRACTION_KERNEL_POIS: + return pmSubtractionKernelsPOIS(size, spatialOrder, mode); + case PM_SUBTRACTION_KERNEL_ISIS: + return pmSubtractionKernelsISIS(size, spatialOrder, fwhms, orders, mode); + case PM_SUBTRACTION_KERNEL_SPAM: + return pmSubtractionKernelsSPAM(size, spatialOrder, inner, binning, mode); + case PM_SUBTRACTION_KERNEL_FRIES: + return pmSubtractionKernelsFRIES(size, spatialOrder, inner, mode); + case PM_SUBTRACTION_KERNEL_GUNK: + return pmSubtractionKernelsGUNK(size, spatialOrder, fwhms, orders, inner, mode); + case PM_SUBTRACTION_KERNEL_RINGS: + return pmSubtractionKernelsRINGS(size, spatialOrder, inner, ringsOrder, mode); + default: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unknown kernel type: %x", type); + return NULL; + } +} + + +// Intermediate string parsing functions required because of different APIs for strtol and strtof +static inline int parseStringInt(const char *string) +{ + return strtol(string, NULL, 10); +} +static inline float parseStringFloat(const char *string) +{ + return strtof(string, NULL); +} + + +// Parse a string of a number, up to some delimiter, and advance past the delimiter +#define PARSE_STRING_NUMBER(TARGET, STRING, DELIM, PARSEFUNC) { \ + char *start = STRING; /* Start of string */ \ + char *end = strchr(STRING, DELIM); /* End of string */ \ + if (!end) { \ + psAbort("End of string encountered"); \ + } \ + int stringSize = end - STRING; /* Size of string with value, NOT including \0 */ \ + char value[stringSize + 1]; /* String to parse */ \ + strncpy(value, start, stringSize); \ + value[stringSize] = '\0'; \ + TARGET = PARSEFUNC(value); \ + STRING += stringSize + 1; /* Advance past delimiter */ \ +} + + +pmSubtractionKernels *pmSubtractionKernelsFromDescription(const char *description, int bgOrder, + pmSubtractionMode mode) +{ + PS_ASSERT_STRING_NON_EMPTY(description, NULL); + + if (bgOrder != 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Background order %d is not yet supported.", bgOrder); + return false; + } + + pmSubtractionKernelsType type = PM_SUBTRACTION_KERNEL_NONE; // Type of kernel + int size = 0; // Half-size of kernel + int spatialOrder = 0; // Order of spatial variations + const psVector *fwhms = NULL; // FWHM of Gaussians + const psVector *orders = NULL; // Polynomial order for each FWHM + int inner = 0; // Size of inner region + int binning = 0; // Binning to use + int ringsOrder = 0; // Polynomial order for rings + + if (strncmp(description, "ISIS", 4) == 0) { + // XXX Support for GUNK + if (strstr(description, "+POIS")) { + type = PM_SUBTRACTION_KERNEL_GUNK; + psAbort("Deciphering GUNK kernels (%s) is not currently supported.", description); + } else { + type = PM_SUBTRACTION_KERNEL_ISIS; + char *ptr = (char*)description + 5; // Eat "ISIS(" + PARSE_STRING_NUMBER(size, ptr, ',', parseStringInt); + + // Count the number of Gaussians + int numGauss = 0; + for (char *string = ptr; string; string = strchr(string, '(')) { + numGauss++; + } + + fwhms = psVectorAlloc(numGauss, PS_TYPE_F32); + orders = psVectorAlloc(numGauss, PS_TYPE_S32); + + for (int i = 0; i < numGauss; i++) { + ptr++; // Eat the '(' + PARSE_STRING_NUMBER(fwhms->data.F32[i], ptr, ',', parseStringFloat); // Eat "1.234," + PARSE_STRING_NUMBER(orders->data.S32[i], ptr, ')', parseStringInt); // Eat "3)" + } + + ptr++; // Eat ',' + spatialOrder = parseStringInt(ptr); + } + } else if (strncmp(description, "RINGS", 5) == 0) { + type = PM_SUBTRACTION_KERNEL_RINGS; + char *ptr = (char*)description + 6; + PARSE_STRING_NUMBER(size, ptr, ',', parseStringInt); + PARSE_STRING_NUMBER(inner, ptr, ',', parseStringInt); + PARSE_STRING_NUMBER(ringsOrder, ptr, ',', parseStringInt); + PARSE_STRING_NUMBER(spatialOrder, ptr, ')', parseStringInt); + } else { + psAbort("Deciphering kernels other than ISIS and RINGS is not currently supported."); + } + + + return pmSubtractionKernelsGenerate(type, size, spatialOrder, fwhms, orders, + inner, binning, ringsOrder, mode); +} + + +pmSubtractionKernelsType pmSubtractionKernelsTypeFromString(const char *type) +{ + if (strcasecmp(type, "POIS") == 0) { + return PM_SUBTRACTION_KERNEL_POIS; + } + if (strcasecmp(type, "ISIS") == 0) { + return PM_SUBTRACTION_KERNEL_ISIS; + } + if (strcasecmp(type, "SPAM") == 0) { + return PM_SUBTRACTION_KERNEL_SPAM; + } + if (strcasecmp(type, "FRIES") == 0) { + return PM_SUBTRACTION_KERNEL_FRIES; + } + if (strcasecmp(type, "GUNK") == 0) { + return PM_SUBTRACTION_KERNEL_GUNK; + } + if (strcasecmp(type, "RINGS") == 0) { + return PM_SUBTRACTION_KERNEL_RINGS; + } + + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unrecognised kernel type: %s", type); + return PM_SUBTRACTION_KERNEL_NONE; +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionKernels.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionKernels.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionKernels.h (revision 18170) @@ -0,0 +1,191 @@ +#ifndef PM_SUBTRACTION_KERNELS_H +#define PM_SUBTRACTION_KERNELS_H + +#include + +/// Type of subtraction kernel +typedef enum { + PM_SUBTRACTION_KERNEL_NONE, ///< Nothing --- an error + PM_SUBTRACTION_KERNEL_POIS, ///< Pan-STARRS Optimal Image Subtraction --- delta functions + PM_SUBTRACTION_KERNEL_ISIS, ///< Traditional kernel --- gaussians modified by polynomials + PM_SUBTRACTION_KERNEL_SPAM, ///< Summed Pixels for Advanced Matching --- summed delta functions + PM_SUBTRACTION_KERNEL_FRIES, ///< Fibonacci Radius Increases Excellence of Subtraction + PM_SUBTRACTION_KERNEL_GUNK, ///< Grid United with Normal Kernel --- POIS and ISIS hybrid + PM_SUBTRACTION_KERNEL_RINGS, ///< Rings Instead of the Normal Gaussian Subtraction +} pmSubtractionKernelsType; + +/// Modes --- specifies which image to convolve +typedef enum { + PM_SUBTRACTION_MODE_ERR, // Error in the mode + PM_SUBTRACTION_MODE_1, // Convolve image 1 + PM_SUBTRACTION_MODE_2, // Convolve image 2 + PM_SUBTRACTION_MODE_UNSURE, // Not sure yet which image to convolve so try to satisfy both + PM_SUBTRACTION_MODE_DUAL, // Dual convolution +} pmSubtractionMode; + +/// Kernels specification +typedef struct { + pmSubtractionKernelsType type; ///< Type of kernels --- allowing the use of multiple kernels + psString description; ///< Description of the kernel parameters + long num; ///< Number of kernel components (not including the spatial ones) + psVector *u, *v; ///< Offset (for POIS) or polynomial order (for ISIS) + psVector *widths; ///< Gaussian FWHMs (ISIS) + psVector *uStop, *vStop; ///< Width of kernel element (SPAM,FRIES only) + psArray *preCalc; ///< Array of images containing pre-calculated kernel (for ISIS) + int size; ///< The half-size of the kernel + int inner; ///< The size of an inner region + int spatialOrder; ///< The spatial order of the kernels + int bgOrder; ///< The order for the background fitting + pmSubtractionMode mode; ///< Mode for subtraction + int numCols, numRows; ///< Size of image (for normalisation), or zero to use image provided + psVector *solution1, *solution2; ///< Solution for the PSF matching +} pmSubtractionKernels; + +// Assertion to check pmSubtractionKernels +#define PM_ASSERT_SUBTRACTION_KERNELS_NON_NULL(KERNELS, RETURNVALUE) { \ + PS_ASSERT_PTR_NON_NULL(KERNELS, RETURNVALUE); \ + PS_ASSERT_STRING_NON_EMPTY((KERNELS)->description, RETURNVALUE); \ + PS_ASSERT_INT_POSITIVE((KERNELS)->num, RETURNVALUE); \ + PS_ASSERT_VECTOR_NON_NULL((KERNELS)->u, RETURNVALUE); \ + PS_ASSERT_VECTOR_NON_NULL((KERNELS)->v, RETURNVALUE); \ + PS_ASSERT_VECTOR_TYPE((KERNELS)->u, PS_TYPE_S32, RETURNVALUE); \ + PS_ASSERT_VECTOR_TYPE((KERNELS)->v, PS_TYPE_S32, RETURNVALUE); \ + PS_ASSERT_VECTOR_SIZE((KERNELS)->u, (KERNELS)->num, RETURNVALUE); \ + PS_ASSERT_VECTOR_SIZE((KERNELS)->v, (KERNELS)->num, RETURNVALUE); \ + if ((KERNELS)->type == PM_SUBTRACTION_KERNEL_ISIS) { \ + PS_ASSERT_VECTOR_NON_NULL((KERNELS)->widths, RETURNVALUE); \ + PS_ASSERT_VECTOR_TYPE((KERNELS)->widths, PS_TYPE_F32, RETURNVALUE); \ + PS_ASSERT_VECTOR_SIZE((KERNELS)->widths, (KERNELS)->num, RETURNVALUE); \ + } \ + if ((KERNELS)->uStop || (KERNELS)->vStop) { \ + PS_ASSERT_VECTOR_NON_NULL((KERNELS)->uStop, RETURNVALUE); \ + PS_ASSERT_VECTOR_NON_NULL((KERNELS)->vStop, RETURNVALUE); \ + PS_ASSERT_VECTOR_TYPE((KERNELS)->uStop, PS_TYPE_S32, RETURNVALUE); \ + PS_ASSERT_VECTOR_TYPE((KERNELS)->vStop, PS_TYPE_S32, RETURNVALUE); \ + PS_ASSERT_VECTOR_SIZE((KERNELS)->uStop, (KERNELS)->num, RETURNVALUE); \ + PS_ASSERT_VECTOR_SIZE((KERNELS)->vStop, (KERNELS)->num, RETURNVALUE); \ + } \ + if ((KERNELS)->preCalc) { \ + PS_ASSERT_ARRAY_NON_NULL((KERNELS)->preCalc, RETURNVALUE); \ + PS_ASSERT_ARRAY_SIZE((KERNELS)->preCalc, (KERNELS)->num, RETURNVALUE); \ + } \ + PS_ASSERT_INT_NONNEGATIVE((KERNELS)->size, RETURNVALUE); \ + PS_ASSERT_INT_NONNEGATIVE((KERNELS)->inner, RETURNVALUE); \ + PS_ASSERT_INT_NONNEGATIVE((KERNELS)->spatialOrder, RETURNVALUE); \ + PS_ASSERT_INT_NONNEGATIVE((KERNELS)->bgOrder, RETURNVALUE); \ +} + +// Assertion to check that the solution is attached +#define PM_ASSERT_SUBTRACTION_KERNELS_SOLUTION(KERNELS, RETURNVALUE) { \ + PS_ASSERT_VECTOR_NON_NULL((KERNELS)->solution1, RETURNVALUE); \ + PS_ASSERT_VECTOR_TYPE((KERNELS)->solution1, PS_TYPE_F64, RETURNVALUE); \ + PS_ASSERT_VECTOR_SIZE((KERNELS)->solution1, \ + (KERNELS)->num * PM_SUBTRACTION_POLYTERMS((KERNELS)->spatialOrder) + 1 + \ + PM_SUBTRACTION_POLYTERMS((KERNELS)->bgOrder), \ + RETURNVALUE); \ + if (kernels->mode == PM_SUBTRACTION_MODE_DUAL) { \ + PS_ASSERT_VECTOR_NON_NULL(kernels->solution2, RETURNVALUE); \ + PS_ASSERT_VECTOR_TYPE((KERNELS)->solution2, PS_TYPE_F64, RETURNVALUE); \ + PS_ASSERT_VECTOR_SIZE((KERNELS)->solution2, \ + (KERNELS)->num * PM_SUBTRACTION_POLYTERMS((KERNELS)->spatialOrder), \ + RETURNVALUE); \ + } \ +} + +/// Generate a delta-function grid for subtraction kernels (like the POIS kernel) +bool p_pmSubtractionKernelsAddGrid(pmSubtractionKernels *kernels, ///< The subtraction kernels to append to + int start, ///< Index at which to start appending + int size ///< Half-size of the grid + ); + +/// General allocator for pmSubtractionKernels +/// +/// Unlike the functions for the specific kernel type, this function does not set up the basis functions, but +/// merely allocates space for their storage. +pmSubtractionKernels *pmSubtractionKernelsAlloc(int numBasisFunctions, ///< Number of basis functions + pmSubtractionKernelsType type, ///< Kernel type + int size, ///< Half-size of kernel + int spatialOrder, ///< Order of spatial variations + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Generate POIS kernels +pmSubtractionKernels *pmSubtractionKernelsPOIS(int size, ///< Half-size of the kernel (in both dims) + int spatialOrder, ///< Order of spatial variations + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Generate ISIS kernels without the flux scaling built in +pmSubtractionKernels *p_pmSubtractionKernelsRawISIS(int size, ///< Half-size of the kernel + int spatialOrder, ///< Order of spatial variations + const psVector *fwhms, ///< Gaussian FWHMs + const psVector *orders, ///< Polynomial order of gaussians + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Generate ISIS kernels +pmSubtractionKernels *pmSubtractionKernelsISIS(int size, ///< Half-size of the kernel + int spatialOrder, ///< Order of spatial variations + const psVector *fwhms, ///< Gaussian FWHMs + const psVector *orders, ///< Polynomial order of gaussians + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Generate SPAM kernels +pmSubtractionKernels *pmSubtractionKernelsSPAM(int size, ///< Half-size of the kernel + int spatialOrder, ///< Order of spatial variations + int inner, ///< Inner radius to preserve unbinned + int binning, ///< Kernel binning factor + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Generate FRIES kernels +pmSubtractionKernels *pmSubtractionKernelsFRIES(int size, ///< Half-size of the kernel + int spatialOrder, ///< Order of spatial variations + int inner, ///< Inner radius to preserve unbinned + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Generate GUNK kernels +pmSubtractionKernels *pmSubtractionKernelsGUNK(int size, ///< Half-size of the kernel + int spatialOrder, ///< Order of spatial variations + const psVector *fwhms, ///< Gaussian FWHMs + const psVector *orders, ///< Polynomial order of gaussians + int inner, ///< Inner radius containing grid of delta functions + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Generate RINGS kernels +pmSubtractionKernels *pmSubtractionKernelsRINGS(int size, ///< Half-size of the kernel + int spatialOrder, ///< Order of spatial variations + int inner, ///< Inner radius to preserve unbinned + int ringsOrder, ///< Polynomial order + pmSubtractionMode mode ///< Mode for subtraction + ); + + +/// Generate a kernel of a specified type +pmSubtractionKernels *pmSubtractionKernelsGenerate(pmSubtractionKernelsType type, ///< Kernel type + int size, ///< Half-size of the kernel + int spatialOrder, ///< Order of spatial variations + const psVector *fwhms, ///< Gaussian FWHMs + const psVector *orders, ///< Polynomial order of gaussians + int inner, ///< Inner radius to preserve unbinned + int binning, ///< Kernel binning factor + int ringsOrder, ///< Polynomial order for RINGS + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Generate a kernel using the description +pmSubtractionKernels *pmSubtractionKernelsFromDescription( + const char *description, ///< Description of kernel + int bgOrder, ///< Polynomial order for background fitting + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Return the appropriate type from a string +pmSubtractionKernelsType pmSubtractionKernelsTypeFromString(const char *string // String name for kernel type + ); + + +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMask.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMask.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMask.c (revision 18170) @@ -0,0 +1,259 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include + +#include "pmSubtraction.h" +#include "pmSubtractionKernels.h" + +#include "pmSubtractionMask.h" + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Private (file-static) functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Mark a pixel as blank in the image, mask and weight +static inline void markBlank(psImage *image, // Image to mark as blank + psImage *mask, // Mask to mark as blank (or NULL) + psImage *weight, // Weight map to mark as blank (or NULL) + int x, int y, // Coordinates to mark blank + psMaskType blank // Blank mask value + ) +{ + image->data.F32[y][x] = NAN; + if (mask) { + mask->data.PS_TYPE_MASK_DATA[y][x] |= blank; + } + if (weight) { + weight->data.F32[y][x] = NAN; + } + return; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +psImage *pmSubtractionMask(const psImage *mask1, const psImage *mask2, psMaskType maskVal, + int size, int footprint, float badFrac, bool useFFT) +{ + PS_ASSERT_IMAGE_NON_NULL(mask1, NULL); + PS_ASSERT_IMAGE_TYPE(mask1, PS_TYPE_MASK, NULL); + if (mask2) { + PS_ASSERT_IMAGE_NON_NULL(mask2, NULL); + PS_ASSERT_IMAGE_TYPE(mask2, PS_TYPE_MASK, NULL); + PS_ASSERT_IMAGES_SIZE_EQUAL(mask2, mask1, NULL); + } + PS_ASSERT_INT_NONNEGATIVE(size, NULL); + PS_ASSERT_INT_NONNEGATIVE(footprint, NULL); + if (isfinite(badFrac)) { + PS_ASSERT_FLOAT_LARGER_THAN(badFrac, 0.0, NULL); + PS_ASSERT_FLOAT_LESS_THAN_OR_EQUAL(badFrac, 1.0, NULL); + } + + int numCols = mask1->numCols, numRows = mask1->numRows; // Size of the images + + // Dereference inputs for convenience + psMaskType **data1 = mask1->data.PS_TYPE_MASK_DATA; + psMaskType **data2 = NULL; + if (mask2) { + data2 = mask2->data.PS_TYPE_MASK_DATA; + } + + // First, a pass through to determine the fraction of bad pixels + if (isfinite(badFrac) && badFrac != 1.0) { + int numBad = 0; // Number of bad pixels + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if (data1[y][x] & maskVal) { + numBad++; + continue; + } + if (data2 && data2[y][x] & maskVal) { + numBad++; + } + } + } + if (numBad > badFrac * numCols * numRows) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Fraction of bad pixels (%d/%d=%f) exceeds limit (%f)\n", + numBad, numCols * numRows, (float)numBad/(float)(numCols * numRows), badFrac); + return NULL; + } + } + + // Worried about the masks for bad pixels and bad stamps colliding, so make our own mask + psImage *mask = psImageAlloc(numCols, numRows, PS_TYPE_MASK); // The global mask + psImageInit(mask, 0); + psMaskType **maskData = mask->data.PS_TYPE_MASK_DATA; // Dereference for convenience + + // Block out a border around the edge of the image + + // Bottom stripe + for (int y = 0; y < PS_MIN(size + footprint, numRows); y++) { + for (int x = 0; x < numCols; x++) { + maskData[y][x] |= PM_SUBTRACTION_MASK_BORDER; + } + } + // Either side + for (int y = PS_MIN(size + footprint, numRows); y < numRows - size - footprint; y++) { + for (int x = 0; x < PS_MIN(size + footprint, numCols); x++) { + maskData[y][x] |= PM_SUBTRACTION_MASK_BORDER; + } + for (int x = PS_MAX(numCols - size - footprint, 0); x < numCols; x++) { + maskData[y][x] |= PM_SUBTRACTION_MASK_BORDER; + } + } + // Top stripe + for (int y = PS_MAX(numRows - size - footprint, 0); y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + maskData[y][x] |= PM_SUBTRACTION_MASK_BORDER; + } + } + + // XXX Could do something smarter here --- we will get images that are predominantly masked (where the + // skycell isn't overlapped by a large fraction by the observation), so that convolving around every bad + // pixel is wasting time. As a first cut, I've put in a check on the fraction of bad pixels, but we could + // imagine looking for the edge of big regions and convolving just at the edge. As a second cut, allow + // use of FFT convolution. + + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if (data1[y][x] & maskVal) { + maskData[y][x] |= PM_SUBTRACTION_MASK_BAD_1; + } + if (data2 && data2[y][x] & maskVal) { + maskData[y][x] |= PM_SUBTRACTION_MASK_BAD_2; + } + } + } + + // Block out the entire stamp footprint around bad input pixels. + + // We want to block out with the CONVOLVE mask anything that would be bad if we convolved with a bad + // reference pixel (within 'size'). Then we want to block out with the FOOTPRINT mask everything within a + // footprint's distance of those (within 'footprint'). + + if (!psImageConvolveMask(mask, mask, PM_SUBTRACTION_MASK_BAD_1, + PM_SUBTRACTION_MASK_CONVOLVE_1 | PM_SUBTRACTION_MASK_FOOTPRINT_1, + -size, size, -size, size)) { + psError(PS_ERR_UNKNOWN, false, "Unable to convolve bad pixels from mask 1."); + psFree(mask); + return NULL; + } + if (!psImageConvolveMask(mask, mask, PM_SUBTRACTION_MASK_BAD_2, + PM_SUBTRACTION_MASK_CONVOLVE_2 | PM_SUBTRACTION_MASK_FOOTPRINT_2, + -size, size, -size, size)) { + psError(PS_ERR_UNKNOWN, false, "Unable to convolve bad pixels from mask 2."); + psFree(mask); + return NULL; + } + if (!psImageConvolveMask(mask, mask, PM_SUBTRACTION_MASK_CONVOLVE_1, + PM_SUBTRACTION_MASK_FOOTPRINT_1, + -footprint, footprint, -footprint, footprint)) { + psError(PS_ERR_UNKNOWN, false, "Unable to reconvolve bad pixels from mask 1."); + psFree(mask); + return NULL; + } + if (!psImageConvolveMask(mask, mask, PM_SUBTRACTION_MASK_CONVOLVE_2, + PM_SUBTRACTION_MASK_FOOTPRINT_2, + -footprint, footprint, -footprint, footprint)) { + psError(PS_ERR_UNKNOWN, false, "Unable to reconvolve bad pixels from mask 2."); + psFree(mask); + return NULL; + } + + return mask; +} + + +bool pmSubtractionBorder(psImage *image, psImage *weight, psImage *mask, + int size, psMaskType blank) +{ + PS_ASSERT_IMAGE_NON_NULL(image, false); + PS_ASSERT_IMAGE_TYPE(image, PS_TYPE_F32, false); + if (mask) { + PS_ASSERT_IMAGE_NON_NULL(mask, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(mask, image, false); + PS_ASSERT_IMAGE_TYPE(mask, PS_TYPE_MASK, false); + } + if (weight) { + PS_ASSERT_IMAGE_NON_NULL(weight, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(weight, image, false); + PS_ASSERT_IMAGE_TYPE(weight, PS_TYPE_F32, false); + } + + int numCols = image->numCols, numRows = image->numRows; // Image dimensions + + for (int y = size; y < numRows - size; y++) { + for (int x = 0; x < size; x++) { + markBlank(image, mask, weight, x, y, blank); + } + for (int x = numCols - size; x < numCols; x++) { + markBlank(image, mask, weight, x, y, blank); + } + } + for (int y = 0; y < size; y++) { + for (int x = 0; x < numCols; x++) { + markBlank(image, mask, weight, x, y, blank); + } + } + for (int y = numRows - size; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + markBlank(image, mask, weight, x, y, blank); + } + } + + return true; +} + + +bool pmSubtractionMaskApply(psImage *image, psImage *weight, const psImage *mask, pmSubtractionMode mode) +{ + PS_ASSERT_IMAGE_NON_NULL(image, false); + PS_ASSERT_IMAGE_TYPE(image, PS_TYPE_F32, false); + if (weight) { + PS_ASSERT_IMAGE_NON_NULL(weight, false); + PS_ASSERT_IMAGE_TYPE(weight, PS_TYPE_F32, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(weight, image, false); + } + PS_ASSERT_IMAGE_NON_NULL(mask, false); + PS_ASSERT_IMAGE_TYPE(mask, PS_TYPE_MASK, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(mask, image, false); + + bool maskVal = PM_SUBTRACTION_MASK_BORDER; // Value to mask + switch (mode) { + case PM_SUBTRACTION_MODE_1: + maskVal |= PM_SUBTRACTION_MASK_CONVOLVE_1; + break; + case PM_SUBTRACTION_MODE_2: + maskVal |= PM_SUBTRACTION_MASK_CONVOLVE_2; + break; + case PM_SUBTRACTION_MODE_DUAL: + maskVal |= PM_SUBTRACTION_MASK_CONVOLVE_2 | PM_SUBTRACTION_MASK_CONVOLVE_2; + break; + case PM_SUBTRACTION_MODE_ERR: + case PM_SUBTRACTION_MODE_UNSURE: + default: + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unsuppored subtraction mode: %x", mode); + return false; + } + + int numCols = image->numCols, numRows = image->numRows; // Size of image + psMaskType **maskData = mask->data.PS_TYPE_MASK_DATA; // Dereference mask + + for (int y = 0; y < numRows; y++) { + for (int x = 0; x < numCols; x++) { + if (maskData[y][x] & maskVal) { + image->data.F32[y][x] = NAN; + if (weight) { + weight->data.F32[y][x] = NAN; + } + } + } + } + + return true; +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMask.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMask.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMask.h (revision 18170) @@ -0,0 +1,36 @@ +#ifndef PM_SUBTRACTION_MASK_H +#define PM_SUBTRACTION_MASK_H + +#include + +/// Generate a mask for use in the subtraction process +psImage *pmSubtractionMask(const psImage *refMask, ///< Mask for the reference image (will be convolved) + const psImage *inMask, ///< Mask for the input image, or NULL + psMaskType maskVal, ///< Value to mask out + int size, ///< Half-size of the kernel (pmSubtractionKernels.size) + int footprint, ///< Half-size of the kernel footprint + float badFrac, ///< Maximum fraction of bad input pixels to accept + bool useFFT ///< Use FFT to do convolution? + ); + +/// Mark the non-convolved part of the image as blank +bool pmSubtractionBorder(psImage *image,///< Image + psImage *weight, ///< Weight map (or NULL) + psImage *mask, ///< Mask (or NULL) + int size, ///< Kernel half-size + psMaskType blank ///< Mask value for blank regions + ); + +/// Apply the subtraction mask to an image and weight. +/// +/// Unfortunately, image subtraction may result in a bi-modal image in masked areas, which can upset image +/// statistics (very important for quantising images so that a product can be written out!). This function +/// sets masked areas to NAN in the image and weight. +bool pmSubtractionMaskApply(psImage *image, ///< Image to which to apply mask + psImage *weight, ///< Weight map to which to apply mask (or NULL) + const psImage *mask, ///< Subtraction mask + pmSubtractionMode mode ///< Subtraction mode + ); + + +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMatch.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMatch.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMatch.c (revision 18170) @@ -0,0 +1,681 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmSubtractionParams.h" +#include "pmSubtractionKernels.h" +#include "pmSubtractionStamps.h" +#include "pmSubtractionEquation.h" +#include "pmSubtraction.h" +#include "pmSubtractionMask.h" +#include "pmSubtractionMatch.h" + +#define KERNEL_MOSAIC 2 // Half-number of kernel instances in the mosaic image +#define BG_STAT PS_STAT_ROBUST_MEDIAN // Statistic to use for background + +static bool useFFT = true; // Do convolutions using FFT + + +//#define TESTING +//#define TESTING_MEMORY + +// Output memory usage information +static void memCheck(const char *where) +{ +#ifdef TESTING_MEMORY + psMemBlock **leaks = NULL; + int numLeaks = psMemCheckLeaks(0, &leaks, NULL, true); + size_t largestSize = 0; + psMemId largest = 0; + size_t totalSize = 0; + for (int i = 0; i < numLeaks; i++) { + psMemBlock *mb = leaks[i]; + totalSize += mb->userMemorySize; + if (mb->userMemorySize > largestSize) { + largestSize = mb->userMemorySize; + largest = mb->id; + } + } + psFree(leaks); + fprintf(stderr, "%s:\n", where); + fprintf(stderr, " Memory in use: %zd\n", totalSize); + fprintf(stderr, " Largest block: %ld\n", largest); + fprintf(stderr, " sbrk(): %zd\n", (size_t)sbrk(0)); +#endif + return; +} + + +static bool getStamps(pmSubtractionStampList **stamps, // Stamps to read + const pmReadout *ro1, // Readout 1 + const pmReadout *ro2, // Readout 2 + const psImage *subMask, // Mask for subtraction, or NULL + psImage *weight, // Weight map + const psRegion *region, // Region of interest, or NULL + float threshold, // Threshold for stamp finding + float stampSpacing, // Spacing between stamps + int size, // Kernel half-size + int footprint, // Convolution footprint for stamps + pmSubtractionMode mode // Mode for subtraction + ) +{ + psTrace("psModules.imcombine", 3, "Finding stamps...\n"); + *stamps = pmSubtractionStampsFind(*stamps, ro1->image, subMask, region, threshold, footprint, + stampSpacing, mode); + if (!*stamps) { + psError(PS_ERR_UNKNOWN, false, "Unable to find stamps."); + return false; + } + + memCheck(" find stamps"); + + psTrace("psModules.imcombine", 3, "Extracting stamps...\n"); + if (!pmSubtractionStampsExtract(*stamps, ro1->image, ro2 ? ro2->image : NULL, weight, size)) { + psError(PS_ERR_UNKNOWN, false, "Unable to extract stamps."); + return false; + } + + memCheck(" extract stamps"); + + return true; +} + + + + + +bool pmSubtractionMatch(pmReadout *conv1, pmReadout *conv2, const pmReadout *ro1, const pmReadout *ro2, + int footprint, float regionSize, float stampSpacing, float threshold, + const psArray *sources, const char *stampsName, + pmSubtractionKernelsType type, int size, int spatialOrder, + const psVector *isisWidths, const psVector *isisOrders, + int inner, int ringsOrder, int binning, bool optimum, const psVector *optFWHMs, + int optOrder, float optThreshold, int iter, float rej, psMaskType maskBad, + psMaskType maskBlank, float badFrac, pmSubtractionMode mode) +{ + if (mode != PM_SUBTRACTION_MODE_2) { + PM_ASSERT_READOUT_NON_NULL(conv1, false); + if (conv1->image) { + psFree(conv1->image); + conv1->image = NULL; + } + if (conv1->mask) { + psFree(conv1->mask); + conv1->mask = NULL; + } + if (conv1->weight) { + psFree(conv1->weight); + conv1->weight = NULL; + } + } + if (mode != PM_SUBTRACTION_MODE_1) { + PM_ASSERT_READOUT_NON_NULL(conv2, false); + if (conv2->image) { + psFree(conv2->image); + conv2->image = NULL; + } + if (conv2->mask) { + psFree(conv2->mask); + conv2->mask = NULL; + } + if (conv2->weight) { + psFree(conv2->weight); + conv2->weight = NULL; + } + } + + PM_ASSERT_READOUT_NON_NULL(ro1, false); + PM_ASSERT_READOUT_NON_NULL(ro2, false); + PM_ASSERT_READOUT_IMAGE(ro1, false); + PM_ASSERT_READOUT_IMAGE(ro2, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(ro1->image, ro2->image, false); + + PS_ASSERT_INT_NONNEGATIVE(footprint, false); + // regionSize can be just about anything (except maybe negative, but it can be NAN) + PS_ASSERT_FLOAT_LARGER_THAN(stampSpacing, 0.0, false); + // Don't care what threshold is + if (sources) { + PS_ASSERT_ARRAY_NON_NULL(sources, false); + } + // stampsName may be anything + // We'll check kernel type when we allocate the kernels + PS_ASSERT_INT_POSITIVE(size, false); + PS_ASSERT_INT_NONNEGATIVE(spatialOrder, false); + if (isisWidths || isisOrders) { + PS_ASSERT_VECTOR_NON_NULL(isisWidths, false); + PS_ASSERT_VECTOR_TYPE(isisWidths, PS_TYPE_F32, false); + PS_ASSERT_VECTOR_NON_NULL(isisOrders, false); + PS_ASSERT_VECTOR_TYPE(isisOrders, PS_TYPE_S32, false); + PS_ASSERT_VECTORS_SIZE_EQUAL(isisWidths, isisOrders, false); + } + PS_ASSERT_INT_NONNEGATIVE(inner, false); + PS_ASSERT_INT_NONNEGATIVE(ringsOrder, false); + PS_ASSERT_INT_POSITIVE(binning, false); + if (optimum) { + PS_ASSERT_VECTOR_NON_NULL(optFWHMs, false); + PS_ASSERT_INT_NONNEGATIVE(optOrder, false); + PS_ASSERT_FLOAT_LARGER_THAN(optThreshold, 0.0, false); + PS_ASSERT_FLOAT_LESS_THAN_OR_EQUAL(optThreshold, 1.0, false); + } + PS_ASSERT_INT_POSITIVE(iter, false); + PS_ASSERT_FLOAT_LARGER_THAN(rej, 0.0, false); + // Don't care about maskBad + // Don't care about maskBlank + if (isfinite(badFrac)) { + PS_ASSERT_FLOAT_LARGER_THAN(badFrac, 0.0, NULL); + PS_ASSERT_FLOAT_LESS_THAN_OR_EQUAL(badFrac, 1.0, NULL); + } + + // If the stamp footprint is smaller than the kernel size, then we won't get much signal in the outer + // parts of the kernel, which can result in bad matching artifacts. + if (footprint < size) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Stamp footprint (%d) should be larger than or equal to the kernel size (%d)", + footprint, size); + return false; + } + + // Where does our weight map come from? + // Getting the weight exactly right is not necessary --- it's just used for weighting. + psImage *weight = NULL; // Weight image to use + if (ro1->weight && ro2->weight) { + weight = (psImage*)psBinaryOp(NULL, ro1->weight, "+", ro2->weight); + } else if (ro1->weight) { + weight = psMemIncrRefCounter(ro1->weight); + } else if (ro2->weight) { + weight = psMemIncrRefCounter(ro2->weight); + } else { + weight = (psImage*)psBinaryOp(NULL, ro1->image, "+", ro2->image); + } + + // Putting important variable declarations here, since they are freed after a "goto" if there is an error. + psImage *subMask = NULL; // Mask for subtraction + psRegion *region = NULL; // Iso-kernel region + psString regionString = NULL; // String for region + pmSubtractionStampList *stamps = NULL; // Stamps for matching PSF + pmSubtractionKernels *kernels = NULL; // Kernel basis functions + + int numCols = ro1->image->numCols, numRows = ro1->image->numRows; // Image dimensions + + memCheck("start"); + + subMask = pmSubtractionMask(ro1->mask, ro2 ? ro2->mask : NULL, maskBad, size, footprint, + badFrac, useFFT); + if (!subMask) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate subtraction mask."); + psFree(weight); + return false; + } + + memCheck("mask"); + + // Get region of interest + int xRegions = 1, yRegions = 1; // Number of iso-kernel regions + float xRegionSize = 0, yRegionSize = 0; // Size of iso-kernel regions + if (isfinite(regionSize) && regionSize != 0.0) { + xRegions = numCols / regionSize + 1; + yRegions = numRows / regionSize + 1; + xRegionSize = (float)numCols / (float)xRegions; + yRegionSize = (float)numRows / (float)yRegions; + region = psRegionAlloc(NAN, NAN, NAN, NAN); + } + + // Iterate over iso-kernel regions + for (int j = 0; j < yRegions; j++) { + for (int i = 0; i < xRegions; i++) { + psTrace("psModules.imcombine", 1, "Subtracting region %d of %d...\n", + j * xRegions + i + 1, xRegions * yRegions); + if (region) { + *region = psRegionSet((int)(i * xRegionSize), (int)((i + 1) * xRegionSize), + (int)(j * yRegionSize), (int)((j + 1) * yRegionSize)); + psFree(regionString); + regionString = psRegionToString(*region); + psTrace("psModules.imcombine", 3, "Iso-kernel region: %s out of %d,%d\n", + regionString, numCols, numRows); + } + + if (sources) { + stamps = pmSubtractionStampsSetFromSources(sources, subMask, region, footprint, + stampSpacing, mode); + } else if (stampsName && strlen(stampsName) > 0) { + stamps = pmSubtractionStampsSetFromFile(stampsName, ro1->image, subMask, region, footprint, + stampSpacing, mode); + } + + // We get the stamps here; we will also attempt to get stamps at the first iteration, but it + // doesn't matter. + if (!getStamps(&stamps, ro1, ro2, subMask, weight, NULL, threshold, stampSpacing, + size, footprint, mode)) { + goto MATCH_ERROR; + } + + if (mode == PM_SUBTRACTION_MODE_UNSURE) { + // Get backgrounds + psStats *bgStats = psStatsAlloc(BG_STAT); // Statistics for background + psRandom *rng = psRandomAlloc(PS_RANDOM_TAUS, 0); // Random number generator + psVector *buffer = NULL;// Buffer for stats + if (!psImageBackground(bgStats, &buffer, ro1->image, ro1->mask, maskBad, rng)) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure background of image 1."); + psFree(bgStats); + psFree(rng); + psFree(buffer); + goto MATCH_ERROR; + } + float bg1 = psStatsGetValue(bgStats, BG_STAT); // Background for image 1 + if (!psImageBackground(bgStats, &buffer, ro2->image, ro2->mask, maskBad, rng)) { + psError(PS_ERR_UNKNOWN, false, "Unable to measure background of image 2."); + psFree(bgStats); + psFree(rng); + psFree(buffer); + goto MATCH_ERROR; + } + float bg2 = psStatsGetValue(bgStats, BG_STAT); // Background for image 2 + psFree(bgStats); + psFree(rng); + psFree(buffer); + + pmSubtractionMode newMode = pmSubtractionOrder(stamps, bg1, bg2); // Subtraction mode to use + switch (newMode) { + case PM_SUBTRACTION_MODE_1: + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Convolving image 1 to match image 2."); + break; + case PM_SUBTRACTION_MODE_2: + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Convolving image 2 to match image 1."); + break; + default: + psError(PS_ERR_UNKNOWN, false, "Unable to determine subtraction order."); + goto MATCH_ERROR; + } + mode = newMode; + } + + // Define kernel basis functions + if (optimum && (type == PM_SUBTRACTION_KERNEL_ISIS || type == PM_SUBTRACTION_KERNEL_GUNK)) { + kernels = pmSubtractionKernelsOptimumISIS(type, size, inner, spatialOrder, optFWHMs, optOrder, + stamps, footprint, optThreshold, mode); + if (!kernels) { + psErrorClear(); + psWarning("Unable to derive optimum ISIS kernel --- switching to default."); + } + } + if (kernels == NULL) { + // Not an ISIS/GUNK kernel, or the optimum kernel search failed + kernels = pmSubtractionKernelsGenerate(type, size, spatialOrder, isisWidths, isisOrders, + inner, binning, ringsOrder, mode); + } + + // Add analysis metadata + { + psRegion *subRegion; // Region over which subtraction was performed + if (region) { + subRegion = psMemIncrRefCounter(region); + } else { + subRegion = psRegionAlloc(0, numCols, 0, numRows); + } + + if (mode == PM_SUBTRACTION_MODE_1 || mode == PM_SUBTRACTION_MODE_DUAL) { + psMetadataAddPtr(conv1->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_KERNEL, + PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK, "Subtraction kernels", kernels); + psMetadataAddS32(conv1->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_MODE, + PS_META_DUPLICATE_OK, "Subtraction kernels", mode); + psMetadataAddPtr(conv1->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_REGION, + PS_DATA_REGION | PS_META_DUPLICATE_OK, + "Region over which subtraction was performed", subRegion); + } + if (mode == PM_SUBTRACTION_MODE_2 || mode == PM_SUBTRACTION_MODE_DUAL) { + psMetadataAddPtr(conv2->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_KERNEL, + PS_DATA_UNKNOWN | PS_META_DUPLICATE_OK, "Subtraction kernels", kernels); + psMetadataAddS32(conv2->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_MODE, + PS_META_DUPLICATE_OK, "Subtraction kernels", mode); + psMetadataAddPtr(conv2->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_REGION, + PS_DATA_REGION | PS_META_DUPLICATE_OK, + "Region over which subtraction was performed", subRegion); + } + psFree(subRegion); + } + + memCheck("kernels"); + + float rmsStamps = NAN; // RMS for stamps + int numStamps = 0; // Number of good stamps + int numRejected = -1; // Number of rejected stamps in each iteration + for (int k = 0; k < iter && numRejected != 0; k++) { + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Iteration %d.", k); + + if (!getStamps(&stamps, ro1, ro2, subMask, weight, region, threshold, stampSpacing, + size, footprint, mode)) { + goto MATCH_ERROR; + } + + psTrace("psModules.imcombine", 3, "Calculating equation...\n"); + if (!pmSubtractionCalculateEquation(stamps, kernels)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate least-squares equation."); + goto MATCH_ERROR; + } + + memCheck(" calculate equation"); + + psTrace("psModules.imcombine", 3, "Solving equation...\n"); + + if (!pmSubtractionSolveEquation(kernels, stamps)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate least-squares equation."); + goto MATCH_ERROR; + } + + memCheck(" solve equation"); + + psVector *deviations = pmSubtractionCalculateDeviations(stamps, kernels); // Stamp deviations + if (!deviations) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate deviations."); + goto MATCH_ERROR; + } + + memCheck(" calculate deviations"); + + psTrace("psModules.imcombine", 3, "Rejecting stamps...\n"); + numRejected = pmSubtractionRejectStamps(&rmsStamps, &numStamps, stamps, deviations, + subMask, rej, footprint); + if (numRejected < 0) { + psError(PS_ERR_UNKNOWN, false, "Unable to reject stamps."); + psFree(deviations); + goto MATCH_ERROR; + } + psFree(deviations); + + memCheck(" reject stamps"); + } + + if (numRejected > 0) { + psTrace("psModules.imcombine", 3, "Solving equation...\n"); + if (!pmSubtractionSolveEquation(kernels, stamps)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate least-squares equation."); + goto MATCH_ERROR; + } + psVector *deviations = pmSubtractionCalculateDeviations(stamps, kernels); // Stamp deviations + if (!deviations) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate deviations."); + goto MATCH_ERROR; + } + pmSubtractionRejectStamps(&rmsStamps, &numStamps, stamps, deviations, + subMask, NAN, footprint); + psFree(deviations); + } + psFree(stamps); + stamps = NULL; + + if (mode == PM_SUBTRACTION_MODE_1 || mode == PM_SUBTRACTION_MODE_DUAL) { + psMetadataAddS32(conv1->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_STAMPS_NUM, + PS_META_DUPLICATE_OK, "Number of good stamps", numStamps); + psMetadataAddF32(conv1->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_STAMPS_RMS, + PS_META_DUPLICATE_OK, "RMS deviation of stamps", rmsStamps); + } + if (mode == PM_SUBTRACTION_MODE_2 || mode == PM_SUBTRACTION_MODE_DUAL) { + psMetadataAddS32(conv2->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_STAMPS_NUM, + PS_META_DUPLICATE_OK, "Number of good stamps", numStamps); + psMetadataAddF32(conv2->analysis, PS_LIST_TAIL, PM_SUBTRACTION_ANALYSIS_STAMPS_RMS, + PS_META_DUPLICATE_OK, "RMS deviation of stamps", rmsStamps); + } + + memCheck("solution"); + + { + psTrace("psModules.imcombine", 2, "Generating diagnostics...\n"); + // Generate image with convolution kernels + int fullSize = 2 * size + 1 + 1; // Full size of kernel + psImage *convKernels = psImageAlloc(5 * fullSize - 1, 5 * fullSize - 1, PS_TYPE_F32); + psImageInit(convKernels, NAN); + for (int j = -KERNEL_MOSAIC; j <= KERNEL_MOSAIC; j++) { + for (int i = -KERNEL_MOSAIC; i <= KERNEL_MOSAIC; i++) { + psImage *kernel = pmSubtractionKernelImage(kernels, (float)i / (float)KERNEL_MOSAIC, + (float)j / (float)KERNEL_MOSAIC, + false); // Image of the kernel + if (!kernel) { + psError(PS_ERR_UNKNOWN, false, "Unable to generate kernel image."); + psFree(convKernels); + goto MATCH_ERROR; + } + + if (psImageOverlaySection(convKernels, kernel, (i + KERNEL_MOSAIC) * fullSize, + (j + KERNEL_MOSAIC) * fullSize, "=") == 0) { + psError(PS_ERR_UNKNOWN, false, "Unable to overlay kernel image."); + psFree(kernel); + psFree(convKernels); + goto MATCH_ERROR; + } + psFree(kernel); + } + } + + psString comment = NULL; // Comment for metadata + psStringAppend(&comment, "Subtraction kernel for region %s", regionString); + psMetadataAddImage(conv1->analysis, PS_LIST_TAIL, "SUBTRACTION.KERNEL.IMAGE", + PS_META_DUPLICATE_OK, comment, convKernels); + psFree(comment); + psFree(convKernels); + } + +#if 0 + { + // Generate images of the kernel components + psMetadata *header = psMetadataAlloc(); // Header + for (int i = 0; i < solution->n; i++) { + psString name = NULL; // Header keyword + psStringAppend(&name, "SOLN%04d", i); + psMetadataAddF64(header, PS_LIST_TAIL, name, 0, NULL, solution->data.F64[i]); + psFree(name); + } + psArray *kernelImages = pmSubtractionKernelSolutions(solution, kernels, 0.0, 0.0); + psFits *kernelFile = psFitsOpen("kernels.fits", "w"); + (void)psFitsWriteImageCube(kernelFile, header, kernelImages, NULL); + psFitsClose(kernelFile); + psFree(kernelImages); + psFree(header); + } +#endif + + memCheck("diag outputs"); + + psTrace("psModules.imcombine", 2, "Convolving...\n"); + if (!pmSubtractionConvolve(conv1, conv2, ro1, ro2, subMask, maskBlank, region, kernels, + true, useFFT)) { + psError(PS_ERR_UNKNOWN, false, "Unable to convolve image."); + goto MATCH_ERROR; + } + psFree(kernels); + kernels = NULL; + + // There is data in the readout now + if (mode == PM_SUBTRACTION_MODE_1 || mode == PM_SUBTRACTION_MODE_DUAL) { + conv1->data_exists = true; + if (conv1->parent) { + conv1->parent->data_exists = true; + conv1->parent->parent->data_exists = true; + } + } + if (mode == PM_SUBTRACTION_MODE_2 || mode == PM_SUBTRACTION_MODE_DUAL) { + conv2->data_exists = true; + if (conv2->parent) { + conv2->parent->data_exists = true; + conv2->parent->parent->data_exists = true; + } + } + } + } + psFree(region); + region = NULL; + psFree(regionString); + regionString = NULL; + psFree(subMask); + subMask = NULL; + psFree(weight); + weight = NULL; + + if (!pmSubtractionBorder(conv1->image, conv1->weight, conv1->mask, size, maskBlank)) { + psError(PS_ERR_UNKNOWN, false, "Unable to set border of convolved image."); + goto MATCH_ERROR; + } + + memCheck("convolution"); + + +#ifdef TESTING + { + if (mode == PM_SUBTRACTION_MODE_1 || mode == PM_SUBTRACTION_MODE_DUAL) { + psFits *fits = psFitsOpen("convolved1.fits", "w"); + psFitsWriteImage(fits, NULL, conv1->image, 0, NULL); + psFitsClose(fits); + } + + if (mode == PM_SUBTRACTION_MODE_2 || mode == PM_SUBTRACTION_MODE_DUAL) { + psFits *fits = psFitsOpen("convolved2.fits", "w"); + psFitsWriteImage(fits, NULL, conv2->image, 0, NULL); + psFitsClose(fits); + } + } +#endif + + return true; + +MATCH_ERROR: + psFree(region); + psFree(regionString); + psFree(subMask); + psFree(kernels); + psFree(stamps); + psFree(weight); + return false; +} + + +// Determine a rough width (integer value) of the star in the image +// XXX Could improve this by using a user-provided list of floating-point widths (or an end point and +// increment). +static int subtractionOrderWidth(const psKernel *kernel, // Image + float bg, // Background in image + int size, // Maximum size + psArray **models, // Buffer of models + psVector **modelSums // Buffer of model sums + ) +{ + assert(kernel); + assert(models); + assert(modelSums); + + int xMin = kernel->xMin, xMax = kernel->xMax; // Bounds in x + int yMin = kernel->yMin, yMax = kernel->yMax; // Bounds in y + + // Generate models + if (!*models) { + assert(!*modelSums); + *models = psArrayAlloc(size); + *modelSums = psVectorAlloc(size, PS_TYPE_F64); + for (int sigma = 0; sigma < size; sigma++) { + psKernel *model = psKernelAlloc(xMin, xMax, yMin, yMax); // Gaussian model + float invSigma2 = 1.0 / (float)PS_SQR(1 + sigma); // Inverse sigma squared + double sumGG = 0.0; // Sum of square of Gaussian + for (int y = yMin; y <= yMax; y++) { + int y2 = PS_SQR(y); // y squared + for (int x = xMin; x <= xMax; x++) { + float rad2 = PS_SQR(x) + y2; // Radius squared + float value = expf(-rad2 * invSigma2); // Model value + model->kernel[y][x] = value; + sumGG += PS_SQR(value); + } + } + (*models)->data[sigma] = model; + (*modelSums)->data.F64[sigma] = sumGG; + } + } + + // Fit gaussians of varying widths to the image, record the chi^2 + psVector *chi2 = psVectorAlloc(size, PS_TYPE_F32); // chi^2 as a function of radius + for (int sigma = 0; sigma < size; sigma++) { + double sumFG = 0.0; // Sum for calculating the normalisation of the Gaussian + psKernel *model = (*models)->data[sigma]; // Model of interest + for (int y = yMin; y <= yMax; y++) { + for (int x = xMin; x <= xMax; x++) { + sumFG += model->kernel[y][x] * (kernel->kernel[y][x] - bg); + } + } + float norm = sumFG / (*modelSums)->data.F64[sigma]; // Normalisation for Gaussian + double sumDev2 = 0.0; // Sum of square deviations + for (int y = yMin; y <= yMax; y++) { + for (int x = xMin; x <= xMax; x++) { + float dev = kernel->kernel[y][x] - bg - norm * model->kernel[y][x]; // Deviation + sumDev2 += PS_SQR(dev); + } + } + chi2->data.F32[sigma] = sumDev2; + } + + // Find the minimum chi^2 + int bestIndex = -1; // Index of best chi^2 + float bestChi2 = INFINITY; // Best chi^2 + for (int i = 0; i < size; i++) { + if (chi2->data.F32[i] < bestChi2) { + bestChi2 = chi2->data.F32[i]; + bestIndex = i; + } + } + psFree(chi2); + + return bestIndex + 1; +} + +pmSubtractionMode pmSubtractionOrder(pmSubtractionStampList *stamps, float bg1, float bg2) +{ + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, PM_SUBTRACTION_MODE_ERR); + + psVector *mask = psVectorAlloc(stamps->num, PS_TYPE_MASK); // Mask for stamps + psVector *ratios = psVectorAlloc(stamps->num, PS_TYPE_F32); // Ratios of widths + psArray *models = NULL; // Gaussian models + psVector *modelSums = NULL; // Gaussian model sums + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (stamp->status != PM_SUBTRACTION_STAMP_CALCULATE && stamp->status != PM_SUBTRACTION_STAMP_USED) { + mask->data.PS_TYPE_MASK_DATA[i] = 0xff; + continue; + } + + // Widths of stars + int width1 = subtractionOrderWidth(stamp->image1, bg1, stamps->footprint, &models, &modelSums); + int width2 = subtractionOrderWidth(stamp->image2, bg2, stamps->footprint, &models, &modelSums); + + if (width1 == 0 || width2 == 0) { + ratios->data.F32[i] = NAN; + mask->data.PS_TYPE_MASK_DATA[i] = 0xff; + } else { + ratios->data.F32[i] = (float)width1 / (float)width2; + mask->data.PS_TYPE_MASK_DATA[i] = 0; + } + } + psFree(models); + psFree(modelSums); + + psStats *stats = psStatsAlloc(PS_STAT_ROBUST_MEDIAN); + if (!psVectorStats(stats, ratios, NULL, mask, 0xff)) { + psError(PS_ERR_UNKNOWN, false, "Unable to calculate statistics for moments ratio."); + psFree(mask); + psFree(ratios); + psFree(stats); + return PM_SUBTRACTION_MODE_ERR; + } + psFree(ratios); + psFree(mask); + + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Median width ratio: %lf", stats->robustMedian); + pmSubtractionMode mode = (stats->robustMedian <= 1.0 ? PM_SUBTRACTION_MODE_1 : PM_SUBTRACTION_MODE_2); + psFree(stats); + + return mode; +} + + + Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMatch.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMatch.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionMatch.h (revision 18170) @@ -0,0 +1,57 @@ +#ifndef PM_SUBTRACTION_MATCH_H +#define PM_SUBTRACTION_MATCH_H + +#include + +#include +#include +#include +#include + +#define PM_SUBTRACTION_ANALYSIS_KERNEL "SUBTRACTION.KERNEL" // Name of kernel in the analysis metadata +#define PM_SUBTRACTION_ANALYSIS_MODE "SUBTRACTION.MODE" // Name of subtraction mode in the analysis metadata +#define PM_SUBTRACTION_ANALYSIS_REGION "SUBTRACTION.REGION" // Name of subtraction region in the analysis MD +#define PM_SUBTRACTION_ANALYSIS_STAMPS_RMS "SUBTRACTION.RMS" // Name of stamp rms in the analysis metadata +#define PM_SUBTRACTION_ANALYSIS_STAMPS_NUM "SUBTRACTION.NUM"// Name of the number of stamps in the analysis MD + +/// Match two images +bool pmSubtractionMatch(pmReadout *conv1, ///< Output convolved data for image 1 + pmReadout *conv2, ///< Output convolved data for image 2 + const pmReadout *ro1, ///< Image 1 + const pmReadout *ro2, ///< Image 2 + // Stamp parameters + int footprint, ///< Stamp half-size + float regionSize, ///< Typical size of iso-kernel regions + float stampSpacing, ///< Typical spacing between stamps + float threshold, ///< Threshold for stamps + const psArray *sources, ///< Sources for stamps + const char *stampsName, ///< Filename for stamps + // Kernel parameters + pmSubtractionKernelsType type, ///< Kernel type + int size, ///< Kernel half-size + int order, ///< Spatial polynomial order + const psVector *widths, ///< ISIS Gaussian widths + const psVector *orders, ///< ISIS Polynomial orders + int inner, ///< Inner radius for various kernel types + int ringsOrder, ///< RINGS polynomial order + int binning, ///< SPAM kernel binning + bool optimum, ///< Search for optimum ISIS kernel? + const psVector *optFWHMs, ///< FWHMs for optimum search + int optOrder, ///< Maximum order for optimum search + float optThreshold, ///< Threshold for optimum search (0..1) + // Operational parameters + int iter, ///< Rejection iterations + float rej, ///< Rejection threshold + psMaskType maskBad, ///< Value to mask + psMaskType maskBlank, ///< Mask for blank region + float badFrac, ///< Maximum fraction of bad input pixels to accept + pmSubtractionMode mode ///< Mode of subtraction; may be modified + ); + +/// Determine which image to convolve +pmSubtractionMode pmSubtractionOrder(pmSubtractionStampList *stamps, ///< Stamps that have been extracted + float bg1, float bg2 // Background for each image + ); + + +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionParams.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionParams.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionParams.c (revision 18170) @@ -0,0 +1,510 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#include "pmSubtractionStamps.h" +#include "pmSubtraction.h" +#include "pmSubtractionParams.h" + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +#if 0 +// Convolve the reference stamp by the kernel +static psKernel *convolveStamp(const pmSubtractionStamp *stamp, // Stamp to be convolved + const psKernel *kernel, // Kernel by which to convolve + int footprint // Size of area to be convolved + ) +{ + psKernel *convolution = psKernelAlloc(-footprint, footprint, -footprint, footprint); // Result + psKernel *reference = stamp->reference; // Reference stamp, to be convolved + + // Range of kernel + int uMin = kernel->xMin; + int uMax = kernel->xMax; + int vMin = kernel->yMin; + int vMax = kernel->yMax; + + for (int y = -footprint; y <= footprint; y++) { + psF32 *conv = &convolution->kernel[y][-footprint]; // Dereference convolution + for (int x = -footprint; x <= footprint; x++, conv++) { + *conv = 0.0; + + int xStart = x + uMin; // Start index for convolution + for (int v = vMin; v <= vMax; v++) { + psF32 *ref = &reference->kernel[y + v][xStart]; // Dereference reference image + psF32 *krnl = &kernel->kernel[v][uMin]; // Dereference kernel in x + for (int u = uMin; u <= uMax; u++, ref++, krnl++) { + *conv += *ref * *krnl; + } + } + } + } + + return convolution; +} +#endif + +/// Select the appropriate convolution, given the kernel basis function and subtraction mode +static inline psKernel *selectConvolution(const pmSubtractionStamp *stamp, // Stamp + int kernelIndex, // Index for kernel component + pmSubtractionMode mode // Mode of subtraction + ) +{ + switch (mode) { + case PM_SUBTRACTION_MODE_1: + return stamp->convolutions1->data[kernelIndex]; + case PM_SUBTRACTION_MODE_2: + return stamp->convolutions2->data[kernelIndex]; + default: + psAbort("Unsupported subtraction mode: %x", mode); + } + return NULL; // Unreached +} + +// Accumulate cross-term sums for a stamp +static void accumulateCross(double *sumI, // Sum of I(x)/sigma(x)^2 + double *sumII, // Sum of I(x)^2/sigma(x)^2 + double *sumIC, // Sum of I(x)conv(x)/sigma(x)^2 + const pmSubtractionStamp *stamp, // Stamp with weight + const psKernel *target, // Target stamp + int kernelIndex, // Index for kernel component + int footprint, // Size of region of interest + pmSubtractionMode mode // Mode of subtraction + ) +{ + psKernel *weight = stamp->weight; // Weight, sigma(x)^2 + psKernel *convolution = selectConvolution(stamp, kernelIndex, mode); // Convolution of interest + + for (int y = -footprint; y <= footprint; y++) { + psF32 *in = &target->kernel[y][-footprint]; // Dereference input + psF32 *wt = &weight->kernel[y][-footprint]; // Dereference weight + psF32 *conv = &convolution->kernel[y][-footprint]; // Dereference convolution + for (int x = -footprint; x <= footprint; x++, in++, wt++, conv++) { + double temp = *in / *wt; // Temporary product + *sumI += temp; + *sumII += *in * temp; + *sumIC += *conv * temp; + } + } + return; +} + +// Accumulate convolution sums for a stamp +static void accumulateConvolutions(double *sumC, // Sum of conv(x)/sigma(x)^2 + double *sumCC, // Sum of conv(x)^2/sigma(x)^2 + const pmSubtractionStamp *stamp, // Stamp with input and weight + int kernelIndex, // Index for kernel component + int footprint, // Size of region of interest + pmSubtractionMode mode // Mode of subtraction + ) +{ + psKernel *weight = stamp->weight; // Weight, sigma(x)^2 + psKernel *convolution = selectConvolution(stamp, kernelIndex, mode); // Convolution of interest + + for (int y = -footprint; y <= footprint; y++) { + psF32 *wt = &weight->kernel[y][-footprint]; // Dereference weight + psF32 *conv = &convolution->kernel[y][-footprint]; // Dereference convolution + for (int x = -footprint; x <= footprint; x++, wt++, conv++) { + double convNoise = *conv / *wt; // Temporary product + *sumC += convNoise; + *sumCC += *conv * convNoise; + } + } + return; +} + +static double accumulateChi2(const psKernel *target, // Target stamp + pmSubtractionStamp *stamp, // Stamp with weight + int kernelIndex, // Index for kernel component + double coeff, // Coefficient of convolution + double bg, // Background term + int footprint, // Size of region of interest + pmSubtractionMode mode // Mode of subtraction + ) +{ + double chi2 = 0.0; + psKernel *weight = stamp->weight; // Weight, sigma(x)^2 + psKernel *convolution = selectConvolution(stamp, kernelIndex, mode); // Convolution of interest + + for (int y = -footprint; y <= footprint; y++) { + psF32 *in = &target->kernel[y][-footprint]; // Dereference input + psF32 *wt = &weight->kernel[y][-footprint]; // Dereference weight + psF32 *conv = &convolution->kernel[y][-footprint]; // Dereference convolution + for (int x = -footprint; x <= footprint; x++, in++, wt++, conv++) { + chi2 += PS_SQR(*in - bg - coeff * *conv) / *wt; + } + } + + return chi2; +} + +// Return the initial value of chi^2 +static double initialChi2(const psKernel *target, // Target stamp + const pmSubtractionStamp *stamp, // Stamp with weight + int footprint, // Size of convolution + pmSubtractionMode mode // Mode of subtraction + ) +{ + psKernel *weight = stamp->weight; // Weight map + psKernel *source; // Source stamp + switch (mode) { + case PM_SUBTRACTION_MODE_1: + source = stamp->image1; + break; + case PM_SUBTRACTION_MODE_2: + source = stamp->image2; + break; + default: + psAbort("Unsupported subtraction mode: %x", mode); + } + + double chi2 = 0.0; // Chi^2 + for (int y = -footprint; y <= footprint; y++) { + psF32 *in = &target->kernel[y][-footprint]; // Dereference input + psF32 *wt = &weight->kernel[y][-footprint]; // Dereference weight + psF32 *ref = &source->kernel[y][-footprint]; // Derference reference + for (int x = -footprint; x <= footprint; x++, in++, wt++, ref++) { + float diff = *in - *ref; // Temporary value + chi2 += PS_SQR(diff) / *wt; + } + } + + return chi2; +} + +// Subtract a convolution from the input +static void subtractConvolution(psKernel *target, // Target stamp + const pmSubtractionStamp *stamp, // Stamp with weight + int kernelIndex, // Index for kernel component + float coeff, // Coefficient of subtraction + float bg, // Background term + int footprint, // Size of region of interest + pmSubtractionMode mode // Mode of subtraction + ) +{ + psKernel *convolution = selectConvolution(stamp, kernelIndex, mode); // Convolution of interest + for (int y = -footprint; y <= footprint; y++) { + psF32 *in = &target->kernel[y][-footprint]; // Dereference input + psF32 *conv = &convolution->kernel[y][-footprint]; // Dereference convolution + for (int x = -footprint; x <= footprint; x++, in++, conv++) { + *in -= *conv * coeff + bg; + } + } + + return; +} + + +pmSubtractionKernels *pmSubtractionKernelsOptimumISIS(pmSubtractionKernelsType type, int size, int inner, + int spatialOrder, const psVector *fwhms, int maxOrder, + const pmSubtractionStampList *stamps, int footprint, + float tolerance, pmSubtractionMode mode) +{ + if (type != PM_SUBTRACTION_KERNEL_ISIS && type != PM_SUBTRACTION_KERNEL_GUNK) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Invalid kernel type: %x\n", type); + return NULL; + } + PS_ASSERT_INT_NONNEGATIVE(size, NULL); + PS_ASSERT_INT_NONNEGATIVE(inner, NULL); + PS_ASSERT_INT_NONNEGATIVE(spatialOrder, NULL); + PS_ASSERT_VECTOR_NON_NULL(fwhms, NULL); + PS_ASSERT_VECTOR_TYPE(fwhms, PS_TYPE_F32, NULL); + PS_ASSERT_INT_NONNEGATIVE(maxOrder, NULL); + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, NULL); + PS_ASSERT_INT_NONNEGATIVE(footprint, NULL); + PS_ASSERT_FLOAT_LARGER_THAN(tolerance, 0.0, NULL); + + // Generate the kernels to test + int numGaussians = fwhms->n; // Number of Gaussians + int numKernels = numGaussians * (maxOrder + 1) * (maxOrder + 2) / 2; // Number of kernel components + psString params = NULL; // Parameter, for description + for (int i = 0; i < numGaussians; i++) { + psStringAppend(¶ms, "%.2f,", fwhms->data.F32[i]); + } + params[strlen(params) - 1] = '\0'; + + psVector *orders = psVectorAlloc(numGaussians, PS_TYPE_S32); // Polynomial orders + psVectorInit(orders, maxOrder); + pmSubtractionKernels *kernels = p_pmSubtractionKernelsRawISIS(size, spatialOrder, + fwhms, orders, mode); // Kernels + psFree(orders); + psFree(kernels->description); + kernels->description = NULL; + psStringAppend(&kernels->description, "OptISIS(%d,(%s),%d)", size, params, spatialOrder); + psFree(params); + + // Need to save the stamp inputs --- we're changing the values! + int numStamps = stamps->num; // Number of stamps + psArray *targets = psArrayAlloc(numStamps); // Deep copies of the targets + psVector *badStamps = psVectorAlloc(numStamps, PS_TYPE_U8); // Mark the bad stamps + psVectorInit(badStamps, 0); + for (int i = 0; i < numStamps; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (stamp->status != PM_SUBTRACTION_STAMP_CALCULATE && stamp->status != PM_SUBTRACTION_STAMP_USED) { + badStamps->data.U8[i] = 0xff; + continue; + } + psKernel *target; // Target image of interest + switch (mode) { + case PM_SUBTRACTION_MODE_1: + target = stamp->image2; + break; + case PM_SUBTRACTION_MODE_2: + target = stamp->image1; + break; + default: + psAbort("Unsupported subtraction mode: %x", mode); + } + psImage *copy = psImageCopy(NULL, target->image, PS_TYPE_F32); // Copy of the image + targets->data[i] = psKernelAllocFromImage(copy, size + footprint, size + footprint); + psFree(copy); // Drop reference + } + + // Generate the convolutions, accumulate sums, and measure initial chi^2 + double sum1 = 0.0; // sum of 1/sigma(x,y)^2 + psVector *sumC = psVectorAlloc(numKernels, PS_TYPE_F64); // sum of R(x)*k(u)/sigma(x)^2 + psVector *sumCC = psVectorAlloc(numKernels, PS_TYPE_F64); // sum of [R(x)*k(u)]^2/sigma(x)^2 + psVectorInit(sumC, 0.0); + psVectorInit(sumCC, 0.0); + double lastChi2 = 0.0; // Chi^2 from last iteration + int numPixels = 0; // Number of pixels contributing to chi^2 + for (int i = 0; i < numStamps; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (badStamps->data.U8[i]) { + continue; + } + if (!pmSubtractionConvolveStamp(stamp, kernels, footprint)) { + psError(PS_ERR_UNKNOWN, false, "Unable to convolve stamp %d.", i); + psFree(targets); + psFree(kernels); + psFree(badStamps); + return NULL; + } + + // This sum is invariant to the kernel + psKernel *weight = stamp->weight; // Weight map for stamp + for (int v = -footprint; v <= footprint; v++) { + psF32 *wt = &weight->kernel[v][-footprint]; // Dereference weight map + for (int u = -footprint; u <= footprint; u++, wt++) { + sum1 += 1.0 / *wt; + } + } + if (!isfinite(sum1)) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, + "Sum of 1/sigma^2 is non-finite for stamp %d (%d,%d)\n", + i, (int)stamp->x, (int)stamp->y); + psFree(targets); + psFree(kernels); + psFree(badStamps); + return NULL; + } + + for (int j = 0; j < numKernels; j++) { + accumulateConvolutions(&sumC->data.F64[j], &sumCC->data.F64[j], stamp, j, footprint, mode); + } + + lastChi2 += initialChi2(targets->data[i], stamp, footprint, mode); + numPixels += PS_SQR(2 * footprint + 1); + } + lastChi2 /= numPixels; + + // Rank the kernel components + psVector *ranking = psVectorAlloc(numKernels, PS_TYPE_S32); // Ranking of the kernel components + psVectorInit(ranking, -1); + int cutIndex = -1; // Index at which to cut off kernels + for (int iter = 0; iter < numKernels; iter++) { + int bestIndex = -1; // Index of best kernel component + double bestChi2 = INFINITY; // Value of best chi^2 + double bestCoeff = 0; // Value of best coefficient + double bestBG = 0; // Value of best background + + for (int i = 0; i < numKernels; i++) { + if (ranking->data.S32[i] >= 0) { + continue; + } + + double sumI = 0.0; // sum of I(x)/sigma(x)^2 + double sumII = 0.0; // sum of I(x)^2/sigma(x)^2 + double sumIC = 0.0; // sum of I(x)C(x)/sigma(x)^2 + + for (int j = 0; j < numStamps; j++) { + if (badStamps->data.U8[j]) { + continue; + } + pmSubtractionStamp *stamp = stamps->stamps->data[j]; // Stamp of interest + accumulateCross(&sumI, &sumII, &sumIC, stamp, targets->data[j], i, footprint, mode); + } + + double invDet = 1.0 / (sum1 * sumCC->data.F64[i] - PS_SQR(sumC->data.F64[i])); // Determinant^-1 + double coeff = invDet * (sum1 * sumIC - sumC->data.F64[i] * sumI); // Coefficient for kernel + double bg = invDet * (sumCC->data.F64[i] * sumI - sumC->data.F64[i] * sumIC); // Background + + double chi2 = 0.0; // Chi^2 + for (int j = 0; j < numStamps; j++) { + if (badStamps->data.U8[j]) { + continue; + } + pmSubtractionStamp *stamp = stamps->stamps->data[j]; // Stamp of interest + chi2 += accumulateChi2(targets->data[j], stamp, i, coeff, bg, footprint, mode); + } + + if (chi2 < bestChi2) { + bestIndex = i; + bestCoeff = coeff; + bestChi2 = chi2; + bestBG = bg; + } + + psTrace("psModules.imcombine", 8, "%d: %lf %lf %lf %lf %lf %lf\n", i, sum1, sumI, sumII, sumIC, + sumC->data.F64[i], sumCC->data.F64[i]); + psTrace("psModules.imcombine", 6, "%d: %lf %lf %lf\n", i, coeff, bg, chi2); + } + bestChi2 /= numPixels; + + if (bestIndex == -1) { + psError(PS_ERR_UNKNOWN, false, "Unable to find best kernel component in round %d.", iter); + psFree(targets); + psFree(sumC); + psFree(sumCC); + psFree(ranking); + psFree(kernels); + psFree(badStamps); + return NULL; + } + + // And the winner is.... + ranking->data.S32[bestIndex] = iter; + // Remove its contribution, and don't include it in the future. + for (int j = 0; j < numStamps; j++) { + if (badStamps->data.U8[j]) { + continue; + } + pmSubtractionStamp *stamp = stamps->stamps->data[j]; // Stamp of interest + subtractConvolution(targets->data[j], stamp, bestIndex, bestCoeff, bestBG, footprint, mode); + } + + double diff = lastChi2 - bestChi2; // Difference in chi^2 between iterations + + psTrace("psModules.imcombine", 3, "The winner of round %d is %d (%f,%d,%d): %lf (%lf) --> %lf\n", + iter, bestIndex, kernels->widths->data.F32[bestIndex], kernels->u->data.S32[bestIndex], + kernels->v->data.S32[bestIndex], bestCoeff, diff, bestChi2); + + if (fabsf(diff) < tolerance) { + cutIndex = iter; + break; + } + + lastChi2 = bestChi2; + } + psFree(targets); + psFree(sumC); + psFree(sumCC); + + if (cutIndex < 0) { + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Unable to converge to tolerance %g\n", tolerance); + psFree(ranking); + psFree(kernels); + psFree(badStamps); + return NULL; + } + + int newSize = cutIndex + 1; // Size of new kernel basis set + psTrace("psModules.imcombine", 2, "Accepting %d kernels.\n", newSize); + psVector *uNew = psVectorAlloc(newSize, PS_TYPE_S32); + psVector *vNew = psVectorAlloc(newSize, PS_TYPE_S32); + psVector *widthsNew = psVectorAlloc(newSize, PS_TYPE_F32); + psArray *preCalcNew = psArrayAlloc(newSize); + psArray *convNew = psArrayAlloc(numStamps); + for (int i = 0; i < numStamps; i++) { + if (badStamps->data.U8[i]) { + continue; + } + convNew->data[i] = psArrayAlloc(newSize); + } + + for (int i = 0; i < numKernels; i++) { + int rank = ranking->data.S32[i]; // This kernel component's ranking + if (rank >= 0 && rank < newSize) { + uNew->data.S32[rank] = kernels->u->data.S32[i]; + vNew->data.S32[rank] = kernels->v->data.S32[i]; + widthsNew->data.F32[rank] = kernels->widths->data.F32[i]; + preCalcNew->data[rank] = psMemIncrRefCounter(kernels->preCalc->data[i]); + + for (int j = 0; j < numStamps; j++) { + if (badStamps->data.U8[j]) { + continue; + } + pmSubtractionStamp *stamp = stamps->stamps->data[j]; // Stamp of interest + psArray *convolutions = convNew->data[j]; // Convolutions for this stamp + convolutions->data[rank] = psMemIncrRefCounter(selectConvolution(stamp, i, mode)); + } + } + } + psFree(kernels->u); + psFree(kernels->v); + psFree(kernels->widths); + psFree(kernels->preCalc); + kernels->u = uNew; + kernels->v = vNew; + kernels->widths = widthsNew; + kernels->preCalc = preCalcNew; + kernels->num = newSize; + + for (int i = 0; i < numStamps; i++) { + if (badStamps->data.U8[i]) { + continue; + } + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + psFree(stamp->convolutions1); + psFree(stamp->convolutions2); + switch (mode) { + case PM_SUBTRACTION_MODE_1: + stamp->convolutions1 = convNew->data[i]; + stamp->convolutions2 = NULL; + break; + case PM_SUBTRACTION_MODE_2: + stamp->convolutions1 = NULL; + stamp->convolutions2 = convNew->data[i]; + break; + default: + psAbort("Unsupported subtraction mode: %x", mode); + } + } + + psFree(badStamps); + psFree(ranking); + + // Maintain photometric scaling + if (type == PM_SUBTRACTION_KERNEL_ISIS) { + psKernel *subtract = kernels->preCalc->data[0]; // Kernel to subtract from the rest + for (int i = 1; i < newSize; i++) { + if (kernels->u->data.S32[i] % 2 == 0 && kernels->v->data.S32[i] % 2 == 0) { + psKernel *kernel = kernels->preCalc->data[i]; // Kernel of interest + psBinaryOp(kernel->image, kernel->image, "-", subtract->image); + } + } + } else if (type == PM_SUBTRACTION_KERNEL_GUNK) { + psStringPrepend(&kernels->description, "GUNK="); + psStringAppend(&kernels->description, "+POIS(%d,%d)", inner, spatialOrder); + + for (int i = 0; i < newSize; i++) { + if (kernels->u->data.S32[i] % 2 == 0 && kernels->v->data.S32[i] % 2 == 0) { + psKernel *kernel = kernels->preCalc->data[i]; // Kernel of interest + kernel->kernel[0][0] -= 1.0; + } + } + + if (!p_pmSubtractionKernelsAddGrid(kernels, numGaussians, inner)) { + psAbort("Should never get here."); + } + + kernels->type = PM_SUBTRACTION_KERNEL_GUNK; + } + + return kernels; +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionParams.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionParams.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionParams.h (revision 18170) @@ -0,0 +1,21 @@ +#ifndef PM_SUBTRACTION_PARAMS_H +#define PM_SUBTRACTION_PARAMS_H + +#include +#include +#include + +/// Generate a set of optimum kernels for ISIS (or GUNK) +pmSubtractionKernels *pmSubtractionKernelsOptimumISIS(pmSubtractionKernelsType type, ///< Kernel type + int size, ///< Half-size of kernel + int inner, ///< Inner radius for GUNK + int spatialOrder, ///< Spatial polynomial order + const psVector *fwhms, ///< Gaussian FWHMs to try + int maxOrder, ///< Maximum polynomial order + const pmSubtractionStampList *stamps, ///< Stamps + int footprint, ///< Convolution footprint for stamps + float tolerance, ///< Maximum difference in chi^2 + pmSubtractionMode mode // Mode for subtraction + ); + +#endif Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionStamps.c =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionStamps.c (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionStamps.c (revision 18170) @@ -0,0 +1,603 @@ +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include + +// All these includes required to get stamps out of an array of pmSources +#include "pmMoments.h" +#include "pmPeaks.h" +#include "pmResiduals.h" +#include "pmHDU.h" +#include "pmFPA.h" +#include "pmGrowthCurve.h" +#include "pmTrend2D.h" +#include "pmPSF.h" +#include "pmModel.h" +#include "pmSource.h" + + +#include "pmSubtraction.h" +#include "pmSubtractionStamps.h" + +#define STAMP_LIST_BUFFER 20 // Number of stamps to add to list at a time + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Private (file-static) functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +// Free function for pmSubtractionStampList +static void subtractionStampListFree(pmSubtractionStampList *list // Stamp list to free + ) +{ + psFree(list->stamps); + psFree(list->regions); + psFree(list->x); + psFree(list->y); + psFree(list->flux); +} + +// Free function for pmSubtractionStamp +static void subtractionStampFree(pmSubtractionStamp *stamp // Stamp to free + ) +{ + psFree(stamp->image1); + psFree(stamp->image2); + psFree(stamp->weight); + psFree(stamp->convolutions1); + psFree(stamp->convolutions2); + + psFree(stamp->matrix1); + psFree(stamp->matrix2); + psFree(stamp->matrixX); + psFree(stamp->vector1); + psFree(stamp->vector2); + +} + +// Is this region OK? +static bool checkStampRegion(int x, int y, // Coordinates of stamp + const psRegion *region // Region of interest + ) +{ + if (!region) { + return true; + } + return (x < region->x0 || x > region->x1 || y < region->y0 || y > region->y1) ? + false : true; +} + +// Is this position unmasked? +static bool checkStampMask(int x, int y, // Coordinates of stamp + const psImage *mask, // Mask + pmSubtractionMode mode // Mode for subtraction + ) +{ + if (!mask) { + return true; + } + if (x < 0 || x >= mask->numCols || y < 0 || y >= mask->numRows) { + return false; + } + + psMaskType maskVal = PM_SUBTRACTION_MASK_BORDER | PM_SUBTRACTION_MASK_REJ; // Mask value + switch (mode) { + case PM_SUBTRACTION_MODE_1: + maskVal |= PM_SUBTRACTION_MASK_FOOTPRINT_1; + break; + case PM_SUBTRACTION_MODE_2: + maskVal |= PM_SUBTRACTION_MASK_FOOTPRINT_2; + break; + case PM_SUBTRACTION_MODE_UNSURE: + case PM_SUBTRACTION_MODE_DUAL: + maskVal |= PM_SUBTRACTION_MASK_FOOTPRINT_1 | PM_SUBTRACTION_MASK_FOOTPRINT_2; + break; + default: + psAbort("Unsupported subtraction mode: %x", mode); + } + + return (mask->data.PS_TYPE_MASK_DATA[y][x] & maskVal) ? false : true; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// Public functions +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +pmSubtractionStampList *pmSubtractionStampListAlloc(int numCols, int numRows, const psRegion *region, + int footprint, float spacing) +{ + pmSubtractionStampList *list = psAlloc(sizeof(pmSubtractionStampList)); // Stamp list to return + psMemSetDeallocator(list, (psFreeFunc)subtractionStampListFree); + + // Get region in which to find stamps: [xMin:xMax,yMin:yMax] + int xMin = 0, xMax = numCols, yMin = 0, yMax = numRows; + if (region) { + xMin = PS_MAX(region->x0, xMin); + xMax = PS_MIN(region->x1, xMax); + yMin = PS_MAX(region->y0, yMin); + yMax = PS_MIN(region->y1, yMax); + } + int xSize = xMax - xMin, ySize = yMax - yMin; // Size of region of interest + int xStamps = (float)xSize / spacing + 1, yStamps = (float)ySize / spacing + 1; // Number of stamps + + list->num = xStamps * yStamps; + list->stamps = psArrayAlloc(list->num); + list->regions = psArrayAlloc(list->num); + + for (int y = 0, index = 0; y < yStamps; y++) { + int yStart = yMin + y * ((float)ySize / (float)(yStamps)); // Subregion starts here + int yStop = yMin + (y + 1) * ((float)ySize / (float)(yStamps)) - 1; // Subregion stops here + assert(yStart >= yMin && yStop < yMax); + + for (int x = 0; x < xStamps; x++, index++) { + int xStart = xMin + x * ((float)xSize / (float)(xStamps)); // Subregion starts here + int xStop = xMin + (x + 1) * ((float)xSize / (float)(xStamps)) - 1; // Subregion stops here + assert(xStart >= xMin && xStop < xMax); + + list->stamps->data[index] = pmSubtractionStampAlloc(); + psTrace("psModules.imcombine", 6, "Stamp region %d: [%d:%d,%d:%d]\n", + index, xStart, xStop, yStart, yStop); + list->regions->data[index] = psRegionAlloc(xStart, xStop, yStart, yStop); + } + } + + list->x = NULL; + list->y = NULL; + list->flux = NULL; + list->footprint = footprint; + + return list; +} + +pmSubtractionStamp *pmSubtractionStampAlloc(void) +{ + pmSubtractionStamp *stamp = psAlloc(sizeof(pmSubtractionStamp)); // Stamp to return + psMemSetDeallocator(stamp, (psFreeFunc)subtractionStampFree); + + stamp->x = NAN; + stamp->y = NAN; + stamp->flux = NAN; + stamp->xNorm = NAN; + stamp->yNorm = NAN; + stamp->status = PM_SUBTRACTION_STAMP_INIT; + + stamp->image1 = NULL; + stamp->image2 = NULL; + stamp->weight = NULL; + stamp->convolutions1 = NULL; + stamp->convolutions2 = NULL; + + stamp->matrix1 = NULL; + stamp->matrix2 = NULL; + stamp->matrixX = NULL; + stamp->vector1 = NULL; + stamp->vector2 = NULL; + + return stamp; +} + + +pmSubtractionStampList *pmSubtractionStampsFind(pmSubtractionStampList *stamps, const psImage *image, + const psImage *subMask, const psRegion *region, + float threshold, int footprint, float spacing, + pmSubtractionMode mode) +{ + PS_ASSERT_IMAGE_NON_NULL(image, NULL); + PS_ASSERT_IMAGE_TYPE(image, PS_TYPE_F32, NULL); + if (subMask) { + PS_ASSERT_IMAGE_NON_NULL(subMask, NULL); + PS_ASSERT_IMAGES_SIZE_EQUAL(image, subMask, NULL); + PS_ASSERT_IMAGE_TYPE(subMask, PS_TYPE_MASK, NULL); + } + PS_ASSERT_INT_NONNEGATIVE(footprint, NULL); + PS_ASSERT_FLOAT_LARGER_THAN(spacing, 0.0, NULL); + if (region) { + if (psRegionIsNaN(*region)) { + psString string = psRegionToString(*region); + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Input region (%s) contains NAN values", string); + psFree(string); + return false; + } + if (region->x0 < 0 || region->x1 > image->numCols || + region->y0 < 0 || region->y1 > image->numRows) { + psString string = psRegionToString(*region); + psError(PS_ERR_BAD_PARAMETER_VALUE, true, "Input region (%s) does not fit in image (%dx%d)", + string, image->numCols, image->numRows); + psFree(string); + return false; + } + } + + int numRows = image->numRows, numCols = image->numCols; // Size of image + + if (!stamps) { + stamps = pmSubtractionStampListAlloc(numCols, numRows, region, footprint, spacing); + } + + int numStamps = stamps->num; // Number of stamp regions + int numFound = 0; // Number of stamps found + int numValid = 0; // Number of valid regions + for (int i = 0; i < numStamps; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + + // Only find a new stamp if we need to + if (stamp->status != PM_SUBTRACTION_STAMP_REJECTED && stamp->status != PM_SUBTRACTION_STAMP_INIT) { + continue; + } + numValid++; + + float xStamp = 0, yStamp = 0; // Coordinates of stamp + float fluxStamp = NAN; // Flux of stamp + bool goodStamp = false; // Found a good stamp? + + // A couple different ways of finding stamps: + if (stamps->x && stamps->y) { + // Get the next stamp from the list + psVector *xList = stamps->x->data[i], *yList = stamps->y->data[i]; // Coordinate lists + psVector *fluxList = stamps->flux->data[i]; // List of stamp fluxes + + // Take stamp off the top of the (sorted) list + if (xList->n > 0) { + int index = xList->n - 1; // Index of new stamp + xStamp = xList->data.F32[index]; + yStamp = yList->data.F32[index]; + fluxStamp = fluxList->data.F32[index]; + + // Chop off the top of the list + xList->n = index; + yList->n = index; + fluxList->n = index; + + goodStamp = true; + } + } else { + // Use a simple method of automatically finding stamps --- take the highest pixel in the subregion + fluxStamp = threshold; + psRegion *subRegion = stamps->regions->data[i]; // Sub-region of interest + for (int y = subRegion->y0; y <= subRegion->y1; y++) { + for (int x = subRegion->x0; x <= subRegion->x1; x++) { + if (checkStampMask(x, y, subMask, mode) && image->data.F32[y][x] > fluxStamp) { + fluxStamp = image->data.F32[y][x]; + xStamp = x; + yStamp = y; + goodStamp = true; + } + } + } + } + + if (goodStamp) { + stamp->x = xStamp; + stamp->y = yStamp; + stamp->flux = fluxStamp; + + // Reset the postage stamps since we're making a new stamp + psFree(stamp->image1); + psFree(stamp->image2); + psFree(stamp->weight); + psFree(stamp->convolutions1); + psFree(stamp->convolutions2); + stamp->image1 = stamp->image2 = stamp->weight = NULL; + stamp->convolutions1 = stamp->convolutions2 = NULL; + + stamp->status = PM_SUBTRACTION_STAMP_FOUND; + numFound++; + psTrace("psModules.imcombine", 5, "Found stamp in subregion %d: %d,%d\n", + i, (int)stamp->x, (int)stamp->y); + } else { + stamp->status = PM_SUBTRACTION_STAMP_NONE; + } + } + + if (numValid > 0) { + psLogMsg("psModules.imcombine", PS_LOG_INFO, "Found %d stamps", numFound); + } + + return stamps; +} + + +pmSubtractionStampList *pmSubtractionStampsSet(const psVector *x, const psVector *y, const psVector *flux, + const psImage *image, const psImage *subMask, + const psRegion *region, int footprint, float spacing, + pmSubtractionMode mode) + +{ + PS_ASSERT_VECTOR_NON_NULL(x, NULL); + PS_ASSERT_VECTOR_TYPE(x, PS_TYPE_F32, NULL); + PS_ASSERT_VECTOR_NON_NULL(y, NULL); + PS_ASSERT_VECTOR_TYPE(y, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(y, x, NULL); + if (flux) { + PS_ASSERT_VECTOR_NON_NULL(flux, NULL); + PS_ASSERT_VECTOR_TYPE(flux, PS_TYPE_F32, NULL); + PS_ASSERT_VECTORS_SIZE_EQUAL(flux, x, NULL); + } else { + PS_ASSERT_IMAGE_NON_NULL(image, NULL); + } + if (subMask) { + PS_ASSERT_IMAGE_NON_NULL(subMask, NULL); + PS_ASSERT_IMAGE_TYPE(subMask, PS_TYPE_MASK, NULL); + if (image) { + PS_ASSERT_IMAGE_NON_NULL(image, NULL); + PS_ASSERT_IMAGES_SIZE_EQUAL(image, subMask, NULL); + } + } + PS_ASSERT_FLOAT_LARGER_THAN(spacing, 0.0, NULL); + + int numStars = x->n; // Number of stars + pmSubtractionStampList *stamps = pmSubtractionStampListAlloc(subMask->numCols, subMask->numRows, + region, footprint, spacing); // Stamp list + int numStamps = stamps->num; // Number of stamps + + // Initialise the lists + stamps->x = psArrayAlloc(numStamps); + stamps->y = psArrayAlloc(numStamps); + stamps->flux = psArrayAlloc(numStamps); + for (int i = 0; i < numStamps; i++) { + stamps->x->data[i] = psVectorAllocEmpty(STAMP_LIST_BUFFER, PS_TYPE_F32); + stamps->y->data[i] = psVectorAllocEmpty(STAMP_LIST_BUFFER, PS_TYPE_F32); + stamps->flux->data[i] = psVectorAllocEmpty(STAMP_LIST_BUFFER, PS_TYPE_F32); + } + + // Put the stars into their appropriate subregions + for (int i = 0; i < numStars; i++) { + float xStamp = x->data.F32[i], yStamp = y->data.F32[i]; // Coordinates of stamp + int xPix = xStamp + 0.5, yPix = yStamp + 0.5; // Pixel coordinate of stamp + if (!checkStampRegion(xPix, yPix, region)) { + // It's not in the big region + psTrace("psModules.imcombine", 9, "Rejecting input stamp (%d,%d) because outside region", + xPix, yPix); + continue; + } + if (!checkStampMask(xPix, yPix, subMask, mode)) { + // Not a good stamp + psTrace("psModules.imcombine", 9, "Rejecting input stamp (%d,%d) because bad mask", + xPix, yPix); + continue; + } + + bool found = false; + for (int j = 0; j < numStamps && !found; j++) { + psRegion *subRegion = stamps->regions->data[j]; // Subregion of interest + if (checkStampRegion(xPix, yPix, subRegion)) { + psVector *xList = stamps->x->data[j], *yList = stamps->y->data[j]; // Pixel lists + psVector *fluxList = stamps->flux->data[j]; // Flux list + + int index = xList->n; // Index of new stamp candidate + + psVectorExtend(xList, STAMP_LIST_BUFFER, 1); + psVectorExtend(yList, STAMP_LIST_BUFFER, 1); + psVectorExtend(fluxList, STAMP_LIST_BUFFER, 1); + + xList->data.F32[index] = xStamp; + yList->data.F32[index] = yStamp; + + if (flux) { + fluxList->data.F32[index] = flux->data.F32[i]; + } else { + fluxList->data.F32[index] = image->data.F32[yPix][xPix]; + } + + found = true; + psTrace("psModules.imcombine", 9, "Putting input stamp (%d,%d) into subregion %d", + xPix, yPix, j); + } + } + + if (!found) { + psTrace("psModules.imcombine", 9, "Unable to find subregion for stamp (%d,%d)", + xPix, yPix); + } + } + + // Sort the list by flux, with the brightest last + for (int i = 0; i < numStamps; i++) { + psVector *xList = stamps->x->data[i], *yList = stamps->y->data[i]; // Pixel lists + psVector *fluxList = stamps->flux->data[i]; // Flux list + + psVector *indexes = psVectorSortIndex(NULL, fluxList); // Indices to sort flux + int num = indexes->n; // Number of candidate stamps in this subregion + + psVector *xSorted = psVectorAlloc(num, PS_TYPE_F32); // Sorted version of x list + psVector *ySorted = psVectorAlloc(num, PS_TYPE_F32); // Sorted version of y list + psVector *fluxSorted = psVectorAlloc(num, PS_TYPE_F32); // Sorted version of flux list + for (int j = 0; j < num; j++) { + int k = indexes->data.S32[j]; // Sorted index + xSorted->data.F32[j] = xList->data.F32[k]; + ySorted->data.F32[j] = yList->data.F32[k]; + fluxSorted->data.F32[j] = fluxList->data.F32[k]; + } + psFree(indexes); + + psFree(stamps->x->data[i]); + psFree(stamps->y->data[i]); + psFree(stamps->flux->data[i]); + + stamps->x->data[i] = xSorted; + stamps->y->data[i] = ySorted; + stamps->flux->data[i] = fluxSorted; + } + + + return stamps; +} + + +bool pmSubtractionStampsExtract(pmSubtractionStampList *stamps, psImage *image1, psImage *image2, + psImage *weight, int kernelSize) +{ + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false); + PS_ASSERT_IMAGE_NON_NULL(image1, false); + PS_ASSERT_IMAGE_TYPE(image1, PS_TYPE_F32, false); + if (image2) { + PS_ASSERT_IMAGE_NON_NULL(image2, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(image2, image1, false); + PS_ASSERT_IMAGE_TYPE(image2, PS_TYPE_F32, false); + } + PS_ASSERT_IMAGE_NON_NULL(weight, false); + PS_ASSERT_IMAGES_SIZE_EQUAL(weight, image1, false); + PS_ASSERT_IMAGE_TYPE(weight, PS_TYPE_F32, false); + PS_ASSERT_INT_NONNEGATIVE(kernelSize, false); + + int numCols = image1->numCols, numRows = image1->numRows; // Size of images + int size = kernelSize + stamps->footprint; // Size of postage stamps + + for (int i = 0; i < stamps->num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (!stamp || stamp->status != PM_SUBTRACTION_STAMP_FOUND) { + continue; + } + + if (isnan(stamp->xNorm)) { + stamp->xNorm = 2.0 * (stamp->x - (float)numCols/2.0) / (float)numCols; + } + if (isnan(stamp->yNorm)) { + stamp->yNorm = 2.0 * (stamp->y - (float)numRows/2.0) / (float)numRows; + } + + int x = stamp->x + 0.5, y = stamp->y + 0.5; // Stamp coordinates + if (x < size || x > numCols - size || y < size || y > numRows - size) { + psError(PS_ERR_UNKNOWN, false, "Stamp %d (%d,%d) is within the image border.\n", i, x, y); + return false; + } + + // Catch memory leaks --- these should have been freed and NULLed before + assert(stamp->image1 == NULL); + assert(stamp->image2 == NULL); + assert(stamp->weight == NULL); + + psRegion region = psRegionSet(x - size, x + size + 1, y - size, y + size + 1); // Region of interest + + psImage *sub1 = psImageSubset(image1, region); // Subimage with stamp + stamp->image1 = psKernelAllocFromImage(sub1, size, size); + psFree(sub1); // Drop reference + + if (image2) { + psImage *sub2 = psImageSubset(image2, region); // Subimage with stamp + stamp->image2 = psKernelAllocFromImage(sub2, size, size); + psFree(sub2); // Drop reference + } + + psImage *wtSub = psImageSubset(weight, region); // Subimage with stamp + stamp->weight = psKernelAllocFromImage(wtSub, size, size); + psFree(wtSub); // Drop reference + + stamp->status = PM_SUBTRACTION_STAMP_CALCULATE; + } + + return true; +} + +#if 0 +bool pmSubtractionStampsGenerate(pmSubtractionStampList *stamps, float fwhm, int kernelSize) +{ + PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(stamps, false); + PS_ASSERT_FLOAT_LARGER_THAN(fwhm, 0.0, false); + PS_ASSERT_INT_NONNEGATIVE(kernelSize, false); + + int size = kernelSize + stamps->footprint; // Size of postage stamps + int num = stamps->num; // Number of stamps + float sigma = fwhm / (2.0 * sqrtf(2.0 * log(2.0))); // Gaussian sigma + + for (int i = 0; i < num; i++) { + pmSubtractionStamp *stamp = stamps->stamps->data[i]; // Stamp of interest + if (!(stamp->status & PM_SUBTRACTION_STAMP_CALCULATE)) { + continue; + } + + float x = stamp->x, y = stamp->y; // Coordinates of stamp + float flux = stamp->flux; // Flux of star + if (!isfinite(flux)) { + psWarning("Unable to generate PSF for stamp %d --- bad flux.", i); + stamp->status = PM_SUBTRACTION_STAMP_REJECTED; + continue; + } + + float xStamp = x - (int)(x + 0.5); // x coordinate of star in stamp frame + float yStamp = y - (int)(y + 0.5); // y coordinate of star in stamp frame + + psFree(stamp->image2); + stamp->image2 = psKernelAlloc(-size, size, -size, size); + psKernel *target = stamp->image2; // Target stamp + + // Put in a Waussian, just for fun! + for (int v = -size; v <= size; v++) { + for (int u = -size; u <= size; u++) { + float z = (PS_SQR(u + xStamp) + PS_SQR(v + yStamp)) / (2.0 * PS_SQR(sigma)); + target->kernel[v][u] = flux / sigma * 0.5 * M_2_SQRTPI * M_SQRT1_2 / (1.0 + z + PS_SQR(z)); + } + } + + } + + return true; +} +#endif + +pmSubtractionStampList *pmSubtractionStampsSetFromSources(const psArray *sources, const psImage *subMask, + const psRegion *region, int footprint, + float spacing, pmSubtractionMode mode) +{ + PS_ASSERT_ARRAY_NON_NULL(sources, NULL); + // Let pmSubtractionStampsSet take care of the rest of the assertions + + int numSources = sources->n; // Number of stars + + psVector *x = psVectorAlloc(numSources, PS_TYPE_F32); // x coordinates + psVector *y = psVectorAlloc(numSources, PS_TYPE_F32); // y coordinates + psVector *flux = psVectorAlloc(numSources, PS_TYPE_F32); // Fluxes + + for (int i = 0; i < numSources; i++) { + pmSource *source = sources->data[i]; // Source of interest + if (source->modelPSF) { + x->data.F32[i] = source->modelPSF->params->data.F32[PM_PAR_XPOS]; + y->data.F32[i] = source->modelPSF->params->data.F32[PM_PAR_YPOS]; + } else { + x->data.F32[i] = source->peak->xf; + y->data.F32[i] = source->peak->yf; + } + flux->data.F32[i] = powf(10.0, -0.4 * source->psfMag); + } + + pmSubtractionStampList *stamps = pmSubtractionStampsSet(x, y, flux, NULL, subMask, region, + footprint, spacing, mode); // Stamps to return + psFree(x); + psFree(y); + psFree(flux); + + if (!stamps) { + psError(PS_ERR_UNKNOWN, false, "Unable to set stamps from sources."); + } + + return stamps; +} + + +pmSubtractionStampList *pmSubtractionStampsSetFromFile(const char *filename, const psImage *image, + const psImage *subMask, const psRegion *region, + int footprint, float spacing, pmSubtractionMode mode) +{ + PS_ASSERT_STRING_NON_EMPTY(filename, NULL); + // Let pmSubtractionStampsSet take care of the rest of the assertions + + psArray *data = psVectorsReadFromFile(filename, "%f %f"); + if (!data) { + psError(PS_ERR_IO, false, "Unable to read stamps file %s", filename); + return NULL; + } + psVector *x = data->data[0], *y = data->data[1]; // Stamp positions + + // Correct for IRAF/FITS (unit-offset) positions to C (zero-offset) positions + psBinaryOp(x, x, "-", psScalarAlloc(1.0, PS_TYPE_F32)); + psBinaryOp(y, y, "-", psScalarAlloc(1.0, PS_TYPE_F32)); + + pmSubtractionStampList *stamps = pmSubtractionStampsSet(x, y, NULL, image, subMask, region, footprint, + spacing, mode); + psFree(data); + + return stamps; + +} Index: /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionStamps.h =================================================================== --- /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionStamps.h (revision 18170) +++ /branches/pap_branch_080617/psModules/src/imcombine/pmSubtractionStamps.h (revision 18170) @@ -0,0 +1,126 @@ +#ifndef PM_SUBTRACTION_STAMPS_H +#define PM_SUBTRACTION_STAMPS_H + +#include + +#include "pmSubtractionKernels.h" + +/// Status of stamp +typedef enum { + PM_SUBTRACTION_STAMP_INIT, ///< Initial state + PM_SUBTRACTION_STAMP_FOUND, ///< Found a suitable source for this stamp + PM_SUBTRACTION_STAMP_CALCULATE, ///< Calculate matrix and vector values for this stamp + PM_SUBTRACTION_STAMP_USED, ///< Use this stamp + PM_SUBTRACTION_STAMP_REJECTED, ///< This stamp has been rejected + PM_SUBTRACTION_STAMP_NONE ///< No stamp in this region +} pmSubtractionStampStatus; + +/// A list of stamps +typedef struct { + long num; ///< Number of stamps + psArray *stamps; ///< The stamps + psArray *regions; ///< Regions for each stamp + psArray *x, *y; ///< Coordinates for possible stamps (or NULL) + psArray *flux; ///< Fluxes for possible stamps (or NULL) + int footprint; ///< Half-size of stamps +} pmSubtractionStampList; + +/// Allocate a list of stamps +pmSubtractionStampList *pmSubtractionStampListAlloc(int numCols, // Number of columns in image + int numRows, // Number of rows in image + const psRegion *region, // Region for stamps, or NULL + int footprint, // Half-size of stamps + float spacing // Rough average spacing between stamps + ); + +/// Assertion for stamp list to be valid +#define PM_ASSERT_SUBTRACTION_STAMP_LIST_NON_NULL(LIST, RETURNVALUE) { \ + PS_ASSERT_PTR_NON_NULL(LIST, RETURNVALUE); \ + PS_ASSERT_ARRAY_NON_NULL((LIST)->stamps, RETURNVALUE); \ + PS_ASSERT_ARRAY_NON_NULL((LIST)->regions, RETURNVALUE); \ + PS_ASSERT_INT_POSITIVE((LIST)->num, RETURNVALUE); \ + PS_ASSERT_ARRAY_SIZE((LIST)->stamps, (LIST)->num, RETURNVALUE); \ + PS_ASSERT_ARRAY_SIZE((LIST)->regions, (LIST)->num, RETURNVALUE); \ + PS_ASSERT_INT_NONNEGATIVE((LIST)->footprint, RETURNVALUE); \ + if ((LIST)->x || (LIST)->y || (LIST)->flux) { \ + PS_ASSERT_ARRAY_NON_NULL((LIST)->x, RETURNVALUE); \ + PS_ASSERT_ARRAY_NON_NULL((LIST)->y, RETURNVALUE); \ + PS_ASSERT_ARRAY_NON_NULL((LIST)->flux, RETURNVALUE); \ + PS_ASSERT_ARRAY_SIZE((LIST)->x, (LIST)->num, RETURNVALUE); \ + PS_ASSERT_ARRAY_SIZE((LIST)->y, (LIST)->num, RETURNVALUE); \ + PS_ASSERT_ARRAY_SIZE((LIST)->flux, (LIST)->num, RETURNVALUE); \ + } \ +} + +/// A stamp for image subtraction +typedef struct { + float x, y; ///< Position + float flux; ///< Flux + float xNorm, yNorm; ///< Normalised position + psKernel *image1; ///< Reference image postage stamp + psKernel *image2; ///< Input image postage stamp + psKernel *weight; ///< Weight image postage stamp, or NULL + psArray *convolutions1; ///< Convolutions of image 1 for each kernel component, or NULL + psArray *convolutions2; ///< Convolutions of image 2 for each kernel component, or NULL + psImage *matrix1, *matrix2; ///< Least-squares matrices for each image, or NULL + psImage *matrixX; ///< Cross-matrix (for mode DUAL), or NULL + psVector *vector1, *vector2; ///< Least-squares vectors for each image, or NULL + pmSubtractionStampStatus status; ///< Status of stamp +} pmSubtractionStamp; + +/// Allocate a stamp +pmSubtractionStamp *pmSubtractionStampAlloc(void); + +/// Find stamps on an image +pmSubtractionStampList *pmSubtractionStampsFind(pmSubtractionStampList *stamps, ///< Output stamps, or NULL + const psImage *image, ///< Image for which to find stamps + const psImage *mask, ///< Mask, or NULL + const psRegion *region, ///< Region to search, or NULL + float threshold, ///< Threshold for stamps in the image + int footprint, ///< Half-size for stamps + float spacing, ///< Rough spacing for stamps + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Set stamps based on a list of x,y +pmSubtractionStampList *pmSubtractionStampsSet(const psVector *x, ///< x coordinates for each stamp + const psVector *y, ///< y coordinates for each stamp + const psVector *flux, ///< Flux for each stamp, or NULL + const psImage *image, ///< Image for flux of stamp + const psImage *mask, ///< Mask, or NULL + const psRegion *region, ///< Region to search, or NULL + int footprint, ///< Half-size for stamps + float spacing, ///< Rough spacing for stamps + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Set stamps based on a list of sources +pmSubtractionStampList *pmSubtractionStampsSetFromSources( + const psArray *sources, ///< Sources for each stamp + const psImage *subMask, ///< Mask, or NULL + const psRegion *region, ///< Region to search, or NULL + int footprint, ///< Half-size for stamps + float spacing, ///< Rough spacing for stamps + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Set stamps based on values in a file +pmSubtractionStampList *pmSubtractionStampsSetFromFile( + const char *filename, ///< Filename of file containing x,y (or x,y,flux) on each line + const psImage *image, ///< Image for flux of stamp + const psImage *subMask, ///< Mask, or NULL + const psRegion *region, ///< Region to search, or NULL + int footprint, ///< Half-size for stamps + float spacing, ///< Rough spacing for stamps + pmSubtractionMode mode ///< Mode for subtraction + ); + +/// Extract stamps from the images +bool pmSubtractionStampsExtract(pmSubtractionStampList *stamps, ///< Stamps + psImage *image1, ///< Reference image + psImage *image2, ///< Input image (or NULL) + psImage *weight, ///< Weight (variance) map + int kernelSize ///< Kernel half-size + ); + +#endif