Index: trunk/Ohana/src/libdvo/Makefile =================================================================== --- trunk/Ohana/src/libdvo/Makefile (revision 38154) +++ trunk/Ohana/src/libdvo/Makefile (revision 38441) @@ -10,4 +10,6 @@ MAN = $(HOME)/doc INC = $(HOME)/include +TEST = $(HOME)/test +TESTBIN = $(HOME)/test ASRC = $(AUTO)/src AINC = $(AUTO)/include @@ -153,4 +155,5 @@ rm -f $@ cp $< $@ + echo "done making autocode.h" $(AINC)/autocode.h: $(AINCS) $(ADEF)/autocode.h $(ADEF)/common.h Index: trunk/Ohana/src/libdvo/include/dvo.h =================================================================== --- trunk/Ohana/src/libdvo/include/dvo.h (revision 38154) +++ trunk/Ohana/src/libdvo/include/dvo.h (revision 38441) @@ -10,9 +10,12 @@ /* DVO table modes */ -typedef enum {DVO_MODE_UNDEF, DVO_MODE_RAW, DVO_MODE_MEF, DVO_MODE_SPLIT, DVO_MODE_MYSQL} DVOTableMode; +typedef enum {DVO_MODE_UNDEF = 0, DVO_MODE_RAW, DVO_MODE_MEF, DVO_MODE_SPLIT, DVO_MODE_MYSQL} DVOCatMode; + +/* DVO table modes */ +typedef enum {DVO_COMPRESS_NONE = 0, DVO_COMPRESS_NONE_1, DVO_COMPRESS_NONE_2, DVO_COMPRESS_AUTO, DVO_COMPRESS_GZIP_1, DVO_COMPRESS_GZIP_2, DVO_COMPRESS_RICE_1} DVOCatCompress; // /* DVO table formats */ typedef enum { - DVO_FORMAT_UNDEF, + DVO_FORMAT_UNDEF = 0, DVO_FORMAT_INTERNAL, DVO_FORMAT_ELIXIR, @@ -30,11 +33,32 @@ DVO_FORMAT_PS1_V4, DVO_FORMAT_PS1_V5, -} DVOTableFormat; +} DVOCatFormat; + +typedef enum {DVO_CAT_OPEN_FAIL, DVO_CAT_OPEN_OK, DVO_CAT_OPEN_EMPTY} DVOCatalogOpenModes; + +/* catalog values to be loaded */ +typedef enum { + DVO_LOAD_NONE = 0x0000, + DVO_LOAD_AVERAGE = 0x0001, + DVO_LOAD_MEASURE = 0x0002, + DVO_LOAD_MISSING = 0x0004, + DVO_LOAD_SECFILT = 0x0008, + DVO_SKIP_AVERAGE = 0x0010, + DVO_SKIP_MEASURE = 0x0020, + DVO_SKIP_MISSING = 0x0040, + DVO_SKIP_SECFILT = 0x0080, + DVO_LOAD_LENSING = 0x0100, + DVO_LOAD_LENSOBJ = 0x0200, + DVO_SKIP_LENSING = 0x0400, + DVO_SKIP_LENSOBJ = 0x0800, + DVO_LOAD_STARPAR = 0x1000, + DVO_SKIP_STARPAR = 0x2000, + DVO_LOAD_GALPHOT = 0x4000, + DVO_SKIP_GALPHOT = 0x8000, +} DVOCatFlags; /* image data modes in RegImage */ typedef enum {T_UNDEF = -1, T_NONE, T_OBJECT, T_DARK, T_BIAS, T_FLAT, T_MASK, T_FRINGE, T_SCATTER, T_MODES, T_FRINGEPTS, T_ANY, N_TYPE} ElixirDetrendTypes; typedef enum {M_UNDEF = -1, M_NONE, M_MEF, M_SPLIT, M_SINGLE, M_CUBE, M_SLICE, M_MODES, N_MODE} ElixirDetrendModes; - -typedef enum {DVO_CAT_OPEN_FAIL, DVO_CAT_OPEN_OK, DVO_CAT_OPEN_EMPTY} DVOCatalogOpenModes; // these are used as NAN for types of int values @@ -54,28 +78,13 @@ # define IMREG_DIST 0x01 /* image distributed, only imregister-3.0 */ -/* catalog values to be loaded */ -# define LOAD_NONE 0x0000 -# define LOAD_AVES 0x0001 -# define LOAD_MEAS 0x0002 -# define LOAD_MISS 0x0004 -# define LOAD_SECF 0x0008 -# define SKIP_AVES 0x0010 -# define SKIP_MEAS 0x0020 -# define SKIP_MISS 0x0040 -# define SKIP_SECF 0x0080 -# define LOAD_LENSING 0x0100 -# define LOAD_LENSOBJ 0x0200 -# define SKIP_LENSING 0x0400 -# define SKIP_LENSOBJ 0x0800 -# define LOAD_STARPAR 0x1000 -# define SKIP_STARPAR 0x2000 - /* photometry code types */ // # define PHOT_PRI 0x01 -# define PHOT_SEC 0x02 -# define PHOT_DEP 0x03 -# define PHOT_REF 0x04 -# define PHOT_ALT 0x05 /* never stored, only for look-ups */ -# define PHOT_MAG 0x06 /* generic magnitude; never stored */ +typedef enum { + PHOT_SEC = 0x02, + PHOT_DEP = 0x03, + PHOT_REF = 0x04, + PHOT_ALT = 0x05, /* never stored, only for look-ups */ + PHOT_MAG = 0x06, /* generic magnitude; never stored */ +} DVOPhotCodeTypes; /* Image.code values -- these values are 32 bit (as of PS1_V1) */ @@ -222,7 +231,8 @@ char lockstate; char mode; /* what data storage mode is used for disk file? */ - char format; /* what data format is used for disk file? */ + DVOCatFormat format; /* what data format is used for disk file? */ char virtual; /* is table in ftable or vtable? */ - char swapped; /* is table in internal byte-order? */ + char nativeOrder; /* is table in internal byte-order? */ + char scaledValue; /* is table in internal byte-order? */ } FITS_DB; @@ -484,5 +494,5 @@ unsigned short Nmeasure; // number of psf measurements unsigned short Nmissing; // number of missings - unsigned short Nextend; // number of extended measurements + unsigned short Ngalphot; // number of extended measurements uint32_t measureOffset; // offset to first psf measurement uint32_t missingOffset; // offset to first missing obs @@ -704,10 +714,10 @@ unsigned short Nlensing; // number of lensing measurements unsigned short Nlensobj; // number of lensing measurements - unsigned short Nextend; // number of extended measurements + unsigned short Ngalphot; // number of galphot measurements int measureOffset; // offset to first psf measurement int missingOffset; // offset to first missing obs int lensingOffset; // offset to first lensing obs int lensobjOffset; // offset to mean lensing data - int extendOffset; // offset to extended object entry + int galphotOffset; // offset to extended object entry int starparOffset; // offset to stellar parameter data float refColorBlue; // color of astrometry ref stars @@ -742,12 +752,13 @@ // lensing data (optional?) - Lensing *lensing; - Lensobj *lensobj; - StarPar *starpar; + Lensing *lensing; + Lensobj *lensobj; + StarPar *starpar; + GalPhot *galphot; int Nsecfilt; /* number of secfilt entries for each average entry */ - off_t Naverage, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar, Nsecf_mem; /* current number of each component in memory */ - off_t Naves_disk, Nmeas_disk, Nmiss_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Nsecf_disk; /* current number of each component on disk */ - off_t Naves_off, Nmeas_off, Nmiss_off, Nlensing_off, Nlensobj_off, Nstarpar_off, Nsecf_off; /* index of first loaded data value */ + off_t Naverage, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar, Ngalphot, Nsecfilt_mem; /* current number of each component in memory */ + off_t Naverage_disk, Nmeasure_disk, Nmissing_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Ngalphot_disk, Nsecfilt_disk; /* current number of each component on disk */ + off_t Naverage_off, Nmeasure_off, Nmissing_off, Nlensing_off, Nlensobj_off, Nstarpar_off, Ngalphot_off, Nsecfilt_off; /* index of first loaded data value */ // note that we use these for the full-sky relphot analysis @@ -765,10 +776,11 @@ /* pointers to split data files */ - struct Catalog *measure_catalog; /* measure catalog data (split) */ - struct Catalog *missing_catalog; /* missing catalog data (split) */ - struct Catalog *secfilt_catalog; /* secfilt catalog data (split) */ - struct Catalog *lensing_catalog; /* lensing catalog data (split) */ - struct Catalog *lensobj_catalog; /* lensobj catalog data (split) */ - struct Catalog *starpar_catalog; /* starpar catalog data (split) */ + struct Catalog *measure_catalog; /* measure catalog data (split) */ + struct Catalog *missing_catalog; /* missing catalog data (split) */ + struct Catalog *secfilt_catalog; /* secfilt catalog data (split) */ + struct Catalog *lensing_catalog; /* lensing catalog data (split) */ + struct Catalog *lensobj_catalog; /* lensobj catalog data (split) */ + struct Catalog *starpar_catalog; /* starpar catalog data (split) */ + struct Catalog *galphot_catalog; /* galphot catalog data (split) */ unsigned int objID; @@ -777,9 +789,11 @@ /* extra catalog information */ char lockmode; - char catmode; /* storage mode (raw, mef, split, mysql) */ - char catformat; /* storage format (elixir, panstarrs, etc) */ + DVOCatMode catmode; /* storage mode (raw, mef, split, mysql) */ + DVOCatFormat catformat; /* storage format (elixir, panstarrs, etc) */ + DVOCatCompress catcompress; // output compress mode + DVOCatFlags catflags; /* choices to be loaded */ + int sorted; /* is measure table average-sorted? (NOTE this is an int only because gfits_scan %t requires it) */ - - short catflags; /* choices to be loaded */ + /* pointers for data manipulation */ @@ -928,6 +942,8 @@ int dvo_catalog_save_complete (Catalog *catalog, char VERBOSE); int dvo_catalog_update (Catalog *catalog, char VERBOSE); -int dvo_catalog_catformat (char *catformat); -int dvo_catalog_catmode (char *catmode); +DVOCatFormat dvo_catalog_catformat (char *catformat); +DVOCatMode dvo_catalog_catmode (char *catmode); +DVOCatCompress dvo_catalog_catcompress (char *catcompress); +char *dvo_catalog_compress_string (DVOCatCompress catcompress); void dvo_catalog_test (Catalog *catalog, int halt); @@ -956,22 +972,28 @@ int WriteRawSecFilt (FILE *f, SecFilt *secfilt, off_t Nsecfilt, char format); -DVOTableFormat FtableGetFormat (FTable *ftable); - -Average *FtableToAverage (FTable *ftable, off_t *Naverage, char *format, SecFilt **primary); -Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, char *format); -SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, char *format); -Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, char *format); -Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, char *format); -StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, char *format); -int FtableToImage (FTable *ftable, Header *theader, char *format); - -int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, char format, SecFilt *primary); -int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, char format); -int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, char format); -int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, char format); -int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, char format); -int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, char format); -int ImageToFtable (FTable *ftable, Header *theader, char format); -int ImageToVtable (VTable *vtable, Header *theader, char format); +DVOCatFormat FtableGetFormat (FTable *ftable); + +Average *FtableToAverage (FTable *ftable, off_t *Naverage, DVOCatFormat *format, SecFilt **primary, char nativeOrder); + +Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeOrder); +Missing *FtableToMissing (FTable *ftable, Average *average, off_t *Nmissing, DVOCatFormat *format, char nativeBytes); +SecFilt *FtableToSecFilt (FTable *ftable, Average *average, off_t *Nsecfilt, DVOCatFormat *format, char nativeOrder); +Lensing *FtableToLensing (FTable *ftable, Average *average, off_t *Nlensing, DVOCatFormat *format, char nativeOrder); +Lensobj *FtableToLensobj (FTable *ftable, Average *average, off_t *Nlensobj, DVOCatFormat *format, char nativeOrder); +StarPar *FtableToStarPar (FTable *ftable, Average *average, off_t *Nstarpar, DVOCatFormat *format, char nativeOrder); +GalPhot *FtableToGalPhot (FTable *ftable, Average *average, off_t *Ngalphot, DVOCatFormat *format, char nativeOrder); + +int FtableToImage (FTable *ftable, Header *theader, DVOCatFormat *format); + +int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, DVOCatFormat format, int swapFromNative); +int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, DVOCatFormat format, SecFilt *primary, int swapFromNative); +int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, DVOCatFormat format, int swapFromNative); +int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, DVOCatFormat format, int swapFromNative); +int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, DVOCatFormat format, int swapFromNative); +int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, DVOCatFormat format, int swapFromNative); +int GalPhotToFtable (FTable *ftable, GalPhot *galphot, off_t Ngalphot, DVOCatFormat format, int swapFromNative); + +int ImageToFtable (FTable *ftable, Header *theader, DVOCatFormat format); +int ImageToVtable (VTable *vtable, Header *theader, DVOCatFormat format); # include "loneos_defs.h" @@ -1005,5 +1027,5 @@ int gfits_table_set_Image (FTable *ftable); int gfits_table_mkheader_Image (Header *header); -Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *swapped); +Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *scaledValue, char *nativeOrder); /* flatcorr APIs */ @@ -1069,5 +1091,7 @@ void InitHost (HostInfo *host); -HostTableGroup *HostTableGroups (HostTable *table, int *ngroups); +HostTableGroup *HostTableGroupsUniqueMachines (HostTable *table, int *ngroups); +HostTableGroup *HostTableGroupsMaxNumber (HostTable *table, int *ngroups, int Nmax); + int HostTableGroupWaitJobsGetIO (HostTableGroup *table, char *file, int lineno, int VERBOSE); @@ -1106,4 +1130,5 @@ void dvo_lensobj_init (Lensobj *lensobj, int toZero); void dvo_starpar_init (StarPar *starpar); +void dvo_galphot_init (GalPhot *galphot); void InitRegionHosts (RegionHostInfo *hosts, int Nhosts, int NHOSTS); Index: trunk/Ohana/src/libdvo/include/ps1_v5_defs.h =================================================================== --- trunk/Ohana/src/libdvo/include/ps1_v5_defs.h (revision 38154) +++ trunk/Ohana/src/libdvo/include/ps1_v5_defs.h (revision 38441) @@ -23,4 +23,7 @@ StarPar_PS1_V5 *StarParInternalTo_PS1_V5 (StarPar *in, off_t Nvalues); +GalPhot *GalPhot_PS1_V5_ToInternal (GalPhot_PS1_V5 *in, off_t Nvalues); +GalPhot_PS1_V5 *GalPhotInternalTo_PS1_V5 (GalPhot *in, off_t Nvalues); + PhotCode *PhotCode_PS1_V5_To_Internal (PhotCode_PS1_V5 *in, off_t Nvalues); PhotCode_PS1_V5 *PhotCode_Internal_To_PS1_V5 (PhotCode *in, off_t Nvalues); Index: trunk/Ohana/src/libdvo/src/AstromOffsetMapIO.c =================================================================== --- trunk/Ohana/src/libdvo/src/AstromOffsetMapIO.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/AstromOffsetMapIO.c (revision 38441) @@ -39,5 +39,5 @@ return (NULL); } - AstromOffsetMap_Disk_6x6 *map_disk = gfits_table_get_AstromOffsetMap_Disk_6x6 (&ftable, &Nmap, NULL); + AstromOffsetMap_Disk_6x6 *map_disk = gfits_table_get_AstromOffsetMap_Disk_6x6 (&ftable, &Nmap, NULL, NULL); if (!map_disk) { fprintf (stderr, "ERROR: failed to read Astrom Offset Map\n"); @@ -78,5 +78,5 @@ ftable.header = &theader; - gfits_table_set_AstromOffsetMap_Disk_6x6 (&ftable, map_disk, Nmap); + gfits_table_set_AstromOffsetMap_Disk_6x6 (&ftable, map_disk, Nmap, TRUE); f = fopen (filename, "w"); Index: trunk/Ohana/src/libdvo/src/HostTable.c =================================================================== --- trunk/Ohana/src/libdvo/src/HostTable.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/HostTable.c (revision 38441) @@ -155,5 +155,5 @@ // split a host table into Ngroups, each with a unique set of hosts -HostTableGroup *HostTableGroups (HostTable *table, int *ngroups) { +HostTableGroup *HostTableGroupsUniqueMachines (HostTable *table, int *ngroups) { // identify the unique host names and the number of times they each exist @@ -210,4 +210,29 @@ foundHost[k] = TRUE; } + } + } + *ngroups = Ngroups; + return groups; +} + +// split a host table into Ngroups, each with a unique set of hosts +HostTableGroup *HostTableGroupsMaxNumber (HostTable *table, int *ngroups, int Nmax) { + + int i, j; + + int Ngroups = (table->Nhosts % Nmax) ? (int)(table->Nhosts / Nmax + 1) : table->Nhosts / Nmax; + + HostTableGroup *groups = NULL; + ALLOCATE (groups, HostTableGroup, Ngroups); + + // in each group, attempt to add one of each unique host + int k = 0; + for (i = 0; i < Ngroups; i++) { + groups[i].Nhosts = 0; + ALLOCATE (groups[i].hosts, HostInfo *, Nmax); + for (j = 0; j < Nmax; j++) { + groups[i].hosts[groups[i].Nhosts] = &table->hosts[k]; + groups[i].Nhosts ++; + k++; } } Index: trunk/Ohana/src/libdvo/src/LoadImages.c =================================================================== --- trunk/Ohana/src/libdvo/src/LoadImages.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/LoadImages.c (revision 38441) @@ -65,5 +65,5 @@ } - image = gfits_table_get_Image (&db.ftable, Nimage, &db.swapped); + image = gfits_table_get_Image (&db.ftable, Nimage, &db.scaledValue, &db.nativeOrder); if (!image) { fprintf (stderr, "ERROR: failed to read images\n"); Index: trunk/Ohana/src/libdvo/src/LoadPhotcodesFITS.c =================================================================== --- trunk/Ohana/src/libdvo/src/LoadPhotcodesFITS.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/LoadPhotcodesFITS.c (revision 38441) @@ -15,7 +15,7 @@ /* XXX choose more sensible lock timeouts! */ + gfits_db_init (&db); db.timeout = 60.0; db.lockstate = LCK_SOFT; - gfits_db_init (&db); /* does this mean the db is empty, non-existent, or has access errors? */ @@ -43,5 +43,5 @@ # define CONVERT_FORMAT(NAME, FORMAT) \ if (!strcmp (extname, NAME)) { \ - PhotCode_##FORMAT *photcode_input = gfits_table_get_PhotCode_##FORMAT (&db.ftable, &Ncode, &db.swapped); \ + PhotCode_##FORMAT *photcode_input = gfits_table_get_PhotCode_##FORMAT (&db.ftable, &Ncode, &db.scaledValue, &db.nativeOrder); \ if (!photcode_input) { \ fprintf (stderr, "ERROR: failed to read photcodes in LoadPhotcodesFITS.c\n"); \ Index: trunk/Ohana/src/libdvo/src/SavePhotcodesFITS.c =================================================================== --- trunk/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/SavePhotcodesFITS.c (revision 38441) @@ -17,7 +17,7 @@ /* XXX choose more sensible lock timeouts! */ + gfits_db_init (&db); db.lockstate = LCK_XCLD; db.timeout = 10.0; - gfits_db_init (&db); /* does this mean the db is empty, non-existent, or has access errors? */ @@ -34,5 +34,5 @@ /* convert FITS format data to internal format (byteswaps & EXTNAME) */ if (!gfits_db_create (&db)) return (FALSE); - if (!gfits_table_set_PhotCode_PS1_V5 (&db.ftable, photcode_output, table[0].Ncode)) return (FALSE); + if (!gfits_table_set_PhotCode_PS1_V5 (&db.ftable, photcode_output, table[0].Ncode, TRUE)) return (FALSE); if (!gfits_db_save (&db)) return (FALSE); if (!gfits_db_close (&db)) return (FALSE); Index: trunk/Ohana/src/libdvo/src/dvo_catalog.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_catalog.c (revision 38441) @@ -42,5 +42,5 @@ } -int dvo_catalog_catformat (char *catformat) { +DVOCatFormat dvo_catalog_catformat (char *catformat) { /* set the specified CATFORMAT */ @@ -62,5 +62,5 @@ } -int dvo_catalog_catmode (char *catmode) { +DVOCatMode dvo_catalog_catmode (char *catmode) { /* set the specified CATMODE */ @@ -69,4 +69,49 @@ if (!strcasecmp (catmode, "SPLIT")) return (DVO_MODE_SPLIT); return (DVO_MODE_UNDEF); +} + +DVOCatCompress dvo_catalog_catcompress (char *catcompress) { + + /* set the specified CATMODE */ + if (!strcasecmp (catcompress, "NONE")) return (DVO_COMPRESS_NONE); + if (!strcasecmp (catcompress, "AUTO")) return (DVO_COMPRESS_AUTO); + if (!strcasecmp (catcompress, "NONE_1")) return (DVO_COMPRESS_NONE_1); + if (!strcasecmp (catcompress, "NONE_2")) return (DVO_COMPRESS_NONE_2); + if (!strcasecmp (catcompress, "GZIP_1")) return (DVO_COMPRESS_GZIP_1); + if (!strcasecmp (catcompress, "GZIP_2")) return (DVO_COMPRESS_GZIP_2); + if (!strcasecmp (catcompress, "RICE_1")) return (DVO_COMPRESS_RICE_1); + return (DVO_COMPRESS_NONE); +} + +// we can return a static string here unless we run multiple outputs threads at once +static char compress_string[16]; +char *dvo_catalog_compress_string (DVOCatCompress catcompress) { + + switch (catcompress) { + case DVO_COMPRESS_NONE: + strcpy (compress_string, "NONE"); + break; + case DVO_COMPRESS_AUTO: + strcpy (compress_string, "AUTO"); + break; + case DVO_COMPRESS_GZIP_1: + strcpy (compress_string, "GZIP_1"); + break; + case DVO_COMPRESS_GZIP_2: + strcpy (compress_string, "GZIP_2"); + break; + case DVO_COMPRESS_NONE_1: + strcpy (compress_string, "NONE_1"); + break; + case DVO_COMPRESS_NONE_2: + strcpy (compress_string, "NONE_2"); + break; + case DVO_COMPRESS_RICE_1: + strcpy (compress_string, "RICE_1"); + break; + default: + myAbort ("error in compress option"); + } + return compress_string; } @@ -131,5 +176,5 @@ average->Nlensobj = 0; average->Nstarpar = 0; - average->Nextend = 0; + average->Ngalphot = 0; average->measureOffset = -1; @@ -138,5 +183,5 @@ average->lensobjOffset = -1; average->starparOffset = -1; - average->extendOffset = -1; + average->galphotOffset = -1; average->refColorBlue = NAN; @@ -466,4 +511,36 @@ // init all data, or just catalog data +void dvo_galphot_init (GalPhot *galphot) { + galphot->Xfit = NAN; + galphot->Yfit = NAN; + galphot->mag = NAN; + galphot->magErr = NAN; + galphot->majorAxis = NAN; + galphot->minorAxis = NAN; + galphot->majorAxisErr = NAN; + galphot->minorAxisErr = NAN; + galphot->theta = NAN; + galphot->theta_err = NAN; + galphot->index = NAN; + galphot->chisq = NAN; + galphot->Npix = 0; + + galphot->detID = -1; + galphot->objID = -1; + galphot->catID = -1; + galphot->imageID = -1; + + galphot->photcode = 0; + galphot->modelType = 0; +} + +# define INIT_TABLE(TABLE) \ + catalog[0].TABLE = NULL; \ + catalog[0].N##TABLE = 0; \ + catalog[0].N##TABLE##_disk = 0; \ + catalog[0].N##TABLE##_off = 0; \ + catalog[0].TABLE##_catalog = NULL; + +// init all data, or just catalog data void dvo_catalog_init (Catalog *catalog, int complete) { @@ -473,68 +550,41 @@ catalog[0].filename = NULL; - catalog[0].lockmode = 0; - catalog[0].catmode = 0; - catalog[0].catformat = 0; - catalog[0].catflags = 0; - catalog[0].Nsecfilt = 0; + catalog[0].lockmode = 0; + catalog[0].catmode = DVO_MODE_UNDEF; + catalog[0].catformat = DVO_FORMAT_UNDEF; + catalog[0].catcompress = DVO_COMPRESS_NONE; + catalog[0].catflags = DVO_LOAD_NONE; + catalog[0].Nsecfilt = 0; } gfits_init_header (&catalog[0].header); - + // the following describe the catalog files on disk catalog[0].average = NULL; - catalog[0].measure = NULL; - catalog[0].missing = NULL; + catalog[0].Naverage = 0; + catalog[0].Naverage_disk = 0; + catalog[0].Naverage_off = 0; + // average lacks average_catalog + catalog[0].secfilt = NULL; - - catalog[0].lensing = NULL; - catalog[0].lensobj = NULL; - - catalog[0].starpar = NULL; + catalog[0].Nsecfilt_mem = 0; + catalog[0].Nsecfilt_disk = 0; + catalog[0].Nsecfilt_off = 0; + catalog[0].secfilt_catalog = NULL; + // secfilt uses Nsecfilt_mem not Nsecfilt catalog[0].averageT = NULL; catalog[0].measureT = NULL; + + INIT_TABLE(measure); + INIT_TABLE(missing); + INIT_TABLE(lensing); + INIT_TABLE(lensobj); + INIT_TABLE(starpar); + INIT_TABLE(galphot); catalog[0].objID = 0; catalog[0].catID = 0; catalog[0].sorted = 0; - - catalog[0].Naverage = 0; - catalog[0].Nmeasure = 0; - catalog[0].Nmissing = 0; - catalog[0].Nsecf_mem = 0; - - catalog[0].Nlensing = 0; - catalog[0].Nlensobj = 0; - - catalog[0].Nstarpar = 0; - - catalog[0].Naves_disk = 0; - catalog[0].Nmeas_disk = 0; - catalog[0].Nmiss_disk = 0; - catalog[0].Nsecf_disk = 0; - - catalog[0].Nlensing_disk = 0; - catalog[0].Nlensobj_disk = 0; - - catalog[0].Nstarpar_disk = 0; - - catalog[0].Naves_off = 0; - catalog[0].Nmeas_off = 0; - catalog[0].Nmiss_off = 0; - catalog[0].Nsecf_off = 0; - - catalog[0].Nlensing_off = 0; - catalog[0].Nlensobj_off = 0; - - catalog[0].Nstarpar_off = 0; - - /* pointers to SPLIT data files */ - catalog[0].measure_catalog = NULL; - catalog[0].missing_catalog = NULL; - catalog[0].secfilt_catalog = NULL; - catalog[0].lensing_catalog = NULL; - catalog[0].lensobj_catalog = NULL; - catalog[0].starpar_catalog = NULL; /* pointers for data manipulation */ @@ -568,5 +618,8 @@ if (catalog[0].f == NULL) return (DVO_CAT_OPEN_FAIL); - fseeko (catalog[0].f, 0, SEEK_SET); + if (fseeko (catalog[0].f, 0, SEEK_SET)) { + perror ("fseeko: "); + exit (1); + } return (DVO_CAT_OPEN_OK); } @@ -580,5 +633,6 @@ if (fflush (catalog[0].f)) { - fprintf (stderr, "failed to flush file\n"); + perror ("fflush: "); + fprintf (stderr, "failed to flush file %s\n", catalog[0].filename); return FALSE; } @@ -607,4 +661,5 @@ if (catalog[0].lensobj_catalog) { if (!dvo_catalog_unlock (catalog[0].lensobj_catalog)) { fprintf (stderr, "failed to unlock lensobj\n"); return (0); }} if (catalog[0].starpar_catalog) { if (!dvo_catalog_unlock (catalog[0].starpar_catalog)) { fprintf (stderr, "failed to unlock starpar\n"); return (0); }} + if (catalog[0].galphot_catalog) { if (!dvo_catalog_unlock (catalog[0].galphot_catalog)) { fprintf (stderr, "failed to unlock galphot\n"); return (0); }} } return (1); @@ -849,5 +904,5 @@ catalog[0].secfilt = outsec; catalog[0].Nsecfilt = Nsecfilt; - catalog[0].Nsecf_mem = Nsecfilt * catalog[0].Naverage; + catalog[0].Nsecfilt_mem = Nsecfilt * catalog[0].Naverage; } return (TRUE); @@ -887,4 +942,9 @@ dvo_catalog_free (catalog[0].starpar_catalog); free (catalog[0].starpar_catalog); + } + if (catalog[0].galphot_catalog) { + free (catalog[0].galphot_catalog[0].filename); + dvo_catalog_free (catalog[0].galphot_catalog); + free (catalog[0].galphot_catalog); } } @@ -914,5 +974,5 @@ if (catalog[0].secfilt) { free (catalog[0].secfilt); - catalog[0].Nsecf_mem = 0; + catalog[0].Nsecfilt_mem = 0; catalog[0].secfilt = NULL; } @@ -932,4 +992,9 @@ catalog[0].starpar = NULL; } + if (catalog[0].galphot) { + free (catalog[0].galphot); + catalog[0].Ngalphot = 0; + catalog[0].galphot = NULL; + } if (catalog[0].nOwn_t) { free (catalog[0].nOwn_t); catalog[0].nOwn_t = NULL; } if (catalog[0].found_t) { free (catalog[0].found_t); catalog[0].found_t = NULL; } @@ -938,5 +1003,5 @@ /* - mode : items to read (LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF) + mode : items to read (DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT) format : what table structure on disk (INTERNAL, LONEOS, etc, ) style : raw, mef, split, mysql @@ -971,5 +1036,5 @@ // skip empty cpt files - if (primary && !catalog->Naves_disk) { + if (primary && !catalog->Naverage_disk) { return TRUE; } @@ -1053,4 +1118,14 @@ } } + if (catalog[0].galphot_catalog != NULL) { + if (catalog[0].Ngalphot_disk == 0) { + // need to relock (and re-open) file for close elsewhere + status = dvo_catalog_lock (catalog[0].galphot_catalog, lockmode); + } else { + if (!dvo_catalog_backup (catalog[0].galphot_catalog, FALSE)) { + return FALSE; + } + } + } } return TRUE; @@ -1060,5 +1135,5 @@ int dvo_catalog_unlink_backup (Catalog *catalog, int primary) { - if (primary && !catalog->Naves_disk) { + if (primary && !catalog->Naverage_disk) { // skip empty files (empty when read, but output may not be empty) return TRUE; @@ -1109,4 +1184,9 @@ } } + if ((catalog[0].galphot_catalog != NULL) && (catalog[0].Ngalphot_disk > 0)) { + if (!dvo_catalog_unlink_backup (catalog[0].galphot_catalog, FALSE)) { + return FALSE; + } + } } return TRUE; Index: trunk/Ohana/src/libdvo/src/dvo_catalog_create.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_create.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_create.c (revision 38441) @@ -1,4 +1,6 @@ # include # define DEBUG 0 + +Catalog *dvo_catalog_create_subcat (Catalog *catalog, char *ext, char *tablename); // create a new dvo catalog file (if split, lock extra files as well?) @@ -9,8 +11,4 @@ // catalog[0].lockmode void dvo_catalog_create (SkyRegion *region, Catalog *catalog) { - - int length; - char *path, *root, *file, *line; - time_t now; if (DEBUG) fprintf (stderr, "new catalog file: %s\n", catalog[0].filename); @@ -29,84 +27,11 @@ if (catalog[0].catmode == DVO_MODE_SPLIT) { - path = pathname (catalog[0].filename); - root = filerootname (catalog[0].filename); - length = strlen(path) + strlen(root) + 6; - - /* define measure catalog file */ - ALLOCATE (catalog[0].measure_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].measure_catalog, TRUE); - - /* create basic data for measure catalog file */ - gfits_create_header (&catalog[0].measure_catalog[0].header); - - ALLOCATE (catalog[0].measure_catalog[0].filename, char, length); - - sprintf (catalog[0].measure_catalog[0].filename, "%s/%s.cpm", path, root); - file = filebasename (catalog[0].measure_catalog[0].filename); - gfits_modify (&catalog[0].header, "MEASURE", "%s", 1, file); - free (file); - - /* define missing catalog file */ - ALLOCATE (catalog[0].missing_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].missing_catalog, TRUE); - - /* create basic data for missing catalog file */ - gfits_create_header (&catalog[0].missing_catalog[0].header); - ALLOCATE (catalog[0].missing_catalog[0].filename, char, length); - sprintf (catalog[0].missing_catalog[0].filename, "%s/%s.cpn", path, root); - file = filebasename (catalog[0].missing_catalog[0].filename); - gfits_modify (&catalog[0].header, "MISSING", "%s", 1, file); - free (file); - - /* define secfilt catalog file */ - ALLOCATE (catalog[0].secfilt_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].secfilt_catalog, TRUE); - - /* create basic data for secfilt catalog file */ - gfits_create_header (&catalog[0].secfilt_catalog[0].header); - ALLOCATE (catalog[0].secfilt_catalog[0].filename, char, length); - sprintf (catalog[0].secfilt_catalog[0].filename, "%s/%s.cps", path, root); - file = filebasename (catalog[0].secfilt_catalog[0].filename); - gfits_modify (&catalog[0].header, "SECFILT", "%s", 1, file); - free (file); - - /* define lensing catalog file */ - ALLOCATE (catalog[0].lensing_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].lensing_catalog, TRUE); - - /* create basic data for lensing catalog file */ - gfits_create_header (&catalog[0].lensing_catalog[0].header); - ALLOCATE (catalog[0].lensing_catalog[0].filename, char, length); - sprintf (catalog[0].lensing_catalog[0].filename, "%s/%s.cpx", path, root); - file = filebasename (catalog[0].lensing_catalog[0].filename); - gfits_modify (&catalog[0].header, "LENSING", "%s", 1, file); - free (file); - - /* define lensobj catalog file */ - ALLOCATE (catalog[0].lensobj_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].lensobj_catalog, TRUE); - - /* create basic data for lensobj catalog file */ - gfits_create_header (&catalog[0].lensobj_catalog[0].header); - ALLOCATE (catalog[0].lensobj_catalog[0].filename, char, length); - sprintf (catalog[0].lensobj_catalog[0].filename, "%s/%s.cpy", path, root); - file = filebasename (catalog[0].lensobj_catalog[0].filename); - gfits_modify (&catalog[0].header, "LENSOBJ", "%s", 1, file); - free (file); - - /* define starpar catalog file */ - ALLOCATE (catalog[0].starpar_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].starpar_catalog, TRUE); - - /* create basic data for starpar catalog file */ - gfits_create_header (&catalog[0].starpar_catalog[0].header); - ALLOCATE (catalog[0].starpar_catalog[0].filename, char, length); - sprintf (catalog[0].starpar_catalog[0].filename, "%s/%s.cpz", path, root); - file = filebasename (catalog[0].starpar_catalog[0].filename); - gfits_modify (&catalog[0].header, "STARPAR", "%s", 1, file); - free (file); - - free (path); - free (root); + catalog[0].measure_catalog = dvo_catalog_create_subcat (catalog, "cpm", "MEASURE"); + catalog[0].missing_catalog = dvo_catalog_create_subcat (catalog, "cpn", "MISSING"); + catalog[0].secfilt_catalog = dvo_catalog_create_subcat (catalog, "cps", "SECFILT"); + catalog[0].lensing_catalog = dvo_catalog_create_subcat (catalog, "cpx", "LENSING"); + catalog[0].lensobj_catalog = dvo_catalog_create_subcat (catalog, "cpy", "LENSOBJ"); + catalog[0].starpar_catalog = dvo_catalog_create_subcat (catalog, "cpz", "STARPAR"); + catalog[0].galphot_catalog = dvo_catalog_create_subcat (catalog, "cpq", "GALPHOT"); // lock the additional split files @@ -136,4 +61,8 @@ exit (2); } + if (dvo_catalog_lock (catalog[0].galphot_catalog, catalog[0].lockmode) != DVO_CAT_OPEN_EMPTY) { + fprintf (stderr, "error with file lock\n"); + exit (2); + } } @@ -150,6 +79,7 @@ /* write creation date in header */ + time_t now; ohana_str_to_time ("now", &now); - line = ohana_sec_to_date (now); + char *line = ohana_sec_to_date (now); gfits_modify (&catalog[0].header, "DATE", "%s", 1, line); free (line); @@ -163,4 +93,5 @@ ALLOCATE (catalog[0].lensobj, Lensobj, 1); ALLOCATE (catalog[0].starpar, StarPar, 1); + ALLOCATE (catalog[0].galphot, GalPhot, 1); /* setup secondary filters to match photcodes: @@ -170,4 +101,35 @@ } +Catalog *dvo_catalog_create_subcat (Catalog *catalog, char *ext, char *tablename) { + + char *path = pathname (catalog[0].filename); + char *root = filerootname (catalog[0].filename); + int length = strlen(path) + strlen(root) + 6; + + Catalog *subcat; + + /* define subcat catalog file */ + ALLOCATE (subcat, Catalog, 1); + dvo_catalog_init (subcat, TRUE); + + /* create basic data for measure catalog file */ + gfits_create_header (&subcat->header); + + subcat->catcompress = catalog->catcompress; + + ALLOCATE (subcat->filename, char, length); + + sprintf (subcat->filename, "%s/%s.%s", path, root, ext); + + char *file = filebasename (subcat->filename); + gfits_modify (&catalog[0].header, tablename, "%s", 1, file); + + free (file); + free (path); + free (root); + + return subcat; +} + int dvo_catalog_set_range (Catalog *catalog) { Index: trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 38441) @@ -26,5 +26,5 @@ } - // NSTARS, average, Naves_disk + // NSTARS, average, Naverage_disk # define GET_TABLE_SIZES(HFIELD, CFIELD, DFIELD, REQUIRED) \ @@ -41,10 +41,11 @@ /* get the components and sizes from the header */ - GET_TABLE_SIZES ("NSTARS", average, Naves_disk, TRUE); - GET_TABLE_SIZES ("NMEAS", measure, Nmeas_disk, TRUE); - GET_TABLE_SIZES ("NMISS", missing, Nmiss_disk, FALSE); - GET_TABLE_SIZES ("NLENSING", lensing, Nlensing_disk, FALSE); - GET_TABLE_SIZES ("NLENSOBJ", lensobj, Nlensobj_disk, FALSE); - GET_TABLE_SIZES ("NSTARPAR", starpar, Nstarpar_disk, FALSE); + GET_TABLE_SIZES ("NSTARS", average, Naverage_disk, TRUE); + GET_TABLE_SIZES ("NMEAS", measure, Nmeasure_disk, TRUE); + GET_TABLE_SIZES ("NMISS", missing, Nmissing_disk, FALSE); + GET_TABLE_SIZES ("NLENSING", lensing, Nlensing_disk, FALSE); + GET_TABLE_SIZES ("NLENSOBJ", lensobj, Nlensobj_disk, FALSE); + GET_TABLE_SIZES ("NSTARPAR", starpar, Nstarpar_disk, FALSE); + GET_TABLE_SIZES ("NGALPHOT", galphot, Ngalphot_disk, FALSE); /** Nsecfilt is unusual: it does not list the number of data items in the table @@ -53,5 +54,5 @@ if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; catalog[0].Nsecfilt = Nsecfilt; - catalog[0].Nsecf_disk = Naverage * Nsecfilt; + catalog[0].Nsecfilt_disk = Naverage * Nsecfilt; catalog[0].secfilt = NULL; @@ -70,5 +71,5 @@ /* read Average table data (or skip) */ - if (catalog[0].catflags & LOAD_AVES) { + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table average data"); @@ -76,10 +77,10 @@ } /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */ - catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); - if (Naverage != catalog[0].Naves_disk) { - fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naves_disk); - } - catalog[0].Naverage = catalog[0].Naves_disk; - catalog[0].Naves_off = 0; + catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, FALSE); + if (Naverage != catalog[0].Naverage_disk) { + fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naverage_disk); + } + catalog[0].Naverage = catalog[0].Naverage_disk; + catalog[0].Naverage_off = 0; } else { catalog[0].catformat = FtableGetFormat (&ftable); @@ -88,5 +89,5 @@ ALLOCATE (catalog[0].average, Average, 1); catalog[0].Naverage = 0; - catalog[0].Naves_off = catalog[0].Naves_disk; + catalog[0].Naverage_off = catalog[0].Naverage_disk; } gfits_free_header (&header); @@ -99,15 +100,15 @@ } /* read Measure table data */ - if (catalog[0].catflags & LOAD_MEAS) { + if (catalog[0].catflags & DVO_LOAD_MEASURE) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table measure data"); return (FALSE); } - catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &catalog[0].Nmeasure, &catalog[0].catformat); - if (Nmeasure != catalog[0].Nmeas_disk) { - fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmeasure, catalog[0].Nmeas_disk); - } - catalog[0].Nmeasure = catalog[0].Nmeas_disk; - catalog[0].Nmeas_off = 0; + catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &catalog[0].Nmeasure, &catalog[0].catformat, FALSE); + if (Nmeasure != catalog[0].Nmeasure_disk) { + fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmeasure, catalog[0].Nmeasure_disk); + } + catalog[0].Nmeasure = catalog[0].Nmeasure_disk; + catalog[0].Nmeasure_off = 0; } else { Nbytes = gfits_data_size (&header); @@ -115,5 +116,5 @@ ALLOCATE (catalog[0].measure, Measure, 1); catalog[0].Nmeasure = 0; - catalog[0].Nmeas_off = catalog[0].Nmeas_disk; + catalog[0].Nmeasure_off = catalog[0].Nmeasure_disk; } @@ -124,5 +125,5 @@ } /* read Missing table data */ - if (catalog[0].catflags & LOAD_MISS) { + if (catalog[0].catflags & DVO_LOAD_MISSING) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table missing data"); @@ -130,14 +131,14 @@ } /* no conversions currently defined */ - catalog[0].missing = gfits_table_get_Missing (&ftable, &catalog[0].Nmissing, NULL); + catalog[0].missing = gfits_table_get_Missing (&ftable, &catalog[0].Nmissing, NULL, NULL); if (!catalog[0].missing) { fprintf (stderr, "ERROR: failed to read missing\n"); exit (2); } - if (Nmissing != catalog[0].Nmiss_disk) { - fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmissing, catalog[0].Nmiss_disk); - } - catalog[0].Nmissing = catalog[0].Nmiss_disk; - catalog[0].Nmiss_off = 0; + if (Nmissing != catalog[0].Nmissing_disk) { + fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmissing, catalog[0].Nmissing_disk); + } + catalog[0].Nmissing = catalog[0].Nmissing_disk; + catalog[0].Nmissing_off = 0; } else { Nbytes = gfits_data_size (&header); @@ -145,5 +146,5 @@ ALLOCATE (catalog[0].missing, Missing, 1); catalog[0].Nmissing = 0; - catalog[0].Nmiss_off = catalog[0].Nmiss_disk; + catalog[0].Nmissing_off = catalog[0].Nmissing_disk; } @@ -154,5 +155,5 @@ } /* read secfilt table data */ - if (catalog[0].catflags & LOAD_SECF) { + if (catalog[0].catflags & DVO_LOAD_SECFILT) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table secfilt data"); @@ -162,5 +163,5 @@ /* how many entries do we expect from the secfilt table? */ Nexpect = catalog[0].Nsecfilt * catalog[0].Naverage; - catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); + catalog[0].secfilt = FtableToSecFilt (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, FALSE); if (Nexpect != Nitems) { fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nexpect, Nitems); @@ -175,7 +176,7 @@ Ntmpfilt = catalog[0].Nsecfilt; Nsecfilt = catalog[0].Nsecfilt + 1; - Ntotal = Nsecfilt * catalog[0].Naves_disk; + Ntotal = Nsecfilt * catalog[0].Naverage_disk; ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal); - for (i = 0; i < catalog[0].Naves_disk; i++) { + for (i = 0; i < catalog[0].Naverage_disk; i++) { catalog[0].secfilt[i*Nsecfilt + 0] = primary[i]; for (j = 0; j < Ntmpfilt; j++) { @@ -184,10 +185,10 @@ } catalog[0].Nsecfilt = Nsecfilt; - catalog[0].Nsecf_disk = Ntotal; + catalog[0].Nsecfilt_disk = Ntotal; free (tmpfilt); free (primary); } - catalog[0].Nsecf_mem = catalog[0].Nsecf_disk; - catalog[0].Nsecf_off = 0; + catalog[0].Nsecfilt_mem = catalog[0].Nsecfilt_disk; + catalog[0].Nsecfilt_off = 0; } else { /* no real need to skip the data array here... */ @@ -197,9 +198,9 @@ free (primary); catalog[0].Nsecfilt ++; - catalog[0].Nsecf_disk = catalog[0].Nsecfilt * catalog[0].Naves_disk; + catalog[0].Nsecfilt_disk = catalog[0].Nsecfilt * catalog[0].Naverage_disk; } ALLOCATE (catalog[0].secfilt, SecFilt, 1); - catalog[0].Nsecf_mem = 0; - catalog[0].Nsecf_off = catalog[0].Nsecf_disk; + catalog[0].Nsecfilt_mem = 0; + catalog[0].Nsecfilt_off = catalog[0].Nsecfilt_disk; } @@ -210,10 +211,10 @@ } /* read Lensing table data */ - if (catalog[0].catflags & LOAD_LENSING) { + if (catalog[0].catflags & DVO_LOAD_LENSING) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table lensing data"); return (FALSE); } - catalog[0].lensing = FtableToLensing (&ftable, &catalog[0].Nlensing, &catalog[0].catformat); + catalog[0].lensing = FtableToLensing (&ftable, catalog[0].average, &catalog[0].Nlensing, &catalog[0].catformat, FALSE); if (Nlensing != catalog[0].Nlensing_disk) { fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nlensing, catalog[0].Nlensing_disk); @@ -235,10 +236,10 @@ } /* read Lensobj table data */ - if (catalog[0].catflags & LOAD_LENSOBJ) { + if (catalog[0].catflags & DVO_LOAD_LENSOBJ) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table lensobj data"); return (FALSE); } - catalog[0].lensobj = FtableToLensobj (&ftable, &catalog[0].Nlensobj, &catalog[0].catformat); + catalog[0].lensobj = FtableToLensobj (&ftable, catalog[0].average, &catalog[0].Nlensobj, &catalog[0].catformat, FALSE); if (Nlensobj != catalog[0].Nlensobj_disk) { fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nlensobj, catalog[0].Nlensobj_disk); @@ -260,10 +261,10 @@ } /* read StarPar table data */ - if (catalog[0].catflags & LOAD_STARPAR) { + if (catalog[0].catflags & DVO_LOAD_STARPAR) { if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { if (VERBOSE) fprintf (stderr, "can't read table starpar data"); return (FALSE); } - catalog[0].starpar = FtableToStarPar (&ftable, &catalog[0].Nstarpar, &catalog[0].catformat); + catalog[0].starpar = FtableToStarPar (&ftable, catalog[0].average, &catalog[0].Nstarpar, &catalog[0].catformat, FALSE); if (Nstarpar != catalog[0].Nstarpar_disk) { fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nstarpar, catalog[0].Nstarpar_disk); @@ -277,4 +278,29 @@ catalog[0].Nstarpar = 0; catalog[0].Nstarpar_off = catalog[0].Nstarpar_disk; + } + + /* read GalPhot table header */ + if (!gfits_fread_header (catalog[0].f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read table galphot header"); + return (FALSE); + } + /* read GalPhot table data */ + if (catalog[0].catflags & DVO_LOAD_GALPHOT) { + if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { + if (VERBOSE) fprintf (stderr, "can't read table galphot data"); + return (FALSE); + } + catalog[0].galphot = FtableToGalPhot (&ftable, catalog[0].average, &catalog[0].Ngalphot, &catalog[0].catformat, FALSE); + if (Ngalphot != catalog[0].Ngalphot_disk) { + fprintf (stderr, "Warning: mismatch between Ngalphot in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Ngalphot, catalog[0].Ngalphot_disk); + } + catalog[0].Ngalphot = catalog[0].Ngalphot_disk; + catalog[0].Ngalphot_off = 0; + } else { + Nbytes = gfits_data_size (&header); + fseeko (catalog[0].f, Nbytes, SEEK_CUR); + ALLOCATE (catalog[0].galphot, GalPhot, 1); + catalog[0].Ngalphot = 0; + catalog[0].Ngalphot_off = catalog[0].Ngalphot_disk; } @@ -330,4 +356,6 @@ gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, catalog[0].Nlensobj); gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, catalog[0].Nstarpar); + gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, catalog[0].Ngalphot); + gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); @@ -357,5 +385,5 @@ /* write out Average table (convert to FITS table format) */ - AverageToFtable (&ftable, catalog[0].average, catalog[0].Naverage, catalog[0].catformat, primary); + AverageToFtable (&ftable, catalog[0].average, catalog[0].Naverage, catalog[0].catformat, primary, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -370,5 +398,5 @@ /* write out Measure table (convert to FITS table format) */ - MeasureToFtable (&ftable, catalog[0].average, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat); + MeasureToFtable (&ftable, catalog[0].average, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -383,5 +411,5 @@ /* write out Missing table (convert to FITS table format) */ - gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing); + gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -397,5 +425,5 @@ /* write out SecFilt table (convert to FITS table format) */ Nitems = catalog[0].Naverage * Nsecfilt; - SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat); + SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -410,5 +438,5 @@ /* write out Lensing table (convert to FITS table format) */ - LensingToFtable (&ftable, catalog[0].lensing, catalog[0].Nlensing, catalog[0].catformat); + LensingToFtable (&ftable, catalog[0].lensing, catalog[0].Nlensing, catalog[0].catformat, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); @@ -423,25 +451,31 @@ /* write out Lensobj table (convert to FITS table format) */ - LensobjToFtable (&ftable, catalog[0].lensobj, catalog[0].Nlensobj, catalog[0].catformat); - if (!gfits_fwrite_Theader (catalog[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_table (catalog[0].f, &ftable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } - gfits_free_table (&ftable); - gfits_free_header (&header); - - /* free temp storage */ - if (primary != NULL) { - free (primary); - free (secfilt); - } - return (TRUE); + LensobjToFtable (&ftable, catalog[0].lensobj, catalog[0].Nlensobj, catalog[0].catformat, TRUE); + if (!gfits_fwrite_Theader (catalog[0].f, &header)) { + fprintf (stderr, "can't write table header"); + goto failure; + } + if (!gfits_fwrite_table (catalog[0].f, &ftable)) { + fprintf (stderr, "can't write table data"); + goto failure; + } + gfits_free_table (&ftable); + gfits_free_header (&header); /* write out StarPar table (convert to FITS table format) */ - StarParToFtable (&ftable, catalog[0].starpar, catalog[0].Nstarpar, catalog[0].catformat); + StarParToFtable (&ftable, catalog[0].starpar, catalog[0].Nstarpar, catalog[0].catformat, TRUE); + if (!gfits_fwrite_Theader (catalog[0].f, &header)) { + fprintf (stderr, "can't write table header"); + goto failure; + } + if (!gfits_fwrite_table (catalog[0].f, &ftable)) { + fprintf (stderr, "can't write table data"); + goto failure; + } + gfits_free_table (&ftable); + gfits_free_header (&header); + + /* write out GalPhot table (convert to FITS table format) */ + GalPhotToFtable (&ftable, catalog[0].galphot, catalog[0].Ngalphot, catalog[0].catformat, TRUE); if (!gfits_fwrite_Theader (catalog[0].f, &header)) { fprintf (stderr, "can't write table header"); Index: trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 38441) @@ -136,5 +136,5 @@ /* read and convert the averages (use a macro to clean this up?) */ /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */ - if (catalog[0].catflags & LOAD_AVES) { + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { catalog[0].average = ReadRawAverage (catalog[0].f, catalog[0].Naverage, catalog[0].catformat, &primary); } else { @@ -145,5 +145,5 @@ /* read and convert the measures (use a macro to clean this up?) */ - if (catalog[0].catflags & LOAD_MEAS) { + if (catalog[0].catflags & DVO_LOAD_MEASURE) { catalog[0].measure = ReadRawMeasure (catalog[0].f, catalog[0].average, catalog[0].Nmeasure, catalog[0].catformat); } else { @@ -154,5 +154,5 @@ /* read and convert missing */ - if (catalog[0].catflags & LOAD_MISS) { + if (catalog[0].catflags & DVO_LOAD_MISSING) { ALLOCATE (catalog[0].missing, Missing, MAX (catalog[0].Nmissing, 1)); Nitems = catalog[0].Nmissing; @@ -170,5 +170,5 @@ /* read and convert secfilt */ - if (catalog[0].catflags & LOAD_SECF) { + if (catalog[0].catflags & DVO_LOAD_SECFILT) { Nitems = catalog[0].Naverage * catalog[0].Nsecfilt; catalog[0].secfilt = ReadRawSecFilt (catalog[0].f, Nitems, catalog[0].catformat); @@ -191,5 +191,5 @@ } catalog[0].Nsecfilt = Nsecfilt; - catalog[0].Nsecf_mem = Ntotal; + catalog[0].Nsecfilt_mem = Ntotal; free (primary); } @@ -210,5 +210,5 @@ /* check data integrity */ - if (catalog[0].catflags & LOAD_AVES) { + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { for (i = Nmeas = Nmiss = 0; i < catalog[0].Naverage; i++) { Nmeas += catalog[0].average[i].Nmeasure; @@ -226,7 +226,7 @@ /* save the current number so we can do partial updates */ - catalog[0].Naves_disk = catalog[0].Naverage; - catalog[0].Nmeas_disk = catalog[0].Nmeasure; - catalog[0].Nmiss_disk = catalog[0].Nmissing; + catalog[0].Naverage_disk = catalog[0].Naverage; + catalog[0].Nmeasure_disk = catalog[0].Nmeasure; + catalog[0].Nmissing_disk = catalog[0].Nmissing; return (TRUE); Index: trunk/Ohana/src/libdvo/src/dvo_catalog_split.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 38441) @@ -1,3 +1,4 @@ # include +# define OHANA_MEMCHECK 0 // return options: @@ -6,4 +7,20 @@ // * ok +// utility function to see if we should compress this table) +int output_is_compressed (int start, int Nrows, int Ntotal, DVOCatCompress catcompress) { + int fullWrite = (start == 0) && (Nrows == Ntotal); + if (fullWrite && catcompress) return TRUE; // we want to compress, so do not swap + return FALSE; +} + +int byteswap_varlength_ftable (Header *header, FTable *table) { + int i, Nfields; + if (!gfits_scan (header, "TFIELDS", "%d", 1, &Nfields)) return FALSE; + for (i = 0; i < Nfields; i++) { + if (!gfits_byteswap_varlength_column (table, i+1)) return FALSE; + } + return TRUE; +} + int dvo_catalog_secfilt_to_primary (Catalog *catalog, SecFilt **myPrimary, SecFilt **mySecfilt, int *myNsecfilt) { @@ -20,5 +37,5 @@ // XXX this translation only works if we have loaded / created a matched average/secfilt set - assert (catalog[0].Nsecf_mem == catalog[0].Nsecfilt*catalog[0].Naverage); + assert (catalog[0].Nsecfilt_mem == catalog[0].Nsecfilt*catalog[0].Naverage); Nallfilt = catalog[0].Nsecfilt; @@ -35,5 +52,5 @@ } catalog[0].Nsecfilt --; - catalog[0].Nsecf_mem = catalog[0].Naverage*catalog[0].Nsecfilt; + catalog[0].Nsecfilt_mem = catalog[0].Naverage*catalog[0].Nsecfilt; *myPrimary = primary; @@ -51,5 +68,5 @@ Ntmpfilt = catalog[0].Nsecfilt; - // we do NOT modify Nsecf_disk; this operation only modifies in in-memory values + // we do NOT modify Nsecfilt_disk; this operation only modifies in in-memory values catalog[0].Nsecfilt ++; @@ -57,5 +74,5 @@ Ntotal = Nsecfilt * Naves; - catalog[0].Nsecf_mem = Ntotal; + catalog[0].Nsecfilt_mem = Ntotal; ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal); @@ -74,62 +91,12 @@ Matrix matrix; - - // ohana_memcheck_func (TRUE); - - if (0) { - // XXX a TEST : is the data corrupted or the file? - char name[1024]; - snprintf (name, 1024, "%s.phu.v2", catalog->filename); - FILE *f = fopen (name, "w"); - fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f); - // fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer); - // fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", - // catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], - // catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]); - fclose (f); - } - if (0) { - // XXX a TEST : is the data corrupted or the file? - char name[1024]; - snprintf (name, 1024, "%s.phu", catalog->filename); - FILE *f = fopen (name, "w"); - fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f); - // fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer); - // fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", - // catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], - // catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]); - fclose (f); - } -# if (0) - { - // XXX a TEST : is the data corrupted or the file? - char name[1024]; - snprintf (name, 1024, "%s.phu.v2", catalog->filename); - FILE *f = fopen (name, "w"); - fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f); - fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer); - fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", - catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], - catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]); - fclose (f); - } - { - // XXX a TEST : is the data corrupted or the file? - char name[1024]; - snprintf (name, 1024, "%s.phu.v3", catalog->filename); - FILE *f = fopen (name, "w"); - fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f); - fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer); - fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n", - catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3], - catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]); - fclose (f); - } -# endif + if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE); + /* rewind file pointers and truncate (file is still open) */ if (fseeko (catalog->f, 0LL, SEEK_SET)) { perror ("dvo_catalog_save_subset: "); fprintf (stderr, "failed to seek to beginning\n"); + return FALSE; } @@ -151,13 +118,63 @@ gfits_free_matrix (&matrix); + FTable *outtable = ftable; + FTable cmptable; + Header cmpheader; + gfits_init_header (&cmpheader); + gfits_init_table (&cmptable); + cmptable.header = &cmpheader; + + int fullWrite = (start == 0) && (Nrows == Ntotal); + int isCompressed = output_is_compressed (start, Nrows, Ntotal, catalog->catcompress); + + if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE); + + if (isCompressed) { + // fprintf (stderr, "compress %s\n", catalog->filename); + char *compressMode = dvo_catalog_compress_string (catalog->catcompress); + // I should test how Ntile affects fpack/funpack and speed + if (!gfits_compress_table (ftable, &cmptable, 1000, compressMode)) { + fprintf (stderr, "compression failure\n"); + return (FALSE); + } + if (!byteswap_varlength_ftable (&cmpheader, &cmptable)) { + fprintf (stderr, "failed to swap varlength column\n"); + return FALSE; + } + if (!gfits_modify (cmptable.header, "DVO_CMP", "%s", 1, compressMode)) { + fprintf (stderr, "can't save compression mode\n"); + return (FALSE); + } + outtable = &cmptable; + } + + if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE); + // write the table data - if (!gfits_fwrite_ftable_range (catalog->f, ftable, start, Nrows, Ndisk, Ntotal)) { - fprintf (stderr, "can't write table data\n"); - return (FALSE); - } - fullsize += ftable->datasize + ftable->header->datasize; + if (fullWrite) { + if (!gfits_fwrite_Theader (catalog->f, outtable->header)) { + fprintf (stderr, "can't write table header\n"); + return (FALSE); + } + if (!gfits_fwrite_table (catalog->f, outtable)) { + fprintf (stderr, "can't write table data\n"); + return (FALSE); + } + } else { + if (!gfits_fwrite_ftable_range (catalog->f, outtable, start, Nrows, Ndisk, Ntotal)) { + fprintf (stderr, "can't write table data (range)\n"); + return (FALSE); + } + } + fullsize += outtable->datasize + outtable->header->datasize; + + if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE); + + // since we init'ed the structures above, these operations are safe whether or not we compressed the table + gfits_free_header (&cmpheader); + gfits_free_table (&cmptable); // if the output file will be completely re-written, truncate to total datasize - if ((start == 0) && (Nrows == Ntotal)) { + if (fullWrite) { int fd = fileno (catalog->f); if (ftruncate (fd, fullsize)) { @@ -192,7 +209,8 @@ // databases created prior to ~2014.07.01 did not have the LENSING, LENSOBJ, STARPAR paths in their headers. // in these cases, we do not try to lock or open the relevant file - if (!strcmp (name, "LENSING")) return (DVO_CAT_OPEN_EMPTY); - if (!strcmp (name, "LENSOBJ")) return (DVO_CAT_OPEN_EMPTY); - if (!strcmp (name, "STARPAR")) return (DVO_CAT_OPEN_EMPTY); + if (!strcmp (name, "LENSING")) return (DVO_CAT_OPEN_EMPTY); + if (!strcmp (name, "LENSOBJ")) return (DVO_CAT_OPEN_EMPTY); + if (!strcmp (name, "STARPAR")) return (DVO_CAT_OPEN_EMPTY); + if (!strcmp (name, "GALPHOT")) return (DVO_CAT_OPEN_EMPTY); return (DVO_CAT_OPEN_FAIL); } @@ -200,4 +218,7 @@ sprintf (subcat[0].filename, "%s/%s", path, string); free (path); + + // inherit compression mode for this subcat; + subcat->catcompress = catalog->catcompress; /* lock & open catalog file */ @@ -220,5 +241,5 @@ } Nskip = gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nskip, SEEK_CUR); + if (fseeko (subcat[0].f, Nskip, SEEK_CUR)) { perror ("fseeko: "); exit (1); } /* read table header */ @@ -230,4 +251,104 @@ return (DVO_CAT_OPEN_OK); } + +// ftable must already exist and have a valid, loaded header +int gfits_fread_uncompressed (Catalog *catalog, FTable *ftable, char *nativeOrder, char VERBOSE) { + + // fread_ftable_data requires the header + + *nativeOrder = FALSE; + if (!gfits_fread_ftable_data (catalog->f, ftable, FALSE)) { + if (VERBOSE) fprintf (stderr, "can't read table data\n"); + return FALSE; + } + // NOTE: fread_ftable returns an unswapped table; uncompress swaps the bytes into native order + + if (gfits_extension_is_compressed_table (ftable->header)) { + FTable rawtable; + Header rawheader; + gfits_init_table (&rawtable); + gfits_init_header (&rawheader); + rawtable.header = &rawheader; + + // NOTE: uncompress swaps the data bytes into native order, but needs the varlength columns native + if (!byteswap_varlength_ftable (ftable->header, ftable)) { + fprintf (stderr, "failed to swap varlength column\n"); + return FALSE; + } + if (!gfits_uncompress_table (ftable, &rawtable)) { + if (VERBOSE) fprintf (stderr, "failed to uncompress table\n"); + gfits_free_table (ftable); + return FALSE; + } + + char compressMode[80]; + if (!gfits_scan (ftable->header, "DVO_CMP", "%s", 1, compressMode)) { + strcpy (compressMode, "AUTO"); + } + catalog->catcompress = dvo_catalog_catcompress (compressMode); // if any table is compressed, set all to a compress state? + + // free the buffers + gfits_free_header (ftable->header); + gfits_free_table (ftable); + + // copies values but does not allocate new memory + Header *outheader = ftable->header; + gfits_copy_header_ptr (&rawheader, outheader); + gfits_copy_ftable_ptr (&rawtable, ftable); // this replaces ftable->header + ftable->header = outheader; + *nativeOrder = TRUE; + } + return TRUE; +} + +// LOAD_SUBCAT(measure,MEASURE,Measure) + +# define LOAD_SUBCAT(FIELD, NAME, STRUCT) \ + { \ + Header header; \ + FTable ftable; \ + off_t Nitems; \ + gfits_init_table (&ftable); \ + gfits_init_header (&header); \ + ftable.header = &header; \ + int status = DVO_CAT_OPEN_EMPTY; \ + if (!(catalog[0].catflags & DVO_SKIP_##NAME)) { \ + status = dvo_catalog_open_subcat (catalog, &catalog[0].FIELD##_catalog, ftable.header, #NAME, VERBOSE); \ + if (status == DVO_CAT_OPEN_FAIL) { \ + return (FALSE); \ + } \ + if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].N##FIELD##_disk > 0)) { \ + return (FALSE); \ + } \ + } \ + if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & DVO_LOAD_##NAME)) { \ + char nativeOrder = FALSE; \ + /* read table data */ \ + if (!gfits_fread_uncompressed (catalog[0].FIELD##_catalog, &ftable, &nativeOrder, VERBOSE)) { \ + if (VERBOSE) fprintf (stderr, "can't read table %s data\n", #FIELD); \ + gfits_free_header (&header); \ + return (FALSE); \ + } \ + /* convert data format to internal : returns number of row read in Nvalues */ \ + catalog[0].FIELD = FtableTo##STRUCT (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, nativeOrder); \ + if (Nitems != catalog[0].N##FIELD##_disk) { \ + fprintf (stderr, "Warning: mismatch between N%s in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", #FIELD, Nitems, catalog[0].N##FIELD##_disk); \ + } \ + catalog[0].N##FIELD = catalog[0].N##FIELD##_disk; \ + catalog[0].N##FIELD##_off = 0; \ + } else { \ + if (catalog[0].FIELD##_catalog) { \ + gfits_free_header (&catalog[0].FIELD##_catalog[0].header); \ + } else { \ + ALLOCATE (catalog[0].FIELD##_catalog, Catalog, 1); \ + dvo_catalog_init (catalog[0].FIELD##_catalog, TRUE); \ + } \ + gfits_create_header (&catalog[0].FIELD##_catalog[0].header); \ + ALLOCATE (catalog[0].FIELD, STRUCT, 1); \ + catalog[0].N##FIELD = 0; \ + catalog[0].N##FIELD##_off = catalog[0].N##FIELD##_disk; \ + } \ + gfits_free_header (&header); \ + } int dvo_catalog_load_split (Catalog *catalog, int VERBOSE) { @@ -240,26 +361,20 @@ off_t Nlensobj; off_t Nstarpar; - off_t Nitems; - int status, Nsecfilt; - Header header; - FTable ftable; + off_t Ngalphot; + int Nsecfilt; SecFilt *primary; - /* ftable header storage for below */ - ftable.header = &header; - ftable.buffer = NULL; - header.buffer = NULL; primary = NULL; - status = FALSE; /* get the components from the header - these duplicate information in the split files (NAXIS2) */ - // NSTARS, NMEAS, NMISS are required; NLENSING, NLENSOBJ, NSTARPAR are not (0 if not found) - if (!gfits_scan (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, &Naverage)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, &Nmeasure)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NMISS", OFF_T_FMT, 1, &Nmissing)) return (FALSE); - if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; - if (!gfits_scan (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, &Nlensing)) Nlensing = 0; - if (!gfits_scan (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, &Nlensobj)) Nlensobj = 0; - if (!gfits_scan (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, &Nstarpar)) Nstarpar = 0; + // NSTARS, NMEAS, NMISS are required; NLENSING, NLENSOBJ, NSTARPAR, NGALPHOT are not (0 if not found) + if (!gfits_scan (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, &Naverage)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, &Nmeasure)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NMISS", OFF_T_FMT, 1, &Nmissing)) return (FALSE); + if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; + if (!gfits_scan (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, &Nlensing)) Nlensing = 0; + if (!gfits_scan (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, &Nlensobj)) Nlensobj = 0; + if (!gfits_scan (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, &Nstarpar)) Nstarpar = 0; + if (!gfits_scan (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, &Ngalphot)) Ngalphot = 0; /* the OBJID is a counter that uniquely defines an average entry and never changes. if @@ -276,24 +391,22 @@ /* save the current number so we can do partial updates */ - catalog[0].Naves_disk = Naverage; - catalog[0].Nmeas_disk = Nmeasure; - catalog[0].Nmiss_disk = Nmissing; - catalog[0].Nsecf_disk = Naverage * Nsecfilt; - catalog[0].Nlensing_disk = Nlensing; - catalog[0].Nlensobj_disk = Nlensobj; - catalog[0].Nstarpar_disk = Nstarpar; - - /** Nsecfilt is unusual: it does not list the number of data items in the table - instead, the number of items is Nsecfilt * Naverage. **/ - catalog[0].Nsecfilt = Nsecfilt; + catalog[0].Naverage_disk = Naverage; + catalog[0].Nmeasure_disk = Nmeasure; + catalog[0].Nmissing_disk = Nmissing; + catalog[0].Nsecfilt_disk = Naverage * Nsecfilt; + catalog[0].Nlensing_disk = Nlensing; + catalog[0].Nlensobj_disk = Nlensobj; + catalog[0].Nstarpar_disk = Nstarpar; + catalog[0].Ngalphot_disk = Ngalphot; /* default values, but we will assign these a valid value before we exit (even if empty) */ - catalog[0].average = NULL; - catalog[0].measure = NULL; - catalog[0].missing = NULL; - catalog[0].secfilt = NULL; - catalog[0].lensing = NULL; - catalog[0].lensobj = NULL; - catalog[0].starpar = NULL; + catalog[0].average = NULL; + catalog[0].measure = NULL; + catalog[0].missing = NULL; + catalog[0].secfilt = NULL; + catalog[0].lensing = NULL; + catalog[0].lensobj = NULL; + catalog[0].starpar = NULL; + catalog[0].galphot = NULL; /*** Average Table ***/ @@ -302,268 +415,71 @@ /* move pointer past PHU header -- must be already read (load_catalog) */ Nbytes = catalog[0].header.datasize + gfits_data_size (&catalog[0].header); - fseeko (catalog[0].f, Nbytes, SEEK_SET); + if (fseeko (catalog[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } + + /* ftable header storage for below */ + char nativeOrder; + Header header; + FTable ftable; + gfits_init_table (&ftable); + gfits_init_header (&header); + ftable.header = &header; /* read Average table header */ if (!gfits_fread_header (catalog[0].f, &header)) { - if (VERBOSE) fprintf (stderr, "can't read table average header"); + if (VERBOSE) fprintf (stderr, "can't read table average header\n"); return (FALSE); } - if (catalog[0].catflags & LOAD_AVES) { - /* read Average table data : format is irrelevant here */ - if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table average data"); - return (FALSE); - } - /* convert the saved version of the table to the internal version. Old versions of DVO stored + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { + if (!gfits_fread_uncompressed (catalog, &ftable, &nativeOrder, VERBOSE)) { + if (VERBOSE) fprintf (stderr, "can't read Average table\n"); + gfits_free_header (&header); + return FALSE; + } + + /* convert the disk version of the table to the internal version. Old versions of DVO stored * one of the average magnitudes in Average. We save this in case it is needed below. NOTE: * primary is only used if we read in the secfilt table, otherwise it should be freed */ - catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); - if (Naverage != catalog[0].Naves_disk) { - fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naves_disk); - } - gfits_free_header (&header); - catalog[0].Naverage = catalog[0].Naves_disk; - catalog[0].Naves_off = 0; + catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, nativeOrder); + if (Naverage != catalog[0].Naverage_disk) { + fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Naverage, catalog[0].Naverage_disk); + } + catalog[0].Naverage = catalog[0].Naverage_disk; + catalog[0].Naverage_off = 0; } else { catalog[0].catformat = FtableGetFormat (&ftable); ALLOCATE (catalog[0].average, Average, 1); catalog[0].Naverage = 0; - catalog[0].Naves_off = catalog[0].Naves_disk; - } + catalog[0].Naverage_off = catalog[0].Naverage_disk; + } + gfits_free_header (&header); /*** Measure Table ***/ - if (!(catalog[0].catflags & SKIP_MEAS)) { - // unless we specify 'skip', we still need to load the - status = dvo_catalog_open_subcat (catalog, &catalog[0].measure_catalog, ftable.header, "MEASURE", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmeas_disk > 0)) { - return (FALSE); - } - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MEAS)) { - // XXX this allows an empty Measure catalog with non-empty Average catalog : is that OK? - /* read Measure table data */ - if (!gfits_fread_ftable_data (catalog[0].measure_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table measure data\n"); - return (FALSE); - } - /* convert data format to internal : returns number of row read in Nmeasure */ - catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat); - if (Nmeasure != catalog[0].Nmeas_disk) { - fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmeasure, catalog[0].Nmeas_disk); - } - catalog[0].Nmeasure = catalog[0].Nmeas_disk; - catalog[0].Nmeas_off = 0; - } else { - // XXX is it necessary to generate a template header here? - // XXX this is a memory leak, right? - if (catalog[0].measure_catalog) { - gfits_free_header (&catalog[0].measure_catalog[0].header); + + LOAD_SUBCAT(measure,MEASURE,Measure); + LOAD_SUBCAT(missing,MISSING,Missing); + LOAD_SUBCAT(secfilt,SECFILT,SecFilt); + LOAD_SUBCAT(lensing,LENSING,Lensing); + LOAD_SUBCAT(lensobj,LENSOBJ,Lensobj); + LOAD_SUBCAT(starpar,STARPAR,StarPar); + LOAD_SUBCAT(galphot,GALPHOT,GalPhot); + + /** catalog->Nsecfilt is unusual: it does not list the number of data items in the + table instead, the number of items is Nsecfilt * Naverage, and is stored in + Nsecfilt_mem. fix these below **/ + + catalog[0].Nsecfilt_mem = catalog[0].Nsecfilt; + catalog[0].Nsecfilt = Nsecfilt; + + /** some old formats stored one of the secfilt values in the average table. + repair this below **/ + + if (primary) { + if (catalog[0].Nsecfilt_mem) { + dvo_catalog_primary_to_secfilt (catalog, primary, catalog[0].Naverage_disk); } else { - ALLOCATE (catalog[0].measure_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].measure_catalog, TRUE); - } - gfits_create_header (&catalog[0].measure_catalog[0].header); - ALLOCATE (catalog[0].measure, Measure, 1); - catalog[0].Nmeasure = 0; - catalog[0].Nmeas_off = catalog[0].Nmeas_disk; - } - gfits_free_header (&header); - - /*** Missing Table ***/ - status = dvo_catalog_open_subcat (catalog, &catalog[0].missing_catalog, ftable.header, "MISSING", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmiss_disk > 0)) { - return (FALSE); - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MISS)) { - /* read Missing table data */ - if (!gfits_fread_ftable_data (catalog[0].missing_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table missing data\n"); - return (FALSE); - } - /* no conversions currently defined : this just does the byte swap */ - catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL); - if (!catalog[0].missing) { - fprintf (stderr, "ERROR: failed to read missing\n"); - exit (2); - } - if (Nmissing != catalog[0].Nmiss_disk) { - fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nmissing, catalog[0].Nmiss_disk); - } - catalog[0].Nmissing = catalog[0].Nmiss_disk; - catalog[0].Nmiss_off = 0; - } else { - // XXX is it necessary to generate a template header here? - gfits_create_header (&catalog[0].missing_catalog[0].header); - ALLOCATE (catalog[0].missing, Missing, 1); - catalog[0].Nmissing = 0; - catalog[0].Nmiss_off = catalog[0].Nmiss_disk; - } - gfits_free_header (ftable.header); - - /*** Secfilt Table ***/ - status = dvo_catalog_open_subcat (catalog, &catalog[0].secfilt_catalog, ftable.header, "SECFILT", VERBOSE); - // fprintf (stderr, "secfilt header %s : %zx\n", catalog[0].secfilt_catalog[0].filename, (size_t) catalog[0].secfilt_catalog[0].header.buffer); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nsecf_disk > 0)) { - return (FALSE); - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_SECF)) { - /* read secfilt table data */ - if (!gfits_fread_ftable_data (catalog[0].secfilt_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table secfilt data\n"); - return (FALSE); - } - catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); - if (Nitems != catalog[0].Nsecf_disk) { - fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nitems, catalog[0].Nsecf_disk); - } - catalog[0].Nsecf_mem = catalog[0].Nsecf_disk; - catalog[0].Nsecf_off = 0; - - /* if primary is defined, we were supplied with one additional average magnitude from Average - we need to interleave these magnitudes with the secfilt entries just loaded */ - if (primary != NULL) { - // this modifies catalog.Nsecf_mem,Nsecfilt - dvo_catalog_primary_to_secfilt (catalog, primary, catalog[0].Naves_disk); - } - } else { - if (primary != NULL) { free (primary); catalog[0].Nsecfilt ++; } - gfits_create_header (&catalog[0].secfilt_catalog[0].header); - ALLOCATE (catalog[0].secfilt, SecFilt, 1); - catalog[0].Nsecf_mem = 0; - catalog[0].Nsecf_off = catalog[0].Nsecf_disk; - } - gfits_free_header (ftable.header); - - /*** Lensing Table ***/ - if (!(catalog[0].catflags & SKIP_LENSING)) { - // unless we specify 'skip', we still need to load the Lensing table - status = dvo_catalog_open_subcat (catalog, &catalog[0].lensing_catalog, ftable.header, "LENSING", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - // the Lensing file need not exist, unless we expect data to exist - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nlensing_disk > 0)) { - return (FALSE); - } - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSING)) { - // only read the Lensing table if the file actually exists - /* read Lensing table data */ - if (!gfits_fread_ftable_data (catalog[0].lensing_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table lensing data\n"); - return (FALSE); - } - /* convert data format to internal : returns number of row read in Nlensing */ - catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat); - if (Nlensing != catalog[0].Nlensing_disk) { - fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nlensing, catalog[0].Nlensing_disk); - } - catalog[0].Nlensing = catalog[0].Nlensing_disk; - catalog[0].Nlensing_off = 0; - } else { - if (catalog[0].lensing_catalog) { - gfits_free_header (&catalog[0].lensing_catalog[0].header); - } else { - ALLOCATE (catalog[0].lensing_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].lensing_catalog, TRUE); - } - gfits_create_header (&catalog[0].lensing_catalog[0].header); - ALLOCATE (catalog[0].lensing, Lensing, 1); - catalog[0].Nlensing = 0; - catalog[0].Nlensing_off = catalog[0].Nlensing_disk; - } - gfits_free_header (&header); - - /*** Lensobj Table ***/ - if (!(catalog[0].catflags & SKIP_LENSOBJ)) { - // unless we specify 'skip', we still need to load the - status = dvo_catalog_open_subcat (catalog, &catalog[0].lensobj_catalog, ftable.header, "LENSOBJ", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - // the Lensobj file need not exist, unless we expect data to exist - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nlensobj_disk > 0)) { - return (FALSE); - } - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSOBJ)) { - // only read the Lensobj table if the file actually exists - /* read Lensobj table data */ - if (!gfits_fread_ftable_data (catalog[0].lensobj_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table lensobj data\n"); - return (FALSE); - } - /* convert data format to internal : returns number of row read in Nlensobj */ - catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat); - if (Nlensobj != catalog[0].Nlensobj_disk) { - fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nlensobj, catalog[0].Nlensobj_disk); - } - catalog[0].Nlensobj = catalog[0].Nlensobj_disk; - catalog[0].Nlensobj_off = 0; - } else { - if (catalog[0].lensobj_catalog) { - gfits_free_header (&catalog[0].lensobj_catalog[0].header); - } else { - ALLOCATE (catalog[0].lensobj_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].lensobj_catalog, TRUE); - } - gfits_create_header (&catalog[0].lensobj_catalog[0].header); - ALLOCATE (catalog[0].lensobj, Lensobj, 1); - catalog[0].Nlensobj = 0; - catalog[0].Nlensobj_off = catalog[0].Nlensobj_disk; - } - gfits_free_header (&header); - - /*** StarPar Table ***/ - if (!(catalog[0].catflags & SKIP_STARPAR)) { - // unless we specify 'skip', we still need to load the - status = dvo_catalog_open_subcat (catalog, &catalog[0].starpar_catalog, ftable.header, "STARPAR", VERBOSE); - if (status == DVO_CAT_OPEN_FAIL) { - return (FALSE); - } - // the StarPar file need not exist, unless we expect data to exist - if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nstarpar_disk > 0)) { - return (FALSE); - } - } - if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_STARPAR)) { - // only read the StarPar table if the file actually exists - /* read StarPar table data */ - if (!gfits_fread_ftable_data (catalog[0].starpar_catalog[0].f, &ftable, FALSE)) { - if (VERBOSE) fprintf (stderr, "can't read table starpar data\n"); - return (FALSE); - } - /* convert data format to internal : returns number of row read in Nstarpar */ - catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat); - if (Nstarpar != catalog[0].Nstarpar_disk) { - fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nstarpar, catalog[0].Nstarpar_disk); - } - catalog[0].Nstarpar = catalog[0].Nstarpar_disk; - catalog[0].Nstarpar_off = 0; - } else { - if (catalog[0].starpar_catalog) { - gfits_free_header (&catalog[0].starpar_catalog[0].header); - } else { - ALLOCATE (catalog[0].starpar_catalog, Catalog, 1); - dvo_catalog_init (catalog[0].starpar_catalog, TRUE); - } - gfits_create_header (&catalog[0].starpar_catalog[0].header); - ALLOCATE (catalog[0].starpar, StarPar, 1); - catalog[0].Nstarpar = 0; - catalog[0].Nstarpar_off = catalog[0].Nstarpar_disk; - } - gfits_free_header (&header); + } return (TRUE); @@ -575,5 +491,5 @@ off_t Nbytes; - off_t Naverage, Nexpect, Nitems, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar; + off_t Naverage, Nexpect, Nitems, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar, Ngalphot; Header header; FTable ftable; @@ -587,10 +503,11 @@ /*** Average (& SecFilt) Table ***/ - if (catalog[0].catflags & LOAD_AVES) { + if (catalog[0].catflags & DVO_LOAD_AVERAGE) { /*** load the Average data ***/ /* move pointer past header and matrix -- must be already read (load_catalog) */ Nbytes = catalog[0].header.datasize + gfits_data_size (&catalog[0].header); - fseeko (catalog[0].f, Nbytes, SEEK_SET); + if (fseeko (catalog[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } + /* read Average table header */ if (!gfits_fread_header (catalog[0].f, &header)) { @@ -607,5 +524,5 @@ * one of the average magnitudes in Average. We save this in case it is needed below. NOTE: * primary is only used if we read in the secfilt table, otherwise it should be freed */ - catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); + catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, FALSE); if (Naverage != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -614,5 +531,5 @@ gfits_free_header (&header); catalog[0].Naverage = Naverage; - catalog[0].Naves_off = start; + catalog[0].Naverage_off = start; /*** load the secfilt data ***/ @@ -621,5 +538,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Secfilt table header */ @@ -635,10 +552,10 @@ Nexpect = catalog[0].Naverage * catalog[0].Nsecfilt; - catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); + catalog[0].secfilt = FtableToSecFilt (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, FALSE); if (Nitems != Nexpect) { fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Nitems, Nexpect); } - catalog[0].Nsecf_mem = catalog[0].Naverage * catalog[0].Nsecfilt; - catalog[0].Nsecf_off = start * catalog[0].Nsecfilt; + catalog[0].Nsecfilt_mem = catalog[0].Naverage * catalog[0].Nsecfilt; + catalog[0].Nsecfilt_off = start * catalog[0].Nsecfilt; /* if primary is defined, we were supplied with one additional average magnitude from Average @@ -651,5 +568,5 @@ // XXX check the open status of the catalog - if (catalog[0].catflags & LOAD_MEAS) { + if (catalog[0].catflags & DVO_LOAD_MEASURE) { Catalog *subcat = catalog[0].measure_catalog; @@ -657,5 +574,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Measure table header */ @@ -671,5 +588,5 @@ /* convert data format to internal : returns number of row read in Nmeasure */ - catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat); + catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat, FALSE); if (Nmeasure != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -678,9 +595,9 @@ gfits_free_header (&header); catalog[0].Nmeasure = Nmeasure; - catalog[0].Nmeas_off = start; + catalog[0].Nmeasure_off = start; } // XXX check the open status of the catalog? - if (catalog[0].catflags & LOAD_MISS) { + if (catalog[0].catflags & DVO_LOAD_MISSING) { Catalog *subcat = catalog[0].missing_catalog; @@ -688,5 +605,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Missing table header */ @@ -702,5 +619,5 @@ /* no conversions currently defined : this just does the byte swap */ - catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL); + catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL, NULL); if (!catalog[0].missing) { fprintf (stderr, "ERROR: failed to read missing\n"); @@ -712,9 +629,9 @@ gfits_free_header (&header); catalog[0].Nmissing = Nmissing; - catalog[0].Nmiss_off = start; + catalog[0].Nmissing_off = start; } // XXX check the open status of the catalog - if (catalog[0].catflags & LOAD_LENSING) { + if (catalog[0].catflags & DVO_LOAD_LENSING) { Catalog *subcat = catalog[0].lensing_catalog; @@ -722,5 +639,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Lensing table header */ @@ -736,5 +653,5 @@ /* convert data format to internal : returns number of row read in Nlensing */ - catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat); + catalog[0].lensing = FtableToLensing (&ftable, catalog[0].average, &Nlensing, &catalog[0].catformat, FALSE); if (Nlensing != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -747,5 +664,5 @@ // XXX check the open status of the catalog - if (catalog[0].catflags & LOAD_LENSOBJ) { + if (catalog[0].catflags & DVO_LOAD_LENSOBJ) { Catalog *subcat = catalog[0].lensobj_catalog; @@ -753,5 +670,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read Lensobj table header */ @@ -767,5 +684,5 @@ /* convert data format to internal : returns number of row read in Nlensobj */ - catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat); + catalog[0].lensobj = FtableToLensobj (&ftable, catalog[0].average, &Nlensobj, &catalog[0].catformat, FALSE); if (Nlensobj != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -778,5 +695,5 @@ // XXX check the open status of the catalog - if (catalog[0].catflags & LOAD_STARPAR) { + if (catalog[0].catflags & DVO_LOAD_STARPAR) { Catalog *subcat = catalog[0].starpar_catalog; @@ -784,5 +701,5 @@ /* move pointer past header -- must be already read (load_catalog) */ Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); - fseeko (subcat[0].f, Nbytes, SEEK_SET); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* read StarPar table header */ @@ -798,5 +715,5 @@ /* convert data format to internal : returns number of row read in Nstarpar */ - catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat); + catalog[0].starpar = FtableToStarPar (&ftable, catalog[0].average, &Nstarpar, &catalog[0].catformat, FALSE); if (Nstarpar != Nrows) { // XXX this condition denotes the eof has been reached; not an error or a warning @@ -808,4 +725,35 @@ } + // XXX check the open status of the catalog + if (catalog[0].catflags & DVO_LOAD_GALPHOT) { + + Catalog *subcat = catalog[0].galphot_catalog; + + /* move pointer past header -- must be already read (load_catalog) */ + Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header); + if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); } + + /* read GalPhot table header */ + if (!gfits_fread_header (subcat[0].f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read table galphot header"); + return (FALSE); + } + /* read GalPhot table data : format is irrelevant here */ + if (!gfits_fread_ftable_range (subcat[0].f, FALSE, &ftable, start, Nrows)) { + if (VERBOSE) fprintf (stderr, "can't read table galphot data"); + return (FALSE); + } + + /* convert data format to internal : returns number of row read in Ngalphot */ + catalog[0].galphot = FtableToGalPhot (&ftable, catalog[0].average, &Ngalphot, &catalog[0].catformat, FALSE); + if (Ngalphot != Nrows) { + // XXX this condition denotes the eof has been reached; not an error or a warning + // fprintf (stderr, "Warning: mismatch between Ngalphot in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", Ngalphot, Nrows); + } + gfits_free_header (&header); + catalog[0].Ngalphot = Ngalphot; + catalog[0].Ngalphot_off = start; + } + return (TRUE); } @@ -814,11 +762,9 @@ int dvo_catalog_save_split (Catalog *catalog, char VERBOSE) { - off_t Nitems; Header header; FTable ftable; SecFilt *primary, *secfilt; int Nsecfilt; - off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new; - off_t first, start, Nrows; + off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalphot_disk_new; ftable.header = &header; @@ -828,6 +774,6 @@ // skip empty catalogs: it is illegal to have Measures without corresponding Averages - Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off); - if (Naves_disk_new == 0) { + Naverage_disk_new = MAX (catalog[0].Naverage_disk, catalog[0].Naverage + catalog[0].Naverage_off); + if (Naverage_disk_new == 0) { if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n"); return (TRUE); @@ -847,19 +793,22 @@ } - Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off); - Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off); - Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off); + Nmeasure_disk_new = MAX (catalog[0].Nmeasure_disk, catalog[0].Nmeasure + catalog[0].Nmeasure_off); + Nmissing_disk_new = MAX (catalog[0].Nmissing_disk, catalog[0].Nmissing + catalog[0].Nmissing_off); + Nsecfilt_disk_new = MAX (catalog[0].Nsecfilt_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecfilt_off); Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off); Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off); Nstarpar_disk_new = MAX (catalog[0].Nstarpar_disk, catalog[0].Nstarpar + catalog[0].Nstarpar_off); + Ngalphot_disk_new = MAX (catalog[0].Ngalphot_disk, catalog[0].Ngalphot + catalog[0].Ngalphot_off); /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naves_disk_new); - gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeas_disk_new); - gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmiss_disk_new); - gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); - gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); - gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); - gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naverage_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeasure_disk_new); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmissing_disk_new); + gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); + gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); + gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); + gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); + gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, Ngalphot_disk_new); + gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); @@ -868,22 +817,29 @@ /*** Average Table ***/ - if ((catalog[0].catflags & LOAD_AVES) && (catalog[0].average != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Naves_off; // first disk row to write - Nrows = catalog[0].Naverage - first; + if ((catalog[0].catflags & DVO_LOAD_AVERAGE) && (catalog[0].average != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Naverage_off; // first disk row to write + off_t Nrows = catalog[0].Naverage - first; assert (Nrows >= 0); assert (first >= 0); assert (first <= catalog[0].Naverage); - assert (catalog[0].Naves_disk >= catalog[0].Naves_off); - - /* convert internal to external format */ - if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) { + assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off); + + /* convert internal to external format : also results in a byte-swapped, scaled output + table. Note that ftable is newly allocated. + */ + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Naverage_disk_new, catalog->catcompress); + + // convert to external table format + if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) { fprintf (stderr, "failure writing Average table\n"); goto failure; @@ -894,5 +850,5 @@ // even if we do not save the average table, we need to keep the header in sync /* rewind file pointers and truncate (file is still open) */ - fseeko (catalog[0].f, 0, SEEK_SET); + if (fseeko (catalog[0].f, 0, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* write table PHU header - always write this out */ @@ -905,8 +861,336 @@ /*** Measure Table ***/ - if ((catalog[0].catflags & LOAD_MEAS) && (catalog[0].measure != NULL)) { + if ((catalog[0].catflags & DVO_LOAD_MEASURE) && (catalog[0].measure != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nmeasure_off; // first disk row to write + off_t Nrows = catalog[0].Nmeasure - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Nmeasure); + assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nmeasure_disk_new, catalog->catcompress); + + // convert to external table format + if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Measure table + catalog->measure_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) { + fprintf (stderr, "trouble writing Measure table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Missing Table ***/ + if ((catalog[0].catflags & DVO_LOAD_MISSING) && (catalog[0].missing != NULL)) { + + if (catalog[0].Nmissing_off != 0) { + fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); + goto failure; + } + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (catalog[0].Nmissing_off, catalog[0].Nmissing, Nmissing_disk_new, catalog->catcompress); + + // convert to external table format + if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Missing table (must write out entire table) + catalog->missing_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].missing_catalog, &ftable, 0, catalog[0].Nmissing, catalog[0].Nmissing, catalog[0].Nmissing)) { + fprintf (stderr, "trouble writing Missing Table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Secfilt Table ***/ + if ((catalog[0].catflags & DVO_LOAD_SECFILT) && (catalog[0].secfilt != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nsecfilt_off; // first disk row to write + off_t Nitems = catalog[0].Naverage*Nsecfilt; + off_t Nrows = Nitems - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= Nitems); + assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off); + // XXX check these for consistency... + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nsecfilt_disk_new, catalog->catcompress); + + // convert to external table format + if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out SecFilt table + catalog->secfilt_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) { + fprintf (stderr, "failure writing SecFilt table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Lensing Table ***/ + if ((catalog[0].catflags & DVO_LOAD_LENSING) && (catalog[0].lensing != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nlensing_off; // first disk row to write + off_t Nrows = catalog[0].Nlensing - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Nlensing); + assert (catalog[0].Nlensing_disk >= catalog[0].Nlensing_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nlensing_disk_new, catalog->catcompress); + + // convert to external table format + if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Lensing table + catalog->lensing_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].lensing_catalog, &ftable, start, Nrows, catalog[0].Nlensing_disk, Nlensing_disk_new)) { + fprintf (stderr, "trouble writing Lensing table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Lensobj Table ***/ + if ((catalog[0].catflags & DVO_LOAD_LENSOBJ) && (catalog[0].lensobj != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nlensobj_off; // first disk row to write + off_t Nrows = catalog[0].Nlensobj - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Nlensobj); + assert (catalog[0].Nlensobj_disk >= catalog[0].Nlensobj_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nlensobj_disk_new, catalog->catcompress); + + // convert to external table format + if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Lensobj table + catalog->lensobj_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].lensobj_catalog, &ftable, start, Nrows, catalog[0].Nlensobj_disk, Nlensobj_disk_new)) { + fprintf (stderr, "trouble writing Lensobj table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** StarPar Table ***/ + if ((catalog[0].catflags & DVO_LOAD_STARPAR) && (catalog[0].starpar != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Nstarpar_off; // first disk row to write + off_t Nrows = catalog[0].Nstarpar - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Nstarpar); + assert (catalog[0].Nstarpar_disk >= catalog[0].Nstarpar_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Nstarpar_disk_new, catalog->catcompress); + + // convert to external table format + if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out StarPar table + catalog->starpar_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) { + fprintf (stderr, "trouble writing StarPar table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** GalPhot Table ***/ + if ((catalog[0].catflags & DVO_LOAD_GALPHOT) && (catalog[0].galphot != NULL)) { + + off_t first = 0; // first row in memory to write + off_t start = catalog[0].Ngalphot_off; // first disk row to write + off_t Nrows = catalog[0].Ngalphot - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Ngalphot); + assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off); + + // if we are going to compress, we need to receive unswapped data -- + int swapFromNative = !output_is_compressed (start, Nrows, Ngalphot_disk_new, catalog->catcompress); + + // convert to external table format + if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, swapFromNative)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out GalPhot table + catalog->galphot_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api + if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) { + fprintf (stderr, "trouble writing GalPhot table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /* free temp storage */ + if (primary != NULL) { + free (primary); + free (secfilt); + } + return (TRUE); + + failure: + /* free temp storage */ + gfits_free_header (&header); + gfits_free_table (&ftable); + if (primary != NULL) { + free (primary); + free (secfilt); + } + return (FALSE); +} + +/* update_catalog_split only writes new lines to file. */ +int dvo_catalog_update_split (Catalog *catalog, char VERBOSE) { + + off_t Nitems; + Header header; + FTable ftable; + SecFilt *primary, *secfilt; + int Nsecfilt; + off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalphot_disk_new; + off_t first, start, Nrows; + + ftable.header = &header; + ftable.buffer = NULL; + header.buffer = NULL; + + // skip empty catalogs: it is illegal to have Measures without corresponding Averages + Naverage_disk_new = MAX (catalog[0].Naverage_disk, catalog[0].Naverage + catalog[0].Naverage_off); + if (Naverage_disk_new == 0) { + if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n"); + return (TRUE); + } + + // for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary + switch (catalog[0].catformat) { + case DVO_FORMAT_ELIXIR: // special case for ELIXIR + case DVO_FORMAT_LONEOS: // special case for LONEOS + dvo_catalog_secfilt_to_primary (catalog, &primary, &secfilt, &Nsecfilt); + break; + default: + primary = NULL; + secfilt = catalog[0].secfilt; + Nsecfilt = catalog[0].Nsecfilt; + break; + } + + Nmeasure_disk_new = MAX (catalog[0].Nmeasure_disk, catalog[0].Nmeasure + catalog[0].Nmeasure_off); + Nmissing_disk_new = MAX (catalog[0].Nmissing_disk, catalog[0].Nmissing + catalog[0].Nmissing_off); + Nsecfilt_disk_new = MAX (catalog[0].Nsecfilt_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecfilt_off); + Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off); + Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off); + Nstarpar_disk_new = MAX (catalog[0].Nstarpar_disk, catalog[0].Nstarpar + catalog[0].Nstarpar_off); + Ngalphot_disk_new = MAX (catalog[0].Ngalphot_disk, catalog[0].Ngalphot + catalog[0].Ngalphot_off); + + /* make sure header is consistent with data */ + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naverage_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeasure_disk_new); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmissing_disk_new); + gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); + gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); + gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); + gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); + gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, Ngalphot_disk_new); + + gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); + gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); + + /* in split mode, we can save only part of the data */ + + /*** Average Table ***/ + if (catalog[0].average != NULL) { + + first = catalog[0].Naverage_disk - catalog[0].Naverage_off; // first row to write (memory) + start = catalog[0].Naverage_disk; // first row to write (disk) + Nrows = catalog[0].Naverage - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Naverage); + assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off); + + /* convert internal to external format */ + if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) { + fprintf (stderr, "failure writing Average table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } else { + // even if we do not save the average table, we need to keep the header in sync + /* rewind file pointers and truncate (file is still open) */ + if (fseeko (catalog[0].f, 0, SEEK_SET)) { perror ("fseeko: "); exit (1); } + + /* write table PHU header - always write this out */ + /* XXX EAM : check if disk file size has changed */ + if (!gfits_fwrite_header (catalog[0].f, &catalog[0].header)) { + fprintf (stderr, "can't write primary header"); + goto failure; + } + } + + /*** Measure Table ***/ + if ((catalog[0].catflags & DVO_LOAD_MEASURE) && (catalog[0].measure != NULL)) { first = 0; // first row in memory to write - start = catalog[0].Nmeas_off; // first disk row to write + start = catalog[0].Nmeasure_off; // first disk row to write Nrows = catalog[0].Nmeasure - first; @@ -914,8 +1198,8 @@ assert (first >= 0); assert (first <= catalog[0].Nmeasure); - assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off); + assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off); // convert to external table format (note that the block above does not damage or free catalog.average) - if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) { + if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -923,5 +1207,5 @@ // write out Measure table - if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) { + if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) { fprintf (stderr, "trouble writing Measure table\n"); goto failure; @@ -932,7 +1216,7 @@ /*** Missing Table ***/ - if ((catalog[0].catflags & LOAD_MISS) && (catalog[0].missing != NULL)) { - - if (catalog[0].Nmiss_off != 0) { + if ((catalog[0].catflags & DVO_LOAD_MISSING) && (catalog[0].missing != NULL)) { + + if (catalog[0].Nmissing_off != 0) { fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); goto failure; @@ -940,5 +1224,5 @@ // convert to external table format - if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) { + if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -955,9 +1239,9 @@ /*** Secfilt Table ***/ - if ((catalog[0].catflags & LOAD_SECF) && (catalog[0].secfilt != NULL)) { + if ((catalog[0].catflags & DVO_LOAD_SECFILT) && (catalog[0].secfilt != NULL)) { first = 0; // first row in memory to write - start = catalog[0].Nsecf_off; // first disk row to write - Nitems = catalog[0].Nsecf_mem; + start = catalog[0].Nsecfilt_off; // first disk row to write + Nitems = catalog[0].Nsecfilt_mem; Nrows = Nitems - first; @@ -965,12 +1249,15 @@ assert (first >= 0); assert (first <= Nitems); - assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off); + assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off); // XXX check these for consistency... // convert to external table format - SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat); + if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } // write out SecFilt table - if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) { + if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) { fprintf (stderr, "failure writing SecFilt table\n"); goto failure; @@ -981,5 +1268,5 @@ /*** Lensing Table (optional, do not save if not loaded) ***/ - if ((catalog[0].catflags & LOAD_LENSING) && catalog[0].lensing_catalog->f && catalog[0].lensing) { + if ((catalog[0].catflags & DVO_LOAD_LENSING) && catalog[0].lensing_catalog->f && catalog[0].lensing) { first = 0; // first row in memory to write @@ -994,5 +1281,5 @@ if (catalog[0].Nlensing) { // convert to external table format (note that the block above does not damage or free catalog.average) - if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) { + if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1010,5 +1297,5 @@ /*** Lensobj Table (optional, do not save if not loaded) ***/ - if ((catalog[0].catflags & LOAD_LENSOBJ) && catalog[0].lensobj_catalog->f && catalog[0].lensobj) { + if ((catalog[0].catflags & DVO_LOAD_LENSOBJ) && catalog[0].lensobj_catalog->f && catalog[0].lensobj) { first = 0; // first row in memory to write @@ -1023,5 +1310,5 @@ if (catalog[0].Nlensobj) { // convert to external table format (note that the block above does not damage or free catalog.average) - if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) { + if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1039,5 +1326,5 @@ /*** StarPar Table (optional, do not save if not loaded) ***/ - if ((catalog[0].catflags & LOAD_STARPAR) && catalog[0].starpar_catalog->f && catalog[0].starpar) { + if ((catalog[0].catflags & DVO_LOAD_STARPAR) && catalog[0].starpar_catalog->f && catalog[0].starpar) { first = 0; // first row in memory to write @@ -1052,5 +1339,5 @@ if (catalog[0].Nstarpar) { // convert to external table format (note that the block above does not damage or free catalog.average) - if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) { + if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1067,4 +1354,33 @@ } + /*** GalPhot Table (optional, do not save if not loaded) ***/ + if ((catalog[0].catflags & DVO_LOAD_GALPHOT) && catalog[0].galphot_catalog->f && catalog[0].galphot) { + + first = 0; // first row in memory to write + start = catalog[0].Ngalphot_off; // first disk row to write + Nrows = catalog[0].Ngalphot - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Ngalphot); + assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off); + + if (catalog[0].Ngalphot) { + // convert to external table format (note that the block above does not damage or free catalog.average) + if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out GalPhot table + if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) { + fprintf (stderr, "trouble writing GalPhot table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + } + /* free temp storage */ if (primary != NULL) { @@ -1074,5 +1390,5 @@ return (TRUE); -failure: + failure: /* free temp storage */ gfits_free_header (&header); @@ -1088,10 +1404,9 @@ int dvo_catalog_save_split_complete (Catalog *catalog, char VERBOSE) { - off_t Nitems; Header header; FTable ftable; SecFilt *primary, *secfilt; int Nsecfilt; - off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new; + off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalphot_disk_new; off_t first, start, Nrows; @@ -1102,17 +1417,17 @@ // skip empty catalogs: it is illegal to have Measures without corresponding Averages - if (catalog[0].Naves_off > 0) { + if (catalog[0].Naverage_off > 0) { fprintf (stderr, "ERROR: only partial catalog (Average) was loaded\n"); goto failure; } - if (catalog[0].Nmeas_off > 0) { + if (catalog[0].Nmeasure_off > 0) { fprintf (stderr, "ERROR: only partial catalog (Measure) was loaded\n"); goto failure; } - if (catalog[0].Nmiss_off > 0) { + if (catalog[0].Nmissing_off > 0) { fprintf (stderr, "ERROR: only partial catalog (Missing) was loaded\n"); goto failure; } - if (catalog[0].Nsecf_off > 0) { + if (catalog[0].Nsecfilt_off > 0) { fprintf (stderr, "ERROR: only partial catalog (Secfilt) was loaded\n"); goto failure; @@ -1130,7 +1445,11 @@ goto failure; } - - Naves_disk_new = catalog[0].Naverage; - if (Naves_disk_new == 0) { + if (catalog[0].Ngalphot_off > 0) { + fprintf (stderr, "ERROR: only partial catalog (GalPhot) was loaded\n"); + goto failure; + } + + Naverage_disk_new = catalog[0].Naverage; + if (Naverage_disk_new == 0) { if (VERBOSE) fprintf (stderr, "resulting catalog is empty; delete it\n"); // unlink (); @@ -1151,19 +1470,21 @@ } - Nmeas_disk_new = catalog[0].Nmeasure; - Nmiss_disk_new = catalog[0].Nmissing; - Nsecf_disk_new = catalog[0].Naverage*Nsecfilt; + Nmeasure_disk_new = catalog[0].Nmeasure; + Nmissing_disk_new = catalog[0].Nmissing; + Nsecfilt_disk_new = catalog[0].Naverage*Nsecfilt; Nlensing_disk_new = catalog[0].Nlensing; Nlensobj_disk_new = catalog[0].Nlensobj; Nstarpar_disk_new = catalog[0].Nstarpar; + Ngalphot_disk_new = catalog[0].Ngalphot; /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naves_disk_new); - gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeas_disk_new); - gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmiss_disk_new); + gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naverage_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeasure_disk_new); + gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmissing_disk_new); gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); + gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, Ngalphot_disk_new); gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); @@ -1172,8 +1493,8 @@ /*** Average Table ***/ - if ((catalog[0].catflags & LOAD_AVES) && (catalog[0].average != NULL)) { + if ((catalog[0].catflags & DVO_LOAD_AVERAGE) && (catalog[0].average != NULL)) { first = 0; // first row in memory to write - start = catalog[0].Naves_off; // first disk row to write + start = catalog[0].Naverage_off; // first disk row to write Nrows = catalog[0].Naverage - first; @@ -1181,13 +1502,13 @@ assert (first >= 0); assert (first <= catalog[0].Naverage); - assert (catalog[0].Naves_disk >= catalog[0].Naves_off); + assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off); /* convert internal to external format */ - if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) { + if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) { fprintf (stderr, "failure writing Average table\n"); goto failure; @@ -1198,5 +1519,5 @@ // even if we do not save the average table, we need to keep the header in sync /* rewind file pointers and truncate (file is still open) */ - fseeko (catalog[0].f, 0, SEEK_SET); + if (fseeko (catalog[0].f, 0, SEEK_SET)) { perror ("fseeko: "); exit (1); } /* write table PHU header - always write this out */ @@ -1209,8 +1530,8 @@ /*** Measure Table ***/ - if ((catalog[0].catflags & LOAD_MEAS) && (catalog[0].measure != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nmeas_off; // first disk row to write + if (catalog[0].measure != NULL) { + + first = catalog[0].Nmeasure_disk - catalog[0].Nmeasure_off; // first row in memory to write + start = catalog[0].Nmeasure_disk; // first disk row to write Nrows = catalog[0].Nmeasure - first; @@ -1218,8 +1539,9 @@ assert (first >= 0); assert (first <= catalog[0].Nmeasure); - assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off); + assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off); // convert to external table format - if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) { + // XXX does catalog.measure have averef correctly set up? + if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1227,5 +1549,5 @@ // write out Measure table - if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) { + if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) { fprintf (stderr, "trouble writing Measure table\n"); goto failure; @@ -1235,8 +1557,12 @@ } + /* missing table CANNOT be written unsorted, thus it is always written + out in full */ + /*** Missing Table ***/ - if ((catalog[0].catflags & LOAD_MISS) && (catalog[0].missing != NULL)) { - - if (catalog[0].Nmiss_off != 0) { + + if (catalog[0].missing != NULL) { + + if (catalog[0].Nmissing_off != 0) { fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); goto failure; @@ -1244,5 +1570,5 @@ // convert to external table format - if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) { + if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1259,22 +1585,22 @@ /*** Secfilt Table ***/ - if ((catalog[0].catflags & LOAD_SECF) && (catalog[0].secfilt != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nsecf_off; // first disk row to write - Nitems = catalog[0].Naverage*Nsecfilt; - Nrows = Nitems - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= Nitems); - assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off); - // XXX check these for consistency... + if (catalog[0].secfilt != NULL) { + + first = catalog[0].Nsecfilt_disk - catalog[0].Nsecfilt_off; // first row in memory to write + start = catalog[0].Nsecfilt_disk; // first disk row to write + Nrows = catalog[0].Nsecfilt_mem - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off); // convert to external table format - SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat); + if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } // write out SecFilt table - if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) { + if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) { fprintf (stderr, "failure writing SecFilt table\n"); goto failure; @@ -1285,8 +1611,8 @@ /*** Lensing Table ***/ - if ((catalog[0].catflags & LOAD_LENSING) && (catalog[0].lensing != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nlensing_off; // first disk row to write + if (catalog[0].lensing != NULL) { + + first = catalog[0].Nlensing_disk - catalog[0].Nlensing_off; // first row in memory to write + start = catalog[0].Nlensing_disk; // first disk row to write Nrows = catalog[0].Nlensing - first; @@ -1297,5 +1623,6 @@ // convert to external table format - if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) { + // XXX does catalog.lensing have averef correctly set up? + if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1312,8 +1639,8 @@ /*** Lensobj Table ***/ - if ((catalog[0].catflags & LOAD_LENSOBJ) && (catalog[0].lensobj != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nlensobj_off; // first disk row to write + if (catalog[0].lensobj != NULL) { + + first = catalog[0].Nlensobj_disk - catalog[0].Nlensobj_off; // first row in memory to write + start = catalog[0].Nlensobj_disk; // first disk row to write Nrows = catalog[0].Nlensobj - first; @@ -1324,5 +1651,6 @@ // convert to external table format - if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) { + // XXX does catalog.lensobj have averef correctly set up? + if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1339,8 +1667,8 @@ /*** StarPar Table ***/ - if ((catalog[0].catflags & LOAD_STARPAR) && (catalog[0].starpar != NULL)) { - - first = 0; // first row in memory to write - start = catalog[0].Nstarpar_off; // first disk row to write + if (catalog[0].starpar != NULL) { + + first = catalog[0].Nstarpar_disk - catalog[0].Nstarpar_off; // first row in memory to write + start = catalog[0].Nstarpar_disk; // first disk row to write Nrows = catalog[0].Nstarpar - first; @@ -1351,5 +1679,6 @@ // convert to external table format - if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) { + // XXX does catalog.starpar have averef correctly set up? + if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) { fprintf (stderr, "trouble converting format\n"); goto failure; @@ -1359,4 +1688,32 @@ if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) { fprintf (stderr, "trouble writing StarPar table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** GalPhot Table ***/ + if (catalog[0].galphot != NULL) { + + first = catalog[0].Ngalphot_disk - catalog[0].Ngalphot_off; // first row in memory to write + start = catalog[0].Ngalphot_disk; // first disk row to write + Nrows = catalog[0].Ngalphot - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first <= catalog[0].Ngalphot); + assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off); + + // convert to external table format + // XXX does catalog.galphot have averef correctly set up? + if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out GalPhot table + if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) { + fprintf (stderr, "trouble writing GalPhot table\n"); goto failure; } @@ -1370,279 +1727,8 @@ free (secfilt); } + return (TRUE); -failure: - /* free temp storage */ - gfits_free_header (&header); - gfits_free_table (&ftable); - if (primary != NULL) { - free (primary); - free (secfilt); - } - return (FALSE); -} - -/* update_catalog_split only writes new lines to file. */ -int dvo_catalog_update_split (Catalog *catalog, char VERBOSE) { - - Header header; - FTable ftable; - SecFilt *primary, *secfilt; - int Nsecfilt; - off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new; - off_t first, start, Nrows; - - ftable.header = &header; - ftable.buffer = NULL; - header.buffer = NULL; - - // skip empty catalogs: it is illegal to have Measures without corresponding Averages - Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off); - if (Naves_disk_new == 0) { - if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n"); - return (TRUE); - } - - // for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary - switch (catalog[0].catformat) { - case DVO_FORMAT_ELIXIR: // special case for ELIXIR - case DVO_FORMAT_LONEOS: // special case for LONEOS - dvo_catalog_secfilt_to_primary (catalog, &primary, &secfilt, &Nsecfilt); - break; - default: - primary = NULL; - secfilt = catalog[0].secfilt; - Nsecfilt = catalog[0].Nsecfilt; - break; - } - - Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off); - Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off); - Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off); - Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off); - Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off); - Nstarpar_disk_new = MAX (catalog[0].Nstarpar_disk, catalog[0].Nstarpar + catalog[0].Nstarpar_off); - - /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", OFF_T_FMT, 1, Naves_disk_new); - gfits_modify (&catalog[0].header, "NMEAS", OFF_T_FMT, 1, Nmeas_disk_new); - gfits_modify (&catalog[0].header, "NMISS", OFF_T_FMT, 1, Nmiss_disk_new); - gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); - gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1, Nlensing_disk_new); - gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1, Nlensobj_disk_new); - gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1, Nstarpar_disk_new); - gfits_modify_alt (&catalog[0].header, "EXTEND", "%t", 1, TRUE); - gfits_modify (&catalog[0].header, "OBJID", "%d", 1, catalog[0].objID); - - /* in split mode, we can save only part of the data */ - - /*** Average Table ***/ - if (catalog[0].average != NULL) { - - first = catalog[0].Naves_disk - catalog[0].Naves_off; // first row to write (memory) - start = catalog[0].Naves_disk; // first row to write (disk) - Nrows = catalog[0].Naverage - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Naverage); - assert (catalog[0].Naves_disk >= catalog[0].Naves_off); - - /* convert internal to external format */ - if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) { - fprintf (stderr, "failure writing Average table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } else { - // even if we do not save the average table, we need to keep the header in sync - /* rewind file pointers and truncate (file is still open) */ - fseeko (catalog[0].f, 0, SEEK_SET); - - /* write table PHU header - always write this out */ - /* XXX EAM : check if disk file size has changed */ - if (!gfits_fwrite_header (catalog[0].f, &catalog[0].header)) { - fprintf (stderr, "can't write primary header"); - goto failure; - } - } - - /*** Measure Table ***/ - if (catalog[0].measure != NULL) { - - first = catalog[0].Nmeas_disk - catalog[0].Nmeas_off; // first row in memory to write - start = catalog[0].Nmeas_disk; // first disk row to write - Nrows = catalog[0].Nmeasure - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Nmeasure); - assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off); - - // convert to external table format - // XXX does catalog.measure have averef correctly set up? - if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out Measure table - if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) { - fprintf (stderr, "trouble writing Measure table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /* missing table CANNOT be written unsorted, thus it is always written - out in full */ - - /*** Missing Table ***/ - - if (catalog[0].missing != NULL) { - - if (catalog[0].Nmiss_off != 0) { - fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); - goto failure; - } - - // convert to external table format - if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out Missing table (must write out entire table) - if (!dvo_catalog_save_subcat (catalog[0].missing_catalog, &ftable, 0, catalog[0].Nmissing, catalog[0].Nmissing, catalog[0].Nmissing)) { - fprintf (stderr, "trouble writing Missing Table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /*** Secfilt Table ***/ - if (catalog[0].secfilt != NULL) { - - first = catalog[0].Nsecf_disk - catalog[0].Nsecf_off; // first row in memory to write - start = catalog[0].Nsecf_disk; // first disk row to write - Nrows = catalog[0].Nsecf_mem - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off); - - // convert to external table format - SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat); - - // write out SecFilt table - if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) { - fprintf (stderr, "failure writing SecFilt table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /*** Lensing Table ***/ - if (catalog[0].lensing != NULL) { - - first = catalog[0].Nlensing_disk - catalog[0].Nlensing_off; // first row in memory to write - start = catalog[0].Nlensing_disk; // first disk row to write - Nrows = catalog[0].Nlensing - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Nlensing); - assert (catalog[0].Nlensing_disk >= catalog[0].Nlensing_off); - - // convert to external table format - // XXX does catalog.lensing have averef correctly set up? - if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out Lensing table - if (!dvo_catalog_save_subcat (catalog[0].lensing_catalog, &ftable, start, Nrows, catalog[0].Nlensing_disk, Nlensing_disk_new)) { - fprintf (stderr, "trouble writing Lensing table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /*** Lensobj Table ***/ - if (catalog[0].lensobj != NULL) { - - first = catalog[0].Nlensobj_disk - catalog[0].Nlensobj_off; // first row in memory to write - start = catalog[0].Nlensobj_disk; // first disk row to write - Nrows = catalog[0].Nlensobj - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Nlensobj); - assert (catalog[0].Nlensobj_disk >= catalog[0].Nlensobj_off); - - // convert to external table format - // XXX does catalog.lensobj have averef correctly set up? - if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out Lensobj table - if (!dvo_catalog_save_subcat (catalog[0].lensobj_catalog, &ftable, start, Nrows, catalog[0].Nlensobj_disk, Nlensobj_disk_new)) { - fprintf (stderr, "trouble writing Lensobj table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /*** StarPar Table ***/ - if (catalog[0].starpar != NULL) { - - first = catalog[0].Nstarpar_disk - catalog[0].Nstarpar_off; // first row in memory to write - start = catalog[0].Nstarpar_disk; // first disk row to write - Nrows = catalog[0].Nstarpar - first; - - assert (Nrows >= 0); - assert (first >= 0); - assert (first <= catalog[0].Nstarpar); - assert (catalog[0].Nstarpar_disk >= catalog[0].Nstarpar_off); - - // convert to external table format - // XXX does catalog.starpar have averef correctly set up? - if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - - // write out StarPar table - if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) { - fprintf (stderr, "trouble writing StarPar table\n"); - goto failure; - } - gfits_free_header (&header); - gfits_free_table (&ftable); - } - - /* free temp storage */ - if (primary != NULL) { - free (primary); - free (secfilt); - } - - return (TRUE); - -failure: + failure: /* free temp storage */ if (primary != NULL) { @@ -1662,5 +1748,5 @@ /* XXX EAM : update is not efficient. MeasureToFtable should only - convert the new rows (Nmeas_disk to Nmeasure). the resulting + convert the new rows (Nmeasure_disk to Nmeasure). the resulting table represents the end rows of the ftable. we need to define the vtable based on the ftable, but with Ny = Nmeasure */ @@ -1668,11 +1754,11 @@ - // * convert to an ftable - // * optionally write the PHU header/matrix - // * advance to the start of the output data block: - // ** Nx * catalog[0].Nmeas_off - // * write out the ftable data block - // * if Nmeas_off + Nmeasure >= Nmeas_disk, update padding - // ** start = Nmeas_off - // ** Nrows = Nmeasure - +// * convert to an ftable +// * optionally write the PHU header/matrix +// * advance to the start of the output data block: +// ** Nx * catalog[0].Nmeasure_off +// * write out the ftable data block +// * if Nmeasure_off + Nmeasure >= Nmeasure_disk, update padding +// ** start = Nmeasure_off +// ** Nrows = Nmeasure + Index: trunk/Ohana/src/libdvo/src/dvo_convert.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_convert.c (revision 38441) @@ -11,7 +11,7 @@ constructions below **/ -DVOTableFormat FtableGetFormat (FTable *ftable) { - - DVOTableFormat format; +DVOCatFormat FtableGetFormat (FTable *ftable) { + + DVOCatFormat format; char extname[80]; @@ -52,5 +52,5 @@ /*** Average / FTable conversion functions ***/ -Average *FtableToAverage (FTable *ftable, off_t *Naverage, char *format, SecFilt **primary) { +Average *FtableToAverage (FTable *ftable, off_t *Naverage, DVOCatFormat *format, SecFilt **primary, char nativeBytes) { Average *average; @@ -98,5 +98,5 @@ if (!strcmp (extname, NAME)) { \ Average_##TYPE *tmpAverage; \ - tmpAverage = gfits_table_get_Average_##TYPE (ftable, Naverage, NULL); \ + tmpAverage = gfits_table_get_Average_##TYPE (ftable, Naverage, NULL, &nativeBytes); \ if (!tmpAverage) { \ fprintf (stderr, "ERROR: failed to read averages\n"); \ @@ -109,5 +109,5 @@ if (!strcmp (extname, "DVO_AVERAGE")) { - average = gfits_table_get_Average (ftable, Naverage, NULL); + average = gfits_table_get_Average (ftable, Naverage, NULL, &nativeBytes); if (!average) { fprintf (stderr, "ERROR: failed to read averages\n"); @@ -139,18 +139,18 @@ } -int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, char format, SecFilt *primary) { - -# define FORMAT_CASE(FORMAT, TYPE) \ - case DVO_FORMAT_##FORMAT: { \ - Average_##TYPE *tmpAverage; \ +int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, DVOCatFormat format, SecFilt *primary, int swapFromNative) { + +# define FORMAT_CASE(FORMAT, TYPE) \ + case DVO_FORMAT_##FORMAT: { \ + Average_##TYPE *tmpAverage; \ tmpAverage = AverageInternalTo_##TYPE (average, Naverage, primary); \ - gfits_table_set_Average_##TYPE (ftable, tmpAverage, Naverage); \ - free (tmpAverage); \ - break; } - + gfits_table_set_Average_##TYPE (ftable, tmpAverage, Naverage, swapFromNative); \ + free (tmpAverage); \ + break; } + /* convert from the internal format */ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_Average (ftable, average, Naverage); + gfits_table_set_Average (ftable, average, Naverage, swapFromNative); break; } @@ -180,5 +180,6 @@ // FtableToMeasure needs the Average since old formats stored measure.dR,dD only -Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, char *format) { +// other FtableToFOO conversions accept the average argument for macro construction +Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeBytes) { Measure *measure; @@ -226,10 +227,10 @@ if (!strcmp (extname, NAME)) { \ Measure_##TYPE *tmpMeasure; \ - tmpMeasure = gfits_table_get_Measure_##TYPE (ftable, Nmeasure, NULL); \ - if (!tmpMeasure) { \ - fprintf (stderr, "ERROR: failed to read measures\n"); \ - exit (2); \ - } \ - myAssert (ABS_COORDS || average, "conversion to internal needs average table"); \ + tmpMeasure = gfits_table_get_Measure_##TYPE (ftable, Nmeasure, NULL, &nativeBytes); \ + if (!tmpMeasure) { \ + fprintf (stderr, "ERROR: failed to read measures\n"); \ + exit (2); \ + } \ + myAssert (ABS_COORDS || average, "conversion to internal needs average table"); \ measure = Measure_##TYPE##_ToInternal (average, tmpMeasure, *Nmeasure); \ free (tmpMeasure); \ @@ -238,5 +239,5 @@ if (!strcmp (extname, "DVO_MEASURE")) { - measure = gfits_table_get_Measure (ftable, Nmeasure, NULL); + measure = gfits_table_get_Measure (ftable, Nmeasure, NULL, &nativeBytes); if (!measure) { fprintf (stderr, "ERROR: failed to read measures\n"); @@ -269,5 +270,5 @@ // MeasureToFtable needs the Average since old formats stored measure.dR,dD only -int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, char format) { +int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE, ABS_COORDS) \ @@ -276,5 +277,5 @@ myAssert (ABS_COORDS || average, "conversion from internal needs average table"); \ tmpMeasure = MeasureInternalTo_##TYPE (average, measure, Nmeasure); \ - gfits_table_set_Measure_##TYPE (ftable, tmpMeasure, Nmeasure); \ + gfits_table_set_Measure_##TYPE (ftable, tmpMeasure, Nmeasure, swapFromNative); \ free (tmpMeasure); \ break; } @@ -283,5 +284,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_Measure (ftable, measure, Nmeasure); + gfits_table_set_Measure (ftable, measure, Nmeasure, swapFromNative); break; } @@ -308,7 +309,83 @@ } +/*** Missing / FTable conversion functions ***/ + +Missing *FtableToMissing (FTable *ftable, Average *average, off_t *Nmissing, DVOCatFormat *format, char nativeBytes) { + + Missing *missing; + char extname[80]; + + /* convert to the internal format */ + if (!gfits_scan (ftable[0].header, "EXTNAME", "%s", 1, extname)) { + fprintf (stderr, "EXTNAME missing for missing table\n"); + return (FALSE); + } + +# define SKIPPING_FORMAT(NAME, FORMAT, TYPE) \ + if (!strcmp (extname, NAME)) { \ + fprintf (stderr, "ERROR: format %s not defined for missing, skipping\n", NAME); \ + *Nmissing = 0; \ + return NULL; \ + } + +# define CONVERT_FORMAT(NAME, FORMAT, TYPE) \ + if (!strcmp (extname, NAME)) { \ + Missing_##TYPE *tmpMissing; \ + tmpMissing = gfits_table_get_Missing_##TYPE (ftable, Nmissing, NULL, &nativeBytes); \ + if (!tmpMissing) { \ + fprintf (stderr, "ERROR: failed to read missing\n"); \ + exit (2); \ + } \ + missing = Missing_##TYPE##_ToInternal (tmpMissing, *Nmissing); \ + free (tmpMissing); \ + *format = DVO_FORMAT_##FORMAT; \ + return (missing); } + + // XXX the structure is here for future expansion, but no transformations are currently defined + if (TRUE) { + missing = gfits_table_get_Missing (ftable, Nmissing, NULL, &nativeBytes); + if (!missing) { + fprintf (stderr, "ERROR: failed to read missing\n"); + exit (2); + } + *format = DVO_FORMAT_INTERNAL; + return (missing); + } + + if (!strcmp (extname, "DVO_MISSING")) { + missing = gfits_table_get_Missing (ftable, Nmissing, NULL, &nativeBytes); + if (!missing) { + fprintf (stderr, "ERROR: failed to read missing\n"); + exit (2); + } + *format = DVO_FORMAT_INTERNAL; + return (missing); + } + + SKIPPING_FORMAT ("DVO_MISSING_PS1_SIM", PS1_SIM, PS1_SIM); + SKIPPING_FORMAT ("DVO_MISSING_PS1_REF", PS1_REF, PS1_REF); + SKIPPING_FORMAT ("DVO_MISSING_ELIXIR", ELIXIR, Elixir); + SKIPPING_FORMAT ("DVO_MISSING_LONEOS", LONEOS, Loneos); + SKIPPING_FORMAT ("DVO_MISSING_PANSTARRS_DEV_0", PANSTARRS_DEV_0, Panstarrs_DEV_0); + SKIPPING_FORMAT ("DVO_MISSING_PANSTARRS_DEV_1", PANSTARRS_DEV_1, Panstarrs_DEV_1); + SKIPPING_FORMAT ("DVO_MISSING_PS1_DEV_1", PS1_DEV_1, PS1_DEV_1); + SKIPPING_FORMAT ("DVO_MISSING_PS1_DEV_2", PS1_DEV_2, PS1_DEV_2); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V1", PS1_V1, PS1_V1); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V2", PS1_V2, PS1_V2); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V3", PS1_V3, PS1_V3); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V4", PS1_V4, PS1_V4); + SKIPPING_FORMAT ("DVO_MISSING_PS1_V5", PS1_V5, PS1_V5); +# undef CONVERT_FORMAT +# undef SKIPPING_FORMAT + + fprintf (stderr, "table format unknown: %s\n", extname); + + *Nmissing = 0; + return (NULL); +} + /*** SecFilt / FTable conversion functions ***/ -SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, char *format) { +SecFilt *FtableToSecFilt (FTable *ftable, Average *average, off_t *Nsecfilt, DVOCatFormat *format, char nativeBytes) { SecFilt *secfilt; @@ -339,5 +416,5 @@ if (!strcmp (extname, NAME)) { \ SecFilt_##TYPE *tmpSecFilt; \ - tmpSecFilt = gfits_table_get_SecFilt_##TYPE (ftable, Nsecfilt, NULL); \ + tmpSecFilt = gfits_table_get_SecFilt_##TYPE (ftable, Nsecfilt, NULL, &nativeBytes); \ if (!tmpSecFilt) { \ fprintf (stderr, "ERROR: failed to read secfilts\n"); \ @@ -350,5 +427,5 @@ if (!strcmp (extname, "DVO_SECFILT")) { - secfilt = gfits_table_get_SecFilt (ftable, Nsecfilt, NULL); + secfilt = gfits_table_get_SecFilt (ftable, Nsecfilt, NULL, &nativeBytes); if (!secfilt) { fprintf (stderr, "ERROR: failed to read secfilts\n"); @@ -380,5 +457,5 @@ } -int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, char format) { +int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE) \ @@ -386,5 +463,5 @@ SecFilt_##TYPE *tmpSecFilt; \ tmpSecFilt = SecFiltInternalTo_##TYPE (secfilt, Nsecfilt); \ - gfits_table_set_SecFilt_##TYPE (ftable, tmpSecFilt, Nsecfilt); \ + gfits_table_set_SecFilt_##TYPE (ftable, tmpSecFilt, Nsecfilt, swapFromNative); \ free (tmpSecFilt); \ break; } @@ -393,5 +470,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_SecFilt (ftable, secfilt, Nsecfilt); + gfits_table_set_SecFilt (ftable, secfilt, Nsecfilt, swapFromNative); break; } @@ -420,5 +497,5 @@ /*** Lensing / FTable conversion functions ***/ -Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, char *format) { +Lensing *FtableToLensing (FTable *ftable, Average *average, off_t *Nlensing, DVOCatFormat *format, char nativeBytes) { Lensing *lensing; @@ -433,5 +510,5 @@ if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 128)) { Lensing_PS1_V5_R0 *tmpLensing; - tmpLensing = gfits_table_get_Lensing_PS1_V5_R0 (ftable, Nlensing, NULL); + tmpLensing = gfits_table_get_Lensing_PS1_V5_R0 (ftable, Nlensing, NULL, &nativeBytes); if (!tmpLensing) { fprintf (stderr, "ERROR: failed to read lensings\n"); @@ -445,5 +522,5 @@ if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 136)) { Lensing_PS1_V5_R1 *tmpLensing; - tmpLensing = gfits_table_get_Lensing_PS1_V5_R1 (ftable, Nlensing, NULL); + tmpLensing = gfits_table_get_Lensing_PS1_V5_R1 (ftable, Nlensing, NULL, &nativeBytes); if (!tmpLensing) { fprintf (stderr, "ERROR: failed to read lensings\n"); @@ -457,5 +534,5 @@ if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 144)) { Lensing_PS1_V5_R2 *tmpLensing; - tmpLensing = gfits_table_get_Lensing_PS1_V5_R2 (ftable, Nlensing, NULL); + tmpLensing = gfits_table_get_Lensing_PS1_V5_R2 (ftable, Nlensing, NULL, &nativeBytes); if (!tmpLensing) { fprintf (stderr, "ERROR: failed to read lensings\n"); @@ -477,5 +554,5 @@ if (!strcmp (extname, NAME)) { \ Lensing_##TYPE *tmpLensing; \ - tmpLensing = gfits_table_get_Lensing_##TYPE (ftable, Nlensing, NULL); \ + tmpLensing = gfits_table_get_Lensing_##TYPE (ftable, Nlensing, NULL, &nativeBytes); \ if (!tmpLensing) { \ fprintf (stderr, "ERROR: failed to read lensings\n"); \ @@ -488,5 +565,5 @@ if (!strcmp (extname, "DVO_LENSING")) { - lensing = gfits_table_get_Lensing (ftable, Nlensing, NULL); + lensing = gfits_table_get_Lensing (ftable, Nlensing, NULL, &nativeBytes); if (!lensing) { fprintf (stderr, "ERROR: failed to read lensings\n"); @@ -520,5 +597,5 @@ // LensingToFtable needs the Average since old formats stored lensing.dR,dD only -int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, char format) { +int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE) \ @@ -526,5 +603,5 @@ Lensing_##TYPE *tmpLensing; \ tmpLensing = LensingInternalTo_##TYPE (lensing, Nlensing); \ - gfits_table_set_Lensing_##TYPE (ftable, tmpLensing, Nlensing); \ + gfits_table_set_Lensing_##TYPE (ftable, tmpLensing, Nlensing, swapFromNative); \ free (tmpLensing); \ break; } @@ -533,5 +610,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_Lensing (ftable, lensing, Nlensing); + gfits_table_set_Lensing (ftable, lensing, Nlensing, swapFromNative); break; } @@ -559,5 +636,5 @@ /*** Lensobj / FTable conversion functions ***/ -Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, char *format) { +Lensobj *FtableToLensobj (FTable *ftable, Average *average, off_t *Nlensobj, DVOCatFormat *format, char nativeBytes) { Lensobj *lensobj; @@ -572,5 +649,5 @@ if (!strcmp (extname, "DVO_LENSOBJ_PS1_V5") && (ftable[0].header[0].Naxis[0] == 136)) { Lensobj_PS1_V5_R0 *tmpLensobj; - tmpLensobj = gfits_table_get_Lensobj_PS1_V5_R0 (ftable, Nlensobj, NULL); + tmpLensobj = gfits_table_get_Lensobj_PS1_V5_R0 (ftable, Nlensobj, NULL, &nativeBytes); if (!tmpLensobj) { fprintf (stderr, "ERROR: failed to read lensobjs\n"); @@ -592,5 +669,5 @@ if (!strcmp (extname, NAME)) { \ Lensobj_##TYPE *tmpLensobj; \ - tmpLensobj = gfits_table_get_Lensobj_##TYPE (ftable, Nlensobj, NULL); \ + tmpLensobj = gfits_table_get_Lensobj_##TYPE (ftable, Nlensobj, NULL, &nativeBytes); \ if (!tmpLensobj) { \ fprintf (stderr, "ERROR: failed to read lensobjs\n"); \ @@ -603,5 +680,5 @@ if (!strcmp (extname, "DVO_LENSOBJ")) { - lensobj = gfits_table_get_Lensobj (ftable, Nlensobj, NULL); + lensobj = gfits_table_get_Lensobj (ftable, Nlensobj, NULL, &nativeBytes); if (!lensobj) { fprintf (stderr, "ERROR: failed to read lensobjs\n"); @@ -635,5 +712,5 @@ // LensobjToFtable needs the Average since old formats stored lensobj.dR,dD only -int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, char format) { +int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE) \ @@ -641,5 +718,5 @@ Lensobj_##TYPE *tmpLensobj; \ tmpLensobj = LensobjInternalTo_##TYPE (lensobj, Nlensobj); \ - gfits_table_set_Lensobj_##TYPE (ftable, tmpLensobj, Nlensobj); \ + gfits_table_set_Lensobj_##TYPE (ftable, tmpLensobj, Nlensobj, swapFromNative); \ free (tmpLensobj); \ break; } @@ -648,5 +725,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_Lensobj (ftable, lensobj, Nlensobj); + gfits_table_set_Lensobj (ftable, lensobj, Nlensobj, swapFromNative); break; } @@ -674,5 +751,5 @@ /*** StarPar / FTable conversion functions ***/ -StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, char *format) { +StarPar *FtableToStarPar (FTable *ftable, Average *average, off_t *Nstarpar, DVOCatFormat *format, char nativeBytes) { StarPar *starpar; @@ -695,5 +772,5 @@ if (!strcmp (extname, NAME)) { \ StarPar_##TYPE *tmpStarPar; \ - tmpStarPar = gfits_table_get_StarPar_##TYPE (ftable, Nstarpar, NULL); \ + tmpStarPar = gfits_table_get_StarPar_##TYPE (ftable, Nstarpar, NULL, &nativeBytes); \ if (!tmpStarPar) { \ fprintf (stderr, "ERROR: failed to read starpar\n"); \ @@ -706,5 +783,5 @@ if (!strcmp (extname, "DVO_STARPAR")) { - starpar = gfits_table_get_StarPar (ftable, Nstarpar, NULL); + starpar = gfits_table_get_StarPar (ftable, Nstarpar, NULL, &nativeBytes); if (!starpar) { fprintf (stderr, "ERROR: failed to read starpar\n"); @@ -738,5 +815,5 @@ // StarParToFtable needs the Average since old formats stored starpar.dR,dD only -int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, char format) { +int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, DVOCatFormat format, int swapFromNative) { # define FORMAT_CASE(FORMAT, TYPE) \ @@ -744,5 +821,5 @@ StarPar_##TYPE *tmpStarPar; \ tmpStarPar = StarParInternalTo_##TYPE (starpar, Nstarpar); \ - gfits_table_set_StarPar_##TYPE (ftable, tmpStarPar, Nstarpar); \ + gfits_table_set_StarPar_##TYPE (ftable, tmpStarPar, Nstarpar, swapFromNative); \ free (tmpStarPar); \ break; } @@ -751,5 +828,5 @@ switch (format) { case DVO_FORMAT_INTERNAL: { - gfits_table_set_StarPar (ftable, starpar, Nstarpar); + gfits_table_set_StarPar (ftable, starpar, Nstarpar, swapFromNative); break; } @@ -776,9 +853,112 @@ } +/*** GalPhot / FTable conversion functions ***/ + +GalPhot *FtableToGalPhot (FTable *ftable, Average *average, off_t *Ngalphot, DVOCatFormat *format, char nativeBytes) { + + GalPhot *galphot; + char extname[80]; + + /* convert to the internal format */ + if (!gfits_scan (ftable[0].header, "EXTNAME", "%s", 1, extname)) { + fprintf (stderr, "EXTNAME missing for galphot table\n"); + return (FALSE); + } + +# define SKIPPING_FORMAT(NAME, FORMAT, TYPE) \ + if (!strcmp (extname, NAME)) { \ + fprintf (stderr, "ERROR: format %s not defined for galphot, skipping\n", NAME); \ + *Ngalphot = 0; \ + return NULL; \ + } + +# define CONVERT_FORMAT(NAME, FORMAT, TYPE) \ + if (!strcmp (extname, NAME)) { \ + GalPhot_##TYPE *tmpGalPhot; \ + tmpGalPhot = gfits_table_get_GalPhot_##TYPE (ftable, Ngalphot, NULL, &nativeBytes); \ + if (!tmpGalPhot) { \ + fprintf (stderr, "ERROR: failed to read galphots\n"); \ + exit (2); \ + } \ + galphot = GalPhot_##TYPE##_ToInternal (tmpGalPhot, *Ngalphot); \ + free (tmpGalPhot); \ + *format = DVO_FORMAT_##FORMAT; \ + return (galphot); } + + if (!strcmp (extname, "DVO_GALPHOT")) { + galphot = gfits_table_get_GalPhot (ftable, Ngalphot, NULL, &nativeBytes); + if (!galphot) { + fprintf (stderr, "ERROR: failed to read galphots\n"); + exit (2); + } + *format = DVO_FORMAT_INTERNAL; + return (galphot); + } + + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_REF", PS1_REF, PS1_REF); + SKIPPING_FORMAT ("DVO_GALPHOT_ELIXIR", ELIXIR, Elixir); + SKIPPING_FORMAT ("DVO_GALPHOT_LONEOS", LONEOS, Loneos); + SKIPPING_FORMAT ("DVO_GALPHOT_PANSTARRS_DEV_0", PANSTARRS_DEV_0, Panstarrs_DEV_0); + SKIPPING_FORMAT ("DVO_GALPHOT_PANSTARRS_DEV_1", PANSTARRS_DEV_1, Panstarrs_DEV_1); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_DEV_1", PS1_DEV_1, PS1_DEV_1); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_DEV_2", PS1_DEV_2, PS1_DEV_2); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V1", PS1_V1, PS1_V1); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V2", PS1_V2, PS1_V2); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V3", PS1_V3, PS1_V3); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V4", PS1_V4, PS1_V4); + CONVERT_FORMAT ("DVO_GALPHOT_PS1_V5", PS1_V5, PS1_V5); + SKIPPING_FORMAT ("DVO_GALPHOT_PS1_SIM", PS1_SIM, PS1_SIM); +# undef CONVERT_FORMAT +# undef SKIPPING_FORMAT + + fprintf (stderr, "table format unknown: %s\n", extname); + + *Ngalphot = 0; + return (NULL); +} + +// GalPhotToFtable needs the Average since old formats stored galphot.dR,dD only +int GalPhotToFtable (FTable *ftable, GalPhot *galphot, off_t Ngalphot, DVOCatFormat format, int swapFromNative) { + +# define FORMAT_CASE(FORMAT, TYPE) \ + case DVO_FORMAT_##FORMAT: { \ + GalPhot_##TYPE *tmpGalPhot; \ + tmpGalPhot = GalPhotInternalTo_##TYPE (galphot, Ngalphot); \ + gfits_table_set_GalPhot_##TYPE (ftable, tmpGalPhot, Ngalphot, swapFromNative); \ + free (tmpGalPhot); \ + break; } + + /* convert from the internal format */ + switch (format) { + case DVO_FORMAT_INTERNAL: { + gfits_table_set_GalPhot (ftable, galphot, Ngalphot, swapFromNative); + break; } + +// FORMAT_CASE (PS1_REF, PS1_REF); +// FORMAT_CASE (ELIXIR, Elixir); +// FORMAT_CASE (LONEOS, Loneos); +// FORMAT_CASE (PANSTARRS_DEV_0, Panstarrs_DEV_0); +// FORMAT_CASE (PANSTARRS_DEV_1, Panstarrs_DEV_1); +// FORMAT_CASE (PS1_DEV_1, PS1_DEV_1); +// FORMAT_CASE (PS1_DEV_2, PS1_DEV_2); +// FORMAT_CASE (PS1_V1, PS1_V1); +// FORMAT_CASE (PS1_V2, PS1_V2); +// FORMAT_CASE (PS1_V3, PS1_V3); +// FORMAT_CASE (PS1_V4, PS1_V4); + FORMAT_CASE (PS1_V5, PS1_V5); +# undef FORMAT_CASE + + default: + fprintf (stderr, "table format unknown (galphot)\n"); + return (FALSE); + } + return (TRUE); +} + /*** Image Conversions ***/ // I have loaded the disk db table and now I want to convert to the internal format // (Image structure), but I onyl -int FtableToImage (FTable *ftable, Header *theader, char *format) { +int FtableToImage (FTable *ftable, Header *theader, DVOCatFormat *format) { off_t Nimage; @@ -788,5 +968,5 @@ if (*format == DVO_FORMAT_ELIXIR) { // special case for ELIXIR Image_Elixir *tmpimage; - tmpimage = gfits_table_get_Image_Elixir (ftable, &Nimage, NULL); + tmpimage = gfits_table_get_Image_Elixir (ftable, &Nimage, NULL, NULL); if (!tmpimage) { fprintf (stderr, "ERROR: failed to read images\n"); @@ -814,5 +994,5 @@ Image_##TYPE *tmpimage; \ *format = DVO_FORMAT_##FORMAT; \ - tmpimage = gfits_table_get_Image_##TYPE (ftable, &Nimage, NULL); \ + tmpimage = gfits_table_get_Image_##TYPE (ftable, &Nimage, NULL, NULL); \ if (!tmpimage) { \ fprintf (stderr, "ERROR: failed to read images\n"); \ @@ -849,5 +1029,5 @@ } -int ImageToFtable (FTable *ftable, Header *theader, char format) { +int ImageToFtable (FTable *ftable, Header *theader, DVOCatFormat format) { off_t Nimage; @@ -860,5 +1040,5 @@ tmpImage = ImageInternalTo_##TYPE ((Image *) ftable[0].buffer, Nimage); \ free (ftable[0].buffer); \ - gfits_table_set_Image_##TYPE (ftable, tmpImage, Nimage); \ + gfits_table_set_Image_##TYPE (ftable, tmpImage, Nimage, TRUE); \ free (tmpImage); \ break; } @@ -889,5 +1069,5 @@ } -int ImageToVtable (VTable *vtable, Header *theader, char format) { +int ImageToVtable (VTable *vtable, Header *theader, DVOCatFormat format) { off_t i, Nrow, Nimage; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c (revision 38441) @@ -118,5 +118,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Nextend; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -152,5 +152,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Nextend = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c (revision 38441) @@ -138,5 +138,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -173,5 +173,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c (revision 38441) @@ -143,5 +143,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -182,5 +182,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c (revision 38441) @@ -143,5 +143,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -184,5 +184,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c (revision 38441) @@ -164,5 +164,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -208,5 +208,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; @@ -690,5 +690,5 @@ out[i].Nmeasure = in[i].Nmeasure; out[i].Nmissing = in[i].Nmissing; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; out[i].missingOffset = in[i].missingOffset; Index: trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c (revision 38441) @@ -196,5 +196,5 @@ out[i].Nlensobj = in[i].Nlensobj; out[i].Nstarpar = in[i].Nstarpar; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; @@ -203,5 +203,5 @@ out[i].lensobjOffset = in[i].lensobjOffset; out[i].starparOffset = in[i].starparOffset; - out[i].extendOffset = in[i].extendOffset; + out[i].galphotOffset = in[i].galphotOffset; out[i].refColorBlue = in[i].refColorBlue; @@ -264,5 +264,5 @@ out[i].Nlensobj = in[i].Nlensobj; out[i].Nstarpar = in[i].Nstarpar; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; @@ -271,5 +271,5 @@ out[i].lensobjOffset = in[i].lensobjOffset; out[i].starparOffset = in[i].starparOffset; - out[i].extendOffset = in[i].extendOffset; + out[i].galphotOffset = in[i].galphotOffset; out[i].refColorBlue = in[i].refColorBlue; @@ -975,4 +975,68 @@ out[i].objID = in[i].objID ; out[i].catID = in[i].catID ; + } + return (out); +} + +GalPhot *GalPhot_PS1_V5_ToInternal (GalPhot_PS1_V5 *in, off_t Nvalues) { + + off_t i; + GalPhot *out; + + ALLOCATE_ZERO (out, GalPhot, Nvalues); + + for (i = 0; i < Nvalues; i++) { + dvo_galphot_init (&out[i]); + + out[i].Xfit = in[i].Xfit; + out[i]. Yfit = in[i]. Yfit; + out[i]. mag = in[i]. mag; + out[i]. magErr = in[i]. magErr; + out[i]. majorAxis = in[i]. majorAxis; + out[i]. minorAxis = in[i]. minorAxis; + out[i]. majorAxisErr = in[i]. majorAxisErr; + out[i]. minorAxisErr = in[i]. minorAxisErr; + out[i]. theta = in[i]. theta; + out[i]. theta_err = in[i]. theta_err; + out[i]. index = in[i]. index; + out[i]. chisq = in[i]. chisq; + out[i]. Npix = in[i]. Npix; + out[i]. objID = in[i]. objID; + out[i]. catID = in[i]. catID; + out[i]. detID = in[i]. detID; + out[i]. imageID = in[i]. imageID; + out[i]. photcode = in[i]. photcode; + out[i]. modelType = in[i]. modelType; + } + return (out); +} + +GalPhot_PS1_V5 *GalPhotInternalTo_PS1_V5 (GalPhot *in, off_t Nvalues) { + + off_t i; + GalPhot_PS1_V5 *out; + + ALLOCATE_ZERO (out, GalPhot_PS1_V5, Nvalues); + + for (i = 0; i < Nvalues; i++) { + out[i].Xfit = in[i].Xfit; + out[i].Yfit = in[i].Yfit; + out[i].mag = in[i].mag; + out[i].magErr = in[i].magErr; + out[i].majorAxis = in[i].majorAxis; + out[i].minorAxis = in[i].minorAxis; + out[i].majorAxisErr = in[i].majorAxisErr; + out[i].minorAxisErr = in[i].minorAxisErr; + out[i].theta = in[i].theta; + out[i].theta_err = in[i].theta_err; + out[i].index = in[i].index; + out[i].chisq = in[i].chisq; + out[i].Npix = in[i].Npix; + out[i].objID = in[i].objID; + out[i].catID = in[i].catID; + out[i].detID = in[i].detID; + out[i].imageID = in[i].imageID; + out[i].photcode = in[i].photcode; + out[i].modelType = in[i].modelType; } return (out); @@ -1234,5 +1298,5 @@ out[i].Nlensing = in[i].Nlensing; out[i].Nlensobj = in[i].Nlensobj; - out[i].Nextend = in[i].Nextend; + out[i].Ngalphot = in[i].Ngalphot; out[i].measureOffset = in[i].measureOffset; @@ -1241,5 +1305,5 @@ out[i].lensobjOffset = in[i].lensobjOffset; out[i].starparOffset = in[i].starparOffset; - out[i].extendOffset = in[i].extendOffset; + out[i].galphotOffset = in[i].galphotOffset; out[i].refColorBlue = in[i].refColorBlue; @@ -1302,10 +1366,10 @@ SWAP_BYTE (110); // NLENSING SWAP_BYTE (112); // NLENSOBJ - SWAP_BYTE (114); // NEXTEND + SWAP_BYTE (114); // NGALPHOT SWAP_WORD (116); // OFF_MEASURE SWAP_WORD (120); // OFF_MISSING SWAP_WORD (124); // OFF_LENSING SWAP_WORD (128); // OFF_LENSOBJ - SWAP_WORD (132); // OFF_EXTEND + SWAP_WORD (132); // OFF_GALPHOT SWAP_WORD (136); // OFF_PARAMS SWAP_WORD (140); // REF_COLOR_BLUE Index: trunk/Ohana/src/libdvo/src/dvo_image.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_image.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_image.c (revision 38441) @@ -19,5 +19,4 @@ db[0].lockstate = lockstate; db[0].timeout = timeout; - gfits_db_init (db); if (!gfits_db_lock (db, filename)) { @@ -89,5 +88,6 @@ // so when we later do a gfits_copy_ftable, we stomp on bad memory FtableToImage (&db[0].ftable, &db[0].theader, &db[0].format); - db[0].swapped = TRUE; /* table has internal byte-order */ + db[0].nativeOrder = TRUE; /* table has internal byte-order */ + db[0].scaledValue = TRUE; /* table has internal byte-order */ return (TRUE); } @@ -100,5 +100,6 @@ /* convert from internal to requested external format */ ImageToFtable (&db[0].ftable, &db[0].theader, db[0].format); - db[0].swapped = FALSE; + db[0].nativeOrder = FALSE; + db[0].scaledValue = FALSE; /* write data in appropriate mode */ @@ -149,5 +150,6 @@ /* theader is modified to match output format */ ImageToVtable (&db[0].vtable, &db[0].theader, db[0].format); - db[0].swapped = FALSE; + db[0].scaledValue = FALSE; + db[0].nativeOrder = FALSE; /* write data in appropriate mode */ @@ -363,5 +365,5 @@ /* return internal structure representation */ -Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *swapped) { +Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *scaledValue, char *nativeOrder) { int Ncols; @@ -377,9 +379,15 @@ data = (Image *) ftable[0].buffer; - if (!swapped) { - myAbort ("invalid to call this without suppying 'swapped'"); - } - if (*swapped == FALSE) { - myAbort ("invalid for table to be swapped"); + if (!scaledValue) { + myAbort ("invalid to call this without suppying 'scaledValue'"); + } + if (*scaledValue == FALSE) { + myAbort ("invalid for table NOT to be scaledValue"); + } + if (!nativeOrder) { + myAbort ("invalid to call this without suppying 'nativeOrder'"); + } + if (*nativeOrder == FALSE) { + myAbort ("invalid for table NOT to be nativeOrder"); } Index: trunk/Ohana/src/libdvo/src/dvo_image_raw.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_image_raw.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_image_raw.c (revision 38441) @@ -107,5 +107,7 @@ db[0].theader.Naxis[1] = Nimage; db[0].ftable.datasize = gfits_data_size (&db[0].theader); - db[0].swapped = FALSE; /* table does not have internal byte-order */ + + db[0].nativeOrder = FALSE; /* table does not have internal byte-order */ + db[0].scaledValue = FALSE; /* table does not have internal byte-order */ return (TRUE); } Index: trunk/Ohana/src/libdvo/src/dvo_util.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_util.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/dvo_util.c (revision 38441) @@ -127,5 +127,5 @@ } - dvoConfig->images = gfits_table_get_Image (&dvoConfig->imageDB.ftable, &dvoConfig->nImages, &dvoConfig->imageDB.swapped); + dvoConfig->images = gfits_table_get_Image (&dvoConfig->imageDB.ftable, &dvoConfig->nImages, &dvoConfig->imageDB.scaledValue, &dvoConfig->imageDB.nativeOrder); if (!dvoConfig->images) { fprintf (stderr, "problem loading images\n"); @@ -207,5 +207,5 @@ Catalog catalog; catalog.filename = skylist->filename[reg]; - catalog.catflags = GetMeasures ? LOAD_AVES | LOAD_MEAS | LOAD_SECF : LOAD_AVES | LOAD_SECF; + catalog.catflags = GetMeasures ? DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT : DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT; catalog.Nsecfilt = 0; Index: trunk/Ohana/src/libdvo/src/fits_db.c =================================================================== --- trunk/Ohana/src/libdvo/src/fits_db.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/fits_db.c (revision 38441) @@ -6,22 +6,18 @@ db[0].f = NULL; db[0].filename = NULL; + db[0].dbstate = 0; + db[0].lockstate = 0; + db[0].mode = 0; + db[0].format = 0; + db[0].virtual = 0; + db[0].nativeOrder = 0; + db[0].scaledValue = 0; -# if (1) gfits_init_header (&db->header); gfits_init_matrix (&db->matrix); gfits_init_header (&db->theader); - gfits_init_table (&db->ftable); + gfits_init_table (&db->ftable); db->ftable.header = &db->theader; gfits_init_vtable (&db->vtable); -# else - db[0].header.buffer = NULL; - db[0].matrix.buffer = NULL; - db[0].theader.buffer = NULL; - db[0].ftable.buffer = NULL; - db[0].ftable.header = &db[0].theader; - db[0].vtable.header = NULL; - db[0].vtable.buffer = NULL; - db[0].vtable.row = NULL; -# endif return (TRUE); } @@ -97,5 +93,6 @@ return (FALSE); } - db[0].swapped = FALSE; /* table does not have internal byte-order */ + db[0].nativeOrder = FALSE; /* table does not have internal byte-order */ + db[0].scaledValue = FALSE; /* table has not been scaled by BZERO,BSCALE */ return (TRUE); } Index: trunk/Ohana/src/libdvo/src/flatcorr_io.c =================================================================== --- trunk/Ohana/src/libdvo/src/flatcorr_io.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/flatcorr_io.c (revision 38441) @@ -49,5 +49,5 @@ return (NULL); } - flatcorrTable->image = gfits_table_get_FlatCorrectionImage (&ftable, &flatcorrTable->Nimage, NULL); + flatcorrTable->image = gfits_table_get_FlatCorrectionImage (&ftable, &flatcorrTable->Nimage, NULL, NULL); if (!flatcorrTable->image) { fprintf (stderr, "ERROR: failed to read Flat Correction Images\n"); @@ -64,5 +64,5 @@ return (NULL); } - flatcorrTable->corr = gfits_table_get_FlatCorrection (&ftable, &flatcorrTable->Ncorr, NULL); + flatcorrTable->corr = gfits_table_get_FlatCorrection (&ftable, &flatcorrTable->Ncorr, NULL, NULL); if (!flatcorrTable->corr) { fprintf (stderr, "ERROR: failed to read Flat Corrections\n"); @@ -93,5 +93,5 @@ for (j = 0; j < flatcorrTable->image[i].Nx; j++) { ALLOCATE (flatcorrTable->offset[i][j], float, flatcorrTable->image[i].Ny); - memset (flatcorrTable->offset[i][j], 0.0, flatcorrTable->image[i].Ny*sizeof(float)); + memset (flatcorrTable->offset[i][j], 0, flatcorrTable->image[i].Ny*sizeof(float)); } } @@ -161,8 +161,8 @@ ftableImage.header = &theaderImage; - gfits_table_set_FlatCorrectionImage (&ftableImage, image, flatcorrTable->Nimage); + gfits_table_set_FlatCorrectionImage (&ftableImage, image, flatcorrTable->Nimage, TRUE); ftableCorr.header = &theaderCorr; - gfits_table_set_FlatCorrection (&ftableCorr, corr, flatcorrTable->Ncorr); + gfits_table_set_FlatCorrection (&ftableCorr, corr, flatcorrTable->Ncorr, TRUE); f = fopen (filename, "w"); Index: trunk/Ohana/src/libdvo/src/skyregion_io.c =================================================================== --- trunk/Ohana/src/libdvo/src/skyregion_io.c (revision 38154) +++ trunk/Ohana/src/libdvo/src/skyregion_io.c (revision 38441) @@ -40,5 +40,5 @@ ALLOCATE (skytable, SkyTable, 1); memset (skytable->hosts, 0, 80); - skytable[0].regions = gfits_table_get_SkyRegion (&ftable, &skytable[0].Nregions, NULL); + skytable[0].regions = gfits_table_get_SkyRegion (&ftable, &skytable[0].Nregions, NULL, NULL); if (!skytable[0].regions) { fprintf (stderr, "ERROR: failed to read sky regions\n"); @@ -99,5 +99,5 @@ gfits_print (&header, "HOSTS", "%s", 1, skytable->hosts); - gfits_table_set_SkyRegion (&ftable, skytable[0].regions, skytable[0].Nregions); + gfits_table_set_SkyRegion (&ftable, skytable[0].regions, skytable[0].Nregions, TRUE); f = fopen (filename, "w");