IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Ignore:
Timestamp:
Jun 12, 2015, 6:18:23 PM (11 years ago)
Author:
eugene
Message:

merge changes from EAM dev branch ohana.20150429

Location:
trunk/Ohana
Files:
27 edited

Legend:

Unmodified
Added
Removed

  • trunk/Ohana

  • trunk/Ohana/src/libdvo/Makefile

    r37992 r38441  
    1010MAN     =       $(HOME)/doc
    1111INC     =       $(HOME)/include
     12TEST    =       $(HOME)/test
     13TESTBIN =       $(HOME)/test
    1214ASRC    =       $(AUTO)/src
    1315AINC    =       $(AUTO)/include
    … …  
    153155        rm -f $@
    154156        cp $< $@
     157        echo "done making autocode.h"
    155158
    156159$(AINC)/autocode.h: $(AINCS) $(ADEF)/autocode.h $(ADEF)/common.h

  • trunk/Ohana/src/libdvo/include/dvo.h

    r38154 r38441  
    1010
    1111/* DVO table modes */
    12 typedef enum {DVO_MODE_UNDEF, DVO_MODE_RAW, DVO_MODE_MEF, DVO_MODE_SPLIT, DVO_MODE_MYSQL} DVOTableMode;
     12typedef enum {DVO_MODE_UNDEF = 0, DVO_MODE_RAW, DVO_MODE_MEF, DVO_MODE_SPLIT, DVO_MODE_MYSQL} DVOCatMode;
     13
     14/* DVO table modes */
     15typedef enum {DVO_COMPRESS_NONE = 0, DVO_COMPRESS_NONE_1, DVO_COMPRESS_NONE_2, DVO_COMPRESS_AUTO, DVO_COMPRESS_GZIP_1, DVO_COMPRESS_GZIP_2, DVO_COMPRESS_RICE_1} DVOCatCompress; //
    1316
    1417/* DVO table formats */
    1518typedef enum {
    16   DVO_FORMAT_UNDEF,
     19  DVO_FORMAT_UNDEF = 0,
    1720  DVO_FORMAT_INTERNAL,
    1821  DVO_FORMAT_ELIXIR,
    … …  
    3033  DVO_FORMAT_PS1_V4,
    3134  DVO_FORMAT_PS1_V5,
    32 } DVOTableFormat;
     35} DVOCatFormat;
     36
     37typedef enum {DVO_CAT_OPEN_FAIL, DVO_CAT_OPEN_OK, DVO_CAT_OPEN_EMPTY} DVOCatalogOpenModes;
     38
     39/* catalog values to be loaded */
     40typedef enum {
     41  DVO_LOAD_NONE     = 0x0000,
     42  DVO_LOAD_AVERAGE  = 0x0001,
     43  DVO_LOAD_MEASURE  = 0x0002,
     44  DVO_LOAD_MISSING  = 0x0004,
     45  DVO_LOAD_SECFILT  = 0x0008,
     46  DVO_SKIP_AVERAGE  = 0x0010,
     47  DVO_SKIP_MEASURE  = 0x0020,
     48  DVO_SKIP_MISSING  = 0x0040,
     49  DVO_SKIP_SECFILT  = 0x0080,
     50  DVO_LOAD_LENSING  = 0x0100,
     51  DVO_LOAD_LENSOBJ  = 0x0200,
     52  DVO_SKIP_LENSING  = 0x0400,
     53  DVO_SKIP_LENSOBJ  = 0x0800,
     54  DVO_LOAD_STARPAR  = 0x1000,
     55  DVO_SKIP_STARPAR  = 0x2000,
     56  DVO_LOAD_GALPHOT = 0x4000,
     57  DVO_SKIP_GALPHOT = 0x8000,
     58} DVOCatFlags;
    3359
    3460/* image data modes in RegImage */
    3561typedef enum {T_UNDEF = -1, T_NONE, T_OBJECT, T_DARK, T_BIAS, T_FLAT, T_MASK, T_FRINGE, T_SCATTER, T_MODES, T_FRINGEPTS, T_ANY, N_TYPE} ElixirDetrendTypes;
    3662typedef enum {M_UNDEF = -1, M_NONE, M_MEF, M_SPLIT, M_SINGLE, M_CUBE, M_SLICE, M_MODES, N_MODE} ElixirDetrendModes;
    37 
    38 typedef enum {DVO_CAT_OPEN_FAIL, DVO_CAT_OPEN_OK, DVO_CAT_OPEN_EMPTY} DVOCatalogOpenModes;
    3963
    4064// these are used as NAN for types of int values
    … …  
    5478# define IMREG_DIST  0x01 /* image distributed, only imregister-3.0 */
    5579
    56 /* catalog values to be loaded */
    57 # define LOAD_NONE      0x0000
    58 # define LOAD_AVES      0x0001
    59 # define LOAD_MEAS      0x0002
    60 # define LOAD_MISS      0x0004
    61 # define LOAD_SECF      0x0008
    62 # define SKIP_AVES      0x0010
    63 # define SKIP_MEAS      0x0020
    64 # define SKIP_MISS      0x0040
    65 # define SKIP_SECF      0x0080
    66 # define LOAD_LENSING   0x0100
    67 # define LOAD_LENSOBJ   0x0200
    68 # define SKIP_LENSING   0x0400
    69 # define SKIP_LENSOBJ   0x0800
    70 # define LOAD_STARPAR   0x1000
    71 # define SKIP_STARPAR   0x2000
    72 
    7380/* photometry code types */
    7481// # define PHOT_PRI 0x01
    75 # define PHOT_SEC 0x02
    76 # define PHOT_DEP 0x03
    77 # define PHOT_REF 0x04
    78 # define PHOT_ALT 0x05  /* never stored, only for look-ups */
    79 # define PHOT_MAG 0x06  /* generic magnitude; never stored */
     82typedef enum {
     83  PHOT_SEC  = 0x02,
     84  PHOT_DEP  = 0x03,
     85  PHOT_REF  = 0x04,
     86  PHOT_ALT  = 0x05,  /* never stored, only for look-ups */
     87  PHOT_MAG  = 0x06,  /* generic magnitude; never stored */
     88} DVOPhotCodeTypes;
    8089
    8190/* Image.code values -- these values are 32 bit (as of PS1_V1) */
    … …  
    222231  char   lockstate;
    223232  char   mode;          /* what data storage mode is used for disk file? */
    224   char   format;        /* what data format is used for disk file? */
     233  DVOCatFormat format;  /* what data format is used for disk file? */
    225234  char   virtual;       /* is table in ftable or vtable? */
    226   char   swapped;       /* is table in internal byte-order? */
     235  char   nativeOrder;   /* is table in internal byte-order? */
     236  char   scaledValue;   /* is table in internal byte-order? */
    227237} FITS_DB;
    228238
    … …  
    484494  unsigned short   Nmeasure;             // number of psf measurements
    485495  unsigned short   Nmissing;             // number of missings
    486   unsigned short   Nextend;              // number of extended measurements
     496  unsigned short   Ngalphot;            // number of extended measurements
    487497  uint32_t         measureOffset;        // offset to first psf measurement
    488498  uint32_t         missingOffset;        // offset to first missing obs
    … …  
    704714  unsigned short   Nlensing;             // number of lensing measurements
    705715  unsigned short   Nlensobj;             // number of lensing measurements
    706   unsigned short   Nextend;              // number of extended measurements
     716  unsigned short   Ngalphot;            // number of galphot measurements
    707717  int              measureOffset;        // offset to first psf measurement
    708718  int              missingOffset;        // offset to first missing obs
    709719  int              lensingOffset;        // offset to first lensing obs
    710720  int              lensobjOffset;        // offset to mean lensing data
    711   int              extendOffset;         // offset to extended object entry
     721  int              galphotOffset;       // offset to extended object entry
    712722  int              starparOffset;        // offset to stellar parameter data
    713723  float            refColorBlue;         // color of astrometry ref stars
    … …  
    742752
    743753  // lensing data (optional?)
    744   Lensing *lensing;
    745   Lensobj *lensobj;
    746   StarPar *starpar;
     754  Lensing  *lensing;
     755  Lensobj  *lensobj;
     756  StarPar  *starpar;
     757  GalPhot *galphot;
    747758
    748759  int Nsecfilt;  /* number of secfilt entries for each average entry */
    749   off_t Naverage,   Nmeasure,   Nmissing,   Nlensing,      Nlensobj,      Nstarpar,      Nsecf_mem;  /* current number of each component in memory */
    750   off_t Naves_disk, Nmeas_disk, Nmiss_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Nsecf_disk; /* current number of each component on disk */
    751   off_t Naves_off,  Nmeas_off,  Nmiss_off,  Nlensing_off,  Nlensobj_off,  Nstarpar_off,  Nsecf_off;  /* index of first loaded data value */
     760  off_t Naverage,      Nmeasure,      Nmissing,      Nlensing,      Nlensobj,      Nstarpar,      Ngalphot,      Nsecfilt_mem;  /* current number of each component in memory */
     761  off_t Naverage_disk, Nmeasure_disk, Nmissing_disk, Nlensing_disk, Nlensobj_disk, Nstarpar_disk, Ngalphot_disk, Nsecfilt_disk; /* current number of each component on disk */
     762  off_t Naverage_off,  Nmeasure_off,  Nmissing_off,  Nlensing_off,  Nlensobj_off,  Nstarpar_off,  Ngalphot_off,  Nsecfilt_off;  /* index of first loaded data value */
    752763
    753764  // note that we use these for the full-sky relphot analysis
    … …  
    765776
    766777  /* pointers to split data files */
    767   struct Catalog *measure_catalog;              /* measure catalog data (split) */
    768   struct Catalog *missing_catalog;              /* missing catalog data (split) */
    769   struct Catalog *secfilt_catalog;              /* secfilt catalog data (split) */
    770   struct Catalog *lensing_catalog;              /* lensing catalog data (split) */
    771   struct Catalog *lensobj_catalog;              /* lensobj catalog data (split) */
    772   struct Catalog *starpar_catalog;              /* starpar catalog data (split) */
     778  struct Catalog *measure_catalog;              /* measure  catalog data (split) */
     779  struct Catalog *missing_catalog;              /* missing  catalog data (split) */
     780  struct Catalog *secfilt_catalog;              /* secfilt  catalog data (split) */
     781  struct Catalog *lensing_catalog;              /* lensing  catalog data (split) */
     782  struct Catalog *lensobj_catalog;              /* lensobj  catalog data (split) */
     783  struct Catalog *starpar_catalog;              /* starpar  catalog data (split) */
     784  struct Catalog *galphot_catalog;              /* galphot catalog data (split) */
    773785
    774786  unsigned int objID;
    … …  
    777789  /* extra catalog information */
    778790  char lockmode;
    779   char catmode;                         /* storage mode (raw, mef, split, mysql) */
    780   char catformat;                       /* storage format (elixir, panstarrs, etc) */
     791  DVOCatMode     catmode;     /* storage mode (raw, mef, split, mysql) */
     792  DVOCatFormat   catformat;                     /* storage format (elixir, panstarrs, etc) */
     793  DVOCatCompress catcompress;                   // output compress mode
     794  DVOCatFlags    catflags;                      /* choices to be loaded */
     795 
    781796  int sorted;                           /* is measure table average-sorted? (NOTE this is an int only because gfits_scan %t requires it) */
    782  
    783   short catflags; /* choices to be loaded */
     797
    784798
    785799  /* pointers for data manipulation */
    … …  
    928942int dvo_catalog_save_complete (Catalog *catalog, char VERBOSE);
    929943int dvo_catalog_update (Catalog *catalog, char VERBOSE);
    930 int dvo_catalog_catformat (char *catformat);
    931 int dvo_catalog_catmode (char *catmode);
     944DVOCatFormat dvo_catalog_catformat (char *catformat);
     945DVOCatMode dvo_catalog_catmode (char *catmode);
     946DVOCatCompress dvo_catalog_catcompress (char *catcompress);
     947char *dvo_catalog_compress_string (DVOCatCompress catcompress);
    932948void dvo_catalog_test (Catalog *catalog, int halt);
    933949
    … …  
    956972int WriteRawSecFilt (FILE *f, SecFilt *secfilt, off_t Nsecfilt, char format);
    957973
    958 DVOTableFormat FtableGetFormat (FTable *ftable);
    959 
    960 Average *FtableToAverage (FTable *ftable, off_t *Naverage, char *format, SecFilt **primary);
    961 Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, char *format);
    962 SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, char *format);
    963 Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, char *format);
    964 Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, char *format);
    965 StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, char *format);
    966 int      FtableToImage   (FTable *ftable, Header *theader, char *format);
    967 
    968 int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, char format, SecFilt *primary);
    969 int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, char format);
    970 int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, char format);
    971 int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, char format);
    972 int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, char format);
    973 int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, char format);
    974 int ImageToFtable (FTable *ftable, Header *theader, char format);
    975 int ImageToVtable (VTable *vtable, Header *theader, char format);
     974DVOCatFormat FtableGetFormat (FTable *ftable);
     975
     976Average *FtableToAverage   (FTable *ftable, off_t *Naverage,  DVOCatFormat *format, SecFilt **primary, char nativeOrder);
     977
     978Measure *FtableToMeasure   (FTable *ftable, Average *average, off_t *Nmeasure,  DVOCatFormat *format, char nativeOrder);
     979Missing *FtableToMissing   (FTable *ftable, Average *average, off_t *Nmissing,  DVOCatFormat *format, char nativeBytes);
     980SecFilt *FtableToSecFilt   (FTable *ftable, Average *average, off_t *Nsecfilt,  DVOCatFormat *format, char nativeOrder);
     981Lensing *FtableToLensing   (FTable *ftable, Average *average, off_t *Nlensing,  DVOCatFormat *format, char nativeOrder);
     982Lensobj *FtableToLensobj   (FTable *ftable, Average *average, off_t *Nlensobj,  DVOCatFormat *format, char nativeOrder);
     983StarPar *FtableToStarPar   (FTable *ftable, Average *average, off_t *Nstarpar,  DVOCatFormat *format, char nativeOrder);
     984GalPhot *FtableToGalPhot (FTable *ftable, Average *average, off_t *Ngalphot, DVOCatFormat *format, char nativeOrder);
     985
     986int      FtableToImage   (FTable *ftable, Header *theader, DVOCatFormat *format);
     987
     988int MeasureToFtable  (FTable *ftable, Average  *average,  Measure *measure, off_t Nmeasure, DVOCatFormat format, int swapFromNative);
     989int AverageToFtable  (FTable *ftable, Average  *average,  off_t Naverage,  DVOCatFormat format, SecFilt *primary, int swapFromNative);
     990int SecFiltToFtable  (FTable *ftable, SecFilt  *secfilt,  off_t Nsecfilt,  DVOCatFormat format, int swapFromNative);
     991int LensingToFtable  (FTable *ftable, Lensing  *lensing,  off_t Nlensing,  DVOCatFormat format, int swapFromNative);
     992int LensobjToFtable  (FTable *ftable, Lensobj  *lensobj,  off_t Nlensobj,  DVOCatFormat format, int swapFromNative);
     993int StarParToFtable  (FTable *ftable, StarPar  *starpar,  off_t Nstarpar,  DVOCatFormat format, int swapFromNative);
     994int GalPhotToFtable (FTable *ftable, GalPhot *galphot, off_t Ngalphot, DVOCatFormat format, int swapFromNative);
     995
     996int ImageToFtable (FTable *ftable, Header *theader, DVOCatFormat format);
     997int ImageToVtable (VTable *vtable, Header *theader, DVOCatFormat format);
    976998
    977999# include "loneos_defs.h"
    … …  
    10051027int gfits_table_set_Image (FTable *ftable);
    10061028int gfits_table_mkheader_Image (Header *header);
    1007 Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *swapped);
     1029Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *scaledValue, char *nativeOrder);
    10081030
    10091031/* flatcorr APIs */
    … …  
    10691091void InitHost (HostInfo *host);
    10701092
    1071 HostTableGroup *HostTableGroups (HostTable *table, int *ngroups);
     1093HostTableGroup *HostTableGroupsUniqueMachines (HostTable *table, int *ngroups);
     1094HostTableGroup *HostTableGroupsMaxNumber (HostTable *table, int *ngroups, int Nmax);
     1095
    10721096int HostTableGroupWaitJobsGetIO (HostTableGroup *table, char *file, int lineno, int VERBOSE);
    10731097
    … …  
    11061130void dvo_lensobj_init (Lensobj *lensobj, int toZero);
    11071131void dvo_starpar_init (StarPar *starpar);
     1132void dvo_galphot_init (GalPhot *galphot);
    11081133
    11091134void InitRegionHosts (RegionHostInfo *hosts, int Nhosts, int NHOSTS);

  • trunk/Ohana/src/libdvo/include/ps1_v5_defs.h

    r38153 r38441  
    2323StarPar_PS1_V5          *StarParInternalTo_PS1_V5 (StarPar *in, off_t Nvalues);
    2424
     25GalPhot                 *GalPhot_PS1_V5_ToInternal (GalPhot_PS1_V5 *in, off_t Nvalues);
     26GalPhot_PS1_V5         *GalPhotInternalTo_PS1_V5 (GalPhot *in, off_t Nvalues);
     27
    2528PhotCode                *PhotCode_PS1_V5_To_Internal (PhotCode_PS1_V5 *in, off_t Nvalues);
    2629PhotCode_PS1_V5         *PhotCode_Internal_To_PS1_V5 (PhotCode *in, off_t Nvalues);

  • trunk/Ohana/src/libdvo/src/AstromOffsetMapIO.c

    r37807 r38441  
    3939    return (NULL);
    4040  }
    41   AstromOffsetMap_Disk_6x6 *map_disk = gfits_table_get_AstromOffsetMap_Disk_6x6 (&ftable, &Nmap, NULL);
     41  AstromOffsetMap_Disk_6x6 *map_disk = gfits_table_get_AstromOffsetMap_Disk_6x6 (&ftable, &Nmap, NULL, NULL);
    4242  if (!map_disk) {
    4343    fprintf (stderr, "ERROR: failed to read Astrom Offset Map\n");
    … …  
    7878
    7979  ftable.header = &theader;
    80   gfits_table_set_AstromOffsetMap_Disk_6x6 (&ftable, map_disk, Nmap);
     80  gfits_table_set_AstromOffsetMap_Disk_6x6 (&ftable, map_disk, Nmap, TRUE);
    8181
    8282  f = fopen (filename, "w");

  • trunk/Ohana/src/libdvo/src/HostTable.c

    r37807 r38441  
    155155
    156156// split a host table into Ngroups, each with a unique set of hosts
    157 HostTableGroup *HostTableGroups (HostTable *table, int *ngroups) {
     157HostTableGroup *HostTableGroupsUniqueMachines (HostTable *table, int *ngroups) {
    158158
    159159  // identify the unique host names and the number of times they each exist
    … …  
    210210        foundHost[k] = TRUE;
    211211      }
     212    }
     213  }
     214  *ngroups = Ngroups;
     215  return groups;
     216}
     217
     218// split a host table into Ngroups, each with a unique set of hosts
     219HostTableGroup *HostTableGroupsMaxNumber (HostTable *table, int *ngroups, int Nmax) {
     220
     221  int i, j;
     222
     223  int Ngroups = (table->Nhosts % Nmax) ? (int)(table->Nhosts / Nmax + 1) : table->Nhosts / Nmax;
     224
     225  HostTableGroup *groups = NULL;
     226  ALLOCATE (groups, HostTableGroup, Ngroups);
     227
     228  // in each group, attempt to add one of each unique host
     229  int k = 0;
     230  for (i = 0; i < Ngroups; i++) {
     231    groups[i].Nhosts = 0;
     232    ALLOCATE (groups[i].hosts, HostInfo *, Nmax);
     233    for (j = 0; j < Nmax; j++) {
     234      groups[i].hosts[groups[i].Nhosts] = &table->hosts[k];
     235      groups[i].Nhosts ++;
     236      k++;
    212237    }
    213238  }

  • trunk/Ohana/src/libdvo/src/LoadImages.c

    r37807 r38441  
    6565  }
    6666
    67   image = gfits_table_get_Image (&db.ftable, Nimage, &db.swapped);
     67  image = gfits_table_get_Image (&db.ftable, Nimage, &db.scaledValue, &db.nativeOrder);
    6868  if (!image) {
    6969    fprintf (stderr, "ERROR: failed to read images\n");

  • trunk/Ohana/src/libdvo/src/LoadPhotcodesFITS.c

    r37035 r38441  
    1515
    1616  /* XXX choose more sensible lock timeouts! */
     17  gfits_db_init (&db);
    1718  db.timeout   = 60.0;
    1819  db.lockstate = LCK_SOFT;
    19   gfits_db_init (&db);
    2020
    2121  /* does this mean the db is empty, non-existent, or has access errors? */
    … …  
    4343# define CONVERT_FORMAT(NAME, FORMAT)                                   \
    4444  if (!strcmp (extname, NAME)) {                                        \
    45     PhotCode_##FORMAT *photcode_input = gfits_table_get_PhotCode_##FORMAT (&db.ftable, &Ncode, &db.swapped); \
     45    PhotCode_##FORMAT *photcode_input = gfits_table_get_PhotCode_##FORMAT (&db.ftable, &Ncode, &db.scaledValue, &db.nativeOrder); \
    4646    if (!photcode_input) {                                                      \
    4747      fprintf (stderr, "ERROR: failed to read photcodes in LoadPhotcodesFITS.c\n");             \

  • trunk/Ohana/src/libdvo/src/SavePhotcodesFITS.c

    r37035 r38441  
    1717
    1818  /* XXX choose more sensible lock timeouts! */
     19  gfits_db_init (&db);
    1920  db.lockstate = LCK_XCLD;
    2021  db.timeout   = 10.0;
    21   gfits_db_init (&db);
    2222
    2323  /* does this mean the db is empty, non-existent, or has access errors? */
    … …  
    3434  /* convert FITS format data to internal format (byteswaps & EXTNAME) */
    3535  if (!gfits_db_create (&db)) return (FALSE);
    36   if (!gfits_table_set_PhotCode_PS1_V5 (&db.ftable, photcode_output, table[0].Ncode)) return (FALSE);
     36  if (!gfits_table_set_PhotCode_PS1_V5 (&db.ftable, photcode_output, table[0].Ncode, TRUE)) return (FALSE);
    3737  if (!gfits_db_save (&db)) return (FALSE);
    3838  if (!gfits_db_close (&db)) return (FALSE);

  • trunk/Ohana/src/libdvo/src/dvo_catalog.c

    r38062 r38441  
    4242}
    4343
    44 int dvo_catalog_catformat (char *catformat) {
     44DVOCatFormat dvo_catalog_catformat (char *catformat) {
    4545 
    4646  /* set the specified CATFORMAT */
    … …  
    6262}
    6363
    64 int dvo_catalog_catmode (char *catmode) {
     64DVOCatMode dvo_catalog_catmode (char *catmode) {
    6565
    6666  /* set the specified CATMODE */
    … …  
    6969  if (!strcasecmp (catmode, "SPLIT")) return (DVO_MODE_SPLIT);
    7070  return (DVO_MODE_UNDEF);
     71}
     72
     73DVOCatCompress dvo_catalog_catcompress (char *catcompress) {
     74
     75  /* set the specified CATMODE */
     76  if (!strcasecmp (catcompress, "NONE"))   return (DVO_COMPRESS_NONE);
     77  if (!strcasecmp (catcompress, "AUTO"))   return (DVO_COMPRESS_AUTO);
     78  if (!strcasecmp (catcompress, "NONE_1")) return (DVO_COMPRESS_NONE_1);
     79  if (!strcasecmp (catcompress, "NONE_2")) return (DVO_COMPRESS_NONE_2);
     80  if (!strcasecmp (catcompress, "GZIP_1")) return (DVO_COMPRESS_GZIP_1);
     81  if (!strcasecmp (catcompress, "GZIP_2")) return (DVO_COMPRESS_GZIP_2);
     82  if (!strcasecmp (catcompress, "RICE_1")) return (DVO_COMPRESS_RICE_1);
     83  return (DVO_COMPRESS_NONE);
     84}
     85
     86// we can return a static string here unless we run multiple outputs threads at once
     87static char compress_string[16];
     88char *dvo_catalog_compress_string (DVOCatCompress catcompress) {
     89
     90  switch (catcompress) {
     91    case DVO_COMPRESS_NONE:
     92      strcpy (compress_string, "NONE");
     93      break;
     94    case DVO_COMPRESS_AUTO:
     95      strcpy (compress_string, "AUTO");
     96      break;
     97    case DVO_COMPRESS_GZIP_1:
     98      strcpy (compress_string, "GZIP_1");
     99      break;
     100    case DVO_COMPRESS_GZIP_2:
     101      strcpy (compress_string, "GZIP_2");
     102      break;
     103    case DVO_COMPRESS_NONE_1:
     104      strcpy (compress_string, "NONE_1");
     105      break;
     106    case DVO_COMPRESS_NONE_2:
     107      strcpy (compress_string, "NONE_2");
     108      break;
     109    case DVO_COMPRESS_RICE_1:
     110      strcpy (compress_string, "RICE_1");
     111      break;
     112    default:
     113      myAbort ("error in compress option");
     114  }
     115  return compress_string;
    71116}
    72117
    … …  
    131176  average->Nlensobj        = 0;
    132177  average->Nstarpar        = 0;
    133   average->Nextend         = 0;
     178  average->Ngalphot       = 0;
    134179
    135180  average->measureOffset   = -1;
    … …  
    138183  average->lensobjOffset   = -1;
    139184  average->starparOffset   = -1;
    140   average->extendOffset    = -1;
     185  average->galphotOffset  = -1;
    141186
    142187  average->refColorBlue    = NAN;
    … …  
    466511
    467512// init all data, or just catalog data
     513void dvo_galphot_init (GalPhot *galphot) {
     514  galphot->Xfit         = NAN;
     515  galphot->Yfit         = NAN;
     516  galphot->mag          = NAN;
     517  galphot->magErr       = NAN;
     518  galphot->majorAxis    = NAN;
     519  galphot->minorAxis    = NAN;
     520  galphot->majorAxisErr = NAN;
     521  galphot->minorAxisErr = NAN;
     522  galphot->theta        = NAN;
     523  galphot->theta_err    = NAN;
     524  galphot->index        = NAN;
     525  galphot->chisq        = NAN;
     526  galphot->Npix         = 0;
     527
     528  galphot->detID   = -1;
     529  galphot->objID   = -1;
     530  galphot->catID   = -1;
     531  galphot->imageID = -1;
     532
     533  galphot->photcode = 0;
     534  galphot->modelType = 0;
     535}
     536
     537# define INIT_TABLE(TABLE)                      \
     538  catalog[0].TABLE = NULL;                      \
     539  catalog[0].N##TABLE = 0;                      \
     540  catalog[0].N##TABLE##_disk = 0;               \
     541  catalog[0].N##TABLE##_off  = 0;               \
     542  catalog[0].TABLE##_catalog  = NULL;
     543
     544// init all data, or just catalog data
    468545void dvo_catalog_init (Catalog *catalog, int complete) {
    469546
    … …  
    473550    catalog[0].filename = NULL;
    474551
    475     catalog[0].lockmode = 0;
    476     catalog[0].catmode  = 0;
    477     catalog[0].catformat = 0;
    478     catalog[0].catflags = 0;
    479     catalog[0].Nsecfilt = 0;
     552    catalog[0].lockmode    = 0;
     553    catalog[0].catmode     = DVO_MODE_UNDEF;
     554    catalog[0].catformat   = DVO_FORMAT_UNDEF;
     555    catalog[0].catcompress = DVO_COMPRESS_NONE;
     556    catalog[0].catflags    = DVO_LOAD_NONE;
     557    catalog[0].Nsecfilt    = 0;
    480558  }
    481559
    482560  gfits_init_header (&catalog[0].header);
    483 
     561 
    484562  // the following describe the catalog files on disk
    485563  catalog[0].average = NULL;
    486   catalog[0].measure = NULL;
    487   catalog[0].missing = NULL;
     564  catalog[0].Naverage = 0;
     565  catalog[0].Naverage_disk = 0;
     566  catalog[0].Naverage_off  = 0;
     567  // average lacks average_catalog
     568
    488569  catalog[0].secfilt = NULL;
    489  
    490   catalog[0].lensing = NULL;
    491   catalog[0].lensobj = NULL;
    492 
    493   catalog[0].starpar = NULL;
     570  catalog[0].Nsecfilt_mem = 0;
     571  catalog[0].Nsecfilt_disk = 0;
     572  catalog[0].Nsecfilt_off  = 0;
     573  catalog[0].secfilt_catalog = NULL;
     574  // secfilt uses Nsecfilt_mem not Nsecfilt
    494575
    495576  catalog[0].averageT = NULL;
    496577  catalog[0].measureT = NULL;
     578
     579  INIT_TABLE(measure);
     580  INIT_TABLE(missing);
     581  INIT_TABLE(lensing);
     582  INIT_TABLE(lensobj);
     583  INIT_TABLE(starpar);
     584  INIT_TABLE(galphot);
    497585
    498586  catalog[0].objID = 0;
    499587  catalog[0].catID = 0;
    500588  catalog[0].sorted = 0;
    501 
    502   catalog[0].Naverage = 0;
    503   catalog[0].Nmeasure = 0;
    504   catalog[0].Nmissing = 0;
    505   catalog[0].Nsecf_mem = 0;
    506 
    507   catalog[0].Nlensing = 0;
    508   catalog[0].Nlensobj = 0;
    509 
    510   catalog[0].Nstarpar = 0;
    511 
    512   catalog[0].Naves_disk = 0;
    513   catalog[0].Nmeas_disk = 0;
    514   catalog[0].Nmiss_disk = 0;
    515   catalog[0].Nsecf_disk = 0;
    516 
    517   catalog[0].Nlensing_disk = 0;
    518   catalog[0].Nlensobj_disk = 0;
    519 
    520   catalog[0].Nstarpar_disk = 0;
    521 
    522   catalog[0].Naves_off  = 0;
    523   catalog[0].Nmeas_off  = 0;
    524   catalog[0].Nmiss_off  = 0;
    525   catalog[0].Nsecf_off  = 0;
    526 
    527   catalog[0].Nlensing_off  = 0;
    528   catalog[0].Nlensobj_off  = 0;
    529 
    530   catalog[0].Nstarpar_off  = 0;
    531 
    532   /* pointers to SPLIT data files */
    533   catalog[0].measure_catalog = NULL;
    534   catalog[0].missing_catalog = NULL;
    535   catalog[0].secfilt_catalog = NULL;
    536   catalog[0].lensing_catalog = NULL;
    537   catalog[0].lensobj_catalog = NULL;
    538   catalog[0].starpar_catalog = NULL;
    539589
    540590  /* pointers for data manipulation */
    … …  
    568618  if (catalog[0].f == NULL)   return (DVO_CAT_OPEN_FAIL);
    569619
    570   fseeko (catalog[0].f, 0, SEEK_SET);
     620  if (fseeko (catalog[0].f, 0, SEEK_SET)) {
     621    perror ("fseeko: ");
     622    exit (1);
     623  }
    571624  return (DVO_CAT_OPEN_OK);
    572625}
    … …  
    580633
    581634  if (fflush (catalog[0].f)) {
    582     fprintf (stderr, "failed to flush file\n");
     635    perror ("fflush: ");
     636    fprintf (stderr, "failed to flush file %s\n", catalog[0].filename);
    583637    return FALSE;
    584638  }
    … …  
    607661    if (catalog[0].lensobj_catalog) { if (!dvo_catalog_unlock (catalog[0].lensobj_catalog)) { fprintf (stderr, "failed to unlock lensobj\n"); return (0); }}
    608662    if (catalog[0].starpar_catalog) { if (!dvo_catalog_unlock (catalog[0].starpar_catalog)) { fprintf (stderr, "failed to unlock starpar\n"); return (0); }}
     663    if (catalog[0].galphot_catalog) { if (!dvo_catalog_unlock (catalog[0].galphot_catalog)) { fprintf (stderr, "failed to unlock galphot\n"); return (0); }}
    609664  }
    610665  return (1);
    … …  
    849904    catalog[0].secfilt = outsec;
    850905    catalog[0].Nsecfilt = Nsecfilt;
    851     catalog[0].Nsecf_mem = Nsecfilt * catalog[0].Naverage;
     906    catalog[0].Nsecfilt_mem = Nsecfilt * catalog[0].Naverage;
    852907  }
    853908  return (TRUE);
    … …  
    887942      dvo_catalog_free (catalog[0].starpar_catalog);
    888943      free (catalog[0].starpar_catalog);
     944    }
     945    if (catalog[0].galphot_catalog) {
     946      free (catalog[0].galphot_catalog[0].filename);
     947      dvo_catalog_free (catalog[0].galphot_catalog);
     948      free (catalog[0].galphot_catalog);
    889949    }
    890950  }
    … …  
    914974  if (catalog[0].secfilt) {
    915975    free (catalog[0].secfilt);
    916     catalog[0].Nsecf_mem = 0;
     976    catalog[0].Nsecfilt_mem = 0;
    917977    catalog[0].secfilt = NULL;
    918978  }
    … …  
    932992    catalog[0].starpar = NULL;
    933993  }
     994  if (catalog[0].galphot) {
     995    free (catalog[0].galphot);
     996    catalog[0].Ngalphot = 0;
     997    catalog[0].galphot = NULL;
     998  }
    934999  if (catalog[0].nOwn_t)      { free (catalog[0].nOwn_t); catalog[0].nOwn_t = NULL; }
    9351000  if (catalog[0].found_t)     { free (catalog[0].found_t); catalog[0].found_t = NULL; }
    … …  
    9381003
    9391004/*
    940   mode   : items to read (LOAD_AVES | LOAD_MEAS | LOAD_MISS | LOAD_SECF)
     1005  mode   : items to read (DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_MISSING | DVO_LOAD_SECFILT)
    9411006  format : what table structure on disk (INTERNAL, LONEOS, etc, )
    9421007  style  : raw, mef, split, mysql
    … …  
    9711036
    9721037  // skip empty cpt files
    973   if (primary && !catalog->Naves_disk) {
     1038  if (primary && !catalog->Naverage_disk) {
    9741039    return TRUE;
    9751040  }
    … …  
    10531118      }
    10541119    }
     1120    if (catalog[0].galphot_catalog != NULL) {
     1121      if (catalog[0].Ngalphot_disk == 0) {
     1122        // need to relock (and re-open) file for close elsewhere
     1123        status = dvo_catalog_lock (catalog[0].galphot_catalog, lockmode);
     1124      } else {
     1125        if (!dvo_catalog_backup (catalog[0].galphot_catalog, FALSE)) {
     1126          return FALSE;
     1127        }
     1128      }
     1129    }
    10551130  }
    10561131  return TRUE;
    … …  
    10601135int dvo_catalog_unlink_backup (Catalog *catalog, int primary) {
    10611136
    1062   if (primary && !catalog->Naves_disk) {
     1137  if (primary && !catalog->Naverage_disk) {
    10631138    // skip empty files (empty when read, but output may not be empty)
    10641139    return TRUE;
    … …  
    11091184      }
    11101185    }
     1186    if ((catalog[0].galphot_catalog != NULL) && (catalog[0].Ngalphot_disk > 0)) {
     1187      if (!dvo_catalog_unlink_backup (catalog[0].galphot_catalog, FALSE)) {
     1188        return FALSE;
     1189      }
     1190    }
    11111191  }
    11121192  return TRUE;

  • trunk/Ohana/src/libdvo/src/dvo_catalog_create.c

    r37807 r38441  
    11# include <dvo.h>
    22# define DEBUG 0
     3
     4Catalog *dvo_catalog_create_subcat (Catalog *catalog, char *ext, char *tablename);
    35
    46// create a new dvo catalog file (if split, lock extra files as well?)
    … …  
    911// catalog[0].lockmode
    1012void dvo_catalog_create (SkyRegion *region, Catalog *catalog) {
    11 
    12   int length;
    13   char *path, *root, *file, *line;
    14   time_t now;
    1513
    1614  if (DEBUG) fprintf (stderr, "new catalog file: %s\n", catalog[0].filename);
    … …  
    2927
    3028  if (catalog[0].catmode == DVO_MODE_SPLIT) {
    31     path = pathname (catalog[0].filename);
    32     root = filerootname (catalog[0].filename);
    33     length = strlen(path) + strlen(root) + 6;
    34 
    35     /* define measure catalog file */
    36     ALLOCATE (catalog[0].measure_catalog, Catalog, 1);
    37     dvo_catalog_init (catalog[0].measure_catalog, TRUE);
    38 
    39     /* create basic data for measure catalog file */
    40     gfits_create_header (&catalog[0].measure_catalog[0].header);
    41 
    42     ALLOCATE (catalog[0].measure_catalog[0].filename, char, length);
    43 
    44     sprintf (catalog[0].measure_catalog[0].filename, "%s/%s.cpm", path, root);
    45     file = filebasename (catalog[0].measure_catalog[0].filename);
    46     gfits_modify (&catalog[0].header, "MEASURE", "%s", 1, file);
    47     free (file);
    48 
    49     /* define missing catalog file */
    50     ALLOCATE (catalog[0].missing_catalog, Catalog, 1);
    51     dvo_catalog_init (catalog[0].missing_catalog, TRUE);
    52 
    53     /* create basic data for missing catalog file */
    54     gfits_create_header (&catalog[0].missing_catalog[0].header);
    55     ALLOCATE (catalog[0].missing_catalog[0].filename, char, length);
    56     sprintf (catalog[0].missing_catalog[0].filename, "%s/%s.cpn", path, root);
    57     file = filebasename (catalog[0].missing_catalog[0].filename);
    58     gfits_modify (&catalog[0].header, "MISSING", "%s", 1, file);
    59     free (file);
    60 
    61     /* define secfilt catalog file */
    62     ALLOCATE (catalog[0].secfilt_catalog, Catalog, 1);
    63     dvo_catalog_init (catalog[0].secfilt_catalog, TRUE);
    64 
    65     /* create basic data for secfilt catalog file */
    66     gfits_create_header (&catalog[0].secfilt_catalog[0].header);
    67     ALLOCATE (catalog[0].secfilt_catalog[0].filename, char, length);
    68     sprintf (catalog[0].secfilt_catalog[0].filename, "%s/%s.cps", path, root);
    69     file = filebasename (catalog[0].secfilt_catalog[0].filename);
    70     gfits_modify (&catalog[0].header, "SECFILT", "%s", 1, file);
    71     free (file);
    72 
    73     /* define lensing catalog file */
    74     ALLOCATE (catalog[0].lensing_catalog, Catalog, 1);
    75     dvo_catalog_init (catalog[0].lensing_catalog, TRUE);
    76 
    77     /* create basic data for lensing catalog file */
    78     gfits_create_header (&catalog[0].lensing_catalog[0].header);
    79     ALLOCATE (catalog[0].lensing_catalog[0].filename, char, length);
    80     sprintf (catalog[0].lensing_catalog[0].filename, "%s/%s.cpx", path, root);
    81     file = filebasename (catalog[0].lensing_catalog[0].filename);
    82     gfits_modify (&catalog[0].header, "LENSING", "%s", 1, file);
    83     free (file);
    84 
    85     /* define lensobj catalog file */
    86     ALLOCATE (catalog[0].lensobj_catalog, Catalog, 1);
    87     dvo_catalog_init (catalog[0].lensobj_catalog, TRUE);
    88 
    89     /* create basic data for lensobj catalog file */
    90     gfits_create_header (&catalog[0].lensobj_catalog[0].header);
    91     ALLOCATE (catalog[0].lensobj_catalog[0].filename, char, length);
    92     sprintf (catalog[0].lensobj_catalog[0].filename, "%s/%s.cpy", path, root);
    93     file = filebasename (catalog[0].lensobj_catalog[0].filename);
    94     gfits_modify (&catalog[0].header, "LENSOBJ", "%s", 1, file);
    95     free (file);
    96 
    97     /* define starpar catalog file */
    98     ALLOCATE (catalog[0].starpar_catalog, Catalog, 1);
    99     dvo_catalog_init (catalog[0].starpar_catalog, TRUE);
    100 
    101     /* create basic data for starpar catalog file */
    102     gfits_create_header (&catalog[0].starpar_catalog[0].header);
    103     ALLOCATE (catalog[0].starpar_catalog[0].filename, char, length);
    104     sprintf (catalog[0].starpar_catalog[0].filename, "%s/%s.cpz", path, root);
    105     file = filebasename (catalog[0].starpar_catalog[0].filename);
    106     gfits_modify (&catalog[0].header, "STARPAR", "%s", 1, file);
    107     free (file);
    108 
    109     free (path);
    110     free (root);
     29    catalog[0].measure_catalog  = dvo_catalog_create_subcat (catalog, "cpm", "MEASURE");
     30    catalog[0].missing_catalog  = dvo_catalog_create_subcat (catalog, "cpn", "MISSING");
     31    catalog[0].secfilt_catalog  = dvo_catalog_create_subcat (catalog, "cps", "SECFILT");
     32    catalog[0].lensing_catalog  = dvo_catalog_create_subcat (catalog, "cpx", "LENSING");
     33    catalog[0].lensobj_catalog  = dvo_catalog_create_subcat (catalog, "cpy", "LENSOBJ");
     34    catalog[0].starpar_catalog  = dvo_catalog_create_subcat (catalog, "cpz", "STARPAR");
     35    catalog[0].galphot_catalog = dvo_catalog_create_subcat (catalog, "cpq", "GALPHOT");
    11136
    11237    // lock the additional split files
    … …  
    13661      exit (2);
    13762    }
     63    if (dvo_catalog_lock (catalog[0].galphot_catalog, catalog[0].lockmode) != DVO_CAT_OPEN_EMPTY) {
     64      fprintf (stderr, "error with file lock\n");
     65      exit (2);
     66    }
    13867  }   
    13968
    … …  
    15079
    15180  /* write creation date in header */
     81  time_t now;
    15282  ohana_str_to_time ("now", &now);
    153   line = ohana_sec_to_date (now);
     83  char *line = ohana_sec_to_date (now);
    15484  gfits_modify (&catalog[0].header, "DATE", "%s", 1, line);
    15585  free (line);
    … …  
    16393  ALLOCATE (catalog[0].lensobj, Lensobj, 1);
    16494  ALLOCATE (catalog[0].starpar, StarPar, 1);
     95  ALLOCATE (catalog[0].galphot, GalPhot, 1);
    16596
    16697  /* setup secondary filters to match photcodes:
    … …  
    170101}
    171102 
     103Catalog *dvo_catalog_create_subcat (Catalog *catalog, char *ext, char *tablename) {
     104 
     105  char *path = pathname (catalog[0].filename);
     106  char *root = filerootname (catalog[0].filename);
     107  int length = strlen(path) + strlen(root) + 6;
     108   
     109  Catalog *subcat;
     110 
     111  /* define subcat catalog file */
     112  ALLOCATE (subcat, Catalog, 1);
     113  dvo_catalog_init (subcat, TRUE);
     114 
     115  /* create basic data for measure catalog file */
     116  gfits_create_header (&subcat->header);
     117
     118  subcat->catcompress = catalog->catcompress;
     119
     120  ALLOCATE (subcat->filename, char, length);
     121
     122  sprintf (subcat->filename, "%s/%s.%s", path, root, ext);
     123
     124  char *file = filebasename (subcat->filename);
     125  gfits_modify (&catalog[0].header, tablename, "%s", 1, file);
     126
     127  free (file);
     128  free (path);
     129  free (root);
     130
     131  return subcat;
     132}
     133
    172134int dvo_catalog_set_range (Catalog *catalog) {
    173135

  • trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c

    r37807 r38441  
    2626  }
    2727
    28   // NSTARS, average, Naves_disk
     28  // NSTARS, average, Naverage_disk
    2929
    3030# define GET_TABLE_SIZES(HFIELD, CFIELD, DFIELD, REQUIRED)              \
    … …  
    4141 
    4242  /* get the components and sizes from the header */
    43   GET_TABLE_SIZES ("NSTARS",   average, Naves_disk,    TRUE);
    44   GET_TABLE_SIZES ("NMEAS",    measure, Nmeas_disk,    TRUE);
    45   GET_TABLE_SIZES ("NMISS",    missing, Nmiss_disk,    FALSE);
    46   GET_TABLE_SIZES ("NLENSING", lensing, Nlensing_disk, FALSE);
    47   GET_TABLE_SIZES ("NLENSOBJ", lensobj, Nlensobj_disk, FALSE);
    48   GET_TABLE_SIZES ("NSTARPAR", starpar, Nstarpar_disk, FALSE);
     43  GET_TABLE_SIZES ("NSTARS",    average,  Naverage_disk,     TRUE);
     44  GET_TABLE_SIZES ("NMEAS",     measure,  Nmeasure_disk,     TRUE);
     45  GET_TABLE_SIZES ("NMISS",     missing,  Nmissing_disk,     FALSE);
     46  GET_TABLE_SIZES ("NLENSING",  lensing,  Nlensing_disk,  FALSE);
     47  GET_TABLE_SIZES ("NLENSOBJ",  lensobj,  Nlensobj_disk,  FALSE);
     48  GET_TABLE_SIZES ("NSTARPAR",  starpar,  Nstarpar_disk,  FALSE);
     49  GET_TABLE_SIZES ("NGALPHOT", galphot, Ngalphot_disk, FALSE);
    4950
    5051  /**  Nsecfilt is unusual: it does not list the number of data items in the table
    … …  
    5354  if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0;
    5455  catalog[0].Nsecfilt = Nsecfilt;
    55   catalog[0].Nsecf_disk = Naverage * Nsecfilt;
     56  catalog[0].Nsecfilt_disk = Naverage * Nsecfilt;
    5657  catalog[0].secfilt = NULL;
    5758
    … …  
    7071
    7172  /* read Average table data (or skip) */
    72   if (catalog[0].catflags & LOAD_AVES) {
     73  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
    7374    if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
    7475      if (VERBOSE) fprintf (stderr, "can't read table average data");
    … …  
    7677    }
    7778    /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */
    78     catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary);
    79     if (Naverage != catalog[0].Naves_disk) {
    80       fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Naverage,  catalog[0].Naves_disk);
    81     }
    82     catalog[0].Naverage = catalog[0].Naves_disk;
    83     catalog[0].Naves_off = 0;
     79    catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, FALSE);
     80    if (Naverage != catalog[0].Naverage_disk) {
     81      fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Naverage,  catalog[0].Naverage_disk);
     82    }
     83    catalog[0].Naverage = catalog[0].Naverage_disk;
     84    catalog[0].Naverage_off = 0;
    8485  } else {
    8586    catalog[0].catformat = FtableGetFormat (&ftable);
    … …  
    8889    ALLOCATE (catalog[0].average, Average, 1);
    8990    catalog[0].Naverage = 0;
    90     catalog[0].Naves_off = catalog[0].Naves_disk;
     91    catalog[0].Naverage_off = catalog[0].Naverage_disk;
    9192  }
    9293  gfits_free_header (&header);
    … …  
    99100  }
    100101  /* read Measure table data */
    101   if (catalog[0].catflags & LOAD_MEAS) {
     102  if (catalog[0].catflags & DVO_LOAD_MEASURE) {
    102103    if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
    103104      if (VERBOSE) fprintf (stderr, "can't read table measure data");
    104105      return (FALSE);
    105106    }
    106     catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &catalog[0].Nmeasure, &catalog[0].catformat);
    107     if (Nmeasure != catalog[0].Nmeas_disk) {
    108       fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmeasure,  catalog[0].Nmeas_disk);
    109     }
    110     catalog[0].Nmeasure = catalog[0].Nmeas_disk;
    111     catalog[0].Nmeas_off = 0;
     107    catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &catalog[0].Nmeasure, &catalog[0].catformat, FALSE);
     108    if (Nmeasure != catalog[0].Nmeasure_disk) {
     109      fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmeasure,  catalog[0].Nmeasure_disk);
     110    }
     111    catalog[0].Nmeasure = catalog[0].Nmeasure_disk;
     112    catalog[0].Nmeasure_off = 0;
    112113  } else {
    113114    Nbytes = gfits_data_size (&header);
    … …  
    115116    ALLOCATE (catalog[0].measure, Measure, 1);
    116117    catalog[0].Nmeasure = 0;
    117     catalog[0].Nmeas_off = catalog[0].Nmeas_disk;
     118    catalog[0].Nmeasure_off = catalog[0].Nmeasure_disk;
    118119  }
    119120
    … …  
    124125  }
    125126  /* read Missing table data */
    126   if (catalog[0].catflags & LOAD_MISS) {
     127  if (catalog[0].catflags & DVO_LOAD_MISSING) {
    127128    if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
    128129      if (VERBOSE) fprintf (stderr, "can't read table missing data");
    … …  
    130131    }
    131132    /* no conversions currently defined */
    132     catalog[0].missing = gfits_table_get_Missing (&ftable, &catalog[0].Nmissing, NULL);
     133    catalog[0].missing = gfits_table_get_Missing (&ftable, &catalog[0].Nmissing, NULL, NULL);
    133134    if (!catalog[0].missing) {
    134135      fprintf (stderr, "ERROR: failed to read missing\n");
    135136      exit (2);
    136137    }
    137     if (Nmissing != catalog[0].Nmiss_disk) {
    138       fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmissing,  catalog[0].Nmiss_disk);
    139     }
    140     catalog[0].Nmissing = catalog[0].Nmiss_disk;
    141     catalog[0].Nmiss_off = 0;
     138    if (Nmissing != catalog[0].Nmissing_disk) {
     139      fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmissing,  catalog[0].Nmissing_disk);
     140    }
     141    catalog[0].Nmissing = catalog[0].Nmissing_disk;
     142    catalog[0].Nmissing_off = 0;
    142143  } else {
    143144    Nbytes = gfits_data_size (&header);
    … …  
    145146    ALLOCATE (catalog[0].missing, Missing, 1);
    146147    catalog[0].Nmissing = 0;
    147     catalog[0].Nmiss_off = catalog[0].Nmiss_disk;
     148    catalog[0].Nmissing_off = catalog[0].Nmissing_disk;
    148149  }
    149150
    … …  
    154155  }
    155156  /* read secfilt table data */
    156   if (catalog[0].catflags & LOAD_SECF) {
     157  if (catalog[0].catflags & DVO_LOAD_SECFILT) {
    157158    if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
    158159      if (VERBOSE) fprintf (stderr, "can't read table secfilt data");
    … …  
    162163    /* how many entries do we expect from the secfilt table? */
    163164    Nexpect = catalog[0].Nsecfilt * catalog[0].Naverage;
    164     catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat);
     165    catalog[0].secfilt = FtableToSecFilt (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, FALSE);
    165166    if (Nexpect != Nitems) {
    166167      fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nexpect,  Nitems);
    … …  
    175176      Ntmpfilt = catalog[0].Nsecfilt;
    176177      Nsecfilt = catalog[0].Nsecfilt + 1;
    177       Ntotal = Nsecfilt * catalog[0].Naves_disk;
     178      Ntotal = Nsecfilt * catalog[0].Naverage_disk;
    178179      ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal);
    179       for (i = 0; i < catalog[0].Naves_disk; i++) {
     180      for (i = 0; i < catalog[0].Naverage_disk; i++) {
    180181        catalog[0].secfilt[i*Nsecfilt + 0] = primary[i];
    181182        for (j = 0; j < Ntmpfilt; j++) {
    … …  
    184185      }         
    185186      catalog[0].Nsecfilt = Nsecfilt;
    186       catalog[0].Nsecf_disk = Ntotal;
     187      catalog[0].Nsecfilt_disk = Ntotal;
    187188      free (tmpfilt);
    188189      free (primary);
    189190    }
    190     catalog[0].Nsecf_mem = catalog[0].Nsecf_disk;
    191     catalog[0].Nsecf_off = 0;
     191    catalog[0].Nsecfilt_mem = catalog[0].Nsecfilt_disk;
     192    catalog[0].Nsecfilt_off = 0;
    192193  } else {
    193194    /* no real need to skip the data array here... */
    … …  
    197198      free (primary);
    198199      catalog[0].Nsecfilt ++;
    199       catalog[0].Nsecf_disk =  catalog[0].Nsecfilt * catalog[0].Naves_disk;
     200      catalog[0].Nsecfilt_disk =  catalog[0].Nsecfilt * catalog[0].Naverage_disk;
    200201    }
    201202    ALLOCATE (catalog[0].secfilt, SecFilt, 1);
    202     catalog[0].Nsecf_mem = 0;
    203     catalog[0].Nsecf_off = catalog[0].Nsecf_disk;
     203    catalog[0].Nsecfilt_mem = 0;
     204    catalog[0].Nsecfilt_off = catalog[0].Nsecfilt_disk;
    204205  }
    205206
    … …  
    210211  }
    211212  /* read Lensing table data */
    212   if (catalog[0].catflags & LOAD_LENSING) {
     213  if (catalog[0].catflags & DVO_LOAD_LENSING) {
    213214    if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
    214215      if (VERBOSE) fprintf (stderr, "can't read table lensing data");
    215216      return (FALSE);
    216217    }
    217     catalog[0].lensing = FtableToLensing (&ftable, &catalog[0].Nlensing, &catalog[0].catformat);
     218    catalog[0].lensing = FtableToLensing (&ftable, catalog[0].average, &catalog[0].Nlensing, &catalog[0].catformat, FALSE);
    218219    if (Nlensing != catalog[0].Nlensing_disk) {
    219220      fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensing,  catalog[0].Nlensing_disk);
    … …  
    235236  }
    236237  /* read Lensobj table data */
    237   if (catalog[0].catflags & LOAD_LENSOBJ) {
     238  if (catalog[0].catflags & DVO_LOAD_LENSOBJ) {
    238239    if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
    239240      if (VERBOSE) fprintf (stderr, "can't read table lensobj data");
    240241      return (FALSE);
    241242    }
    242     catalog[0].lensobj = FtableToLensobj (&ftable, &catalog[0].Nlensobj, &catalog[0].catformat);
     243    catalog[0].lensobj = FtableToLensobj (&ftable, catalog[0].average, &catalog[0].Nlensobj, &catalog[0].catformat, FALSE);
    243244    if (Nlensobj != catalog[0].Nlensobj_disk) {
    244245      fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensobj,  catalog[0].Nlensobj_disk);
    … …  
    260261  }
    261262  /* read StarPar table data */
    262   if (catalog[0].catflags & LOAD_STARPAR) {
     263  if (catalog[0].catflags & DVO_LOAD_STARPAR) {
    263264    if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
    264265      if (VERBOSE) fprintf (stderr, "can't read table starpar data");
    265266      return (FALSE);
    266267    }
    267     catalog[0].starpar = FtableToStarPar (&ftable, &catalog[0].Nstarpar, &catalog[0].catformat);
     268    catalog[0].starpar = FtableToStarPar (&ftable, catalog[0].average, &catalog[0].Nstarpar, &catalog[0].catformat, FALSE);
    268269    if (Nstarpar != catalog[0].Nstarpar_disk) {
    269270      fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nstarpar,  catalog[0].Nstarpar_disk);
    … …  
    277278    catalog[0].Nstarpar = 0;
    278279    catalog[0].Nstarpar_off = catalog[0].Nstarpar_disk;
     280  }
     281
     282  /* read GalPhot table header */
     283  if (!gfits_fread_header (catalog[0].f, &header)) {
     284    if (VERBOSE) fprintf (stderr, "can't read table galphot header");
     285    return (FALSE);
     286  }
     287  /* read GalPhot table data */
     288  if (catalog[0].catflags & DVO_LOAD_GALPHOT) {
     289    if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
     290      if (VERBOSE) fprintf (stderr, "can't read table galphot data");
     291      return (FALSE);
     292    }
     293    catalog[0].galphot = FtableToGalPhot (&ftable, catalog[0].average, &catalog[0].Ngalphot, &catalog[0].catformat, FALSE);
     294    if (Ngalphot != catalog[0].Ngalphot_disk) {
     295      fprintf (stderr, "Warning: mismatch between Ngalphot in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Ngalphot,  catalog[0].Ngalphot_disk);
     296    }
     297    catalog[0].Ngalphot = catalog[0].Ngalphot_disk;
     298    catalog[0].Ngalphot_off = 0;
     299  } else {
     300    Nbytes = gfits_data_size (&header);
     301    fseeko (catalog[0].f, Nbytes, SEEK_CUR);
     302    ALLOCATE (catalog[0].galphot, GalPhot, 1);
     303    catalog[0].Ngalphot = 0;
     304    catalog[0].Ngalphot_off = catalog[0].Ngalphot_disk;
    279305  }
    280306
    … …  
    330356  gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1,  catalog[0].Nlensobj);
    331357  gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1,  catalog[0].Nstarpar);
     358  gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1,  catalog[0].Ngalphot);
     359
    332360  gfits_modify_alt (&catalog[0].header, "EXTEND",   "%t", 1, TRUE);
    333361  gfits_modify (&catalog[0].header, "OBJID",    "%d", 1, catalog[0].objID);
    … …  
    357385
    358386  /* write out Average table (convert to FITS table format) */
    359   AverageToFtable (&ftable, catalog[0].average, catalog[0].Naverage, catalog[0].catformat, primary);
     387  AverageToFtable (&ftable, catalog[0].average, catalog[0].Naverage, catalog[0].catformat, primary, TRUE);
    360388  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    361389    fprintf (stderr, "can't write table header");
    … …  
    370398
    371399  /* write out Measure table (convert to FITS table format) */
    372   MeasureToFtable (&ftable, catalog[0].average, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat);
     400  MeasureToFtable (&ftable, catalog[0].average, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat, TRUE);
    373401  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    374402    fprintf (stderr, "can't write table header");
    … …  
    383411
    384412  /* write out Missing table (convert to FITS table format) */
    385   gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing);
     413  gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE);
    386414  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    387415    fprintf (stderr, "can't write table header");
    … …  
    397425  /* write out SecFilt table (convert to FITS table format) */
    398426  Nitems = catalog[0].Naverage * Nsecfilt;
    399   SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat);
     427  SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat, TRUE);
    400428  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    401429    fprintf (stderr, "can't write table header");
    … …  
    410438
    411439  /* write out Lensing table (convert to FITS table format) */
    412   LensingToFtable (&ftable, catalog[0].lensing, catalog[0].Nlensing, catalog[0].catformat);
     440  LensingToFtable (&ftable, catalog[0].lensing, catalog[0].Nlensing, catalog[0].catformat, TRUE);
    413441  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    414442    fprintf (stderr, "can't write table header");
    … …  
    423451
    424452  /* write out Lensobj table (convert to FITS table format) */
    425   LensobjToFtable (&ftable, catalog[0].lensobj, catalog[0].Nlensobj, catalog[0].catformat);
    426   if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    427     fprintf (stderr, "can't write table header");
    428     goto failure;
    429   }
    430   if (!gfits_fwrite_table (catalog[0].f, &ftable)) {
    431     fprintf (stderr, "can't write table data");
    432     goto failure;
    433   }
    434   gfits_free_table (&ftable);
    435   gfits_free_header (&header);
    436 
    437   /* free temp storage */
    438   if (primary != NULL) {
    439     free (primary);
    440     free (secfilt);
    441   }
    442   return (TRUE);
     453  LensobjToFtable (&ftable, catalog[0].lensobj, catalog[0].Nlensobj, catalog[0].catformat, TRUE);
     454  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
     455    fprintf (stderr, "can't write table header");
     456    goto failure;
     457  }
     458  if (!gfits_fwrite_table (catalog[0].f, &ftable)) {
     459    fprintf (stderr, "can't write table data");
     460    goto failure;
     461  }
     462  gfits_free_table (&ftable);
     463  gfits_free_header (&header);
    443464
    444465  /* write out StarPar table (convert to FITS table format) */
    445   StarParToFtable (&ftable, catalog[0].starpar, catalog[0].Nstarpar, catalog[0].catformat);
     466  StarParToFtable (&ftable, catalog[0].starpar, catalog[0].Nstarpar, catalog[0].catformat, TRUE);
     467  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
     468    fprintf (stderr, "can't write table header");
     469    goto failure;
     470  }
     471  if (!gfits_fwrite_table (catalog[0].f, &ftable)) {
     472    fprintf (stderr, "can't write table data");
     473    goto failure;
     474  }
     475  gfits_free_table (&ftable);
     476  gfits_free_header (&header);
     477
     478  /* write out GalPhot table (convert to FITS table format) */
     479  GalPhotToFtable (&ftable, catalog[0].galphot, catalog[0].Ngalphot, catalog[0].catformat, TRUE);
    446480  if (!gfits_fwrite_Theader (catalog[0].f, &header)) {
    447481    fprintf (stderr, "can't write table header");

  • trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c

    r37807 r38441  
    136136  /* read and convert the averages (use a macro to clean this up?) */
    137137  /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */
    138   if (catalog[0].catflags & LOAD_AVES) {
     138  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
    139139    catalog[0].average = ReadRawAverage (catalog[0].f, catalog[0].Naverage, catalog[0].catformat, &primary);
    140140  } else {
    … …  
    145145 
    146146  /* read and convert the measures (use a macro to clean this up?) */
    147   if (catalog[0].catflags & LOAD_MEAS) {
     147  if (catalog[0].catflags & DVO_LOAD_MEASURE) {
    148148    catalog[0].measure = ReadRawMeasure (catalog[0].f, catalog[0].average, catalog[0].Nmeasure, catalog[0].catformat);
    149149  } else {
    … …  
    154154
    155155  /* read and convert missing */
    156   if (catalog[0].catflags & LOAD_MISS) {
     156  if (catalog[0].catflags & DVO_LOAD_MISSING) {
    157157    ALLOCATE (catalog[0].missing, Missing, MAX (catalog[0].Nmissing, 1));
    158158    Nitems = catalog[0].Nmissing;
    … …  
    170170 
    171171  /* read and convert secfilt */
    172   if (catalog[0].catflags & LOAD_SECF) {
     172  if (catalog[0].catflags & DVO_LOAD_SECFILT) {
    173173    Nitems = catalog[0].Naverage * catalog[0].Nsecfilt;
    174174    catalog[0].secfilt = ReadRawSecFilt (catalog[0].f, Nitems, catalog[0].catformat);
    … …  
    191191      }         
    192192      catalog[0].Nsecfilt = Nsecfilt;
    193       catalog[0].Nsecf_mem = Ntotal;
     193      catalog[0].Nsecfilt_mem = Ntotal;
    194194      free (primary);
    195195    }
    … …  
    210210
    211211  /* check data integrity */
    212   if (catalog[0].catflags & LOAD_AVES) {
     212  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
    213213    for (i = Nmeas = Nmiss = 0; i < catalog[0].Naverage; i++) {
    214214      Nmeas += catalog[0].average[i].Nmeasure;
    … …  
    226226
    227227  /* save the current number so we can do partial updates */
    228   catalog[0].Naves_disk = catalog[0].Naverage;
    229   catalog[0].Nmeas_disk = catalog[0].Nmeasure;
    230   catalog[0].Nmiss_disk = catalog[0].Nmissing;
     228  catalog[0].Naverage_disk = catalog[0].Naverage;
     229  catalog[0].Nmeasure_disk = catalog[0].Nmeasure;
     230  catalog[0].Nmissing_disk = catalog[0].Nmissing;
    231231
    232232  return (TRUE);

  • trunk/Ohana/src/libdvo/src/dvo_catalog_split.c

    r38153 r38441  
    11# include <dvo.h>
     2# define OHANA_MEMCHECK 0
    23
    34// return options:
    … …  
    67// * ok
    78
     9// utility function to see if we should compress this table)
     10int output_is_compressed (int start, int Nrows, int Ntotal, DVOCatCompress catcompress) {
     11  int fullWrite = (start == 0) && (Nrows == Ntotal);
     12  if (fullWrite && catcompress) return TRUE;  // we want to compress, so do not swap
     13  return FALSE;
     14}
     15
     16int byteswap_varlength_ftable (Header *header, FTable *table) {
     17  int i, Nfields;
     18  if (!gfits_scan (header, "TFIELDS", "%d", 1, &Nfields)) return FALSE;
     19  for (i = 0; i < Nfields; i++) {
     20    if (!gfits_byteswap_varlength_column (table, i+1)) return FALSE;
     21  }
     22  return TRUE;
     23}
     24
    825int dvo_catalog_secfilt_to_primary (Catalog *catalog, SecFilt **myPrimary, SecFilt **mySecfilt, int *myNsecfilt) {
    926
    … …  
    2037
    2138  // XXX this translation only works if we have loaded / created a matched average/secfilt set
    22   assert (catalog[0].Nsecf_mem == catalog[0].Nsecfilt*catalog[0].Naverage);
     39  assert (catalog[0].Nsecfilt_mem == catalog[0].Nsecfilt*catalog[0].Naverage);
    2340
    2441  Nallfilt = catalog[0].Nsecfilt;
    … …  
    3552  }             
    3653  catalog[0].Nsecfilt --;
    37   catalog[0].Nsecf_mem = catalog[0].Naverage*catalog[0].Nsecfilt;
     54  catalog[0].Nsecfilt_mem = catalog[0].Naverage*catalog[0].Nsecfilt;
    3855
    3956  *myPrimary = primary;
    … …  
    5168  Ntmpfilt = catalog[0].Nsecfilt;
    5269
    53   // we do NOT modify Nsecf_disk; this operation only modifies in in-memory values
     70  // we do NOT modify Nsecfilt_disk; this operation only modifies in in-memory values
    5471
    5572  catalog[0].Nsecfilt ++;
    … …  
    5774  Ntotal = Nsecfilt * Naves;
    5875
    59   catalog[0].Nsecf_mem = Ntotal;
     76  catalog[0].Nsecfilt_mem = Ntotal;
    6077
    6178  ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal);
    … …  
    7491
    7592  Matrix matrix;
    76 
    77   // ohana_memcheck_func (TRUE);
    7893 
    79 
    80   if (0) {
    81     // XXX a TEST : is the data corrupted or the file?
    82     char name[1024];
    83     snprintf (name, 1024, "%s.phu.v2", catalog->filename);
    84     FILE *f = fopen (name, "w");
    85     fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f);
    86     // fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer);
    87     // fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n",
    88     //       catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3],
    89     //       catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]);
    90     fclose (f);
    91   }
    92   if (0) {
    93     // XXX a TEST : is the data corrupted or the file?
    94     char name[1024];
    95     snprintf (name, 1024, "%s.phu", catalog->filename);
    96     FILE *f = fopen (name, "w");
    97     fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f);
    98     // fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer);
    99     // fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n",
    100     //       catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3],
    101     //       catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]);
    102     fclose (f);
    103   }
    104 # if (0)
    105   {
    106     // XXX a TEST : is the data corrupted or the file?
    107     char name[1024];
    108     snprintf (name, 1024, "%s.phu.v2", catalog->filename);
    109     FILE *f = fopen (name, "w");
    110     fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f);
    111     fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer);
    112     fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n",
    113              catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3],
    114              catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]);
    115     fclose (f);
    116   }
    117   {
    118     // XXX a TEST : is the data corrupted or the file?
    119     char name[1024];
    120     snprintf (name, 1024, "%s.phu.v3", catalog->filename);
    121     FILE *f = fopen (name, "w");
    122     fwrite (catalog->header.buffer, sizeof(char), catalog->header.datasize, f);
    123     fprintf (stderr, "save %d bytes in %s (%zx)\n", (int) catalog->header.datasize, name, (size_t) catalog->header.buffer);
    124     fprintf (stderr, "first bytes are %c%c%c%c%c%c%c%c\n",
    125              catalog->header.buffer[0], catalog->header.buffer[1], catalog->header.buffer[2], catalog->header.buffer[3],
    126              catalog->header.buffer[4], catalog->header.buffer[5], catalog->header.buffer[6], catalog->header.buffer[7]);
    127     fclose (f);
    128   }
    129 # endif
     94  if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE);
     95 
    13096  /* rewind file pointers and truncate (file is still open) */
    13197  if (fseeko (catalog->f, 0LL, SEEK_SET)) {
    13298    perror ("dvo_catalog_save_subset: ");
    13399    fprintf (stderr, "failed to seek to beginning\n");
     100    return FALSE;
    134101  }
    135102
    … …  
    151118  gfits_free_matrix (&matrix);
    152119
     120  FTable *outtable = ftable;
     121  FTable cmptable;
     122  Header cmpheader;
     123  gfits_init_header (&cmpheader);
     124  gfits_init_table (&cmptable);
     125  cmptable.header = &cmpheader;
     126 
     127  int fullWrite = (start == 0) && (Nrows == Ntotal);
     128  int isCompressed = output_is_compressed (start, Nrows, Ntotal, catalog->catcompress);
     129
     130  if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE);
     131
     132  if (isCompressed) {
     133    // fprintf (stderr, "compress %s\n", catalog->filename);
     134    char *compressMode = dvo_catalog_compress_string (catalog->catcompress);
     135    // I should test how Ntile affects fpack/funpack and speed
     136    if (!gfits_compress_table (ftable, &cmptable, 1000, compressMode)) {
     137      fprintf (stderr, "compression failure\n");
     138      return (FALSE);
     139    }
     140    if (!byteswap_varlength_ftable (&cmpheader, &cmptable)) {
     141      fprintf (stderr, "failed to swap varlength column\n");
     142      return FALSE;
     143    }
     144    if (!gfits_modify (cmptable.header, "DVO_CMP", "%s", 1, compressMode)) {
     145      fprintf (stderr, "can't save compression mode\n");
     146      return (FALSE);
     147    }
     148    outtable = &cmptable;
     149  }
     150
     151  if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE);
     152
    153153  // write the table data
    154   if (!gfits_fwrite_ftable_range (catalog->f, ftable, start, Nrows, Ndisk, Ntotal)) {
    155     fprintf (stderr, "can't write table data\n");
    156     return (FALSE);
    157   }
    158   fullsize += ftable->datasize + ftable->header->datasize;
     154  if (fullWrite) {
     155    if (!gfits_fwrite_Theader (catalog->f, outtable->header)) {
     156      fprintf (stderr, "can't write table header\n");
     157      return (FALSE);
     158    }
     159    if (!gfits_fwrite_table (catalog->f, outtable)) {
     160      fprintf (stderr, "can't write table data\n");
     161      return (FALSE);
     162    }
     163  } else {
     164    if (!gfits_fwrite_ftable_range (catalog->f, outtable, start, Nrows, Ndisk, Ntotal)) {
     165      fprintf (stderr, "can't write table data (range)\n");
     166      return (FALSE);
     167    }
     168  }
     169  fullsize += outtable->datasize + outtable->header->datasize;
     170
     171  if (OHANA_MEMCHECK) ohana_memcheck_func (TRUE);
     172
     173  // since we init'ed the structures above, these operations are safe whether or not we compressed the table
     174  gfits_free_header (&cmpheader);
     175  gfits_free_table (&cmptable);
    159176
    160177  // if the output file will be completely re-written, truncate to total datasize
    161   if ((start == 0) && (Nrows == Ntotal)) {
     178  if (fullWrite) {
    162179    int fd = fileno (catalog->f);
    163180    if (ftruncate (fd, fullsize)) {
    … …  
    192209    // databases created prior to ~2014.07.01 did not have the LENSING, LENSOBJ, STARPAR paths in their headers.
    193210    // in these cases, we do not try to lock or open the relevant file
    194     if (!strcmp (name, "LENSING")) return (DVO_CAT_OPEN_EMPTY);
    195     if (!strcmp (name, "LENSOBJ")) return (DVO_CAT_OPEN_EMPTY);
    196     if (!strcmp (name, "STARPAR")) return (DVO_CAT_OPEN_EMPTY);
     211    if (!strcmp (name, "LENSING"))  return (DVO_CAT_OPEN_EMPTY);
     212    if (!strcmp (name, "LENSOBJ"))  return (DVO_CAT_OPEN_EMPTY);
     213    if (!strcmp (name, "STARPAR"))  return (DVO_CAT_OPEN_EMPTY);
     214    if (!strcmp (name, "GALPHOT")) return (DVO_CAT_OPEN_EMPTY);
    197215    return (DVO_CAT_OPEN_FAIL);
    198216  }
    … …  
    200218  sprintf (subcat[0].filename, "%s/%s", path, string);
    201219  free (path);
     220
     221  // inherit compression mode for this subcat;
     222  subcat->catcompress = catalog->catcompress;
    202223
    203224  /* lock & open catalog file */
    … …  
    220241  }
    221242  Nskip = gfits_data_size (&subcat[0].header);
    222   fseeko (subcat[0].f, Nskip, SEEK_CUR);
     243  if (fseeko (subcat[0].f, Nskip, SEEK_CUR)) { perror ("fseeko: "); exit (1); }
    223244
    224245  /* read table header */
    … …  
    230251  return (DVO_CAT_OPEN_OK);
    231252}
     253
     254// ftable must already exist and have a valid, loaded header
     255int gfits_fread_uncompressed (Catalog *catalog, FTable *ftable, char *nativeOrder, char VERBOSE) {
     256
     257  // fread_ftable_data requires the header
     258
     259  *nativeOrder = FALSE;
     260  if (!gfits_fread_ftable_data (catalog->f, ftable, FALSE)) {
     261    if (VERBOSE) fprintf (stderr, "can't read table data\n");
     262    return FALSE;
     263  }
     264  // NOTE: fread_ftable returns an unswapped table; uncompress swaps the bytes into native order
     265   
     266  if (gfits_extension_is_compressed_table (ftable->header)) {
     267    FTable rawtable;
     268    Header rawheader;
     269    gfits_init_table (&rawtable);
     270    gfits_init_header (&rawheader);
     271    rawtable.header = &rawheader;
     272
     273    // NOTE: uncompress swaps the data bytes into native order, but needs the varlength columns native
     274    if (!byteswap_varlength_ftable (ftable->header, ftable)) {
     275      fprintf (stderr, "failed to swap varlength column\n");
     276      return FALSE;
     277    }
     278    if (!gfits_uncompress_table (ftable, &rawtable)) {
     279      if (VERBOSE) fprintf (stderr, "failed to uncompress table\n");
     280      gfits_free_table (ftable);
     281      return FALSE;
     282    }
     283
     284    char compressMode[80];
     285    if (!gfits_scan (ftable->header, "DVO_CMP", "%s", 1, compressMode)) {
     286      strcpy (compressMode, "AUTO");
     287    }
     288    catalog->catcompress = dvo_catalog_catcompress (compressMode); // if any table is compressed, set all to a compress state?
     289
     290    // free the buffers
     291    gfits_free_header (ftable->header);
     292    gfits_free_table (ftable);
     293   
     294    // copies values but does not allocate new memory
     295    Header *outheader = ftable->header;
     296    gfits_copy_header_ptr (&rawheader, outheader);
     297    gfits_copy_ftable_ptr (&rawtable, ftable); // this replaces ftable->header
     298    ftable->header = outheader;
     299    *nativeOrder = TRUE;
     300  }
     301  return TRUE;
     302}
     303
     304// LOAD_SUBCAT(measure,MEASURE,Measure)
     305
     306# define LOAD_SUBCAT(FIELD, NAME, STRUCT)                               \
     307  {                                                                     \
     308    Header header;                                                      \
     309    FTable ftable;                                                      \
     310    off_t Nitems;                                                       \
     311    gfits_init_table  (&ftable);                                        \
     312    gfits_init_header (&header);                                        \
     313    ftable.header = &header;                                            \
     314    int status = DVO_CAT_OPEN_EMPTY;                                    \
     315    if (!(catalog[0].catflags & DVO_SKIP_##NAME)) {                     \
     316      status = dvo_catalog_open_subcat (catalog, &catalog[0].FIELD##_catalog, ftable.header, #NAME, VERBOSE); \
     317      if (status == DVO_CAT_OPEN_FAIL) {                                \
     318        return (FALSE);                                                 \
     319      }                                                                 \
     320      if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].N##FIELD##_disk > 0)) { \
     321        return (FALSE);                                                 \
     322      }                                                                 \
     323    }                                                                   \
     324    if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & DVO_LOAD_##NAME)) { \
     325      char nativeOrder = FALSE;                                         \
     326      /* read table data */                                             \
     327      if (!gfits_fread_uncompressed (catalog[0].FIELD##_catalog, &ftable, &nativeOrder, VERBOSE)) { \
     328        if (VERBOSE) fprintf (stderr, "can't read table %s data\n", #FIELD); \
     329        gfits_free_header (&header);                                    \
     330        return (FALSE);                                                 \
     331      }                                                                 \
     332      /* convert data format to internal : returns number of row read in Nvalues */ \
     333      catalog[0].FIELD = FtableTo##STRUCT (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, nativeOrder); \
     334        if (Nitems != catalog[0].N##FIELD##_disk) {                     \
     335          fprintf (stderr, "Warning: mismatch between N%s in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n", #FIELD, Nitems, catalog[0].N##FIELD##_disk); \
     336        }                                                               \
     337        catalog[0].N##FIELD = catalog[0].N##FIELD##_disk;               \
     338          catalog[0].N##FIELD##_off = 0;                                \
     339    } else {                                                            \
     340      if (catalog[0].FIELD##_catalog) {                                 \
     341        gfits_free_header (&catalog[0].FIELD##_catalog[0].header);      \
     342      } else {                                                          \
     343        ALLOCATE (catalog[0].FIELD##_catalog, Catalog, 1);              \
     344        dvo_catalog_init (catalog[0].FIELD##_catalog, TRUE);            \
     345      }                                                                 \
     346      gfits_create_header (&catalog[0].FIELD##_catalog[0].header);      \
     347      ALLOCATE (catalog[0].FIELD, STRUCT, 1);                           \
     348      catalog[0].N##FIELD = 0;                                          \
     349        catalog[0].N##FIELD##_off = catalog[0].N##FIELD##_disk;         \
     350    }                                                                   \
     351    gfits_free_header (&header);                                        \
     352  }
    232353
    233354int dvo_catalog_load_split (Catalog *catalog, int VERBOSE) {
    … …  
    240361  off_t Nlensobj;
    241362  off_t Nstarpar;
    242   off_t Nitems;
    243   int status, Nsecfilt;
    244   Header header;
    245   FTable ftable;
     363  off_t Ngalphot;
     364  int Nsecfilt;
    246365  SecFilt *primary;
    247366
    248   /* ftable header storage for below */
    249   ftable.header = &header;
    250   ftable.buffer = NULL;
    251   header.buffer = NULL;
    252367  primary = NULL;
    253   status = FALSE;
    254368
    255369  /* get the components from the header - these duplicate information in the split files (NAXIS2) */
    256   // NSTARS, NMEAS, NMISS are required; NLENSING, NLENSOBJ, NSTARPAR are not (0 if not found)
    257   if (!gfits_scan (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  &Naverage)) return (FALSE);
    258   if (!gfits_scan (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  &Nmeasure)) return (FALSE);
    259   if (!gfits_scan (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  &Nmissing)) return (FALSE);
    260   if (!gfits_scan (&catalog[0].header, "NSECFILT",      "%d", 1,  &Nsecfilt)) Nsecfilt = 0;
    261   if (!gfits_scan (&catalog[0].header, "NLENSING", OFF_T_FMT, 1,  &Nlensing)) Nlensing = 0;
    262   if (!gfits_scan (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1,  &Nlensobj)) Nlensobj = 0;
    263   if (!gfits_scan (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1,  &Nstarpar)) Nstarpar = 0;
     370  // NSTARS, NMEAS, NMISS are required; NLENSING, NLENSOBJ, NSTARPAR, NGALPHOT are not (0 if not found)
     371  if (!gfits_scan (&catalog[0].header, "NSTARS",    OFF_T_FMT, 1,  &Naverage))  return (FALSE);
     372  if (!gfits_scan (&catalog[0].header, "NMEAS",     OFF_T_FMT, 1,  &Nmeasure))  return (FALSE);
     373  if (!gfits_scan (&catalog[0].header, "NMISS",     OFF_T_FMT, 1,  &Nmissing))  return (FALSE);
     374  if (!gfits_scan (&catalog[0].header, "NSECFILT",       "%d", 1,  &Nsecfilt))  Nsecfilt  = 0;
     375  if (!gfits_scan (&catalog[0].header, "NLENSING",  OFF_T_FMT, 1,  &Nlensing))  Nlensing  = 0;
     376  if (!gfits_scan (&catalog[0].header, "NLENSOBJ",  OFF_T_FMT, 1,  &Nlensobj))  Nlensobj  = 0;
     377  if (!gfits_scan (&catalog[0].header, "NSTARPAR",  OFF_T_FMT, 1,  &Nstarpar))  Nstarpar  = 0;
     378  if (!gfits_scan (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1,  &Ngalphot)) Ngalphot = 0;
    264379
    265380  /* the OBJID is a counter that uniquely defines an average entry and never changes.  if
    … …  
    276391
    277392  /* save the current number so we can do partial updates */
    278   catalog[0].Naves_disk = Naverage;
    279   catalog[0].Nmeas_disk = Nmeasure;
    280   catalog[0].Nmiss_disk = Nmissing;
    281   catalog[0].Nsecf_disk = Naverage * Nsecfilt;
    282   catalog[0].Nlensing_disk = Nlensing;
    283   catalog[0].Nlensobj_disk = Nlensobj;
    284   catalog[0].Nstarpar_disk = Nstarpar;
    285 
    286   /**  Nsecfilt is unusual: it does not list the number of data items in the table
    287        instead, the number of items is Nsecfilt * Naverage.  **/
    288   catalog[0].Nsecfilt  = Nsecfilt;
     393  catalog[0].Naverage_disk = Naverage;
     394  catalog[0].Nmeasure_disk = Nmeasure;
     395  catalog[0].Nmissing_disk = Nmissing;
     396  catalog[0].Nsecfilt_disk = Naverage * Nsecfilt;
     397  catalog[0].Nlensing_disk  = Nlensing;
     398  catalog[0].Nlensobj_disk  = Nlensobj;
     399  catalog[0].Nstarpar_disk  = Nstarpar;
     400  catalog[0].Ngalphot_disk = Ngalphot;
    289401
    290402  /* default values, but we will assign these a valid value before we exit (even if empty) */
    291   catalog[0].average = NULL;
    292   catalog[0].measure = NULL;
    293   catalog[0].missing = NULL;
    294   catalog[0].secfilt = NULL;
    295   catalog[0].lensing = NULL;
    296   catalog[0].lensobj = NULL;
    297   catalog[0].starpar = NULL;
     403  catalog[0].average  = NULL;
     404  catalog[0].measure  = NULL;
     405  catalog[0].missing  = NULL;
     406  catalog[0].secfilt  = NULL;
     407  catalog[0].lensing  = NULL;
     408  catalog[0].lensobj  = NULL;
     409  catalog[0].starpar  = NULL;
     410  catalog[0].galphot = NULL;
    298411
    299412  /*** Average Table ***/
    … …  
    302415  /* move pointer past PHU header -- must be already read (load_catalog) */
    303416  Nbytes = catalog[0].header.datasize + gfits_data_size (&catalog[0].header);
    304   fseeko (catalog[0].f, Nbytes, SEEK_SET);
     417  if (fseeko (catalog[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); }
     418
     419  /* ftable header storage for below */
     420  char nativeOrder;
     421  Header header;
     422  FTable ftable;
     423  gfits_init_table  (&ftable);
     424  gfits_init_header (&header);
     425  ftable.header = &header;
    305426
    306427  /* read Average table header */
    307428  if (!gfits_fread_header (catalog[0].f, &header)) {
    308     if (VERBOSE) fprintf (stderr, "can't read table average header");
     429    if (VERBOSE) fprintf (stderr, "can't read table average header\n");
    309430    return (FALSE);
    310431  }
    311   if (catalog[0].catflags & LOAD_AVES) {
    312     /* read Average table data : format is irrelevant here */
    313     if (!gfits_fread_ftable_data (catalog[0].f, &ftable, FALSE)) {
    314       if (VERBOSE) fprintf (stderr, "can't read table average data");
    315       return (FALSE);
    316     }
    317     /* convert the saved version of the table to the internal version.  Old versions of DVO stored
     432  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
     433    if (!gfits_fread_uncompressed (catalog, &ftable, &nativeOrder, VERBOSE)) {
     434      if (VERBOSE) fprintf (stderr, "can't read Average table\n");
     435      gfits_free_header (&header);
     436      return FALSE;
     437    }
     438
     439    /* convert the disk version of the table to the internal version.  Old versions of DVO stored
    318440     * one of the average magnitudes in Average.  We save this in case it is needed below.  NOTE:
    319441     * primary is only used if we read in the secfilt table, otherwise it should be freed */
    320     catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary);
    321     if (Naverage != catalog[0].Naves_disk) {
    322       fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Naverage,  catalog[0].Naves_disk);
    323     }
    324     gfits_free_header (&header);
    325     catalog[0].Naverage = catalog[0].Naves_disk;
    326     catalog[0].Naves_off = 0;
     442    catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, nativeOrder);
     443    if (Naverage != catalog[0].Naverage_disk) {
     444      fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Naverage,  catalog[0].Naverage_disk);
     445    }
     446    catalog[0].Naverage = catalog[0].Naverage_disk;
     447    catalog[0].Naverage_off = 0;
    327448  } else {
    328449    catalog[0].catformat = FtableGetFormat (&ftable);
    329450    ALLOCATE (catalog[0].average, Average, 1);
    330451    catalog[0].Naverage = 0;
    331     catalog[0].Naves_off = catalog[0].Naves_disk;
    332   }
     452    catalog[0].Naverage_off = catalog[0].Naverage_disk;
     453  }
     454  gfits_free_header (&header);
    333455
    334456  /*** Measure Table ***/
    335   if (!(catalog[0].catflags & SKIP_MEAS)) {
    336     // unless we specify 'skip', we still need to load the
    337     status = dvo_catalog_open_subcat (catalog, &catalog[0].measure_catalog, ftable.header, "MEASURE", VERBOSE);
    338     if (status == DVO_CAT_OPEN_FAIL) {
    339       return (FALSE);
    340     }
    341     if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmeas_disk > 0)) {
    342       return (FALSE);
    343     }
    344   }
    345   if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MEAS)) {
    346     // XXX this allows an empty Measure catalog with non-empty Average catalog : is that OK?
    347     /* read Measure table data */
    348     if (!gfits_fread_ftable_data (catalog[0].measure_catalog[0].f, &ftable, FALSE)) {
    349       if (VERBOSE) fprintf (stderr, "can't read table measure data\n");
    350       return (FALSE);
    351     }
    352     /* convert data format to internal : returns number of row read in Nmeasure */
    353     catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat);
    354     if (Nmeasure != catalog[0].Nmeas_disk) {
    355       fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmeasure,  catalog[0].Nmeas_disk);
    356     }
    357     catalog[0].Nmeasure = catalog[0].Nmeas_disk;
    358     catalog[0].Nmeas_off = 0;
    359   } else {
    360     // XXX is it necessary to generate a template header here?
    361     // XXX this is a memory leak, right?
    362     if (catalog[0].measure_catalog) {
    363       gfits_free_header (&catalog[0].measure_catalog[0].header);
     457 
     458  LOAD_SUBCAT(measure,MEASURE,Measure);
     459  LOAD_SUBCAT(missing,MISSING,Missing);
     460  LOAD_SUBCAT(secfilt,SECFILT,SecFilt);
     461  LOAD_SUBCAT(lensing,LENSING,Lensing);
     462  LOAD_SUBCAT(lensobj,LENSOBJ,Lensobj);
     463  LOAD_SUBCAT(starpar,STARPAR,StarPar);
     464  LOAD_SUBCAT(galphot,GALPHOT,GalPhot);
     465
     466  /**  catalog->Nsecfilt is unusual: it does not list the number of data items in the
     467       table instead, the number of items is Nsecfilt * Naverage, and is stored in
     468       Nsecfilt_mem. fix these below **/
     469
     470  catalog[0].Nsecfilt_mem = catalog[0].Nsecfilt;
     471  catalog[0].Nsecfilt  = Nsecfilt;
     472
     473  /** some old formats stored one of the secfilt values in the average table.
     474      repair this below **/
     475
     476  if (primary) {
     477    if (catalog[0].Nsecfilt_mem) {
     478      dvo_catalog_primary_to_secfilt (catalog, primary, catalog[0].Naverage_disk);
    364479    } else {
    365       ALLOCATE (catalog[0].measure_catalog, Catalog, 1);
    366       dvo_catalog_init (catalog[0].measure_catalog, TRUE);
    367     }
    368     gfits_create_header (&catalog[0].measure_catalog[0].header);
    369     ALLOCATE (catalog[0].measure, Measure, 1);
    370     catalog[0].Nmeasure = 0;
    371     catalog[0].Nmeas_off = catalog[0].Nmeas_disk;
    372   }
    373   gfits_free_header (&header);
    374 
    375   /*** Missing Table ***/
    376   status = dvo_catalog_open_subcat (catalog, &catalog[0].missing_catalog, ftable.header, "MISSING", VERBOSE);
    377   if (status == DVO_CAT_OPEN_FAIL) {
    378     return (FALSE);
    379   }
    380   if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmiss_disk > 0)) {
    381     return (FALSE);
    382   }
    383   if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MISS)) {
    384     /* read Missing table data */
    385     if (!gfits_fread_ftable_data (catalog[0].missing_catalog[0].f, &ftable, FALSE)) {
    386       if (VERBOSE) fprintf (stderr, "can't read table missing data\n");
    387       return (FALSE);
    388     }
    389     /* no conversions currently defined : this just does the byte swap */
    390     catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL);
    391     if (!catalog[0].missing) {
    392       fprintf (stderr, "ERROR: failed to read missing\n");
    393       exit (2);
    394     }
    395     if (Nmissing != catalog[0].Nmiss_disk) {
    396       fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nmissing,  catalog[0].Nmiss_disk);
    397     }
    398     catalog[0].Nmissing = catalog[0].Nmiss_disk;
    399     catalog[0].Nmiss_off = 0;
    400   } else {
    401     // XXX is it necessary to generate a template header here?
    402     gfits_create_header (&catalog[0].missing_catalog[0].header);
    403     ALLOCATE (catalog[0].missing, Missing, 1);
    404     catalog[0].Nmissing = 0;
    405     catalog[0].Nmiss_off = catalog[0].Nmiss_disk;
    406   }
    407   gfits_free_header (ftable.header);
    408 
    409   /*** Secfilt Table ***/
    410   status = dvo_catalog_open_subcat (catalog, &catalog[0].secfilt_catalog, ftable.header, "SECFILT", VERBOSE);
    411   // fprintf (stderr, "secfilt header %s : %zx\n", catalog[0].secfilt_catalog[0].filename, (size_t) catalog[0].secfilt_catalog[0].header.buffer);
    412   if (status == DVO_CAT_OPEN_FAIL) {
    413     return (FALSE);
    414   }
    415   if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nsecf_disk > 0)) {
    416     return (FALSE);
    417   }
    418   if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_SECF)) {
    419     /* read secfilt table data */
    420     if (!gfits_fread_ftable_data (catalog[0].secfilt_catalog[0].f, &ftable, FALSE)) {
    421       if (VERBOSE) fprintf (stderr, "can't read table secfilt data\n");
    422       return (FALSE);
    423     }
    424     catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat);
    425     if (Nitems != catalog[0].Nsecf_disk) {
    426       fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nitems,  catalog[0].Nsecf_disk);
    427     }
    428     catalog[0].Nsecf_mem = catalog[0].Nsecf_disk;
    429     catalog[0].Nsecf_off = 0;
    430 
    431     /* if primary is defined, we were supplied with one additional average magnitude from Average
    432        we need to interleave these magnitudes with the secfilt entries just loaded */
    433     if (primary != NULL) {
    434       // this modifies catalog.Nsecf_mem,Nsecfilt
    435       dvo_catalog_primary_to_secfilt (catalog, primary, catalog[0].Naves_disk);
    436     }
    437   } else {
    438     if (primary != NULL) {
    439480      free (primary);
    440481      catalog[0].Nsecfilt ++;
    441482    }
    442     gfits_create_header (&catalog[0].secfilt_catalog[0].header);
    443     ALLOCATE (catalog[0].secfilt, SecFilt, 1);
    444     catalog[0].Nsecf_mem = 0;
    445     catalog[0].Nsecf_off = catalog[0].Nsecf_disk;
    446   }
    447   gfits_free_header (ftable.header);
    448 
    449   /*** Lensing Table ***/
    450   if (!(catalog[0].catflags & SKIP_LENSING)) {
    451     // unless we specify 'skip', we still need to load the Lensing table
    452     status = dvo_catalog_open_subcat (catalog, &catalog[0].lensing_catalog, ftable.header, "LENSING", VERBOSE);
    453     if (status == DVO_CAT_OPEN_FAIL) {
    454       return (FALSE);
    455     }
    456     // the Lensing file need not exist, unless we expect data to exist
    457     if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nlensing_disk > 0)) {
    458       return (FALSE);
    459     }
    460   }
    461   if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSING)) {
    462     // only read the Lensing table if the file actually exists
    463     /* read Lensing table data */
    464     if (!gfits_fread_ftable_data (catalog[0].lensing_catalog[0].f, &ftable, FALSE)) {
    465       if (VERBOSE) fprintf (stderr, "can't read table lensing data\n");
    466       return (FALSE);
    467     }
    468     /* convert data format to internal : returns number of row read in Nlensing */
    469     catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat);
    470     if (Nlensing != catalog[0].Nlensing_disk) {
    471       fprintf (stderr, "Warning: mismatch between Nlensing in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensing,  catalog[0].Nlensing_disk);
    472     }
    473     catalog[0].Nlensing = catalog[0].Nlensing_disk;
    474     catalog[0].Nlensing_off = 0;
    475   } else {
    476     if (catalog[0].lensing_catalog) {
    477       gfits_free_header (&catalog[0].lensing_catalog[0].header);
    478     } else {
    479       ALLOCATE (catalog[0].lensing_catalog, Catalog, 1);
    480       dvo_catalog_init (catalog[0].lensing_catalog, TRUE);
    481     }
    482     gfits_create_header (&catalog[0].lensing_catalog[0].header);
    483     ALLOCATE (catalog[0].lensing, Lensing, 1);
    484     catalog[0].Nlensing = 0;
    485     catalog[0].Nlensing_off = catalog[0].Nlensing_disk;
    486   }
    487   gfits_free_header (&header);
    488 
    489   /*** Lensobj Table ***/
    490   if (!(catalog[0].catflags & SKIP_LENSOBJ)) {
    491     // unless we specify 'skip', we still need to load the
    492     status = dvo_catalog_open_subcat (catalog, &catalog[0].lensobj_catalog, ftable.header, "LENSOBJ", VERBOSE);
    493     if (status == DVO_CAT_OPEN_FAIL) {
    494       return (FALSE);
    495     }
    496     // the Lensobj file need not exist, unless we expect data to exist
    497     if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nlensobj_disk > 0)) {
    498       return (FALSE);
    499     }
    500   }
    501   if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_LENSOBJ)) {
    502     // only read the Lensobj table if the file actually exists
    503     /* read Lensobj table data */
    504     if (!gfits_fread_ftable_data (catalog[0].lensobj_catalog[0].f, &ftable, FALSE)) {
    505       if (VERBOSE) fprintf (stderr, "can't read table lensobj data\n");
    506       return (FALSE);
    507     }
    508     /* convert data format to internal : returns number of row read in Nlensobj */
    509     catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat);
    510     if (Nlensobj != catalog[0].Nlensobj_disk) {
    511       fprintf (stderr, "Warning: mismatch between Nlensobj in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nlensobj,  catalog[0].Nlensobj_disk);
    512     }
    513     catalog[0].Nlensobj = catalog[0].Nlensobj_disk;
    514     catalog[0].Nlensobj_off = 0;
    515   } else {
    516     if (catalog[0].lensobj_catalog) {
    517       gfits_free_header (&catalog[0].lensobj_catalog[0].header);
    518     } else {
    519       ALLOCATE (catalog[0].lensobj_catalog, Catalog, 1);
    520       dvo_catalog_init (catalog[0].lensobj_catalog, TRUE);
    521     }
    522     gfits_create_header (&catalog[0].lensobj_catalog[0].header);
    523     ALLOCATE (catalog[0].lensobj, Lensobj, 1);
    524     catalog[0].Nlensobj = 0;
    525     catalog[0].Nlensobj_off = catalog[0].Nlensobj_disk;
    526   }
    527   gfits_free_header (&header);
    528 
    529   /*** StarPar Table ***/
    530   if (!(catalog[0].catflags & SKIP_STARPAR)) {
    531     // unless we specify 'skip', we still need to load the
    532     status = dvo_catalog_open_subcat (catalog, &catalog[0].starpar_catalog, ftable.header, "STARPAR", VERBOSE);
    533     if (status == DVO_CAT_OPEN_FAIL) {
    534       return (FALSE);
    535     }
    536     // the StarPar file need not exist, unless we expect data to exist
    537     if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nstarpar_disk > 0)) {
    538       return (FALSE);
    539     }
    540   }
    541   if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_STARPAR)) {
    542     // only read the StarPar table if the file actually exists
    543     /* read StarPar table data */
    544     if (!gfits_fread_ftable_data (catalog[0].starpar_catalog[0].f, &ftable, FALSE)) {
    545       if (VERBOSE) fprintf (stderr, "can't read table starpar data\n");
    546       return (FALSE);
    547     }
    548     /* convert data format to internal : returns number of row read in Nstarpar */
    549     catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat);
    550     if (Nstarpar != catalog[0].Nstarpar_disk) {
    551       fprintf (stderr, "Warning: mismatch between Nstarpar in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nstarpar,  catalog[0].Nstarpar_disk);
    552     }
    553     catalog[0].Nstarpar = catalog[0].Nstarpar_disk;
    554     catalog[0].Nstarpar_off = 0;
    555   } else {
    556     if (catalog[0].starpar_catalog) {
    557       gfits_free_header (&catalog[0].starpar_catalog[0].header);
    558     } else {
    559       ALLOCATE (catalog[0].starpar_catalog, Catalog, 1);
    560       dvo_catalog_init (catalog[0].starpar_catalog, TRUE);
    561     }
    562     gfits_create_header (&catalog[0].starpar_catalog[0].header);
    563     ALLOCATE (catalog[0].starpar, StarPar, 1);
    564     catalog[0].Nstarpar = 0;
    565     catalog[0].Nstarpar_off = catalog[0].Nstarpar_disk;
    566   }
    567   gfits_free_header (&header);
     483  }
    568484
    569485  return (TRUE);
    … …  
    575491
    576492  off_t Nbytes;
    577   off_t Naverage, Nexpect, Nitems, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar;
     493  off_t Naverage, Nexpect, Nitems, Nmeasure, Nmissing, Nlensing, Nlensobj, Nstarpar, Ngalphot;
    578494  Header header;
    579495  FTable ftable;
    … …  
    587503
    588504  /*** Average (& SecFilt) Table ***/
    589   if (catalog[0].catflags & LOAD_AVES) {
     505  if (catalog[0].catflags & DVO_LOAD_AVERAGE) {
    590506
    591507    /*** load the Average data ***/
    592508    /* move pointer past header and matrix -- must be already read (load_catalog) */
    593509    Nbytes = catalog[0].header.datasize + gfits_data_size (&catalog[0].header);
    594     fseeko (catalog[0].f, Nbytes, SEEK_SET);
     510    if (fseeko (catalog[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); }
     511
    595512    /* read Average table header */
    596513    if (!gfits_fread_header (catalog[0].f, &header)) {
    … …  
    607524     * one of the average magnitudes in Average.  We save this in case it is needed below.  NOTE:
    608525     * primary is only used if we read in the secfilt table, otherwise it should be freed */
    609     catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary);
     526    catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary, FALSE);
    610527    if (Naverage != Nrows) {
    611528      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    614531    gfits_free_header (&header);
    615532    catalog[0].Naverage = Naverage;
    616     catalog[0].Naves_off = start;
     533    catalog[0].Naverage_off = start;
    617534
    618535    /*** load the secfilt data ***/
    … …  
    621538    /* move pointer past header -- must be already read (load_catalog) */
    622539    Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header);
    623     fseeko (subcat[0].f, Nbytes, SEEK_SET);
     540    if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); }
    624541
    625542    /* read Secfilt table header */
    … …  
    635552
    636553    Nexpect = catalog[0].Naverage * catalog[0].Nsecfilt;
    637     catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat);
     554    catalog[0].secfilt = FtableToSecFilt (&ftable, catalog[0].average, &Nitems, &catalog[0].catformat, FALSE);
    638555    if (Nitems != Nexpect) {
    639556      fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Nitems,  Nexpect);
    640557    }
    641     catalog[0].Nsecf_mem = catalog[0].Naverage * catalog[0].Nsecfilt;
    642     catalog[0].Nsecf_off = start               * catalog[0].Nsecfilt;
     558    catalog[0].Nsecfilt_mem = catalog[0].Naverage * catalog[0].Nsecfilt;
     559    catalog[0].Nsecfilt_off = start               * catalog[0].Nsecfilt;
    643560
    644561    /* if primary is defined, we were supplied with one additional average magnitude from Average
    … …  
    651568
    652569  // XXX check the open status of the catalog
    653   if (catalog[0].catflags & LOAD_MEAS) {
     570  if (catalog[0].catflags & DVO_LOAD_MEASURE) {
    654571
    655572    Catalog *subcat = catalog[0].measure_catalog;
    … …  
    657574    /* move pointer past header -- must be already read (load_catalog) */
    658575    Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header);
    659     fseeko (subcat[0].f, Nbytes, SEEK_SET);
     576    if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); }
    660577
    661578    /* read Measure table header */
    … …  
    671588
    672589    /* convert data format to internal : returns number of row read in Nmeasure */
    673     catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat);
     590    catalog[0].measure = FtableToMeasure (&ftable, catalog[0].average, &Nmeasure, &catalog[0].catformat, FALSE);
    674591    if (Nmeasure != Nrows) {
    675592      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    678595    gfits_free_header (&header);
    679596    catalog[0].Nmeasure = Nmeasure;
    680     catalog[0].Nmeas_off = start;
     597    catalog[0].Nmeasure_off = start;
    681598  }
    682599
    683600  // XXX check the open status of the catalog?
    684   if (catalog[0].catflags & LOAD_MISS) {
     601  if (catalog[0].catflags & DVO_LOAD_MISSING) {
    685602
    686603    Catalog *subcat = catalog[0].missing_catalog;
    … …  
    688605    /* move pointer past header -- must be already read (load_catalog) */
    689606    Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header);
    690     fseeko (subcat[0].f, Nbytes, SEEK_SET);
     607    if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); }
    691608
    692609    /* read Missing table header */
    … …  
    702619
    703620    /* no conversions currently defined : this just does the byte swap */
    704     catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL);
     621    catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL, NULL);
    705622    if (!catalog[0].missing) {
    706623      fprintf (stderr, "ERROR: failed to read missing\n");
    … …  
    712629    gfits_free_header (&header);
    713630    catalog[0].Nmissing = Nmissing;
    714     catalog[0].Nmiss_off = start;
     631    catalog[0].Nmissing_off = start;
    715632  }
    716633
    717634  // XXX check the open status of the catalog
    718   if (catalog[0].catflags & LOAD_LENSING) {
     635  if (catalog[0].catflags & DVO_LOAD_LENSING) {
    719636
    720637    Catalog *subcat = catalog[0].lensing_catalog;
    … …  
    722639    /* move pointer past header -- must be already read (load_catalog) */
    723640    Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header);
    724     fseeko (subcat[0].f, Nbytes, SEEK_SET);
     641    if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); }
    725642
    726643    /* read Lensing table header */
    … …  
    736653
    737654    /* convert data format to internal : returns number of row read in Nlensing */
    738     catalog[0].lensing = FtableToLensing (&ftable, &Nlensing, &catalog[0].catformat);
     655    catalog[0].lensing = FtableToLensing (&ftable, catalog[0].average, &Nlensing, &catalog[0].catformat, FALSE);
    739656    if (Nlensing != Nrows) {
    740657      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    747664
    748665  // XXX check the open status of the catalog
    749   if (catalog[0].catflags & LOAD_LENSOBJ) {
     666  if (catalog[0].catflags & DVO_LOAD_LENSOBJ) {
    750667
    751668    Catalog *subcat = catalog[0].lensobj_catalog;
    … …  
    753670    /* move pointer past header -- must be already read (load_catalog) */
    754671    Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header);
    755     fseeko (subcat[0].f, Nbytes, SEEK_SET);
     672    if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); }
    756673
    757674    /* read Lensobj table header */
    … …  
    767684
    768685    /* convert data format to internal : returns number of row read in Nlensobj */
    769     catalog[0].lensobj = FtableToLensobj (&ftable, &Nlensobj, &catalog[0].catformat);
     686    catalog[0].lensobj = FtableToLensobj (&ftable, catalog[0].average, &Nlensobj, &catalog[0].catformat, FALSE);
    770687    if (Nlensobj != Nrows) {
    771688      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    778695
    779696  // XXX check the open status of the catalog
    780   if (catalog[0].catflags & LOAD_STARPAR) {
     697  if (catalog[0].catflags & DVO_LOAD_STARPAR) {
    781698
    782699    Catalog *subcat = catalog[0].starpar_catalog;
    … …  
    784701    /* move pointer past header -- must be already read (load_catalog) */
    785702    Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header);
    786     fseeko (subcat[0].f, Nbytes, SEEK_SET);
     703    if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); }
    787704
    788705    /* read StarPar table header */
    … …  
    798715
    799716    /* convert data format to internal : returns number of row read in Nstarpar */
    800     catalog[0].starpar = FtableToStarPar (&ftable, &Nstarpar, &catalog[0].catformat);
     717    catalog[0].starpar = FtableToStarPar (&ftable, catalog[0].average, &Nstarpar, &catalog[0].catformat, FALSE);
    801718    if (Nstarpar != Nrows) {
    802719      // XXX this condition denotes the eof has been reached; not an error or a warning
    … …  
    808725  }
    809726
     727  // XXX check the open status of the catalog
     728  if (catalog[0].catflags & DVO_LOAD_GALPHOT) {
     729
     730    Catalog *subcat = catalog[0].galphot_catalog;
     731
     732    /* move pointer past header -- must be already read (load_catalog) */
     733    Nbytes = subcat[0].header.datasize + gfits_data_size (&subcat[0].header);
     734    if (fseeko (subcat[0].f, Nbytes, SEEK_SET)) { perror ("fseeko: "); exit (1); }
     735
     736    /* read GalPhot table header */
     737    if (!gfits_fread_header (subcat[0].f, &header)) {
     738      if (VERBOSE) fprintf (stderr, "can't read table galphot header");
     739      return (FALSE);
     740    }
     741    /* read GalPhot table data : format is irrelevant here */
     742    if (!gfits_fread_ftable_range (subcat[0].f, FALSE, &ftable, start, Nrows)) {
     743      if (VERBOSE) fprintf (stderr, "can't read table galphot data");
     744      return (FALSE);
     745    }
     746
     747    /* convert data format to internal : returns number of row read in Ngalphot */
     748    catalog[0].galphot = FtableToGalPhot (&ftable, catalog[0].average, &Ngalphot, &catalog[0].catformat, FALSE);
     749    if (Ngalphot != Nrows) {
     750      // XXX this condition denotes the eof has been reached; not an error or a warning
     751      // fprintf (stderr, "Warning: mismatch between Ngalphot in PHU and Table headers ("OFF_T_FMT" vs "OFF_T_FMT")\n",  Ngalphot,  Nrows);
     752    }
     753    gfits_free_header (&header);
     754    catalog[0].Ngalphot = Ngalphot;
     755    catalog[0].Ngalphot_off = start;
     756  }
     757
    810758  return (TRUE);
    811759}
    … …  
    814762int dvo_catalog_save_split (Catalog *catalog, char VERBOSE) {
    815763
    816   off_t Nitems;
    817764  Header header;
    818765  FTable ftable;
    819766  SecFilt *primary, *secfilt;
    820767  int Nsecfilt;
    821   off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new;
    822   off_t first, start, Nrows;
     768  off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalphot_disk_new;
    823769
    824770  ftable.header = &header;
    … …  
    828774 
    829775  // skip empty catalogs: it is illegal to have Measures without corresponding Averages
    830   Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off);
    831   if (Naves_disk_new == 0) {
     776  Naverage_disk_new = MAX (catalog[0].Naverage_disk, catalog[0].Naverage + catalog[0].Naverage_off);
     777  if (Naverage_disk_new == 0) {
    832778    if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n");
    833779    return (TRUE);
    … …  
    847793  }
    848794
    849   Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off);
    850   Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off);
    851   Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off);
     795  Nmeasure_disk_new = MAX (catalog[0].Nmeasure_disk, catalog[0].Nmeasure + catalog[0].Nmeasure_off);
     796  Nmissing_disk_new = MAX (catalog[0].Nmissing_disk, catalog[0].Nmissing + catalog[0].Nmissing_off);
     797  Nsecfilt_disk_new = MAX (catalog[0].Nsecfilt_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecfilt_off);
    852798  Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off);
    853799  Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off);
    854800  Nstarpar_disk_new = MAX (catalog[0].Nstarpar_disk, catalog[0].Nstarpar + catalog[0].Nstarpar_off);
     801  Ngalphot_disk_new = MAX (catalog[0].Ngalphot_disk, catalog[0].Ngalphot + catalog[0].Ngalphot_off);
    855802
    856803  /* make sure header is consistent with data */
    857   gfits_modify (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  Naves_disk_new);
    858   gfits_modify (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  Nmeas_disk_new);
    859   gfits_modify (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  Nmiss_disk_new);
    860   gfits_modify (&catalog[0].header, "NSECFILT", "%d",   1, Nsecfilt);
    861   gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1,  Nlensing_disk_new);
    862   gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1,  Nlensobj_disk_new);
    863   gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1,  Nstarpar_disk_new);
     804  gfits_modify (&catalog[0].header, "NSTARS",    OFF_T_FMT, 1,  Naverage_disk_new);
     805  gfits_modify (&catalog[0].header, "NMEAS",     OFF_T_FMT, 1,  Nmeasure_disk_new);
     806  gfits_modify (&catalog[0].header, "NMISS",     OFF_T_FMT, 1,  Nmissing_disk_new);
     807  gfits_modify (&catalog[0].header, "NSECFILT",  "%d",      1,  Nsecfilt);
     808  gfits_modify (&catalog[0].header, "NLENSING",  OFF_T_FMT, 1,  Nlensing_disk_new);
     809  gfits_modify (&catalog[0].header, "NLENSOBJ",  OFF_T_FMT, 1,  Nlensobj_disk_new);
     810  gfits_modify (&catalog[0].header, "NSTARPAR",  OFF_T_FMT, 1,  Nstarpar_disk_new);
     811  gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1,  Ngalphot_disk_new);
     812
    864813  gfits_modify_alt (&catalog[0].header, "EXTEND",   "%t", 1, TRUE);
    865814  gfits_modify (&catalog[0].header, "OBJID",    "%d", 1, catalog[0].objID);
    … …  
    868817
    869818  /*** Average Table ***/
    870   if ((catalog[0].catflags & LOAD_AVES) && (catalog[0].average != NULL)) {
    871 
    872     first  = 0;                    // first row in memory to write
    873     start  = catalog[0].Naves_off; // first disk row to write
    874     Nrows  = catalog[0].Naverage - first;
     819  if ((catalog[0].catflags & DVO_LOAD_AVERAGE) && (catalog[0].average != NULL)) {
     820
     821    off_t first  = 0;                    // first row in memory to write
     822    off_t start  = catalog[0].Naverage_off; // first disk row to write
     823    off_t Nrows  = catalog[0].Naverage - first;
    875824
    876825    assert (Nrows >= 0);
    877826    assert (first >= 0);
    878827    assert (first <= catalog[0].Naverage);
    879     assert (catalog[0].Naves_disk >= catalog[0].Naves_off);
    880 
    881     /* convert internal to external format */
    882     if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) {
    883       fprintf (stderr, "trouble converting format\n");
    884       goto failure;
    885     }
    886 
    887     if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) {
     828    assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off);
     829
     830    /* convert internal to external format : also results in a byte-swapped, scaled output
     831       table.  Note that ftable is newly allocated.
     832    */
     833   
     834    // if we are going to compress, we need to receive unswapped data --
     835    int swapFromNative = !output_is_compressed (start, Nrows, Naverage_disk_new, catalog->catcompress);
     836
     837    // convert to external table format
     838    if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, swapFromNative)) {
     839      fprintf (stderr, "trouble converting format\n");
     840      goto failure;
     841    }
     842
     843    if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) {
    888844      fprintf (stderr, "failure writing Average table\n");
    889845      goto failure;
    … …  
    894850    // even if we do not save the average table, we need to keep the header in sync
    895851    /* rewind file pointers and truncate (file is still open) */
    896     fseeko (catalog[0].f, 0, SEEK_SET);
     852    if (fseeko (catalog[0].f, 0, SEEK_SET)) { perror ("fseeko: "); exit (1); }
    897853
    898854    /* write table PHU header - always write this out */
    … …  
    905861
    906862  /*** Measure Table ***/
    907   if ((catalog[0].catflags & LOAD_MEAS) && (catalog[0].measure != NULL)) {
     863  if ((catalog[0].catflags & DVO_LOAD_MEASURE) && (catalog[0].measure != NULL)) {
     864
     865    off_t first  = 0;                    // first row in memory to write
     866    off_t start  = catalog[0].Nmeasure_off; // first disk row to write
     867    off_t Nrows  = catalog[0].Nmeasure - first;
     868
     869    assert (Nrows >= 0);
     870    assert (first >= 0);
     871    assert (first <= catalog[0].Nmeasure);
     872    assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
     873
     874    // if we are going to compress, we need to receive unswapped data --
     875    int swapFromNative = !output_is_compressed (start, Nrows, Nmeasure_disk_new, catalog->catcompress);
     876
     877    // convert to external table format
     878    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, swapFromNative)) {
     879      fprintf (stderr, "trouble converting format\n");
     880      goto failure;
     881    }
     882
     883    // write out Measure table
     884    catalog->measure_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api
     885    if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) {
     886      fprintf (stderr, "trouble writing Measure table\n");
     887      goto failure;
     888    }
     889    gfits_free_header (&header);
     890    gfits_free_table (&ftable);
     891  }
     892
     893  /*** Missing Table ***/
     894  if ((catalog[0].catflags & DVO_LOAD_MISSING) && (catalog[0].missing != NULL)) {
     895
     896    if (catalog[0].Nmissing_off != 0) {
     897      fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n");
     898      goto failure;
     899    }
     900
     901    // if we are going to compress, we need to receive unswapped data --
     902    int swapFromNative = !output_is_compressed (catalog[0].Nmissing_off, catalog[0].Nmissing, Nmissing_disk_new, catalog->catcompress);
     903
     904    // convert to external table format
     905    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, swapFromNative)) {
     906      fprintf (stderr, "trouble converting format\n");
     907      goto failure;
     908    }
     909
     910    // write out Missing table (must write out entire table)
     911    catalog->missing_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api
     912    if (!dvo_catalog_save_subcat (catalog[0].missing_catalog, &ftable, 0, catalog[0].Nmissing, catalog[0].Nmissing, catalog[0].Nmissing)) {
     913      fprintf (stderr, "trouble writing Missing Table\n");
     914      goto failure;
     915    }
     916    gfits_free_header (&header);
     917    gfits_free_table (&ftable);
     918  }
     919
     920  /*** Secfilt Table ***/
     921  if ((catalog[0].catflags & DVO_LOAD_SECFILT) && (catalog[0].secfilt != NULL)) {
     922
     923    off_t first  = 0;                    // first row in memory to write
     924    off_t start  = catalog[0].Nsecfilt_off; // first disk row to write
     925    off_t Nitems = catalog[0].Naverage*Nsecfilt;
     926    off_t Nrows  = Nitems - first;
     927
     928    assert (Nrows >= 0);
     929    assert (first >= 0);
     930    assert (first <= Nitems);
     931    assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off);
     932    // XXX check these for consistency...
     933
     934    // if we are going to compress, we need to receive unswapped data --
     935    int swapFromNative = !output_is_compressed (start, Nrows, Nsecfilt_disk_new, catalog->catcompress);
     936
     937    // convert to external table format
     938    if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, swapFromNative)) {
     939      fprintf (stderr, "trouble converting format\n");
     940      goto failure;
     941    }
     942
     943    // write out SecFilt table
     944    catalog->secfilt_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api
     945    if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) {
     946      fprintf (stderr, "failure writing SecFilt table\n");
     947      goto failure;
     948    }
     949    gfits_free_header (&header);
     950    gfits_free_table (&ftable);
     951  }
     952
     953  /*** Lensing Table ***/
     954  if ((catalog[0].catflags & DVO_LOAD_LENSING) && (catalog[0].lensing != NULL)) {
     955
     956    off_t first  = 0;                    // first row in memory to write
     957    off_t start  = catalog[0].Nlensing_off; // first disk row to write
     958    off_t Nrows  = catalog[0].Nlensing - first;
     959
     960    assert (Nrows >= 0);
     961    assert (first >= 0);
     962    assert (first <= catalog[0].Nlensing);
     963    assert (catalog[0].Nlensing_disk >= catalog[0].Nlensing_off);
     964
     965    // if we are going to compress, we need to receive unswapped data --
     966    int swapFromNative = !output_is_compressed (start, Nrows, Nlensing_disk_new, catalog->catcompress);
     967
     968    // convert to external table format
     969    if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, swapFromNative)) {
     970      fprintf (stderr, "trouble converting format\n");
     971      goto failure;
     972    }
     973
     974    // write out Lensing table
     975    catalog->lensing_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api
     976    if (!dvo_catalog_save_subcat (catalog[0].lensing_catalog, &ftable, start, Nrows, catalog[0].Nlensing_disk, Nlensing_disk_new)) {
     977      fprintf (stderr, "trouble writing Lensing table\n");
     978      goto failure;
     979    }
     980    gfits_free_header (&header);
     981    gfits_free_table (&ftable);
     982  }
     983
     984  /*** Lensobj Table ***/
     985  if ((catalog[0].catflags & DVO_LOAD_LENSOBJ) && (catalog[0].lensobj != NULL)) {
     986
     987    off_t first  = 0;                    // first row in memory to write
     988    off_t start  = catalog[0].Nlensobj_off; // first disk row to write
     989    off_t Nrows  = catalog[0].Nlensobj - first;
     990
     991    assert (Nrows >= 0);
     992    assert (first >= 0);
     993    assert (first <= catalog[0].Nlensobj);
     994    assert (catalog[0].Nlensobj_disk >= catalog[0].Nlensobj_off);
     995
     996    // if we are going to compress, we need to receive unswapped data --
     997    int swapFromNative = !output_is_compressed (start, Nrows, Nlensobj_disk_new, catalog->catcompress);
     998
     999    // convert to external table format
     1000    if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, swapFromNative)) {
     1001      fprintf (stderr, "trouble converting format\n");
     1002      goto failure;
     1003    }
     1004
     1005    // write out Lensobj table
     1006    catalog->lensobj_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api
     1007    if (!dvo_catalog_save_subcat (catalog[0].lensobj_catalog, &ftable, start, Nrows, catalog[0].Nlensobj_disk, Nlensobj_disk_new)) {
     1008      fprintf (stderr, "trouble writing Lensobj table\n");
     1009      goto failure;
     1010    }
     1011    gfits_free_header (&header);
     1012    gfits_free_table (&ftable);
     1013  }
     1014
     1015  /*** StarPar Table ***/
     1016  if ((catalog[0].catflags & DVO_LOAD_STARPAR) && (catalog[0].starpar != NULL)) {
     1017
     1018    off_t first  = 0;                    // first row in memory to write
     1019    off_t start  = catalog[0].Nstarpar_off; // first disk row to write
     1020    off_t Nrows  = catalog[0].Nstarpar - first;
     1021
     1022    assert (Nrows >= 0);
     1023    assert (first >= 0);
     1024    assert (first <= catalog[0].Nstarpar);
     1025    assert (catalog[0].Nstarpar_disk >= catalog[0].Nstarpar_off);
     1026
     1027    // if we are going to compress, we need to receive unswapped data --
     1028    int swapFromNative = !output_is_compressed (start, Nrows, Nstarpar_disk_new, catalog->catcompress);
     1029
     1030    // convert to external table format
     1031    if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, swapFromNative)) {
     1032      fprintf (stderr, "trouble converting format\n");
     1033      goto failure;
     1034    }
     1035
     1036    // write out StarPar table
     1037    catalog->starpar_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api
     1038    if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) {
     1039      fprintf (stderr, "trouble writing StarPar table\n");
     1040      goto failure;
     1041    }
     1042    gfits_free_header (&header);
     1043    gfits_free_table (&ftable);
     1044  }
     1045
     1046  /*** GalPhot Table ***/
     1047  if ((catalog[0].catflags & DVO_LOAD_GALPHOT) && (catalog[0].galphot != NULL)) {
     1048
     1049    off_t first  = 0;                    // first row in memory to write
     1050    off_t start  = catalog[0].Ngalphot_off; // first disk row to write
     1051    off_t Nrows  = catalog[0].Ngalphot - first;
     1052
     1053    assert (Nrows >= 0);
     1054    assert (first >= 0);
     1055    assert (first <= catalog[0].Ngalphot);
     1056    assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off);
     1057
     1058    // if we are going to compress, we need to receive unswapped data --
     1059    int swapFromNative = !output_is_compressed (start, Nrows, Ngalphot_disk_new, catalog->catcompress);
     1060
     1061    // convert to external table format
     1062    if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, swapFromNative)) {
     1063      fprintf (stderr, "trouble converting format\n");
     1064      goto failure;
     1065    }
     1066
     1067    // write out GalPhot table
     1068    catalog->galphot_catalog->catcompress = catalog->catcompress; // XXX this is a bit of a hack, should be done in an api
     1069    if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) {
     1070      fprintf (stderr, "trouble writing GalPhot table\n");
     1071      goto failure;
     1072    }
     1073    gfits_free_header (&header);
     1074    gfits_free_table (&ftable);
     1075  }
     1076
     1077  /* free temp storage */
     1078  if (primary != NULL) {
     1079    free (primary);
     1080    free (secfilt);
     1081  }
     1082  return (TRUE);
     1083
     1084 failure:
     1085  /* free temp storage */
     1086  gfits_free_header (&header);
     1087  gfits_free_table (&ftable);
     1088  if (primary != NULL) {
     1089    free (primary);
     1090    free (secfilt);
     1091  }
     1092  return (FALSE);
     1093}
     1094
     1095/* update_catalog_split only writes new lines to file. */
     1096int dvo_catalog_update_split (Catalog *catalog, char VERBOSE) {
     1097
     1098  off_t Nitems;
     1099  Header header;
     1100  FTable ftable;
     1101  SecFilt *primary, *secfilt;
     1102  int Nsecfilt;
     1103  off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalphot_disk_new;
     1104  off_t first, start, Nrows;
     1105
     1106  ftable.header = &header;
     1107  ftable.buffer = NULL;
     1108  header.buffer = NULL;
     1109
     1110  // skip empty catalogs: it is illegal to have Measures without corresponding Averages
     1111  Naverage_disk_new = MAX (catalog[0].Naverage_disk, catalog[0].Naverage + catalog[0].Naverage_off);
     1112  if (Naverage_disk_new == 0) {
     1113    if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n");
     1114    return (TRUE);
     1115  }
     1116
     1117  // for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary
     1118  switch (catalog[0].catformat) {
     1119    case DVO_FORMAT_ELIXIR: // special case for ELIXIR
     1120    case DVO_FORMAT_LONEOS: // special case for LONEOS
     1121      dvo_catalog_secfilt_to_primary (catalog, &primary, &secfilt, &Nsecfilt);
     1122      break;
     1123    default:
     1124      primary = NULL;
     1125      secfilt = catalog[0].secfilt;
     1126      Nsecfilt = catalog[0].Nsecfilt;
     1127      break;
     1128  }
     1129
     1130  Nmeasure_disk_new = MAX (catalog[0].Nmeasure_disk, catalog[0].Nmeasure + catalog[0].Nmeasure_off);
     1131  Nmissing_disk_new = MAX (catalog[0].Nmissing_disk, catalog[0].Nmissing + catalog[0].Nmissing_off);
     1132  Nsecfilt_disk_new = MAX (catalog[0].Nsecfilt_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecfilt_off);
     1133  Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off);
     1134  Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off);
     1135  Nstarpar_disk_new = MAX (catalog[0].Nstarpar_disk, catalog[0].Nstarpar + catalog[0].Nstarpar_off);
     1136  Ngalphot_disk_new = MAX (catalog[0].Ngalphot_disk, catalog[0].Ngalphot + catalog[0].Ngalphot_off);
     1137
     1138  /* make sure header is consistent with data */
     1139  gfits_modify (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  Naverage_disk_new);
     1140  gfits_modify (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  Nmeasure_disk_new);
     1141  gfits_modify (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  Nmissing_disk_new);
     1142  gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt);
     1143  gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1,  Nlensing_disk_new);
     1144  gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1,  Nlensobj_disk_new);
     1145  gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1,  Nstarpar_disk_new);
     1146  gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1,  Ngalphot_disk_new);
     1147
     1148  gfits_modify_alt (&catalog[0].header, "EXTEND",   "%t", 1, TRUE);
     1149  gfits_modify (&catalog[0].header, "OBJID",    "%d", 1, catalog[0].objID);
     1150
     1151  /* in split mode, we can save only part of the data */
     1152
     1153  /*** Average Table ***/
     1154  if (catalog[0].average != NULL) {
     1155
     1156    first  = catalog[0].Naverage_disk - catalog[0].Naverage_off; // first row to write (memory)
     1157    start  = catalog[0].Naverage_disk;                        // first row to write (disk)
     1158    Nrows  = catalog[0].Naverage - first;
     1159
     1160    assert (Nrows >= 0);
     1161    assert (first >= 0);
     1162    assert (first <= catalog[0].Naverage);
     1163    assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off);
     1164
     1165    /* convert internal to external format */
     1166    if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, TRUE)) {
     1167      fprintf (stderr, "trouble converting format\n");
     1168      goto failure;
     1169    }
     1170
     1171    if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) {
     1172      fprintf (stderr, "failure writing Average table\n");
     1173      goto failure;
     1174    }
     1175    gfits_free_header (&header);
     1176    gfits_free_table (&ftable);
     1177  } else {
     1178    // even if we do not save the average table, we need to keep the header in sync
     1179    /* rewind file pointers and truncate (file is still open) */
     1180    if (fseeko (catalog[0].f, 0, SEEK_SET)) { perror ("fseeko: "); exit (1); }
     1181
     1182    /* write table PHU header - always write this out */
     1183    /* XXX EAM : check if disk file size has changed */
     1184    if (!gfits_fwrite_header  (catalog[0].f, &catalog[0].header)) {
     1185      fprintf (stderr, "can't write primary header");
     1186      goto failure;
     1187    }
     1188  }
     1189
     1190  /*** Measure Table ***/
     1191  if ((catalog[0].catflags & DVO_LOAD_MEASURE) && (catalog[0].measure != NULL)) {
    9081192
    9091193    first  = 0;                    // first row in memory to write
    910     start  = catalog[0].Nmeas_off; // first disk row to write
     1194    start  = catalog[0].Nmeasure_off; // first disk row to write
    9111195    Nrows  = catalog[0].Nmeasure - first;
    9121196
    … …  
    9141198    assert (first >= 0);
    9151199    assert (first <= catalog[0].Nmeasure);
    916     assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off);
     1200    assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
    9171201
    9181202    // convert to external table format (note that the block above does not damage or free catalog.average)
    919     if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) {
     1203    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) {
    9201204      fprintf (stderr, "trouble converting format\n");
    9211205      goto failure;
    … …  
    9231207
    9241208    // write out Measure table
    925     if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) {
     1209    if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) {
    9261210      fprintf (stderr, "trouble writing Measure table\n");
    9271211      goto failure;
    … …  
    9321216
    9331217  /*** Missing Table ***/
    934   if ((catalog[0].catflags & LOAD_MISS) && (catalog[0].missing != NULL)) {
    935 
    936     if (catalog[0].Nmiss_off != 0) {
     1218  if ((catalog[0].catflags & DVO_LOAD_MISSING) && (catalog[0].missing != NULL)) {
     1219
     1220    if (catalog[0].Nmissing_off != 0) {
    9371221      fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n");
    9381222      goto failure;
    … …  
    9401224
    9411225    // convert to external table format
    942     if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) {
     1226    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) {
    9431227      fprintf (stderr, "trouble converting format\n");
    9441228      goto failure;
    … …  
    9551239
    9561240  /*** Secfilt Table ***/
    957   if ((catalog[0].catflags & LOAD_SECF) && (catalog[0].secfilt != NULL)) {
     1241  if ((catalog[0].catflags & DVO_LOAD_SECFILT) && (catalog[0].secfilt != NULL)) {
    9581242
    9591243    first  = 0;                    // first row in memory to write
    960     start  = catalog[0].Nsecf_off; // first disk row to write
    961     Nitems = catalog[0].Nsecf_mem;
     1244    start  = catalog[0].Nsecfilt_off; // first disk row to write
     1245    Nitems = catalog[0].Nsecfilt_mem;
    9621246    Nrows  = Nitems - first;
    9631247
    … …  
    9651249    assert (first >= 0);
    9661250    assert (first <= Nitems);
    967     assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off);
     1251    assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off);
    9681252    // XXX check these for consistency...
    9691253
    9701254    // convert to external table format
    971     SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat);
     1255    if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) {
     1256      fprintf (stderr, "trouble converting format\n");
     1257      goto failure;
     1258    }
    9721259
    9731260    // write out SecFilt table
    974     if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) {
     1261    if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) {
    9751262      fprintf (stderr, "failure writing SecFilt table\n");
    9761263      goto failure;
    … …  
    9811268
    9821269  /*** Lensing Table (optional, do not save if not loaded) ***/
    983   if ((catalog[0].catflags & LOAD_LENSING) && catalog[0].lensing_catalog->f && catalog[0].lensing) {
     1270  if ((catalog[0].catflags & DVO_LOAD_LENSING) && catalog[0].lensing_catalog->f && catalog[0].lensing) {
    9841271
    9851272    first  = 0;                    // first row in memory to write
    … …  
    9941281    if (catalog[0].Nlensing) {
    9951282      // convert to external table format (note that the block above does not damage or free catalog.average)
    996       if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) {
     1283      if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) {
    9971284        fprintf (stderr, "trouble converting format\n");
    9981285        goto failure;
    … …  
    10101297
    10111298  /*** Lensobj Table (optional, do not save if not loaded) ***/
    1012   if ((catalog[0].catflags & LOAD_LENSOBJ) && catalog[0].lensobj_catalog->f && catalog[0].lensobj) {
     1299  if ((catalog[0].catflags & DVO_LOAD_LENSOBJ) && catalog[0].lensobj_catalog->f && catalog[0].lensobj) {
    10131300
    10141301    first  = 0;                    // first row in memory to write
    … …  
    10231310    if (catalog[0].Nlensobj) {
    10241311      // convert to external table format (note that the block above does not damage or free catalog.average)
    1025       if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) {
     1312      if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) {
    10261313        fprintf (stderr, "trouble converting format\n");
    10271314        goto failure;
    … …  
    10391326
    10401327  /*** StarPar Table (optional, do not save if not loaded) ***/
    1041     if ((catalog[0].catflags & LOAD_STARPAR) && catalog[0].starpar_catalog->f && catalog[0].starpar) {
     1328  if ((catalog[0].catflags & DVO_LOAD_STARPAR) && catalog[0].starpar_catalog->f && catalog[0].starpar) {
    10421329
    10431330    first  = 0;                    // first row in memory to write
    … …  
    10521339    if (catalog[0].Nstarpar) {
    10531340      // convert to external table format (note that the block above does not damage or free catalog.average)
    1054       if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) {
     1341      if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) {
    10551342        fprintf (stderr, "trouble converting format\n");
    10561343        goto failure;
    … …  
    10671354  }
    10681355
     1356  /*** GalPhot Table (optional, do not save if not loaded) ***/
     1357  if ((catalog[0].catflags & DVO_LOAD_GALPHOT) && catalog[0].galphot_catalog->f && catalog[0].galphot) {
     1358
     1359    first  = 0;                    // first row in memory to write
     1360    start  = catalog[0].Ngalphot_off; // first disk row to write
     1361    Nrows  = catalog[0].Ngalphot - first;
     1362
     1363    assert (Nrows >= 0);
     1364    assert (first >= 0);
     1365    assert (first <= catalog[0].Ngalphot);
     1366    assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off);
     1367
     1368    if (catalog[0].Ngalphot) {
     1369      // convert to external table format (note that the block above does not damage or free catalog.average)
     1370      if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) {
     1371        fprintf (stderr, "trouble converting format\n");
     1372        goto failure;
     1373      }
     1374
     1375      // write out GalPhot table
     1376      if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) {
     1377        fprintf (stderr, "trouble writing GalPhot table\n");
     1378        goto failure;
     1379      }
     1380      gfits_free_header (&header);
     1381      gfits_free_table (&ftable);
     1382    }
     1383  }
     1384
    10691385  /* free temp storage */
    10701386  if (primary != NULL) {
    … …  
    10741390  return (TRUE);
    10751391
    1076 failure:
     1392 failure:
    10771393  /* free temp storage */
    10781394  gfits_free_header (&header);
    … …  
    10881404int dvo_catalog_save_split_complete (Catalog *catalog, char VERBOSE) {
    10891405
    1090   off_t Nitems;
    10911406  Header header;
    10921407  FTable ftable;
    10931408  SecFilt *primary, *secfilt;
    10941409  int Nsecfilt;
    1095   off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new;
     1410  off_t Naverage_disk_new, Nmeasure_disk_new, Nmissing_disk_new, Nsecfilt_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new, Ngalphot_disk_new;
    10961411  off_t first, start, Nrows;
    10971412
    … …  
    11021417 
    11031418  // skip empty catalogs: it is illegal to have Measures without corresponding Averages
    1104   if (catalog[0].Naves_off > 0) {
     1419  if (catalog[0].Naverage_off > 0) {
    11051420    fprintf (stderr, "ERROR: only partial catalog (Average) was loaded\n");
    11061421    goto failure;
    11071422  }
    1108   if (catalog[0].Nmeas_off > 0) {
     1423  if (catalog[0].Nmeasure_off > 0) {
    11091424    fprintf (stderr, "ERROR: only partial catalog (Measure) was loaded\n");
    11101425    goto failure;
    11111426  }
    1112   if (catalog[0].Nmiss_off > 0) {
     1427  if (catalog[0].Nmissing_off > 0) {
    11131428    fprintf (stderr, "ERROR: only partial catalog (Missing) was loaded\n");
    11141429    goto failure;
    11151430  }
    1116   if (catalog[0].Nsecf_off > 0) {
     1431  if (catalog[0].Nsecfilt_off > 0) {
    11171432    fprintf (stderr, "ERROR: only partial catalog (Secfilt) was loaded\n");
    11181433    goto failure;
    … …  
    11301445    goto failure;
    11311446  }
    1132 
    1133   Naves_disk_new = catalog[0].Naverage;
    1134   if (Naves_disk_new == 0) {
     1447  if (catalog[0].Ngalphot_off > 0) {
     1448    fprintf (stderr, "ERROR: only partial catalog (GalPhot) was loaded\n");
     1449    goto failure;
     1450  }
     1451
     1452  Naverage_disk_new = catalog[0].Naverage;
     1453  if (Naverage_disk_new == 0) {
    11351454    if (VERBOSE) fprintf (stderr, "resulting catalog is empty; delete it\n");
    11361455    // unlink ();
    … …  
    11511470  }
    11521471
    1153   Nmeas_disk_new = catalog[0].Nmeasure;
    1154   Nmiss_disk_new = catalog[0].Nmissing;
    1155   Nsecf_disk_new = catalog[0].Naverage*Nsecfilt;
     1472  Nmeasure_disk_new = catalog[0].Nmeasure;
     1473  Nmissing_disk_new = catalog[0].Nmissing;
     1474  Nsecfilt_disk_new = catalog[0].Naverage*Nsecfilt;
    11561475  Nlensing_disk_new = catalog[0].Nlensing;
    11571476  Nlensobj_disk_new = catalog[0].Nlensobj;
    11581477  Nstarpar_disk_new = catalog[0].Nstarpar;
     1478  Ngalphot_disk_new = catalog[0].Ngalphot;
    11591479
    11601480  /* make sure header is consistent with data */
    1161   gfits_modify (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  Naves_disk_new);
    1162   gfits_modify (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  Nmeas_disk_new);
    1163   gfits_modify (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  Nmiss_disk_new);
     1481  gfits_modify (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  Naverage_disk_new);
     1482  gfits_modify (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  Nmeasure_disk_new);
     1483  gfits_modify (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  Nmissing_disk_new);
    11641484  gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt);
    11651485  gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1,  Nlensing_disk_new);
    11661486  gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1,  Nlensobj_disk_new);
    11671487  gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1,  Nstarpar_disk_new);
     1488  gfits_modify (&catalog[0].header, "NGALPHOT", OFF_T_FMT, 1, Ngalphot_disk_new);
    11681489  gfits_modify_alt (&catalog[0].header, "EXTEND",   "%t", 1, TRUE);
    11691490  gfits_modify (&catalog[0].header, "OBJID",    "%d", 1, catalog[0].objID);
    … …  
    11721493
    11731494  /*** Average Table ***/
    1174   if ((catalog[0].catflags & LOAD_AVES) && (catalog[0].average != NULL)) {
     1495  if ((catalog[0].catflags & DVO_LOAD_AVERAGE) && (catalog[0].average != NULL)) {
    11751496
    11761497    first  = 0;                    // first row in memory to write
    1177     start  = catalog[0].Naves_off; // first disk row to write
     1498    start  = catalog[0].Naverage_off; // first disk row to write
    11781499    Nrows  = catalog[0].Naverage - first;
    11791500
    … …  
    11811502    assert (first >= 0);
    11821503    assert (first <= catalog[0].Naverage);
    1183     assert (catalog[0].Naves_disk >= catalog[0].Naves_off);
     1504    assert (catalog[0].Naverage_disk >= catalog[0].Naverage_off);
    11841505
    11851506    /* convert internal to external format */
    1186     if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) {
    1187       fprintf (stderr, "trouble converting format\n");
    1188       goto failure;
    1189     }
    1190 
    1191     if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) {
     1507    if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary, TRUE)) {
     1508      fprintf (stderr, "trouble converting format\n");
     1509      goto failure;
     1510    }
     1511
     1512    if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naverage_disk, Naverage_disk_new)) {
    11921513      fprintf (stderr, "failure writing Average table\n");
    11931514      goto failure;
    … …  
    11981519    // even if we do not save the average table, we need to keep the header in sync
    11991520    /* rewind file pointers and truncate (file is still open) */
    1200     fseeko (catalog[0].f, 0, SEEK_SET);
     1521    if (fseeko (catalog[0].f, 0, SEEK_SET)) { perror ("fseeko: "); exit (1); }
    12011522
    12021523    /* write table PHU header - always write this out */
    … …  
    12091530
    12101531  /*** Measure Table ***/
    1211   if ((catalog[0].catflags & LOAD_MEAS) && (catalog[0].measure != NULL)) {
    1212 
    1213     first  = 0;                    // first row in memory to write
    1214     start  = catalog[0].Nmeas_off; // first disk row to write
     1532  if (catalog[0].measure != NULL) {
     1533
     1534    first  = catalog[0].Nmeasure_disk - catalog[0].Nmeasure_off;  // first row in memory to write
     1535    start  = catalog[0].Nmeasure_disk; // first disk row to write
    12151536    Nrows  = catalog[0].Nmeasure - first;
    12161537
    … …  
    12181539    assert (first >= 0);
    12191540    assert (first <= catalog[0].Nmeasure);
    1220     assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off);
     1541    assert (catalog[0].Nmeasure_disk >= catalog[0].Nmeasure_off);
    12211542
    12221543    // convert to external table format
    1223     if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) {
     1544    // XXX does catalog.measure have averef correctly set up?
     1545    if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat, TRUE)) {
    12241546      fprintf (stderr, "trouble converting format\n");
    12251547      goto failure;
    … …  
    12271549
    12281550    // write out Measure table
    1229     if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) {
     1551    if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeasure_disk, Nmeasure_disk_new)) {
    12301552      fprintf (stderr, "trouble writing Measure table\n");
    12311553      goto failure;
    … …  
    12351557  }
    12361558
     1559  /* missing table CANNOT be written unsorted, thus it is always written
     1560     out in full */
     1561
    12371562  /*** Missing Table ***/
    1238   if ((catalog[0].catflags & LOAD_MISS) && (catalog[0].missing != NULL)) {
    1239 
    1240     if (catalog[0].Nmiss_off != 0) {
     1563
     1564  if (catalog[0].missing != NULL) {
     1565
     1566    if (catalog[0].Nmissing_off != 0) {
    12411567      fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n");
    12421568      goto failure;
    … …  
    12441570
    12451571    // convert to external table format
    1246     if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) {
     1572    if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing, TRUE)) {
    12471573      fprintf (stderr, "trouble converting format\n");
    12481574      goto failure;
    … …  
    12591585
    12601586  /*** Secfilt Table ***/
    1261   if ((catalog[0].catflags & LOAD_SECF) && (catalog[0].secfilt != NULL)) {
    1262 
    1263     first  = 0;                    // first row in memory to write
    1264     start  = catalog[0].Nsecf_off; // first disk row to write
    1265     Nitems = catalog[0].Naverage*Nsecfilt;
    1266     Nrows  = Nitems - first;
    1267 
    1268     assert (Nrows >= 0);
    1269     assert (first >= 0);
    1270     assert (first <= Nitems);
    1271     assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off);
    1272     // XXX check these for consistency...
     1587  if (catalog[0].secfilt != NULL) {
     1588
     1589    first  = catalog[0].Nsecfilt_disk - catalog[0].Nsecfilt_off;  // first row in memory to write
     1590    start  = catalog[0].Nsecfilt_disk; // first disk row to write
     1591    Nrows  = catalog[0].Nsecfilt_mem - first;
     1592
     1593    assert (Nrows >= 0);
     1594    assert (first >= 0);
     1595    assert (catalog[0].Nsecfilt_disk >= catalog[0].Nsecfilt_off);
    12731596
    12741597    // convert to external table format
    1275     SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat);
     1598    if (!SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat, TRUE)) {
     1599      fprintf (stderr, "trouble converting format\n");
     1600      goto failure;
     1601    }
    12761602
    12771603    // write out SecFilt table
    1278     if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) {
     1604    if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecfilt_disk, Nsecfilt_disk_new)) {
    12791605      fprintf (stderr, "failure writing SecFilt table\n");
    12801606      goto failure;
    … …  
    12851611
    12861612  /*** Lensing Table ***/
    1287   if ((catalog[0].catflags & LOAD_LENSING) && (catalog[0].lensing != NULL)) {
    1288 
    1289     first  = 0;                    // first row in memory to write
    1290     start  = catalog[0].Nlensing_off; // first disk row to write
     1613  if (catalog[0].lensing != NULL) {
     1614
     1615    first  = catalog[0].Nlensing_disk - catalog[0].Nlensing_off;  // first row in memory to write
     1616    start  = catalog[0].Nlensing_disk; // first disk row to write
    12911617    Nrows  = catalog[0].Nlensing - first;
    12921618
    … …  
    12971623
    12981624    // convert to external table format
    1299     if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) {
     1625    // XXX does catalog.lensing have averef correctly set up?
     1626    if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat, TRUE)) {
    13001627      fprintf (stderr, "trouble converting format\n");
    13011628      goto failure;
    … …  
    13121639
    13131640  /*** Lensobj Table ***/
    1314   if ((catalog[0].catflags & LOAD_LENSOBJ) && (catalog[0].lensobj != NULL)) {
    1315 
    1316     first  = 0;                    // first row in memory to write
    1317     start  = catalog[0].Nlensobj_off; // first disk row to write
     1641  if (catalog[0].lensobj != NULL) {
     1642
     1643    first  = catalog[0].Nlensobj_disk - catalog[0].Nlensobj_off;  // first row in memory to write
     1644    start  = catalog[0].Nlensobj_disk; // first disk row to write
    13181645    Nrows  = catalog[0].Nlensobj - first;
    13191646
    … …  
    13241651
    13251652    // convert to external table format
    1326     if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) {
     1653    // XXX does catalog.lensobj have averef correctly set up?
     1654    if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat, TRUE)) {
    13271655      fprintf (stderr, "trouble converting format\n");
    13281656      goto failure;
    … …  
    13391667
    13401668  /*** StarPar Table ***/
    1341   if ((catalog[0].catflags & LOAD_STARPAR) && (catalog[0].starpar != NULL)) {
    1342 
    1343     first  = 0;                    // first row in memory to write
    1344     start  = catalog[0].Nstarpar_off; // first disk row to write
     1669  if (catalog[0].starpar != NULL) {
     1670
     1671    first  = catalog[0].Nstarpar_disk - catalog[0].Nstarpar_off;  // first row in memory to write
     1672    start  = catalog[0].Nstarpar_disk; // first disk row to write
    13451673    Nrows  = catalog[0].Nstarpar - first;
    13461674
    … …  
    13511679
    13521680    // convert to external table format
    1353     if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) {
     1681    // XXX does catalog.starpar have averef correctly set up?
     1682    if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat, TRUE)) {
    13541683      fprintf (stderr, "trouble converting format\n");
    13551684      goto failure;
    … …  
    13591688    if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) {
    13601689      fprintf (stderr, "trouble writing StarPar table\n");
     1690      goto failure;
     1691    }
     1692    gfits_free_header (&header);
     1693    gfits_free_table (&ftable);
     1694  }
     1695
     1696  /*** GalPhot Table ***/
     1697  if (catalog[0].galphot != NULL) {
     1698
     1699    first  = catalog[0].Ngalphot_disk - catalog[0].Ngalphot_off;  // first row in memory to write
     1700    start  = catalog[0].Ngalphot_disk; // first disk row to write
     1701    Nrows  = catalog[0].Ngalphot - first;
     1702
     1703    assert (Nrows >= 0);
     1704    assert (first >= 0);
     1705    assert (first <= catalog[0].Ngalphot);
     1706    assert (catalog[0].Ngalphot_disk >= catalog[0].Ngalphot_off);
     1707
     1708    // convert to external table format
     1709    // XXX does catalog.galphot have averef correctly set up?
     1710    if (!GalPhotToFtable (&ftable, &catalog[0].galphot[first], Nrows, catalog[0].catformat, TRUE)) {
     1711      fprintf (stderr, "trouble converting format\n");
     1712      goto failure;
     1713    }
     1714
     1715    // write out GalPhot table
     1716    if (!dvo_catalog_save_subcat (catalog[0].galphot_catalog, &ftable, start, Nrows, catalog[0].Ngalphot_disk, Ngalphot_disk_new)) {
     1717      fprintf (stderr, "trouble writing GalPhot table\n");
    13611718      goto failure;
    13621719    }
    … …  
    13701727    free (secfilt);
    13711728  }
     1729
    13721730  return (TRUE);
    13731731
    1374 failure:
    1375   /* free temp storage */
    1376   gfits_free_header (&header);
    1377   gfits_free_table (&ftable);
    1378   if (primary != NULL) {
    1379     free (primary);
    1380     free (secfilt);
    1381   }
    1382   return (FALSE);
    1383 }
    1384 
    1385 /* update_catalog_split only writes new lines to file. */
    1386 int dvo_catalog_update_split (Catalog *catalog, char VERBOSE) {
    1387 
    1388   Header header;
    1389   FTable ftable;
    1390   SecFilt *primary, *secfilt;
    1391   int Nsecfilt;
    1392   off_t Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new, Nlensing_disk_new, Nlensobj_disk_new, Nstarpar_disk_new;
    1393   off_t first, start, Nrows;
    1394 
    1395   ftable.header = &header;
    1396   ftable.buffer = NULL;
    1397   header.buffer = NULL;
    1398 
    1399   // skip empty catalogs: it is illegal to have Measures without corresponding Averages
    1400   Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off);
    1401   if (Naves_disk_new == 0) {
    1402     if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n");
    1403     return (TRUE);
    1404   }
    1405 
    1406   // for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary
    1407   switch (catalog[0].catformat) {
    1408     case DVO_FORMAT_ELIXIR: // special case for ELIXIR
    1409     case DVO_FORMAT_LONEOS: // special case for LONEOS
    1410       dvo_catalog_secfilt_to_primary (catalog, &primary, &secfilt, &Nsecfilt);
    1411       break;
    1412     default:
    1413       primary = NULL;
    1414       secfilt = catalog[0].secfilt;
    1415       Nsecfilt = catalog[0].Nsecfilt;
    1416       break;
    1417   }
    1418 
    1419   Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off);
    1420   Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off);
    1421   Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off);
    1422   Nlensing_disk_new = MAX (catalog[0].Nlensing_disk, catalog[0].Nlensing + catalog[0].Nlensing_off);
    1423   Nlensobj_disk_new = MAX (catalog[0].Nlensobj_disk, catalog[0].Nlensobj + catalog[0].Nlensobj_off);
    1424   Nstarpar_disk_new = MAX (catalog[0].Nstarpar_disk, catalog[0].Nstarpar + catalog[0].Nstarpar_off);
    1425 
    1426   /* make sure header is consistent with data */
    1427   gfits_modify (&catalog[0].header, "NSTARS",   OFF_T_FMT, 1,  Naves_disk_new);
    1428   gfits_modify (&catalog[0].header, "NMEAS",    OFF_T_FMT, 1,  Nmeas_disk_new);
    1429   gfits_modify (&catalog[0].header, "NMISS",    OFF_T_FMT, 1,  Nmiss_disk_new);
    1430   gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt);
    1431   gfits_modify (&catalog[0].header, "NLENSING", OFF_T_FMT, 1,  Nlensing_disk_new);
    1432   gfits_modify (&catalog[0].header, "NLENSOBJ", OFF_T_FMT, 1,  Nlensobj_disk_new);
    1433   gfits_modify (&catalog[0].header, "NSTARPAR", OFF_T_FMT, 1,  Nstarpar_disk_new);
    1434   gfits_modify_alt (&catalog[0].header, "EXTEND",   "%t", 1, TRUE);
    1435   gfits_modify (&catalog[0].header, "OBJID",    "%d", 1, catalog[0].objID);
    1436 
    1437   /* in split mode, we can save only part of the data */
    1438 
    1439   /*** Average Table ***/
    1440   if (catalog[0].average != NULL) {
    1441 
    1442     first  = catalog[0].Naves_disk - catalog[0].Naves_off; // first row to write (memory)
    1443     start  = catalog[0].Naves_disk;                        // first row to write (disk)
    1444     Nrows  = catalog[0].Naverage - first;
    1445 
    1446     assert (Nrows >= 0);
    1447     assert (first >= 0);
    1448     assert (first <= catalog[0].Naverage);
    1449     assert (catalog[0].Naves_disk >= catalog[0].Naves_off);
    1450 
    1451     /* convert internal to external format */
    1452     if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) {
    1453       fprintf (stderr, "trouble converting format\n");
    1454       goto failure;
    1455     }
    1456 
    1457     if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) {
    1458       fprintf (stderr, "failure writing Average table\n");
    1459       goto failure;
    1460     }
    1461     gfits_free_header (&header);
    1462     gfits_free_table (&ftable);
    1463   } else {
    1464     // even if we do not save the average table, we need to keep the header in sync
    1465     /* rewind file pointers and truncate (file is still open) */
    1466     fseeko (catalog[0].f, 0, SEEK_SET);
    1467 
    1468     /* write table PHU header - always write this out */
    1469     /* XXX EAM : check if disk file size has changed */
    1470     if (!gfits_fwrite_header  (catalog[0].f, &catalog[0].header)) {
    1471       fprintf (stderr, "can't write primary header");
    1472       goto failure;
    1473     }
    1474   }
    1475 
    1476   /*** Measure Table ***/
    1477   if (catalog[0].measure != NULL) {
    1478 
    1479     first  = catalog[0].Nmeas_disk - catalog[0].Nmeas_off;  // first row in memory to write
    1480     start  = catalog[0].Nmeas_disk; // first disk row to write
    1481     Nrows  = catalog[0].Nmeasure - first;
    1482 
    1483     assert (Nrows >= 0);
    1484     assert (first >= 0);
    1485     assert (first <= catalog[0].Nmeasure);
    1486     assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off);
    1487 
    1488     // convert to external table format
    1489     // XXX does catalog.measure have averef correctly set up?
    1490     if (!MeasureToFtable (&ftable, catalog[0].average, &catalog[0].measure[first], Nrows, catalog[0].catformat)) {
    1491       fprintf (stderr, "trouble converting format\n");
    1492       goto failure;
    1493     }
    1494 
    1495     // write out Measure table
    1496     if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) {
    1497       fprintf (stderr, "trouble writing Measure table\n");
    1498       goto failure;
    1499     }
    1500     gfits_free_header (&header);
    1501     gfits_free_table (&ftable);
    1502   }
    1503 
    1504   /* missing table CANNOT be written unsorted, thus it is always written
    1505      out in full */
    1506 
    1507   /*** Missing Table ***/
    1508 
    1509   if (catalog[0].missing != NULL) {
    1510 
    1511     if (catalog[0].Nmiss_off != 0) {
    1512       fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n");
    1513       goto failure;
    1514     }
    1515 
    1516     // convert to external table format
    1517     if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) {
    1518       fprintf (stderr, "trouble converting format\n");
    1519       goto failure;
    1520     }
    1521 
    1522     // write out Missing table (must write out entire table)
    1523     if (!dvo_catalog_save_subcat (catalog[0].missing_catalog, &ftable, 0, catalog[0].Nmissing, catalog[0].Nmissing, catalog[0].Nmissing)) {
    1524       fprintf (stderr, "trouble writing Missing Table\n");
    1525       goto failure;
    1526     }
    1527     gfits_free_header (&header);
    1528     gfits_free_table (&ftable);
    1529   }
    1530 
    1531   /*** Secfilt Table ***/
    1532   if (catalog[0].secfilt != NULL) {
    1533 
    1534     first  = catalog[0].Nsecf_disk - catalog[0].Nsecf_off;  // first row in memory to write
    1535     start  = catalog[0].Nsecf_disk; // first disk row to write
    1536     Nrows  = catalog[0].Nsecf_mem - first;
    1537 
    1538     assert (Nrows >= 0);
    1539     assert (first >= 0);
    1540     assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off);
    1541 
    1542     // convert to external table format
    1543     SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat);
    1544 
    1545     // write out SecFilt table
    1546     if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) {
    1547       fprintf (stderr, "failure writing SecFilt table\n");
    1548       goto failure;
    1549     }
    1550     gfits_free_header (&header);
    1551     gfits_free_table (&ftable);
    1552   }
    1553 
    1554   /*** Lensing Table ***/
    1555   if (catalog[0].lensing != NULL) {
    1556 
    1557     first  = catalog[0].Nlensing_disk - catalog[0].Nlensing_off;  // first row in memory to write
    1558     start  = catalog[0].Nlensing_disk; // first disk row to write
    1559     Nrows  = catalog[0].Nlensing - first;
    1560 
    1561     assert (Nrows >= 0);
    1562     assert (first >= 0);
    1563     assert (first <= catalog[0].Nlensing);
    1564     assert (catalog[0].Nlensing_disk >= catalog[0].Nlensing_off);
    1565 
    1566     // convert to external table format
    1567     // XXX does catalog.lensing have averef correctly set up?
    1568     if (!LensingToFtable (&ftable, &catalog[0].lensing[first], Nrows, catalog[0].catformat)) {
    1569       fprintf (stderr, "trouble converting format\n");
    1570       goto failure;
    1571     }
    1572 
    1573     // write out Lensing table
    1574     if (!dvo_catalog_save_subcat (catalog[0].lensing_catalog, &ftable, start, Nrows, catalog[0].Nlensing_disk, Nlensing_disk_new)) {
    1575       fprintf (stderr, "trouble writing Lensing table\n");
    1576       goto failure;
    1577     }
    1578     gfits_free_header (&header);
    1579     gfits_free_table (&ftable);
    1580   }
    1581 
    1582   /*** Lensobj Table ***/
    1583   if (catalog[0].lensobj != NULL) {
    1584 
    1585     first  = catalog[0].Nlensobj_disk - catalog[0].Nlensobj_off;  // first row in memory to write
    1586     start  = catalog[0].Nlensobj_disk; // first disk row to write
    1587     Nrows  = catalog[0].Nlensobj - first;
    1588 
    1589     assert (Nrows >= 0);
    1590     assert (first >= 0);
    1591     assert (first <= catalog[0].Nlensobj);
    1592     assert (catalog[0].Nlensobj_disk >= catalog[0].Nlensobj_off);
    1593 
    1594     // convert to external table format
    1595     // XXX does catalog.lensobj have averef correctly set up?
    1596     if (!LensobjToFtable (&ftable, &catalog[0].lensobj[first], Nrows, catalog[0].catformat)) {
    1597       fprintf (stderr, "trouble converting format\n");
    1598       goto failure;
    1599     }
    1600 
    1601     // write out Lensobj table
    1602     if (!dvo_catalog_save_subcat (catalog[0].lensobj_catalog, &ftable, start, Nrows, catalog[0].Nlensobj_disk, Nlensobj_disk_new)) {
    1603       fprintf (stderr, "trouble writing Lensobj table\n");
    1604       goto failure;
    1605     }
    1606     gfits_free_header (&header);
    1607     gfits_free_table (&ftable);
    1608   }
    1609 
    1610   /*** StarPar Table ***/
    1611   if (catalog[0].starpar != NULL) {
    1612 
    1613     first  = catalog[0].Nstarpar_disk - catalog[0].Nstarpar_off;  // first row in memory to write
    1614     start  = catalog[0].Nstarpar_disk; // first disk row to write
    1615     Nrows  = catalog[0].Nstarpar - first;
    1616 
    1617     assert (Nrows >= 0);
    1618     assert (first >= 0);
    1619     assert (first <= catalog[0].Nstarpar);
    1620     assert (catalog[0].Nstarpar_disk >= catalog[0].Nstarpar_off);
    1621 
    1622     // convert to external table format
    1623     // XXX does catalog.starpar have averef correctly set up?
    1624     if (!StarParToFtable (&ftable, &catalog[0].starpar[first], Nrows, catalog[0].catformat)) {
    1625       fprintf (stderr, "trouble converting format\n");
    1626       goto failure;
    1627     }
    1628 
    1629     // write out StarPar table
    1630     if (!dvo_catalog_save_subcat (catalog[0].starpar_catalog, &ftable, start, Nrows, catalog[0].Nstarpar_disk, Nstarpar_disk_new)) {
    1631       fprintf (stderr, "trouble writing StarPar table\n");
    1632       goto failure;
    1633     }
    1634     gfits_free_header (&header);
    1635     gfits_free_table (&ftable);
    1636   }
    1637 
    1638   /* free temp storage */
    1639   if (primary != NULL) {
    1640     free (primary);
    1641     free (secfilt);
    1642   }
    1643 
    1644   return (TRUE);
    1645 
    1646 failure:
     1732 failure:
    16471733  /* free temp storage */
    16481734  if (primary != NULL) {
    … …  
    16621748
    16631749/* XXX EAM : update is not efficient.  MeasureToFtable should only
    1664    convert the new rows (Nmeas_disk to Nmeasure). the resulting
     1750   convert the new rows (Nmeasure_disk to Nmeasure). the resulting
    16651751   table represents the end rows of the ftable.  we need to define
    16661752   the vtable based on the ftable, but with Ny = Nmeasure */ 
    … …  
    16681754
    16691755
    1670     // * convert to an ftable
    1671     // * optionally write the PHU header/matrix
    1672     // * advance to the start of the output data block:
    1673     // ** Nx * catalog[0].Nmeas_off
    1674     // * write out the ftable data block
    1675     // * if Nmeas_off + Nmeasure >= Nmeas_disk, update padding
    1676     // ** start = Nmeas_off
    1677     // ** Nrows = Nmeasure
    1678 
     1756// * convert to an ftable
     1757// * optionally write the PHU header/matrix
     1758// * advance to the start of the output data block:
     1759// ** Nx * catalog[0].Nmeasure_off
     1760// * write out the ftable data block
     1761// * if Nmeasure_off + Nmeasure >= Nmeasure_disk, update padding
     1762// ** start = Nmeasure_off
     1763// ** Nrows = Nmeasure
     1764

  • trunk/Ohana/src/libdvo/src/dvo_convert.c

    r38153 r38441  
    1111    constructions below **/
    1212
    13 DVOTableFormat FtableGetFormat (FTable *ftable) {
    14 
    15   DVOTableFormat format;
     13DVOCatFormat FtableGetFormat (FTable *ftable) {
     14
     15  DVOCatFormat format;
    1616  char extname[80];
    1717
    … …  
    5252/*** Average / FTable conversion functions ***/
    5353
    54 Average *FtableToAverage (FTable *ftable, off_t *Naverage, char *format, SecFilt **primary) {
     54Average *FtableToAverage (FTable *ftable, off_t *Naverage, DVOCatFormat *format, SecFilt **primary, char nativeBytes) {
    5555
    5656  Average *average;
    … …  
    9898  if (!strcmp (extname, NAME)) { \
    9999    Average_##TYPE *tmpAverage; \
    100     tmpAverage = gfits_table_get_Average_##TYPE (ftable, Naverage, NULL); \
     100      tmpAverage = gfits_table_get_Average_##TYPE (ftable, Naverage, NULL, &nativeBytes); \
    101101    if (!tmpAverage) { \
    102102      fprintf (stderr, "ERROR: failed to read averages\n"); \
    … …  
    109109
    110110  if (!strcmp (extname, "DVO_AVERAGE")) {
    111     average = gfits_table_get_Average (ftable, Naverage, NULL);
     111    average = gfits_table_get_Average (ftable, Naverage, NULL, &nativeBytes);
    112112    if (!average) {
    113113      fprintf (stderr, "ERROR: failed to read averages\n");
    … …  
    139139}
    140140
    141 int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, char format, SecFilt *primary) {
    142 
    143 # define FORMAT_CASE(FORMAT, TYPE) \
    144     case DVO_FORMAT_##FORMAT: { \
    145       Average_##TYPE *tmpAverage; \
     141int AverageToFtable (FTable *ftable, Average *average, off_t Naverage, DVOCatFormat format, SecFilt *primary, int swapFromNative) {
     142 
     143# define FORMAT_CASE(FORMAT, TYPE)              \
     144  case DVO_FORMAT_##FORMAT: {                                           \
     145    Average_##TYPE *tmpAverage;                                         \
    146146      tmpAverage = AverageInternalTo_##TYPE (average, Naverage, primary); \
    147       gfits_table_set_Average_##TYPE (ftable, tmpAverage, Naverage); \
    148       free (tmpAverage); \
    149       break; }
    150 
     147        gfits_table_set_Average_##TYPE (ftable, tmpAverage, Naverage, swapFromNative); \
     148          free (tmpAverage);                                            \
     149          break; }
     150 
    151151  /* convert from the internal format */
    152152  switch (format) {
    153153    case DVO_FORMAT_INTERNAL: {
    154       gfits_table_set_Average (ftable, average, Naverage);
     154      gfits_table_set_Average (ftable, average, Naverage, swapFromNative);
    155155      break; }
    156156
    … …  
    180180
    181181// FtableToMeasure needs the Average since old formats stored measure.dR,dD only
    182 Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, char *format) {
     182// other FtableToFOO conversions accept the average argument for macro construction
     183Measure *FtableToMeasure (FTable *ftable, Average *average, off_t *Nmeasure, DVOCatFormat *format, char nativeBytes) {
    183184
    184185  Measure *measure;
    … …  
    226227  if (!strcmp (extname, NAME)) {                                        \
    227228    Measure_##TYPE *tmpMeasure;                                         \
    228     tmpMeasure = gfits_table_get_Measure_##TYPE (ftable, Nmeasure, NULL); \
    229     if (!tmpMeasure) {                                                  \
    230       fprintf (stderr, "ERROR: failed to read measures\n");             \
    231       exit (2);                                                         \
    232     }                                                                   \
    233     myAssert (ABS_COORDS || average, "conversion to internal needs average table"); \
     229      tmpMeasure = gfits_table_get_Measure_##TYPE (ftable, Nmeasure, NULL, &nativeBytes); \
     230        if (!tmpMeasure) {                                              \
     231          fprintf (stderr, "ERROR: failed to read measures\n");         \
     232          exit (2);                                                     \
     233        }                                                               \
     234        myAssert (ABS_COORDS || average, "conversion to internal needs average table"); \
    234235    measure = Measure_##TYPE##_ToInternal (average, tmpMeasure, *Nmeasure); \
    235236    free (tmpMeasure);                                                  \
    … …  
    238239
    239240  if (!strcmp (extname, "DVO_MEASURE")) {
    240     measure = gfits_table_get_Measure (ftable, Nmeasure, NULL);
     241    measure = gfits_table_get_Measure (ftable, Nmeasure, NULL, &nativeBytes);
    241242    if (!measure) {
    242243      fprintf (stderr, "ERROR: failed to read measures\n");
    … …  
    269270
    270271// MeasureToFtable needs the Average since old formats stored measure.dR,dD only
    271 int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, char format) {
     272int MeasureToFtable (FTable *ftable, Average *average, Measure *measure, off_t Nmeasure, DVOCatFormat format, int swapFromNative) {
    272273
    273274# define FORMAT_CASE(FORMAT, TYPE, ABS_COORDS)          \
    … …  
    276277      myAssert (ABS_COORDS || average, "conversion from internal needs average table"); \
    277278      tmpMeasure = MeasureInternalTo_##TYPE (average, measure, Nmeasure); \
    278       gfits_table_set_Measure_##TYPE (ftable, tmpMeasure, Nmeasure); \
     279        gfits_table_set_Measure_##TYPE (ftable, tmpMeasure, Nmeasure, swapFromNative); \
    279280      free (tmpMeasure); \
    280281      break; }
    … …  
    283284  switch (format) {
    284285    case DVO_FORMAT_INTERNAL: {
    285       gfits_table_set_Measure (ftable, measure, Nmeasure);
     286      gfits_table_set_Measure (ftable, measure, Nmeasure, swapFromNative);
    286287      break; }
    287288
    … …  
    308309}
    309310
     311/*** Missing / FTable conversion functions ***/
     312
     313Missing *FtableToMissing (FTable *ftable, Average *average, off_t *Nmissing, DVOCatFormat *format, char nativeBytes) {
     314
     315  Missing *missing;
     316  char extname[80];
     317
     318  /* convert to the internal format */
     319  if (!gfits_scan (ftable[0].header, "EXTNAME", "%s", 1, extname)) {
     320    fprintf (stderr, "EXTNAME missing for missing table\n");
     321    return (FALSE);
     322  }
     323
     324# define SKIPPING_FORMAT(NAME, FORMAT, TYPE)            \
     325  if (!strcmp (extname, NAME)) {                                        \
     326    fprintf (stderr, "ERROR: format %s not defined for missing, skipping\n", NAME); \
     327    *Nmissing = 0; \
     328    return NULL; \
     329  }
     330
     331# define CONVERT_FORMAT(NAME, FORMAT, TYPE)             \
     332  if (!strcmp (extname, NAME)) {                                        \
     333    Missing_##TYPE *tmpMissing;                                         \
     334    tmpMissing = gfits_table_get_Missing_##TYPE (ftable, Nmissing, NULL, &nativeBytes); \
     335    if (!tmpMissing) {                                                  \
     336      fprintf (stderr, "ERROR: failed to read missing\n");              \
     337      exit (2);                                                         \
     338    }                                                                   \
     339    missing = Missing_##TYPE##_ToInternal (tmpMissing, *Nmissing); \
     340    free (tmpMissing);                                                  \
     341    *format = DVO_FORMAT_##FORMAT;                                      \
     342    return (missing); }
     343
     344  // XXX the structure is here for future expansion, but no transformations are currently defined
     345  if (TRUE) {
     346    missing = gfits_table_get_Missing (ftable, Nmissing, NULL, &nativeBytes);
     347    if (!missing) {
     348      fprintf (stderr, "ERROR: failed to read missing\n");
     349      exit (2);
     350    }
     351    *format = DVO_FORMAT_INTERNAL;
     352    return (missing);
     353  }
     354
     355  if (!strcmp (extname, "DVO_MISSING")) {
     356    missing = gfits_table_get_Missing (ftable, Nmissing, NULL, &nativeBytes);
     357    if (!missing) {
     358      fprintf (stderr, "ERROR: failed to read missing\n");
     359      exit (2);
     360    }
     361    *format = DVO_FORMAT_INTERNAL;
     362    return (missing);
     363  }
     364
     365  SKIPPING_FORMAT ("DVO_MISSING_PS1_SIM",         PS1_SIM,         PS1_SIM);
     366  SKIPPING_FORMAT ("DVO_MISSING_PS1_REF",         PS1_REF,         PS1_REF);
     367  SKIPPING_FORMAT ("DVO_MISSING_ELIXIR",          ELIXIR,          Elixir);
     368  SKIPPING_FORMAT ("DVO_MISSING_LONEOS",          LONEOS,          Loneos);
     369  SKIPPING_FORMAT ("DVO_MISSING_PANSTARRS_DEV_0", PANSTARRS_DEV_0, Panstarrs_DEV_0);
     370  SKIPPING_FORMAT ("DVO_MISSING_PANSTARRS_DEV_1", PANSTARRS_DEV_1, Panstarrs_DEV_1);
     371  SKIPPING_FORMAT ("DVO_MISSING_PS1_DEV_1",       PS1_DEV_1,       PS1_DEV_1);
     372  SKIPPING_FORMAT ("DVO_MISSING_PS1_DEV_2",       PS1_DEV_2,       PS1_DEV_2);
     373  SKIPPING_FORMAT ("DVO_MISSING_PS1_V1",          PS1_V1,          PS1_V1);
     374  SKIPPING_FORMAT ("DVO_MISSING_PS1_V2",          PS1_V2,          PS1_V2);
     375  SKIPPING_FORMAT ("DVO_MISSING_PS1_V3",          PS1_V3,          PS1_V3);
     376  SKIPPING_FORMAT ("DVO_MISSING_PS1_V4",          PS1_V4,          PS1_V4);
     377  SKIPPING_FORMAT ("DVO_MISSING_PS1_V5",          PS1_V5,          PS1_V5);
     378# undef CONVERT_FORMAT
     379# undef SKIPPING_FORMAT
     380
     381  fprintf (stderr, "table format unknown: %s\n", extname);
     382
     383  *Nmissing = 0;
     384  return (NULL);
     385}
     386
    310387/*** SecFilt / FTable conversion functions ***/
    311388
    312 SecFilt *FtableToSecFilt (FTable *ftable, off_t *Nsecfilt, char *format) {
     389SecFilt *FtableToSecFilt (FTable *ftable, Average *average, off_t *Nsecfilt, DVOCatFormat *format, char nativeBytes) {
    313390
    314391  SecFilt *secfilt;
    … …  
    339416  if (!strcmp (extname, NAME)) { \
    340417    SecFilt_##TYPE *tmpSecFilt; \
    341     tmpSecFilt = gfits_table_get_SecFilt_##TYPE (ftable, Nsecfilt, NULL); \
     418    tmpSecFilt = gfits_table_get_SecFilt_##TYPE (ftable, Nsecfilt, NULL, &nativeBytes); \
    342419    if (!tmpSecFilt) { \
    343420      fprintf (stderr, "ERROR: failed to read secfilts\n"); \
    … …  
    350427
    351428  if (!strcmp (extname, "DVO_SECFILT")) {
    352     secfilt = gfits_table_get_SecFilt (ftable, Nsecfilt, NULL);
     429    secfilt = gfits_table_get_SecFilt (ftable, Nsecfilt, NULL, &nativeBytes);
    353430    if (!secfilt) {
    354431      fprintf (stderr, "ERROR: failed to read secfilts\n");
    … …  
    380457}
    381458
    382 int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, char format) {
     459int SecFiltToFtable (FTable *ftable, SecFilt *secfilt, off_t Nsecfilt, DVOCatFormat format, int swapFromNative) {
    383460
    384461# define FORMAT_CASE(FORMAT, TYPE) \
    … …  
    386463      SecFilt_##TYPE *tmpSecFilt; \
    387464      tmpSecFilt = SecFiltInternalTo_##TYPE (secfilt, Nsecfilt); \
    388       gfits_table_set_SecFilt_##TYPE (ftable, tmpSecFilt, Nsecfilt); \
     465        gfits_table_set_SecFilt_##TYPE (ftable, tmpSecFilt, Nsecfilt, swapFromNative); \
    389466      free (tmpSecFilt); \
    390467      break; }
    … …  
    393470  switch (format) {
    394471    case DVO_FORMAT_INTERNAL: {
    395       gfits_table_set_SecFilt (ftable, secfilt, Nsecfilt);
     472      gfits_table_set_SecFilt (ftable, secfilt, Nsecfilt, swapFromNative);
    396473      break; }
    397474
    … …  
    420497/*** Lensing / FTable conversion functions ***/
    421498
    422 Lensing *FtableToLensing (FTable *ftable, off_t *Nlensing, char *format) {
     499Lensing *FtableToLensing (FTable *ftable, Average *average, off_t *Nlensing, DVOCatFormat *format, char nativeBytes) {
    423500
    424501  Lensing *lensing;
    … …  
    433510  if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 128)) {
    434511    Lensing_PS1_V5_R0 *tmpLensing;                                             
    435     tmpLensing = gfits_table_get_Lensing_PS1_V5_R0 (ftable, Nlensing, NULL);
     512    tmpLensing = gfits_table_get_Lensing_PS1_V5_R0 (ftable, Nlensing, NULL, &nativeBytes);
    436513    if (!tmpLensing) {                                                 
    437514      fprintf (stderr, "ERROR: failed to read lensings\n");             
    … …  
    445522  if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 136)) {
    446523    Lensing_PS1_V5_R1 *tmpLensing;                                             
    447     tmpLensing = gfits_table_get_Lensing_PS1_V5_R1 (ftable, Nlensing, NULL);
     524    tmpLensing = gfits_table_get_Lensing_PS1_V5_R1 (ftable, Nlensing, NULL, &nativeBytes);
    448525    if (!tmpLensing) {                                                 
    449526      fprintf (stderr, "ERROR: failed to read lensings\n");             
    … …  
    457534  if (!strcmp (extname, "DVO_LENSING_PS1_V5") && (ftable[0].header[0].Naxis[0] == 144)) {
    458535    Lensing_PS1_V5_R2 *tmpLensing;                                             
    459     tmpLensing = gfits_table_get_Lensing_PS1_V5_R2 (ftable, Nlensing, NULL);
     536    tmpLensing = gfits_table_get_Lensing_PS1_V5_R2 (ftable, Nlensing, NULL, &nativeBytes);
    460537    if (!tmpLensing) {                                                 
    461538      fprintf (stderr, "ERROR: failed to read lensings\n");             
    … …  
    477554  if (!strcmp (extname, NAME)) {                                        \
    478555    Lensing_##TYPE *tmpLensing;                                         \
    479     tmpLensing = gfits_table_get_Lensing_##TYPE (ftable, Nlensing, NULL); \
     556    tmpLensing = gfits_table_get_Lensing_##TYPE (ftable, Nlensing, NULL, &nativeBytes); \
    480557    if (!tmpLensing) {                                                  \
    481558      fprintf (stderr, "ERROR: failed to read lensings\n");             \
    … …  
    488565
    489566  if (!strcmp (extname, "DVO_LENSING")) {
    490     lensing = gfits_table_get_Lensing (ftable, Nlensing, NULL);
     567    lensing = gfits_table_get_Lensing (ftable, Nlensing, NULL, &nativeBytes);
    491568    if (!lensing) {
    492569      fprintf (stderr, "ERROR: failed to read lensings\n");
    … …  
    520597
    521598// LensingToFtable needs the Average since old formats stored lensing.dR,dD only
    522 int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, char format) {
     599int LensingToFtable (FTable *ftable, Lensing *lensing, off_t Nlensing, DVOCatFormat format, int swapFromNative) {
    523600
    524601# define FORMAT_CASE(FORMAT, TYPE)              \
    … …  
    526603      Lensing_##TYPE *tmpLensing; \
    527604      tmpLensing = LensingInternalTo_##TYPE (lensing, Nlensing); \
    528       gfits_table_set_Lensing_##TYPE (ftable, tmpLensing, Nlensing); \
     605        gfits_table_set_Lensing_##TYPE (ftable, tmpLensing, Nlensing, swapFromNative); \
    529606      free (tmpLensing); \
    530607      break; }
    … …  
    533610  switch (format) {
    534611    case DVO_FORMAT_INTERNAL: {
    535       gfits_table_set_Lensing (ftable, lensing, Nlensing);
     612      gfits_table_set_Lensing (ftable, lensing, Nlensing, swapFromNative);
    536613      break; }
    537614
    … …  
    559636/*** Lensobj / FTable conversion functions ***/
    560637
    561 Lensobj *FtableToLensobj (FTable *ftable, off_t *Nlensobj, char *format) {
     638Lensobj *FtableToLensobj (FTable *ftable, Average *average, off_t *Nlensobj, DVOCatFormat *format, char nativeBytes) {
    562639
    563640  Lensobj *lensobj;
    … …  
    572649  if (!strcmp (extname, "DVO_LENSOBJ_PS1_V5") && (ftable[0].header[0].Naxis[0] == 136)) {
    573650    Lensobj_PS1_V5_R0 *tmpLensobj;                                             
    574     tmpLensobj = gfits_table_get_Lensobj_PS1_V5_R0 (ftable, Nlensobj, NULL);
     651    tmpLensobj = gfits_table_get_Lensobj_PS1_V5_R0 (ftable, Nlensobj, NULL, &nativeBytes);
    575652    if (!tmpLensobj) {                                                 
    576653      fprintf (stderr, "ERROR: failed to read lensobjs\n");             
    … …  
    592669  if (!strcmp (extname, NAME)) {                                        \
    593670    Lensobj_##TYPE *tmpLensobj;                                         \
    594     tmpLensobj = gfits_table_get_Lensobj_##TYPE (ftable, Nlensobj, NULL); \
     671    tmpLensobj = gfits_table_get_Lensobj_##TYPE (ftable, Nlensobj, NULL, &nativeBytes); \
    595672    if (!tmpLensobj) {                                                  \
    596673      fprintf (stderr, "ERROR: failed to read lensobjs\n");             \
    … …  
    603680
    604681  if (!strcmp (extname, "DVO_LENSOBJ")) {
    605     lensobj = gfits_table_get_Lensobj (ftable, Nlensobj, NULL);
     682    lensobj = gfits_table_get_Lensobj (ftable, Nlensobj, NULL, &nativeBytes);
    606683    if (!lensobj) {
    607684      fprintf (stderr, "ERROR: failed to read lensobjs\n");
    … …  
    635712
    636713// LensobjToFtable needs the Average since old formats stored lensobj.dR,dD only
    637 int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, char format) {
     714int LensobjToFtable (FTable *ftable, Lensobj *lensobj, off_t Nlensobj, DVOCatFormat format, int swapFromNative) {
    638715
    639716# define FORMAT_CASE(FORMAT, TYPE)              \
    … …  
    641718      Lensobj_##TYPE *tmpLensobj; \
    642719      tmpLensobj = LensobjInternalTo_##TYPE (lensobj, Nlensobj); \
    643       gfits_table_set_Lensobj_##TYPE (ftable, tmpLensobj, Nlensobj); \
     720        gfits_table_set_Lensobj_##TYPE (ftable, tmpLensobj, Nlensobj, swapFromNative); \
    644721      free (tmpLensobj); \
    645722      break; }
    … …  
    648725  switch (format) {
    649726    case DVO_FORMAT_INTERNAL: {
    650       gfits_table_set_Lensobj (ftable, lensobj, Nlensobj);
     727      gfits_table_set_Lensobj (ftable, lensobj, Nlensobj, swapFromNative);
    651728      break; }
    652729
    … …  
    674751/*** StarPar / FTable conversion functions ***/
    675752
    676 StarPar *FtableToStarPar (FTable *ftable, off_t *Nstarpar, char *format) {
     753StarPar *FtableToStarPar (FTable *ftable, Average *average, off_t *Nstarpar, DVOCatFormat *format, char nativeBytes) {
    677754
    678755  StarPar *starpar;
    … …  
    695772  if (!strcmp (extname, NAME)) {                                        \
    696773    StarPar_##TYPE *tmpStarPar;                                         \
    697     tmpStarPar = gfits_table_get_StarPar_##TYPE (ftable, Nstarpar, NULL); \
     774    tmpStarPar = gfits_table_get_StarPar_##TYPE (ftable, Nstarpar, NULL, &nativeBytes); \
    698775    if (!tmpStarPar) {                                                  \
    699776      fprintf (stderr, "ERROR: failed to read starpar\n");              \
    … …  
    706783
    707784  if (!strcmp (extname, "DVO_STARPAR")) {
    708     starpar = gfits_table_get_StarPar (ftable, Nstarpar, NULL);
     785    starpar = gfits_table_get_StarPar (ftable, Nstarpar, NULL, &nativeBytes);
    709786    if (!starpar) {
    710787      fprintf (stderr, "ERROR: failed to read starpar\n");
    … …  
    738815
    739816// StarParToFtable needs the Average since old formats stored starpar.dR,dD only
    740 int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, char format) {
     817int StarParToFtable (FTable *ftable, StarPar *starpar, off_t Nstarpar, DVOCatFormat format, int swapFromNative) {
    741818
    742819# define FORMAT_CASE(FORMAT, TYPE)              \
    … …  
    744821      StarPar_##TYPE *tmpStarPar; \
    745822      tmpStarPar = StarParInternalTo_##TYPE (starpar, Nstarpar); \
    746       gfits_table_set_StarPar_##TYPE (ftable, tmpStarPar, Nstarpar); \
     823        gfits_table_set_StarPar_##TYPE (ftable, tmpStarPar, Nstarpar, swapFromNative); \
    747824      free (tmpStarPar); \
    748825      break; }
    … …  
    751828  switch (format) {
    752829    case DVO_FORMAT_INTERNAL: {
    753       gfits_table_set_StarPar (ftable, starpar, Nstarpar);
     830      gfits_table_set_StarPar (ftable, starpar, Nstarpar, swapFromNative);
    754831      break; }
    755832
    … …  
    776853}
    777854
     855/*** GalPhot / FTable conversion functions ***/
     856
     857GalPhot *FtableToGalPhot (FTable *ftable, Average *average, off_t *Ngalphot, DVOCatFormat *format, char nativeBytes) {
     858
     859  GalPhot *galphot;
     860  char extname[80];
     861
     862  /* convert to the internal format */
     863  if (!gfits_scan (ftable[0].header, "EXTNAME", "%s", 1, extname)) {
     864    fprintf (stderr, "EXTNAME missing for galphot table\n");
     865    return (FALSE);
     866  }
     867
     868# define SKIPPING_FORMAT(NAME, FORMAT, TYPE)            \
     869  if (!strcmp (extname, NAME)) {                                        \
     870    fprintf (stderr, "ERROR: format %s not defined for galphot, skipping\n", NAME); \
     871    *Ngalphot = 0; \
     872    return NULL; \
     873  }
     874
     875# define CONVERT_FORMAT(NAME, FORMAT, TYPE)             \
     876  if (!strcmp (extname, NAME)) {                                        \
     877    GalPhot_##TYPE *tmpGalPhot;                                         \
     878    tmpGalPhot = gfits_table_get_GalPhot_##TYPE (ftable, Ngalphot, NULL, &nativeBytes); \
     879    if (!tmpGalPhot) {                                                  \
     880      fprintf (stderr, "ERROR: failed to read galphots\n");             \
     881      exit (2);                                                         \
     882    }                                                                   \
     883    galphot = GalPhot_##TYPE##_ToInternal (tmpGalPhot, *Ngalphot); \
     884    free (tmpGalPhot);                                                  \
     885    *format = DVO_FORMAT_##FORMAT;                                      \
     886    return (galphot); }
     887
     888  if (!strcmp (extname, "DVO_GALPHOT")) {
     889    galphot = gfits_table_get_GalPhot (ftable, Ngalphot, NULL, &nativeBytes);
     890    if (!galphot) {
     891      fprintf (stderr, "ERROR: failed to read galphots\n");
     892      exit (2);
     893    }
     894    *format = DVO_FORMAT_INTERNAL;
     895    return (galphot);
     896  }
     897
     898  SKIPPING_FORMAT ("DVO_GALPHOT_PS1_REF",         PS1_REF,         PS1_REF);
     899  SKIPPING_FORMAT ("DVO_GALPHOT_ELIXIR",          ELIXIR,          Elixir);
     900  SKIPPING_FORMAT ("DVO_GALPHOT_LONEOS",          LONEOS,          Loneos);
     901  SKIPPING_FORMAT ("DVO_GALPHOT_PANSTARRS_DEV_0", PANSTARRS_DEV_0, Panstarrs_DEV_0);
     902  SKIPPING_FORMAT ("DVO_GALPHOT_PANSTARRS_DEV_1", PANSTARRS_DEV_1, Panstarrs_DEV_1);
     903  SKIPPING_FORMAT ("DVO_GALPHOT_PS1_DEV_1",       PS1_DEV_1,       PS1_DEV_1);
     904  SKIPPING_FORMAT ("DVO_GALPHOT_PS1_DEV_2",       PS1_DEV_2,       PS1_DEV_2);
     905  SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V1",          PS1_V1,          PS1_V1);
     906  SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V2",          PS1_V2,          PS1_V2);
     907  SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V3",          PS1_V3,          PS1_V3);
     908  SKIPPING_FORMAT ("DVO_GALPHOT_PS1_V4",          PS1_V4,          PS1_V4);
     909  CONVERT_FORMAT  ("DVO_GALPHOT_PS1_V5",          PS1_V5,          PS1_V5);
     910  SKIPPING_FORMAT ("DVO_GALPHOT_PS1_SIM",         PS1_SIM,         PS1_SIM);
     911# undef CONVERT_FORMAT
     912# undef SKIPPING_FORMAT
     913
     914  fprintf (stderr, "table format unknown: %s\n", extname);
     915
     916  *Ngalphot = 0;
     917  return (NULL);
     918}
     919
     920// GalPhotToFtable needs the Average since old formats stored galphot.dR,dD only
     921int GalPhotToFtable (FTable *ftable, GalPhot *galphot, off_t Ngalphot, DVOCatFormat format, int swapFromNative) {
     922
     923# define FORMAT_CASE(FORMAT, TYPE)              \
     924    case DVO_FORMAT_##FORMAT: { \
     925      GalPhot_##TYPE *tmpGalPhot; \
     926      tmpGalPhot = GalPhotInternalTo_##TYPE (galphot, Ngalphot); \
     927        gfits_table_set_GalPhot_##TYPE (ftable, tmpGalPhot, Ngalphot, swapFromNative); \
     928      free (tmpGalPhot); \
     929      break; }
     930
     931  /* convert from the internal format */
     932  switch (format) {
     933    case DVO_FORMAT_INTERNAL: {
     934      gfits_table_set_GalPhot (ftable, galphot, Ngalphot, swapFromNative);
     935      break; }
     936
     937//    FORMAT_CASE (PS1_REF,         PS1_REF);
     938//    FORMAT_CASE (ELIXIR,          Elixir);
     939//    FORMAT_CASE (LONEOS,          Loneos);
     940//    FORMAT_CASE (PANSTARRS_DEV_0, Panstarrs_DEV_0);
     941//    FORMAT_CASE (PANSTARRS_DEV_1, Panstarrs_DEV_1);
     942//    FORMAT_CASE (PS1_DEV_1,       PS1_DEV_1);
     943//    FORMAT_CASE (PS1_DEV_2,       PS1_DEV_2);
     944//    FORMAT_CASE (PS1_V1,          PS1_V1);
     945//    FORMAT_CASE (PS1_V2,          PS1_V2);
     946//    FORMAT_CASE (PS1_V3,          PS1_V3);
     947//    FORMAT_CASE (PS1_V4,          PS1_V4);
     948      FORMAT_CASE (PS1_V5,          PS1_V5);
     949# undef FORMAT_CASE
     950
     951    default:
     952      fprintf (stderr, "table format unknown (galphot)\n");
     953      return (FALSE);
     954  }
     955  return (TRUE);
     956}
     957
    778958/*** Image Conversions ***/
    779959
    780960// I have loaded the disk db table and now I want to convert to the internal format
    781961// (Image structure), but I onyl
    782 int FtableToImage (FTable *ftable, Header *theader, char *format) {
     962int FtableToImage (FTable *ftable, Header *theader, DVOCatFormat *format) {
    783963
    784964  off_t Nimage;
    … …  
    788968  if (*format == DVO_FORMAT_ELIXIR) {             // special case for ELIXIR
    789969    Image_Elixir *tmpimage;
    790     tmpimage = gfits_table_get_Image_Elixir (ftable, &Nimage, NULL);
     970    tmpimage = gfits_table_get_Image_Elixir (ftable, &Nimage, NULL, NULL);
    791971    if (!tmpimage) {
    792972      fprintf (stderr, "ERROR: failed to read images\n");
    … …  
    814994    Image_##TYPE *tmpimage; \
    815995    *format = DVO_FORMAT_##FORMAT; \
    816     tmpimage = gfits_table_get_Image_##TYPE (ftable, &Nimage, NULL); \
     996    tmpimage = gfits_table_get_Image_##TYPE (ftable, &Nimage, NULL, NULL); \
    817997    if (!tmpimage) { \
    818998      fprintf (stderr, "ERROR: failed to read images\n"); \
    … …  
    8491029}
    8501030
    851 int ImageToFtable (FTable *ftable, Header *theader, char format) {
     1031int ImageToFtable (FTable *ftable, Header *theader, DVOCatFormat format) {
    8521032
    8531033  off_t Nimage;
    … …  
    8601040      tmpImage = ImageInternalTo_##TYPE ((Image *) ftable[0].buffer, Nimage); \
    8611041      free (ftable[0].buffer); \
    862       gfits_table_set_Image_##TYPE (ftable, tmpImage, Nimage); \
     1042      gfits_table_set_Image_##TYPE (ftable, tmpImage, Nimage, TRUE); \
    8631043      free (tmpImage); \
    8641044      break; }
    … …  
    8891069}
    8901070
    891 int ImageToVtable (VTable *vtable, Header *theader, char format) {
     1071int ImageToVtable (VTable *vtable, Header *theader, DVOCatFormat format) {
    8921072
    8931073  off_t i, Nrow, Nimage;

  • trunk/Ohana/src/libdvo/src/dvo_convert_PS1_DEV_2.c

    r37807 r38441  
    118118    out[i].Nmeasure      = in[i].Nmeasure;     
    119119    out[i].Nmissing      = in[i].Nmissing;     
    120     out[i].Nextend       = in[i].Nextend;     
     120    out[i].Ngalphot     = in[i].Nextend;     
    121121    out[i].measureOffset = in[i].measureOffset;
    122122    out[i].missingOffset = in[i].missingOffset;
    … …  
    152152    out[i].Nmeasure      = in[i].Nmeasure;     
    153153    out[i].Nmissing      = in[i].Nmissing;     
    154     out[i].Nextend       = in[i].Nextend;     
     154    out[i].Nextend       = in[i].Ngalphot;     
    155155    out[i].measureOffset = in[i].measureOffset;
    156156    out[i].missingOffset = in[i].missingOffset;

  • trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V1.c

    r37807 r38441  
    138138    out[i].Nmeasure      = in[i].Nmeasure;     
    139139    out[i].Nmissing      = in[i].Nmissing;     
    140     out[i].Nextend       = in[i].Nextend;     
     140    out[i].Ngalphot     = in[i].Ngalphot;     
    141141    out[i].measureOffset = in[i].measureOffset;
    142142    out[i].missingOffset = in[i].missingOffset;
    … …  
    173173    out[i].Nmeasure      = in[i].Nmeasure;     
    174174    out[i].Nmissing      = in[i].Nmissing;     
    175     out[i].Nextend       = in[i].Nextend;     
     175    out[i].Ngalphot     = in[i].Ngalphot;     
    176176    out[i].measureOffset = in[i].measureOffset;
    177177    out[i].missingOffset = in[i].missingOffset;

  • trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V2.c

    r37807 r38441  
    143143    out[i].Nmeasure      = in[i].Nmeasure;     
    144144    out[i].Nmissing      = in[i].Nmissing;     
    145     out[i].Nextend       = in[i].Nextend;     
     145    out[i].Ngalphot     = in[i].Ngalphot;     
    146146    out[i].measureOffset = in[i].measureOffset;
    147147    out[i].missingOffset = in[i].missingOffset;
    … …  
    182182    out[i].Nmeasure      = in[i].Nmeasure;     
    183183    out[i].Nmissing      = in[i].Nmissing;     
    184     out[i].Nextend       = in[i].Nextend;     
     184    out[i].Ngalphot     = in[i].Ngalphot;     
    185185    out[i].measureOffset = in[i].measureOffset;
    186186    out[i].missingOffset = in[i].missingOffset;

  • trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V3.c

    r37807 r38441  
    143143    out[i].Nmeasure      = in[i].Nmeasure;     
    144144    out[i].Nmissing      = in[i].Nmissing;     
    145     out[i].Nextend       = in[i].Nextend;     
     145    out[i].Ngalphot     = in[i].Ngalphot;     
    146146    out[i].measureOffset = in[i].measureOffset;
    147147    out[i].missingOffset = in[i].missingOffset;
    … …  
    184184    out[i].Nmeasure      = in[i].Nmeasure;     
    185185    out[i].Nmissing      = in[i].Nmissing;     
    186     out[i].Nextend       = in[i].Nextend;     
     186    out[i].Ngalphot     = in[i].Ngalphot;     
    187187    out[i].measureOffset = in[i].measureOffset;
    188188    out[i].missingOffset = in[i].missingOffset;

  • trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V4.c

    r37807 r38441  
    164164    out[i].Nmeasure      = in[i].Nmeasure;     
    165165    out[i].Nmissing      = in[i].Nmissing;     
    166     out[i].Nextend       = in[i].Nextend;     
     166    out[i].Ngalphot     = in[i].Ngalphot;     
    167167    out[i].measureOffset = in[i].measureOffset;
    168168    out[i].missingOffset = in[i].missingOffset;
    … …  
    208208    out[i].Nmeasure      = in[i].Nmeasure;     
    209209    out[i].Nmissing      = in[i].Nmissing;     
    210     out[i].Nextend       = in[i].Nextend;     
     210    out[i].Ngalphot     = in[i].Ngalphot;     
    211211    out[i].measureOffset = in[i].measureOffset;
    212212    out[i].missingOffset = in[i].missingOffset;
    … …  
    690690    out[i].Nmeasure      = in[i].Nmeasure;     
    691691    out[i].Nmissing      = in[i].Nmissing;     
    692     out[i].Nextend       = in[i].Nextend;     
     692    out[i].Ngalphot     = in[i].Ngalphot;     
    693693    out[i].measureOffset = in[i].measureOffset;
    694694    out[i].missingOffset = in[i].missingOffset;

  • trunk/Ohana/src/libdvo/src/dvo_convert_PS1_V5.c

    r38153 r38441  
    196196    out[i].Nlensobj      = in[i].Nlensobj;     
    197197    out[i].Nstarpar      = in[i].Nstarpar;     
    198     out[i].Nextend       = in[i].Nextend;     
     198    out[i].Ngalphot     = in[i].Ngalphot;     
    199199
    200200    out[i].measureOffset = in[i].measureOffset;
    … …  
    203203    out[i].lensobjOffset = in[i].lensobjOffset;
    204204    out[i].starparOffset = in[i].starparOffset;
    205     out[i].extendOffset  = in[i].extendOffset;
     205    out[i].galphotOffset  = in[i].galphotOffset;
    206206
    207207    out[i].refColorBlue  = in[i].refColorBlue;
    … …  
    264264    out[i].Nlensobj       = in[i].Nlensobj;     
    265265    out[i].Nstarpar       = in[i].Nstarpar;     
    266     out[i].Nextend        = in[i].Nextend;     
     266    out[i].Ngalphot        = in[i].Ngalphot;     
    267267                         
    268268    out[i].measureOffset  = in[i].measureOffset;
    … …  
    271271    out[i].lensobjOffset  = in[i].lensobjOffset;
    272272    out[i].starparOffset  = in[i].starparOffset;
    273     out[i].extendOffset   = in[i].extendOffset;
     273    out[i].galphotOffset   = in[i].galphotOffset;
    274274                         
    275275    out[i].refColorBlue   = in[i].refColorBlue;
    … …  
    975975    out[i].objID   = in[i].objID ;
    976976    out[i].catID   = in[i].catID ;
     977  }
     978  return (out);
     979}
     980
     981GalPhot *GalPhot_PS1_V5_ToInternal (GalPhot_PS1_V5 *in, off_t Nvalues) {
     982
     983  off_t i;
     984  GalPhot *out;
     985
     986  ALLOCATE_ZERO (out, GalPhot, Nvalues);
     987
     988  for (i = 0; i < Nvalues; i++) {
     989    dvo_galphot_init (&out[i]);
     990
     991    out[i].Xfit          = in[i].Xfit;
     992    out[i]. Yfit         = in[i]. Yfit;
     993    out[i]. mag          = in[i]. mag;
     994    out[i]. magErr       = in[i]. magErr;
     995    out[i]. majorAxis    = in[i]. majorAxis;
     996    out[i]. minorAxis    = in[i]. minorAxis;
     997    out[i]. majorAxisErr = in[i]. majorAxisErr;
     998    out[i]. minorAxisErr = in[i]. minorAxisErr;
     999    out[i]. theta        = in[i]. theta;
     1000    out[i]. theta_err    = in[i]. theta_err;
     1001    out[i]. index        = in[i]. index;
     1002    out[i]. chisq        = in[i]. chisq;
     1003    out[i]. Npix         = in[i]. Npix;
     1004    out[i]. objID        = in[i]. objID;
     1005    out[i]. catID        = in[i]. catID;
     1006    out[i]. detID        = in[i]. detID;
     1007    out[i]. imageID      = in[i]. imageID;
     1008    out[i]. photcode     = in[i]. photcode;
     1009    out[i]. modelType    = in[i]. modelType;
     1010  }
     1011  return (out);
     1012}
     1013
     1014GalPhot_PS1_V5 *GalPhotInternalTo_PS1_V5 (GalPhot *in, off_t Nvalues) {
     1015
     1016  off_t i;
     1017  GalPhot_PS1_V5 *out;
     1018
     1019  ALLOCATE_ZERO (out, GalPhot_PS1_V5, Nvalues);
     1020
     1021  for (i = 0; i < Nvalues; i++) {
     1022    out[i].Xfit          = in[i].Xfit;
     1023    out[i].Yfit          = in[i].Yfit;
     1024    out[i].mag           = in[i].mag;
     1025    out[i].magErr        = in[i].magErr;
     1026    out[i].majorAxis     = in[i].majorAxis;
     1027    out[i].minorAxis     = in[i].minorAxis;
     1028    out[i].majorAxisErr  = in[i].majorAxisErr;
     1029    out[i].minorAxisErr  = in[i].minorAxisErr;
     1030    out[i].theta         = in[i].theta;
     1031    out[i].theta_err     = in[i].theta_err;
     1032    out[i].index         = in[i].index;
     1033    out[i].chisq         = in[i].chisq;
     1034    out[i].Npix          = in[i].Npix;
     1035    out[i].objID         = in[i].objID;
     1036    out[i].catID         = in[i].catID;
     1037    out[i].detID         = in[i].detID;
     1038    out[i].imageID       = in[i].imageID;
     1039    out[i].photcode      = in[i].photcode;
     1040    out[i].modelType     = in[i].modelType;
    9771041  }
    9781042  return (out);
    … …  
    12341298    out[i].Nlensing       = in[i].Nlensing;     
    12351299    out[i].Nlensobj       = in[i].Nlensobj;     
    1236     out[i].Nextend        = in[i].Nextend;     
     1300    out[i].Ngalphot        = in[i].Ngalphot;     
    12371301                         
    12381302    out[i].measureOffset  = in[i].measureOffset;
    … …  
    12411305    out[i].lensobjOffset  = in[i].lensobjOffset;
    12421306    out[i].starparOffset  = in[i].starparOffset;
    1243     out[i].extendOffset   = in[i].extendOffset;
     1307    out[i].galphotOffset   = in[i].galphotOffset;
    12441308                         
    12451309    out[i].refColorBlue   = in[i].refColorBlue;
    … …  
    13021366    SWAP_BYTE (110); // NLENSING
    13031367    SWAP_BYTE (112); // NLENSOBJ
    1304     SWAP_BYTE (114); // NEXTEND
     1368    SWAP_BYTE (114); // NGALPHOT
    13051369    SWAP_WORD (116); // OFF_MEASURE
    13061370    SWAP_WORD (120); // OFF_MISSING
    13071371    SWAP_WORD (124); // OFF_LENSING
    13081372    SWAP_WORD (128); // OFF_LENSOBJ
    1309     SWAP_WORD (132); // OFF_EXTEND
     1373    SWAP_WORD (132); // OFF_GALPHOT
    13101374    SWAP_WORD (136); // OFF_PARAMS
    13111375    SWAP_WORD (140); // REF_COLOR_BLUE

  • trunk/Ohana/src/libdvo/src/dvo_image.c

    r37807 r38441  
    1919  db[0].lockstate = lockstate;
    2020  db[0].timeout   = timeout;
    21   gfits_db_init (db);
    2221
    2322  if (!gfits_db_lock (db, filename)) {
    … …  
    8988  // so when we later do a gfits_copy_ftable, we stomp on bad memory
    9089  FtableToImage (&db[0].ftable, &db[0].theader, &db[0].format);
    91   db[0].swapped = TRUE;  /* table has internal byte-order */
     90  db[0].nativeOrder = TRUE;  /* table has internal byte-order */
     91  db[0].scaledValue = TRUE;  /* table has internal byte-order */
    9292  return (TRUE);
    9393}
    … …  
    100100  /* convert from internal to requested external format */
    101101  ImageToFtable (&db[0].ftable, &db[0].theader, db[0].format);
    102   db[0].swapped = FALSE;
     102  db[0].nativeOrder = FALSE;
     103  db[0].scaledValue = FALSE;
    103104
    104105  /* write data in appropriate mode */
    … …  
    149150  /* theader is modified to match output format */
    150151  ImageToVtable (&db[0].vtable, &db[0].theader, db[0].format);
    151   db[0].swapped = FALSE;
     152  db[0].scaledValue = FALSE;
     153  db[0].nativeOrder = FALSE;
    152154
    153155  /* write data in appropriate mode */
    … …  
    363365
    364366/* return internal structure representation */
    365 Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *swapped) {
     367Image *gfits_table_get_Image (FTable *ftable, off_t *Ndata, char *scaledValue, char *nativeOrder) {
    366368
    367369  int Ncols;
    … …  
    377379  data = (Image *) ftable[0].buffer;
    378380
    379   if (!swapped) {
    380     myAbort ("invalid to call this without suppying 'swapped'");
    381   }
    382   if (*swapped == FALSE) {
    383     myAbort ("invalid for table to be swapped");
     381  if (!scaledValue) {
     382    myAbort ("invalid to call this without suppying 'scaledValue'");
     383  }
     384  if (*scaledValue == FALSE) {
     385    myAbort ("invalid for table NOT to be scaledValue");
     386  }
     387  if (!nativeOrder) {
     388    myAbort ("invalid to call this without suppying 'nativeOrder'");
     389  }
     390  if (*nativeOrder == FALSE) {
     391    myAbort ("invalid for table NOT to be nativeOrder");
    384392  }
    385393   

  • trunk/Ohana/src/libdvo/src/dvo_image_raw.c

    r37807 r38441  
    107107  db[0].theader.Naxis[1] = Nimage;
    108108  db[0].ftable.datasize = gfits_data_size (&db[0].theader);
    109   db[0].swapped = FALSE;  /* table does not have internal byte-order */
     109
     110  db[0].nativeOrder = FALSE;  /* table does not have internal byte-order */
     111  db[0].scaledValue = FALSE;  /* table does not have internal byte-order */
    110112  return (TRUE);
    111113}

  • trunk/Ohana/src/libdvo/src/dvo_util.c

    r37807 r38441  
    127127    }
    128128
    129     dvoConfig->images = gfits_table_get_Image (&dvoConfig->imageDB.ftable, &dvoConfig->nImages, &dvoConfig->imageDB.swapped);
     129    dvoConfig->images = gfits_table_get_Image (&dvoConfig->imageDB.ftable, &dvoConfig->nImages, &dvoConfig->imageDB.scaledValue, &dvoConfig->imageDB.nativeOrder);
    130130    if (!dvoConfig->images) {
    131131        fprintf (stderr, "problem loading images\n");
    … …  
    207207      Catalog catalog;
    208208      catalog.filename = skylist->filename[reg];
    209       catalog.catflags = GetMeasures ? LOAD_AVES | LOAD_MEAS | LOAD_SECF : LOAD_AVES | LOAD_SECF;
     209      catalog.catflags = GetMeasures ? DVO_LOAD_AVERAGE | DVO_LOAD_MEASURE | DVO_LOAD_SECFILT : DVO_LOAD_AVERAGE | DVO_LOAD_SECFILT;
    210210      catalog.Nsecfilt = 0;
    211211

  • trunk/Ohana/src/libdvo/src/fits_db.c

    r35755 r38441  
    66  db[0].f             = NULL;
    77  db[0].filename      = NULL;
     8  db[0].dbstate       = 0;
     9  db[0].lockstate     = 0;
     10  db[0].mode          = 0;
     11  db[0].format        = 0;
     12  db[0].virtual       = 0;
     13  db[0].nativeOrder   = 0;
     14  db[0].scaledValue   = 0;
    815 
    9 # if (1)
    1016  gfits_init_header (&db->header);
    1117  gfits_init_matrix (&db->matrix);
    1218  gfits_init_header (&db->theader);
    13   gfits_init_table (&db->ftable);
     19  gfits_init_table  (&db->ftable);
    1420  db->ftable.header = &db->theader;
    1521  gfits_init_vtable (&db->vtable);
    16 # else 
    17   db[0].header.buffer = NULL;
    18   db[0].matrix.buffer = NULL;
    19   db[0].theader.buffer = NULL;
    20   db[0].ftable.buffer = NULL;
    21   db[0].ftable.header = &db[0].theader;
    22   db[0].vtable.header = NULL;
    23   db[0].vtable.buffer = NULL;
    24   db[0].vtable.row    = NULL;
    25 # endif
    2622  return (TRUE);
    2723}
    … …  
    9793    return (FALSE);
    9894  }
    99   db[0].swapped = FALSE;  /* table does not have internal byte-order */
     95  db[0].nativeOrder = FALSE;  /* table does not have internal byte-order */
     96  db[0].scaledValue = FALSE;  /* table has not been scaled by BZERO,BSCALE */
    10097  return (TRUE);
    10198}

  • trunk/Ohana/src/libdvo/src/flatcorr_io.c

    r37807 r38441  
    4949    return (NULL);
    5050  }
    51   flatcorrTable->image = gfits_table_get_FlatCorrectionImage (&ftable, &flatcorrTable->Nimage, NULL);
     51  flatcorrTable->image = gfits_table_get_FlatCorrectionImage (&ftable, &flatcorrTable->Nimage, NULL, NULL);
    5252  if (!flatcorrTable->image) {
    5353    fprintf (stderr, "ERROR: failed to read Flat Correction Images\n");
    … …  
    6464    return (NULL);
    6565  }
    66   flatcorrTable->corr = gfits_table_get_FlatCorrection (&ftable, &flatcorrTable->Ncorr, NULL);
     66  flatcorrTable->corr = gfits_table_get_FlatCorrection (&ftable, &flatcorrTable->Ncorr, NULL, NULL);
    6767  if (!flatcorrTable->corr) {
    6868    fprintf (stderr, "ERROR: failed to read Flat Corrections\n");
    … …  
    9393    for (j = 0; j < flatcorrTable->image[i].Nx; j++) {
    9494      ALLOCATE (flatcorrTable->offset[i][j], float, flatcorrTable->image[i].Ny);
    95       memset (flatcorrTable->offset[i][j], 0.0, flatcorrTable->image[i].Ny*sizeof(float));
     95      memset (flatcorrTable->offset[i][j], 0, flatcorrTable->image[i].Ny*sizeof(float));
    9696    }
    9797  }
    … …  
    161161
    162162  ftableImage.header = &theaderImage;
    163   gfits_table_set_FlatCorrectionImage (&ftableImage, image, flatcorrTable->Nimage);
     163  gfits_table_set_FlatCorrectionImage (&ftableImage, image, flatcorrTable->Nimage, TRUE);
    164164
    165165  ftableCorr.header = &theaderCorr;
    166   gfits_table_set_FlatCorrection (&ftableCorr, corr, flatcorrTable->Ncorr);
     166  gfits_table_set_FlatCorrection (&ftableCorr, corr, flatcorrTable->Ncorr, TRUE);
    167167
    168168  f = fopen (filename, "w");

  • trunk/Ohana/src/libdvo/src/skyregion_io.c

    r36833 r38441  
    4040  ALLOCATE (skytable, SkyTable, 1);
    4141  memset (skytable->hosts, 0, 80);
    42   skytable[0].regions = gfits_table_get_SkyRegion (&ftable, &skytable[0].Nregions, NULL);
     42  skytable[0].regions = gfits_table_get_SkyRegion (&ftable, &skytable[0].Nregions, NULL, NULL);
    4343  if (!skytable[0].regions) {
    4444    fprintf (stderr, "ERROR: failed to read sky regions\n");
    … …  
    9999  gfits_print (&header, "HOSTS", "%s", 1, skytable->hosts);
    100100
    101   gfits_table_set_SkyRegion (&ftable, skytable[0].regions, skytable[0].Nregions);
     101  gfits_table_set_SkyRegion (&ftable, skytable[0].regions, skytable[0].Nregions, TRUE);
    102102
    103103  f = fopen (filename, "w");
Note: See TracChangeset for help on using the changeset viewer.