Index: trunk/Ohana/src/libdvo/doc/dvo-catalogs.txt =================================================================== --- trunk/Ohana/src/libdvo/doc/dvo-catalogs.txt (revision 15722) +++ trunk/Ohana/src/libdvo/doc/dvo-catalogs.txt (revision 15743) @@ -2,12 +2,5 @@ // APIs related to the DVO catalogs: -// XXX change name -// XXX add SORTED test to load -// XXX add Nsecfilt test? or add an API? load_catalog (Catalog *catalog, int VERBOSE); -// returns -// 0: failure to lock catalog -// 1: success: file is locked and opened -// 2: success: file is locked but empty dvo_catalog_lock (Catalog *catalog, int lockmode); @@ -20,4 +13,16 @@ // layout: RAW, MEF, SPLIT // mode: READ, WRITE + +// returns +// DVO_CAT_OPEN_FAIL: failure to lock catalog +// DVO_CAT_OPEN_OK: success: file is locked and opened +// DVO_CAT_OPEN_EMPTY: success: file is locked but empty + + the catalog is opened and the data for each of the elements (average, + measure, etc) is loaded if requested. Whether or not the data is + loaded, the values of catalog.Naves_disk,Naves_off,Naverage are set + to match the segment of data loaded. this API always loads either + the entire data set, or none. If none, the offset value is set to + point at the end of the table. dvo_catalog_save (Catalog *catalog, int VERBOSE); @@ -43,4 +48,23 @@ - open and read an existing catalog / error if missing (can be ignored) +dvo_catalog_open (Catalog *catalog, int mode); -dvo_catalog_open (Catalog *catalog, int mode); +dvo_catalog_load_segment_average (Catalog *catalog, int start, int Nrows) + + this function reads in the requested number of rows from the catalog + for the Average and SecFilt tables only. It is possible within the + DVO framework to examine the average values in slices. + +dvo_catalog_load_segment_measure (Catalog *catalog, int start, int Nrows) + + this function reads in the requested number of rows from the catalog + for the Measure table only. It is not guaranteed to be possible to + examine the measure values in slices if reference is needed to their + average values. + +dvo_catalog_load_segment_missing (Catalog *catalog, int start, int Nrows) + + this function reads in the requested number of rows from the catalog + for the Missing table only. It is not guaranteed to be possible to + examine the missing values in slices if reference is needed to their + average values. Index: trunk/Ohana/src/libdvo/include/dvo.h =================================================================== --- trunk/Ohana/src/libdvo/include/dvo.h (revision 15722) +++ trunk/Ohana/src/libdvo/include/dvo.h (revision 15743) @@ -22,4 +22,6 @@ typedef enum {T_UNDEF = -1, T_NONE, T_OBJECT, T_DARK, T_BIAS, T_FLAT, T_MASK, T_FRINGE, T_SCATTER, T_MODES, T_FRINGEPTS, T_ANY, N_TYPE} ElixirDetrendTypes; typedef enum {M_UNDEF = -1, M_NONE, M_MEF, M_SPLIT, M_SINGLE, M_CUBE, M_SLICE, M_MODES, N_MODE} ElixirDetrendModes; + +typedef enum {DVO_CAT_OPEN_FAIL, DVO_CAT_OPEN_OK, DVO_CAT_OPEN_EMPTY} DVOCatalogOpenModes; typedef enum { @@ -195,8 +197,16 @@ Missing *missing; SecFilt *secfilt; - int Naverage, Nmeasure, Nmissing, Nsecfilt; /* current number of each component */ - int Nave_disk, Nmeas_disk, Nmiss_disk; /* number of component on disk */ - int Nmeas_off; /* dist seq of first loaded data value */ - /* note the different counting for Nsecfilt */ + + int Nsecfilt; /* number of secfilt entries for each average entry */ + int Naverage, Nmeasure, Nmissing, Nsecf_mem; /* current number of each component in memory */ + int Naves_disk, Nmeas_disk, Nmiss_disk, Nsecf_disk; /* current number of each component on disk */ + int Naves_off, Nmeas_off, Nmiss_off, Nsecf_off; /* index of first loaded data value */ + + /* note the different counting for Nsecfilt: + number of secfilt rows on disk is: Nave_disk * Nsecfilt + number of secfilt rows in mem is: Naverage * Nsecfilt + *** that is just silly, and bad: convert to using Nsec_mem, Nsec_disk, Nsec_off. + *** unless we always require the secfilt and average entries to be loaded sychronously. + */ /* pointers to split data files */ Index: trunk/Ohana/src/libdvo/src/dvo_catalog.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog.c (revision 15722) +++ trunk/Ohana/src/libdvo/src/dvo_catalog.c (revision 15743) @@ -90,9 +90,15 @@ catalog[0].Nmeasure = 0; catalog[0].Nmissing = 0; - - catalog[0].Nave_disk = 0; + catalog[0].Nsecf_mem = 0; + + catalog[0].Naves_disk = 0; catalog[0].Nmeas_disk = 0; catalog[0].Nmiss_disk = 0; + catalog[0].Nsecf_disk = 0; + + catalog[0].Naves_off = 0; catalog[0].Nmeas_off = 0; + catalog[0].Nmiss_off = 0; + catalog[0].Nsecf_off = 0; /* pointers to SPLIT data files */ @@ -110,7 +116,7 @@ /* possible exit status for lock_catalog: - 0 - failure (including lock failure) - 1 - success - 2 - empty file (file may be open or closed!) + DVO_CAT_OPEN_FAIL - failure (including lock failure) + DVO_CAT_OPEN_OK - success + DVO_CAT_OPEN_EMPTY - empty file (file may be open or closed!) */ int dvo_catalog_lock (Catalog *catalog, int lockmode) { @@ -125,10 +131,10 @@ catalog[0].f = fsetlockfile (catalog[0].filename, 3600.0, catalog[0].lockmode, &dbstate); - if (dbstate == LCK_MISSING) return (2); - if (dbstate == LCK_EMPTY) return (2); - if (catalog[0].f == NULL) return (0); + if (dbstate == LCK_MISSING) return (DVO_CAT_OPEN_EMPTY); + if (dbstate == LCK_EMPTY) return (DVO_CAT_OPEN_EMPTY); + if (catalog[0].f == NULL) return (DVO_CAT_OPEN_FAIL); fseek (catalog[0].f, 0, SEEK_SET); - return (1); + return (DVO_CAT_OPEN_OK); } @@ -192,8 +198,8 @@ switch (dvo_catalog_lock (catalog, catalog[0].lockmode)) { - case 0: + case DVO_CAT_OPEN_FAIL: if (VERBOSE) fprintf (stderr, "can't lock file %s\n", catalog[0].filename); return (FALSE); - case 1: + case DVO_CAT_OPEN_OK: if (!dvo_catalog_load (catalog, VERBOSE)) { if (VERBOSE) fprintf (stderr, "failure loading catalog\n"); @@ -206,5 +212,5 @@ if (VERBOSE) fprintf (stderr, "loaded existing file %s\n", catalog[0].filename); break; - case 2: + case DVO_CAT_OPEN_EMPTY: if ((mode == DVO_OPEN_READ) || (mode == DVO_OPEN_UPDATE)) return (TRUE); catalog[0].Nsecfilt = Nsecfilt; @@ -235,10 +241,4 @@ if (!status) catalog[0].sorted = TRUE; - // even if the data is sorted on disk, if we only load the MEAS_META, we are - // treating it as unsorted (eg, append only) - if (catalog[0].catflags & LOAD_MEAS_META) { - catalog[0].sorted = FALSE; - } - // determine catmode if (!gfits_scan (&catalog[0].header, "NAXIS", "%d", 1, &Naxis)) { @@ -275,5 +275,7 @@ } -// write out the data, unlink if empty? +// write out the data, unlink if empty? 'save' means: write out all data currently in +// memory. NOTE: this is currently not always possible: for non-SPLIT moe files, this +// operation may require expanding the file size, which does not automatically happen int dvo_catalog_save (Catalog *catalog, char VERBOSE) { @@ -281,11 +283,5 @@ gfits_modify (&catalog[0].header, "SORTED", "%t", 1, catalog[0].sorted); - if (catalog[0].Nmeas_off != 0) { - if (!dvo_catalog_update (catalog, VERBOSE)) { - return (FALSE); - } - return (TRUE); - } - + // XXX handle return status switch (catalog[0].catmode) { case DVO_MODE_RAW: @@ -305,21 +301,22 @@ } +// write out the in-memory data which extends beyond the data on disk. NOTE: this is +// currently only possible for the SPLIT mode. int dvo_catalog_update (Catalog *catalog, char VERBOSE) { + + // set the 'sorted' header keyword + catalog[0].sorted = FALSE; + gfits_modify (&catalog[0].header, "SORTED", "%t", 1, catalog[0].sorted); /* update is only valid for catmode SPLIT */ switch (catalog[0].catmode) { case DVO_MODE_RAW: - dvo_catalog_save_raw (catalog, VERBOSE); + fprintf (stderr, "not allowed for RAW mode\n"); break; case DVO_MODE_MEF: - dvo_catalog_save_mef (catalog, VERBOSE); + fprintf (stderr, "not allowed for MEF mode\n"); break; case DVO_MODE_SPLIT: - /* new file needs to use save_catalog_split */ - if (catalog[0].Nave_disk == 0) { - dvo_catalog_save_split (catalog, VERBOSE); - } else { - dvo_catalog_update_split (catalog, VERBOSE); - } + dvo_catalog_update_split (catalog, VERBOSE); break; default: @@ -360,4 +357,5 @@ catalog[0].secfilt = outsec; catalog[0].Nsecfilt = Nsecfilt; + catalog[0].Nsecf_mem = Nsecfilt * catalog[0].Naverage; } return (TRUE); @@ -396,2 +394,33 @@ style : raw, mef, split, mysql */ + +int dvo_catalog_load_segment (Catalog *catalog, int VERBOSE, int start, int Nrows) { + + int Naxis, split, status; + char measure[80]; + + // catformat determined in dvo_catalog_load_XXX function + catalog[0].catformat = DVO_FORMAT_UNDEF; + + switch (catalog[0].catmode) { + case DVO_MODE_RAW: + if (VERBOSE) fprintf (stderr, "reading catalog (mode DVO_MODE_RAW)\n"); + fprintf (stderr, "cannot do this in raw mode\n"); + // dvo_catalog_load_segment_raw (catalog, VERBOSE, start, Nrows); + break; + case DVO_MODE_MEF: + if (VERBOSE) fprintf (stderr, "reading catalog (mode DVO_MODE_MEF)\n"); + fprintf (stderr, "cannot do this in mef mode\n"); + // dvo_catalog_load_segment_mef (catalog, VERBOSE, start, Nrows); + break; + case DVO_MODE_SPLIT: + if (VERBOSE) fprintf (stderr, "reading catalog (mode DVO_MODE_SPLIT)\n"); + dvo_catalog_load_segment_split (catalog, VERBOSE, start, Nrows); + break; + default: + fprintf (stderr, "error getting catalog mode\n"); + exit (2); + } + return (TRUE); +} + Index: trunk/Ohana/src/libdvo/src/dvo_catalog_create.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_create.c (revision 15722) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_create.c (revision 15743) @@ -73,13 +73,13 @@ // lock the additional split files // XXX clear residual locks if we fail - if (dvo_catalog_lock (catalog[0].measure_catalog, catalog[0].lockmode) != 2) { + if (dvo_catalog_lock (catalog[0].measure_catalog, catalog[0].lockmode) != DVO_CAT_OPEN_EMPTY) { fprintf (stderr, "error with file lock\n"); exit (2); } - if (dvo_catalog_lock (catalog[0].missing_catalog, catalog[0].lockmode) != 2) { + if (dvo_catalog_lock (catalog[0].missing_catalog, catalog[0].lockmode) != DVO_CAT_OPEN_EMPTY) { fprintf (stderr, "error with file lock\n"); exit (2); } - if (dvo_catalog_lock (catalog[0].secfilt_catalog, catalog[0].lockmode) != 2) { + if (dvo_catalog_lock (catalog[0].secfilt_catalog, catalog[0].lockmode) != DVO_CAT_OPEN_EMPTY) { fprintf (stderr, "error with file lock\n"); exit (2); Index: trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 15722) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_mef.c (revision 15743) @@ -28,7 +28,19 @@ if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; + /* save the current number so we can do partial updates */ + catalog[0].Naves_disk = Naverage; + catalog[0].Nmeas_disk = Nmeasure; + catalog[0].Nmiss_disk = Nmissing; + catalog[0].Nsecf_disk = Naverage * Nsecfilt; + /** Nsecfilt is unusual: it does not list the number of data items in the table instead, the number of items is Nsecfilt * Naverage. **/ catalog[0].Nsecfilt = Nsecfilt; + + /* default values, but we will assign these a valid value before we exit (even if empty) */ + catalog[0].average = NULL; + catalog[0].measure = NULL; + catalog[0].missing = NULL; + catalog[0].secfilt = NULL; /* validate table mode */ @@ -46,11 +58,16 @@ } /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */ - catalog[0].average = FtableToAverage (&ftable, &catalog[0].Naverage, &catalog[0].catformat, &primary); - if (Naverage != catalog[0].Naverage) { - fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%d vs %d)\n", Naverage, catalog[0].Naverage); - } + catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); + if (Naverage != catalog[0].Naves_disk) { + fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%d vs %d)\n", Naverage, catalog[0].Naves_disk); + } + catalog[0].Naverage = catalog[0].Naves_disk; + catalog[0].Naves_off = 0; } else { Nbytes = gfits_data_size (&header); fseek (f, Nbytes, SEEK_CUR); + ALLOCATE (catalog[0].average, Average, 1); + catalog[0].Naverage = 0; + catalog[0].Naves_off = catalog[0].Naves_disk; } gfits_free_header (&header); @@ -69,10 +86,15 @@ } catalog[0].measure = FtableToMeasure (&ftable, &catalog[0].Nmeasure, &catalog[0].catformat); - if (Nmeasure != catalog[0].Nmeasure) { - fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, catalog[0].Nmeasure); - } + if (Nmeasure != catalog[0].Nmeas_disk) { + fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, catalog[0].Nmeas_disk); + } + catalog[0].Nmeasure = catalog[0].Nmeas_disk; + catalog[0].Nmeas_off = 0; } else { Nbytes = gfits_data_size (&header); fseek (f, Nbytes, SEEK_CUR); + ALLOCATE (catalog[0].measure, Measure, 1); + catalog[0].Nmeasure = 0; + catalog[0].Nmeas_off = catalog[0].Nmeas_disk; } @@ -90,10 +112,15 @@ /* no conversions currently defined */ catalog[0].missing = gfits_table_get_Missing (&ftable, &catalog[0].Nmissing, NULL); - if (Nmissing != catalog[0].Nmissing) { - fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%d vs %d)\n", Nmissing, catalog[0].Nmissing); - } + if (Nmissing != catalog[0].Nmiss_disk) { + fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%d vs %d)\n", Nmissing, catalog[0].Nmiss_disk); + } + catalog[0].Nmissing = catalog[0].Nmiss_disk; + catalog[0].Nmiss_off = 0; } else { Nbytes = gfits_data_size (&header); fseek (f, Nbytes, SEEK_CUR); + ALLOCATE (catalog[0].missing, Missing, 1); + catalog[0].Nmissing = 0; + catalog[0].Nmiss_off = catalog[0].Nmiss_disk; } @@ -125,7 +152,7 @@ Ntmpfilt = catalog[0].Nsecfilt; Nsecfilt = catalog[0].Nsecfilt + 1; - Ntotal = Nsecfilt * catalog[0].Naverage; + Ntotal = Nsecfilt * catalog[0].Naves_disk; ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal); - for (i = 0; i < catalog[0].Naverage; i++) { + for (i = 0; i < catalog[0].Naves_disk; i++) { catalog[0].secfilt[i*Nsecfilt + 0] = primary[i]; for (j = 0; j < Ntmpfilt; j++) { @@ -134,18 +161,22 @@ } catalog[0].Nsecfilt = Nsecfilt; + catalog[0].Nsecf_disk = Ntotal; free (primary); } - + catalog[0].Nsecf_mem = catalog[0].Nsecf_disk; + catalog[0].Nsecf_off = 0; } else { /* no real need to skip the data array here... */ Nbytes = gfits_data_size (&header); fseek (f, Nbytes, SEEK_CUR); - if (primary != NULL) free (primary); - } - - /* save the current number so we can do partial updates */ - catalog[0].Nave_disk = catalog[0].Naverage; - catalog[0].Nmeas_disk = catalog[0].Nmeasure; - catalog[0].Nmiss_disk = catalog[0].Nmissing; + if (primary != NULL) { + free (primary); + catalog[0].Nsecfilt ++; + catalog[0].Nsecf_disk = catalog[0].Nsecfilt * catalog[0].Naves_disk; + } + ALLOCATE (catalog[0].secfilt, SecFilt, 1); + catalog[0].Nsecf_mem = 0; + catalog[0].Nsecf_off = catalog[0].Nsecf_disk; + } return (TRUE); Index: trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 15722) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_raw.c (revision 15743) @@ -171,4 +171,5 @@ } catalog[0].Nsecfilt = Nsecfilt; + catalog[0].Nsecf_mem = Ntotal; free (primary); } @@ -201,5 +202,5 @@ /* save the current number so we can do partial updates */ - catalog[0].Nave_disk = catalog[0].Naverage; + catalog[0].Naves_disk = catalog[0].Naverage; catalog[0].Nmeas_disk = catalog[0].Nmeasure; catalog[0].Nmiss_disk = catalog[0].Nmissing; Index: trunk/Ohana/src/libdvo/src/dvo_catalog_split.c =================================================================== --- trunk/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 15722) +++ trunk/Ohana/src/libdvo/src/dvo_catalog_split.c (revision 15743) @@ -1,3 +1,155 @@ # include + +// return options: +// * error (cannot lock, open, read, etc) +// * empty (file is not found) +// * ok + +int dvo_catalog_secfilt_to_primary (Catalog *catalog, SecFilt **myPrimary, SecFilt **mySecfilt, int *myNsecfilt) { + + int i, j, Nallfilt, Nsecfilt, Ntotal; + + SecFilt *primary; + SecFilt *secfilt; + + if (catalog[0].secfilt == NULL) { + fprintf (stderr, "missing secfilt, cannot build output averages (dvo_catalog_split.c)\n"); + exit (1); + } + secfilt = catalog[0].secfilt; + + // XXX this translation only works if we have loaded / created a matched average/secfilt set + assert (catalog[0].Nsecf_mem == catalog[0].Nsecfilt*catalog[0].Naverage); + + Nallfilt = catalog[0].Nsecfilt; + Nsecfilt = catalog[0].Nsecfilt - 1; + Ntotal = Nsecfilt * catalog[0].Naverage; + ALLOCATE (primary, SecFilt, catalog[0].Naverage); + ALLOCATE (secfilt, SecFilt, Ntotal); + + for (i = 0; i < catalog[0].Naverage; i++) { + primary[i] = secfilt[i*Nallfilt + 0]; + for (j = 0; j < Nsecfilt; j++) { + secfilt[i*Nsecfilt + j] = catalog[0].secfilt[i*Nallfilt + j + 1]; + } + } + catalog[0].Nsecfilt --; + catalog[0].Nsecf_mem = catalog[0].Naverage*catalog[0].Nsecfilt; + + *myPrimary = primary; + *mySecfilt = secfilt; + *myNsecfilt = Nsecfilt; +} + +int dvo_catalog_primary_to_secfilt (Catalog *catalog, SecFilt *primary, int Naves) { + + int Ntmpfilt, Nsecfilt, Ntotal, i, j; + SecFilt *tmpfilt; + + tmpfilt = catalog[0].secfilt; + Ntmpfilt = catalog[0].Nsecfilt; + + // we do NOT modify Nsecf_disk; this operation only modifies in in-memory values + + catalog[0].Nsecfilt ++; + Nsecfilt = catalog[0].Nsecfilt; + Ntotal = Nsecfilt * Naves; + + catalog[0].Nsecf_mem = Ntotal; + + ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal); + for (i = 0; i < Naves; i++) { + catalog[0].secfilt[i*Nsecfilt + 0] = primary[i]; + for (j = 0; j < Ntmpfilt; j++) { + catalog[0].secfilt[i*Nsecfilt + j + 1] = tmpfilt[i*Ntmpfilt + j]; + } + } + free (primary); + return (TRUE); +} + +int dvo_catalog_save_subcat (Catalog *catalog, FTable *ftable, int start, int Nrows, int Ndisk, int Ntotal) { + + Matrix matrix; + + /* rewind file pointers and truncate (file is still open) */ + fseek (catalog->f, 0, SEEK_SET); + + // write PHU header + if (!gfits_fwrite_header (catalog->f, &catalog->header)) { + fprintf (stderr, "can't write primary header"); + return (FALSE); + } + + // write the PHU matrix; this is probably a NOP, do I have to keep it in? + gfits_create_matrix (&catalog->header, &matrix); + if (!gfits_fwrite_matrix (catalog->f, &matrix)) { + fprintf (stderr, "can't write primary matrix"); + gfits_free_matrix (&matrix); + return (FALSE); + } + gfits_free_matrix (&matrix); + + // write the table data + if (!gfits_fwrite_ftable_range (catalog->f, ftable, start, Nrows, Ndisk, Ntotal)) { + fprintf (stderr, "can't write table data"); + return (FALSE); + } + return (TRUE); +} + +int dvo_catalog_open_subcat (Catalog *catalog, Catalog **Subcat, Header *header, char *name, int VERBOSE) { + + int status; + char *path, string[80]; + Catalog *subcat; + Matrix matrix; + + /* in split mode, we need to init & open the corresponding measure file (even if we do not read + * any data in at this stage) */ + ALLOCATE (subcat, Catalog, 1); + dvo_catalog_init (subcat, TRUE); + + /* needed to find the split files below */ + path = pathname (catalog[0].filename); + + /* get split filename from main header (paths relative to cpt file) */ + if (!gfits_scan (&catalog[0].header, name, "%s", 1, string)) return (DVO_CAT_OPEN_FAIL); + ALLOCATE (subcat[0].filename, char, strlen(path) + strlen(string) + 2); + sprintf (subcat[0].filename, "%s/%s", path, string); + + /* lock & open catalog file */ + status = dvo_catalog_lock (subcat, catalog[0].lockmode); + if (status != DVO_CAT_OPEN_OK) { + if (VERBOSE) { + if (status == DVO_CAT_OPEN_EMPTY) { + fprintf (stderr, "%s (%s) is empty\n", name, subcat[0].filename); + } else { + fprintf (stderr, "failure to lock %s (%s)\n", name, subcat[0].filename); + } + } + return (status); + } + + /* read PHU */ + if (!gfits_load_header (subcat[0].f, &subcat[0].header)) { + if (VERBOSE) fprintf (stderr, "error reading %s header: %s\n", name, subcat[0].filename); + return (DVO_CAT_OPEN_FAIL); + } + /* matrix should be empty : XXX skip the matrix data? */ + if (!gfits_fread_matrix (subcat[0].f, &matrix, &subcat[0].header)) { + if (VERBOSE) fprintf (stderr, "can't read primary matrix for %s\n", name); + return (DVO_CAT_OPEN_FAIL); + } + gfits_free_matrix (&matrix); + /* read Measure table header */ + if (!gfits_fread_header (subcat[0].f, header)) { + if (VERBOSE) fprintf (stderr, "can't read %s PHU header\n", name); + return (DVO_CAT_OPEN_FAIL); + } + + *Subcat = subcat; + return (DVO_CAT_OPEN_OK); +} int dvo_catalog_load_split (Catalog *catalog, int VERBOSE) { @@ -9,8 +161,10 @@ FTable ftable; SecFilt *primary; - Catalog *measure, *missing, *secfilt; /* ftable header storage for below */ ftable.header = &header; + ftable.buffer = NULL; + header.buffer = NULL; + primary = NULL; /* needed to find the split files below */ @@ -23,9 +177,15 @@ if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0; + /* save the current number so we can do partial updates */ + catalog[0].Naves_disk = Naverage; + catalog[0].Nmeas_disk = Nmeasure; + catalog[0].Nmiss_disk = Nmissing; + catalog[0].Nsecf_disk = Naverage * Nsecfilt; + /** Nsecfilt is unusual: it does not list the number of data items in the table instead, the number of items is Nsecfilt * Naverage. **/ - catalog[0].Nsecfilt = Nsecfilt; - - /* default values */ + catalog[0].Nsecfilt = Nsecfilt; + + /* default values, but we will assign these a valid value before we exit (even if empty) */ catalog[0].average = NULL; catalog[0].measure = NULL; @@ -33,12 +193,9 @@ catalog[0].secfilt = NULL; - /* XXX EAM : validate table mode */ - /*** Average Table ***/ - if (catalog[0].catflags & LOAD_AVES) { /* move pointer past header -- must be already read (load_catalog) */ fseek (catalog[0].f, catalog[0].header.size, SEEK_SET); - /* matrix should be empty */ + /* matrix should be empty : XXX should we drop this step and skip the data? */ if (!gfits_fread_matrix (catalog[0].f, &matrix, &catalog[0].header)) { if (VERBOSE) fprintf (stderr, "can't read primary matrix"); @@ -50,235 +207,287 @@ return (FALSE); } - /* read Average table data */ + /* read Average table data : format is irrelevant here */ if (!gfits_fread_ftable_data (catalog[0].f, &ftable)) { if (VERBOSE) fprintf (stderr, "can't read table average data"); return (FALSE); } - /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */ - catalog[0].average = FtableToAverage (&ftable, &catalog[0].Naverage, &catalog[0].catformat, &primary); - if (Naverage != catalog[0].Naverage) { - fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%d vs %d)\n", Naverage, catalog[0].Naverage); - } - gfits_free_header (&header); - } + /* convert the saved version of the table to the internal version. Old versions of DVO stored + * one of the average magnitudes in Average. We save this in case it is needed below. NOTE: + * primary is only used if we read in the secfilt table, otherwise it should be freed */ + catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); + if (Naverage != catalog[0].Naves_disk) { + fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%d vs %d)\n", Naverage, catalog[0].Naves_disk); + } + gfits_free_header (&header); + catalog[0].Naverage = catalog[0].Naves_disk; + catalog[0].Naves_off = 0; + } else { + ALLOCATE (catalog[0].average, Average, 1); + catalog[0].Naverage = 0; + catalog[0].Naves_off = catalog[0].Naves_disk; + } /*** Measure Table ***/ - - measure = NULL; - - /* (Full Load) */ - if (catalog[0].catflags & LOAD_MEAS) { - ALLOCATE (measure, Catalog, 1); - dvo_catalog_init (measure, TRUE); - - /* get split filename from main header (paths relative to cpt file) */ - if (!gfits_scan (&catalog[0].header, "MEASURE", "%s", 1, string)) return (FALSE); - ALLOCATE (measure[0].filename, char, strlen(path) + strlen(string) + 2); - sprintf (measure[0].filename, "%s/%s", path, string); - - /* lock & open catalog file */ - if (dvo_catalog_lock (measure, catalog[0].lockmode) != 1) { - fprintf (stderr, "cannot access measure file %s\n", measure[0].filename); - exit (2); - } - - /* read PHU */ - if (!gfits_load_header (measure[0].f, &measure[0].header)) { - if (VERBOSE) fprintf (stderr, "catalog file does not exist: %s\n", measure[0].filename); - return (FALSE); - } - /* matrix should be empty */ - if (!gfits_fread_matrix (measure[0].f, &matrix, &measure[0].header)) { - if (VERBOSE) fprintf (stderr, "can't read primary matrix\n"); - return (FALSE); - } - /* read Measure table header */ - if (!gfits_fread_header (measure[0].f, &header)) { - if (VERBOSE) fprintf (stderr, "can't read measure PHU header\n"); - return (FALSE); - } + status = dvo_catalog_open_subcat (catalog, &catalog[0].measure_catalog, ftable.header, "MEASURE", VERBOSE); + if (status == DVO_CAT_OPEN_FAIL) { + return (FALSE); + } + if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmeas_disk > 0)) { + return (FALSE); + } + if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MEAS)) { + // XXX this allows an empty Measure catalog with non-empty Average catalog : is that OK? /* read Measure table data */ - if (!gfits_fread_ftable_data (measure[0].f, &ftable)) { + if (!gfits_fread_ftable_data (catalog[0].measure_catalog[0].f, &ftable)) { if (VERBOSE) fprintf (stderr, "can't read table measure data\n"); return (FALSE); } - /* convert data format to internal */ - catalog[0].measure = FtableToMeasure (&ftable, &catalog[0].Nmeasure, &catalog[0].catformat); - if (Nmeasure != catalog[0].Nmeasure) { - fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, catalog[0].Nmeasure); - } + /* convert data format to internal : returns number of row read in Nmeasure */ + catalog[0].measure = FtableToMeasure (&ftable, &Nmeasure, &catalog[0].catformat); + if (Nmeasure != catalog[0].Nmeas_disk) { + fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, catalog[0].Nmeas_disk); + } + catalog[0].Nmeasure = catalog[0].Nmeas_disk; catalog[0].Nmeas_off = 0; - gfits_free_header (&header); - gfits_free_matrix (&matrix); - } - - /* (Meta Load) */ - if (catalog[0].catflags & LOAD_MEAS_META) { - ALLOCATE (measure, Catalog, 1); - dvo_catalog_init (measure, TRUE); - - /* get split filename from main header (paths relative to cpt file) */ - if (!gfits_scan (&catalog[0].header, "MEASURE", "%s", 1, string)) return (FALSE); - ALLOCATE (measure[0].filename, char, strlen(path) + strlen(string) + 2); - sprintf (measure[0].filename, "%s/%s", path, string); - - /* lock & open catalog file */ - if (dvo_catalog_lock (measure, catalog[0].lockmode) != 1) { - fprintf (stderr, "cannot access measure file %s\n", measure[0].filename); - exit (2); - } - - /* read PHU */ - if (!gfits_load_header (measure[0].f, &measure[0].header)) { - if (VERBOSE) fprintf (stderr, "catalog file does not exist: %s\n", measure[0].filename); - return (FALSE); - } - - /* allocate dummy data for valid realloc */ + } else { + // XXX is it necessary to generate a template header here? + gfits_create_header (&catalog[0].measure_catalog[0].header); ALLOCATE (catalog[0].measure, Measure, 1); - - /* set up the table size information */ - catalog[0].Nmeasure = 0; /* no rows loaded */ - catalog[0].Nmeas_off = Nmeasure; /* start of new entries */ - } - catalog[0].measure_catalog = measure; + catalog[0].Nmeasure = 0; + catalog[0].Nmeas_off = catalog[0].Nmeas_disk; + } + gfits_free_header (ftable.header); /*** Missing Table ***/ - - missing = NULL; - if (catalog[0].catflags & LOAD_MISS) { - ALLOCATE (missing, Catalog, 1); - dvo_catalog_init (missing, TRUE); - - /* get split filename from main header (paths relative to cpt file) */ - if (!gfits_scan (&catalog[0].header, "MISSING", "%s", 1, string)) return (FALSE); - ALLOCATE (missing[0].filename, char, strlen(path) + strlen(string) + 2); - sprintf (missing[0].filename, "%s/%s", path, string); - - /* lock & open catalog file */ - status = dvo_catalog_lock (missing, catalog[0].lockmode); - if (!status) { - fprintf (stderr, "ERROR: cannot access missing file %s\n", missing[0].filename); - exit (2); - } - if (status == 2) { - /* MISSING table is empty (this is not an error) */ - gfits_create_header (&missing[0].header); - goto missing_empty; - } - - /* read PHU */ - if (!gfits_load_header (missing[0].f, &missing[0].header)) { - if (VERBOSE) fprintf (stderr, "catalog file does not exist: %s\n", missing[0].filename); - return (FALSE); - } - /* matrix should be empty */ - if (!gfits_fread_matrix (missing[0].f, &matrix, &missing[0].header)) { - if (VERBOSE) fprintf (stderr, "can't read primary matrix\n"); - return (FALSE); - } - /* read Missing table header */ - if (!gfits_fread_header (missing[0].f, &header)) { - if (VERBOSE) fprintf (stderr, "can't read table missing header\n"); - return (FALSE); - } + status = dvo_catalog_open_subcat (catalog, &catalog[0].missing_catalog, ftable.header, "MISSING", VERBOSE); + if (status == DVO_CAT_OPEN_FAIL) { + return (FALSE); + } + if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nmiss_disk > 0)) { + return (FALSE); + } + if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_MISS)) { /* read Missing table data */ - if (!gfits_fread_ftable_data (missing[0].f, &ftable)) { + if (!gfits_fread_ftable_data (catalog[0].missing_catalog[0].f, &ftable)) { if (VERBOSE) fprintf (stderr, "can't read table missing data\n"); return (FALSE); } - /* no conversions currently defined */ - catalog[0].missing = gfits_table_get_Missing (&ftable, &catalog[0].Nmissing, NULL); - if (Nmissing != catalog[0].Nmissing) { - fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%d vs %d)\n", Nmissing, catalog[0].Nmissing); - } - gfits_free_header (&header); - gfits_free_matrix (&matrix); - } -missing_empty: - catalog[0].missing_catalog = missing; + /* no conversions currently defined : this just does the byte swap */ + catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL); + if (Nmissing != catalog[0].Nmiss_disk) { + fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%d vs %d)\n", Nmissing, catalog[0].Nmiss_disk); + } + catalog[0].Nmissing = catalog[0].Nmiss_disk; + catalog[0].Nmiss_off = 0; + } else { + // XXX is it necessary to generate a template header here? + gfits_create_header (&catalog[0].missing_catalog[0].header); + ALLOCATE (catalog[0].missing, Missing, 1); + catalog[0].Nmissing = 0; + catalog[0].Nmiss_off = catalog[0].Nmiss_disk; + } + gfits_free_header (ftable.header); /*** Secfilt Table ***/ - - secfilt = NULL; - if (catalog[0].catflags & LOAD_SECF) { - ALLOCATE (secfilt, Catalog, 1); - dvo_catalog_init (secfilt, TRUE); - - /* get split filename from main header (paths relative to cpt file) */ - if (!gfits_scan (&catalog[0].header, "SECFILT", "%s", 1, string)) return (FALSE); - ALLOCATE (secfilt[0].filename, char, strlen(path) + strlen(string) + 2); - sprintf (secfilt[0].filename, "%s/%s", path, string); - - /* lock & open catalog file */ - if (dvo_catalog_lock (secfilt, catalog[0].lockmode) != 1) { - fprintf (stderr, "cannot access secfilt file %s\n", secfilt[0].filename); - exit (2); - } - - /* read PHU */ - if (!gfits_load_header (secfilt[0].f, &secfilt[0].header)) { - if (VERBOSE) fprintf (stderr, "catalog file does not exist: %s\n", secfilt[0].filename); - return (FALSE); - } - /* matrix should be empty */ - if (!gfits_fread_matrix (secfilt[0].f, &matrix, &secfilt[0].header)) { - if (VERBOSE) fprintf (stderr, "can't read primary matrix\n"); - return (FALSE); - } - /* read secfilt table header */ - if (!gfits_fread_header (secfilt[0].f, &header)) { - if (VERBOSE) fprintf (stderr, "can't read table secfilt header\n"); - return (FALSE); - } + status = dvo_catalog_open_subcat (catalog, &catalog[0].secfilt_catalog, ftable.header, "SECFILT", VERBOSE); + if (status == DVO_CAT_OPEN_FAIL) { + return (FALSE); + } + if ((status == DVO_CAT_OPEN_EMPTY) && (catalog[0].Nsecf_disk > 0)) { + return (FALSE); + } + if ((status != DVO_CAT_OPEN_EMPTY) && (catalog[0].catflags & LOAD_SECF)) { /* read secfilt table data */ - if (!gfits_fread_ftable_data (secfilt[0].f, &ftable)) { + if (!gfits_fread_ftable_data (catalog[0].secfilt_catalog[0].f, &ftable)) { if (VERBOSE) fprintf (stderr, "can't read table secfilt data\n"); return (FALSE); } - Nexpect = catalog[0].Nsecfilt * catalog[0].Naverage; catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); - if (Nexpect != Nitems) { - fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers (%d vs %d)\n", Nexpect, Nitems); - } + if (Nitems != catalog[0].Nsecf_disk) { + fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers (%d vs %d)\n", Nitems, catalog[0].Nsecf_disk); + } + catalog[0].Nsecf_mem = catalog[0].Nsecf_disk; + catalog[0].Nsecf_off = 0; /* if primary is defined, we were supplied with one additional average magnitude from Average we need to interleave these magnitudes with the secfilt entries just loaded */ if (primary != NULL) { - int Ntmpfilt, Ntotal, i, j; - SecFilt *tmpfilt; - tmpfilt = catalog[0].secfilt; - Ntmpfilt = catalog[0].Nsecfilt; - Nsecfilt = catalog[0].Nsecfilt + 1; - Ntotal = Nsecfilt * catalog[0].Naverage; - ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal); - for (i = 0; i < catalog[0].Naverage; i++) { - catalog[0].secfilt[i*Nsecfilt + 0] = primary[i]; - for (j = 0; j < Ntmpfilt; j++) { - catalog[0].secfilt[i*Nsecfilt + j + 1] = tmpfilt[i*Ntmpfilt + j]; - } - } - catalog[0].Nsecfilt = Nsecfilt; + // this modifies catalog.Nsecf_mem,Nsecfilt + dvo_catalog_primary_to_secfilt (catalog, primary, catalog[0].Naves_disk); + } + } else { + if (primary != NULL) { free (primary); - } - - - gfits_free_header (&header); - gfits_free_matrix (&matrix); - } else { - if (primary != NULL) free (primary); - } - - catalog[0].secfilt_catalog = secfilt; - - /* save the current number so we can do partial updates */ - catalog[0].Nave_disk = Naverage; - catalog[0].Nmeas_disk = Nmeasure; - catalog[0].Nmiss_disk = Nmissing; + catalog[0].Nsecfilt ++; + } + gfits_create_header (&catalog[0].secfilt_catalog[0].header); + ALLOCATE (catalog[0].secfilt, SecFilt, 1); + catalog[0].Nsecf_mem = 0; + catalog[0].Nsecf_off = catalog[0].Nsecf_disk; + } + gfits_free_header (ftable.header); return (TRUE); } -/* save_catalog_split writes complete new files from scratch */ +// I need to always read both average and secfilt at the same time to correctly manage the +// primary secfilt values... +int dvo_catalog_load_segment_split (Catalog *catalog, int VERBOSE, int start, int Nrows) { + + int Naverage, Nsecfilt, Nexpect, Nitems, Nmeasure, Nmissing; + Header header; + Matrix matrix; + FTable ftable; + SecFilt *primary; + + /* ftable header storage for below */ + ftable.header = &header; + ftable.buffer = NULL; + header.buffer = NULL; + primary = NULL; + + /*** Average (& SecFilt) Table ***/ + if (catalog[0].catflags & LOAD_AVES) { + + /*** load the Average data ***/ + /* move pointer past header -- must be already read (load_catalog) */ + fseek (catalog[0].f, catalog[0].header.size, SEEK_SET); + /* matrix should be empty : XXX should we drop this step and skip the data? */ + if (!gfits_fread_matrix (catalog[0].f, &matrix, &catalog[0].header)) { + if (VERBOSE) fprintf (stderr, "can't read primary matrix"); + return (FALSE); + } + /* read Average table header */ + if (!gfits_fread_header (catalog[0].f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read table average header"); + return (FALSE); + } + /* read Average table data : format is irrelevant here */ + if (!gfits_fread_ftable_range (catalog[0].f, &ftable, start, Nrows)) { + if (VERBOSE) fprintf (stderr, "can't read table average data"); + return (FALSE); + } + + /* convert the saved version of the table to the internal version. Old versions of DVO stored + * one of the average magnitudes in Average. We save this in case it is needed below. NOTE: + * primary is only used if we read in the secfilt table, otherwise it should be freed */ + catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary); + if (Naverage != Nrows) { + // XXX this condition denotes the eof has been reached; not an error or a warning + // fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%d vs %d)\n", Naverage, Nrows); + } + gfits_free_header (&header); + catalog[0].Naverage = Naverage; + catalog[0].Naves_off = start; + + /*** load the secfilt data ***/ + Catalog *subcat = catalog[0].secfilt_catalog; + + /* move pointer past header -- must be already read (load_catalog) */ + fseek (subcat[0].f, subcat[0].header.size, SEEK_SET); + /* matrix should be empty : XXX should we drop this step and skip the data? */ + if (!gfits_fread_matrix (subcat[0].f, &matrix, &subcat[0].header)) { + if (VERBOSE) fprintf (stderr, "can't read primary matrix"); + return (FALSE); + } + /* read Secfilt table header */ + if (!gfits_fread_header (subcat[0].f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read table measure header"); + return (FALSE); + } + /* read Secfilt table data : format is irrelevant here */ + if (!gfits_fread_ftable_range (subcat[0].f, &ftable, start*catalog[0].Nsecfilt, catalog[0].Naverage*catalog[0].Nsecfilt)) { + if (VERBOSE) fprintf (stderr, "can't read table measure data"); + return (FALSE); + } + + Nexpect = catalog[0].Naverage * catalog[0].Nsecfilt; + catalog[0].secfilt = FtableToSecFilt (&ftable, &Nitems, &catalog[0].catformat); + if (Nitems != Nexpect) { + fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers (%d vs %d)\n", Nitems, Nexpect); + } + catalog[0].Nsecf_mem = catalog[0].Naverage * catalog[0].Nsecfilt; + catalog[0].Nsecf_off = start * catalog[0].Nsecfilt; + + /* if primary is defined, we were supplied with one additional average magnitude from Average + we need to interleave these magnitudes with the secfilt entries just loaded */ + if (primary != NULL) { + dvo_catalog_primary_to_secfilt (catalog, primary, Nrows); + } + gfits_free_header (&header); + gfits_free_matrix (&matrix); + } + + // XXX check the open status of the catalog + if (catalog[0].catflags & LOAD_MEAS) { + + Catalog *subcat = catalog[0].measure_catalog; + + /* move pointer past header -- must be already read (load_catalog) */ + fseek (subcat[0].f, subcat[0].header.size, SEEK_SET); + /* matrix should be empty : XXX should we drop this step and skip the data? */ + if (!gfits_fread_matrix (subcat[0].f, &matrix, &subcat[0].header)) { + if (VERBOSE) fprintf (stderr, "can't read primary matrix"); + return (FALSE); + } + /* read Measure table header */ + if (!gfits_fread_header (subcat[0].f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read table measure header"); + return (FALSE); + } + /* read Measure table data : format is irrelevant here */ + if (!gfits_fread_ftable_range (subcat[0].f, &ftable, start, Nrows)) { + if (VERBOSE) fprintf (stderr, "can't read table measure data"); + return (FALSE); + } + + /* convert data format to internal : returns number of row read in Nmeasure */ + catalog[0].measure = FtableToMeasure (&ftable, &Nmeasure, &catalog[0].catformat); + if (Nmeasure != Nrows) { + fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, Nrows); + } + gfits_free_header (&header); + gfits_free_matrix (&matrix); + catalog[0].Nmeasure = Nmeasure; + catalog[0].Nmeas_off = start; + } + + // XXX check the open status of the catalog? + if (catalog[0].catflags & LOAD_MISS) { + + Catalog *subcat = catalog[0].missing_catalog; + + /* move pointer past header -- must be already read (load_catalog) */ + fseek (subcat[0].f, subcat[0].header.size, SEEK_SET); + /* matrix should be empty : XXX should we drop this step and skip the data? */ + if (!gfits_fread_matrix (subcat[0].f, &matrix, &subcat[0].header)) { + if (VERBOSE) fprintf (stderr, "can't read primary matrix"); + return (FALSE); + } + /* read Missing table header */ + if (!gfits_fread_header (subcat[0].f, &header)) { + if (VERBOSE) fprintf (stderr, "can't read table missing header"); + return (FALSE); + } + /* read Missing table data : format is irrelevant here */ + if (!gfits_fread_ftable_range (subcat[0].f, &ftable, start, Nrows)) { + if (VERBOSE) fprintf (stderr, "can't read table missing data"); + return (FALSE); + } + + /* no conversions currently defined : this just does the byte swap */ + catalog[0].missing = gfits_table_get_Missing (&ftable, &Nmissing, NULL); + if (Nmissing != Nrows) { + fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%d vs %d)\n", Nmissing, Nrows); + } + gfits_free_header (&header); + gfits_free_matrix (&matrix); + catalog[0].Nmissing = Nmissing; + catalog[0].Nmiss_off = start; + } + return (TRUE); +} + +/* save_catalog_split writes all data currently in memory to disk */ int dvo_catalog_save_split (Catalog *catalog, char VERBOSE) { @@ -290,86 +499,262 @@ SecFilt *primary, *secfilt; int i, j, Nsecfilt, Nallfilt, Ntotal; + int Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new; + int first, start, Nrows; ftable.header = &header; - - if (catalog[0].Naverage == 0) { + ftable.buffer = NULL; + header.buffer = NULL; + primary = NULL; + + // skip empty catalogs: it is illegal to have Measures without corresponding Averages + Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off); + if (Naves_disk_new == 0) { if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n"); return (TRUE); } - /* for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary */ - if ((catalog[0].catformat == DVO_FORMAT_ELIXIR) || // special case for ELIXIR - (catalog[0].catformat == DVO_FORMAT_LONEOS)) { // special case for LONEOS - if (catalog[0].secfilt == NULL) { - fprintf (stderr, "missing secfilt, cannot build output averages (dvo_catalog_split.c:544)\n"); - exit (1); - } - secfilt = catalog[0].secfilt; - - Nallfilt = catalog[0].Nsecfilt; - Nsecfilt = catalog[0].Nsecfilt - 1; - Ntotal = Nsecfilt * catalog[0].Naverage; - ALLOCATE (primary, SecFilt, catalog[0].Naverage); - ALLOCATE (secfilt, SecFilt, Ntotal); - - for (i = 0; i < catalog[0].Naverage; i++) { - primary[i] = secfilt[i*Nallfilt + 0]; - for (j = 0; j < Nsecfilt; j++) { - secfilt[i*Nsecfilt + j] = catalog[0].secfilt[i*Nallfilt + j + 1]; - } - } - } else { - primary = NULL; - secfilt = catalog[0].secfilt; - Nsecfilt = catalog[0].Nsecfilt; - } + // for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary + switch (catalog[0].catformat) { + case DVO_FORMAT_ELIXIR: // special case for ELIXIR + case DVO_FORMAT_LONEOS: // special case for LONEOS + dvo_catalog_secfilt_to_primary (catalog, &primary, &secfilt, &Nsecfilt); + break; + default: + primary = NULL; + secfilt = catalog[0].secfilt; + Nsecfilt = catalog[0].Nsecfilt; + break; + } + + Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off); + Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off); + Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off); /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", "%d", 1, catalog[0].Naverage); - gfits_modify (&catalog[0].header, "NMEAS", "%d", 1, catalog[0].Nmeasure); - gfits_modify (&catalog[0].header, "NMISS", "%d", 1, catalog[0].Nmissing); + gfits_modify (&catalog[0].header, "NSTARS", "%d", 1, Naves_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", "%d", 1, Nmeas_disk_new); + gfits_modify (&catalog[0].header, "NMISS", "%d", 1, Nmiss_disk_new); gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); gfits_modify (&catalog[0].header, "EXTEND", "%t", 1, TRUE); - /* rewind file pointers and truncate (file is still open) */ - fseek (catalog[0].f, 0, SEEK_SET); - - /* write table PHU header - always write this out */ - /* XXX EAM : check if disk file size has changed */ - if (!gfits_fwrite_header (catalog[0].f, &catalog[0].header)) { - fprintf (stderr, "can't write primary header"); - goto failure; - } - /* in split mode, we can save only part of the data */ /*** Average Table ***/ - + if ((catalog[0].catflags & LOAD_AVES) && (catalog[0].average != NULL)) { + + first = 0; // first row in memory to write + start = catalog[0].Naves_off; // first disk row to write + Nrows = catalog[0].Naverage - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first < catalog[0].Naverage); + assert (catalog[0].Naves_disk >= catalog[0].Naves_off); + + /* convert internal to external format */ + if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) { + fprintf (stderr, "failure writing Average table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } else { + // even if we do not save the average table, we need to keep the header in sync + /* rewind file pointers and truncate (file is still open) */ + fseek (catalog[0].f, 0, SEEK_SET); + + /* write table PHU header - always write this out */ + /* XXX EAM : check if disk file size has changed */ + if (!gfits_fwrite_header (catalog[0].f, &catalog[0].header)) { + fprintf (stderr, "can't write primary header"); + goto failure; + } + } + + /*** Measure Table ***/ + if ((catalog[0].catflags & LOAD_MEAS) && (catalog[0].measure != NULL)) { + + first = 0; // first row in memory to write + start = catalog[0].Nmeas_off; // first disk row to write + Nrows = catalog[0].Nmeasure - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first < catalog[0].Nmeasure); + assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off); + + // convert to external table format + if (!MeasureToFtable (&ftable, &catalog[0].measure[first], Nrows, catalog[0].catformat)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Measure table + if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) { + fprintf (stderr, "trouble writing Measure table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Missing Table ***/ + if ((catalog[0].catflags & LOAD_MISS) && (catalog[0].missing != NULL)) { + + if (catalog[0].Nmiss_off != 0) { + fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); + goto failure; + } + + // convert to external table format + if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Missing table (must write out entire table) + if (!dvo_catalog_save_subcat (catalog[0].missing_catalog, &ftable, 0, catalog[0].Nmissing, catalog[0].Nmissing, catalog[0].Nmissing)) { + fprintf (stderr, "trouble writing Missing Table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /*** Secfilt Table ***/ + if ((catalog[0].catflags & LOAD_SECF) && (catalog[0].secfilt != NULL)) { + + first = 0; // first row in memory to write + start = catalog[0].Nsecf_off; // first disk row to write + Nitems = catalog[0].Nsecf_mem; + Nrows = Nitems - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first < Nitems); + assert (catalog[0].Nsecf_disk >= catalog[0].Nsecf_off); + // XXX check these for consistency... + + // convert to external table format + SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat); + + // write out SecFilt table + if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) { + fprintf (stderr, "failure writing SecFilt table\n"); + goto failure; + } + gfits_free_header (&header); + gfits_free_table (&ftable); + } + + /* free temp storage */ + if (primary != NULL) { + free (primary); + free (secfilt); + } + return (TRUE); + +failure: + /* free temp storage */ + gfits_free_header (&header); + gfits_free_table (&ftable); + if (primary != NULL) { + free (primary); + free (secfilt); + } + return (FALSE); +} + +/* update_catalog_split only writes new lines to file. */ +int dvo_catalog_update_split (Catalog *catalog, char VERBOSE) { + + int i, Nx, Ny, Nlines; + int Nitems, Nskip, Nout, Ndisk, Nstart; + Matrix matrix; + Header header; + FTable ftable; + Catalog *catfile; + SecFilt *primary, *secfilt; + int j, Nsecfilt, Nallfilt, Ntotal; + int Naves_disk_new, Nmeas_disk_new, Nmiss_disk_new, Nsecf_disk_new; + int first, start, Nrows; + + ftable.header = &header; + ftable.buffer = NULL; + header.buffer = NULL; + + // skip empty catalogs: it is illegal to have Measures without corresponding Averages + Naves_disk_new = MAX (catalog[0].Naves_disk, catalog[0].Naverage + catalog[0].Naves_off); + if (Naves_disk_new == 0) { + if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n"); + return (TRUE); + } + + // for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary + switch (catalog[0].catformat) { + case DVO_FORMAT_ELIXIR: // special case for ELIXIR + case DVO_FORMAT_LONEOS: // special case for LONEOS + dvo_catalog_secfilt_to_primary (catalog, &primary, &secfilt, &Nsecfilt); + break; + default: + primary = NULL; + secfilt = catalog[0].secfilt; + Nsecfilt = catalog[0].Nsecfilt; + break; + } + + Nmeas_disk_new = MAX (catalog[0].Nmeas_disk, catalog[0].Nmeasure + catalog[0].Nmeas_off); + Nmiss_disk_new = MAX (catalog[0].Nmiss_disk, catalog[0].Nmissing + catalog[0].Nmiss_off); + Nsecf_disk_new = MAX (catalog[0].Nsecf_disk, catalog[0].Naverage*Nsecfilt + catalog[0].Nsecf_off); + + /* make sure header is consistent with data */ + gfits_modify (&catalog[0].header, "NSTARS", "%d", 1, Naves_disk_new); + gfits_modify (&catalog[0].header, "NMEAS", "%d", 1, Nmeas_disk_new); + gfits_modify (&catalog[0].header, "NMISS", "%d", 1, Nmiss_disk_new); + gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); + gfits_modify (&catalog[0].header, "EXTEND", "%t", 1, TRUE); + + /* in split mode, we can save only part of the data */ + + /*** Average Table ***/ if (catalog[0].average != NULL) { - ftruncate (fileno (catalog[0].f), catalog[0].header.size); - - /* this is probably a NOP, do I have to keep it in? */ - gfits_create_matrix (&catalog[0].header, &matrix); - if (!gfits_fwrite_matrix (catalog[0].f, &matrix)) { - fprintf (stderr, "can't write primary matrix"); - goto failure; - } - gfits_free_matrix (&matrix); - - /* write out Average table (convert to FITS table format) */ - if (!AverageToFtable (&ftable, catalog[0].average, catalog[0].Naverage, catalog[0].catformat, primary)) { - fprintf (stderr, "trouble converting format\n"); - goto failure; - } - if (!gfits_fwrite_Theader (catalog[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_table (catalog[0].f, &ftable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } + + first = catalog[0].Naves_disk - catalog[0].Naves_off; // first row to write (memory) + start = catalog[0].Naves_disk; // first row to write (disk) + Nrows = catalog[0].Naverage - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first < catalog[0].Naverage); + assert (catalog[0].Naves_disk >= catalog[0].Naves_off); + + /* convert internal to external format */ + if (!AverageToFtable (&ftable, &catalog[0].average[first], Nrows, catalog[0].catformat, primary)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + if (!dvo_catalog_save_subcat (catalog, &ftable, start, Nrows, catalog[0].Naves_disk, Naves_disk_new)) { + fprintf (stderr, "failure writing Average table\n"); + goto failure; + } + gfits_free_header (&header); gfits_free_table (&ftable); - gfits_free_header (&header); + } else { + // even if we do not save the average table, we need to keep the header in sync + /* rewind file pointers and truncate (file is still open) */ + fseek (catalog[0].f, 0, SEEK_SET); + + /* write table PHU header - always write this out */ + /* XXX EAM : check if disk file size has changed */ + if (!gfits_fwrite_header (catalog[0].f, &catalog[0].header)) { + fprintf (stderr, "can't write primary header"); + goto failure; + } } @@ -377,81 +762,53 @@ if (catalog[0].measure != NULL) { - /* catalog file data is stored in separate structure */ - catfile = catalog[0].measure_catalog; - - /* XXX EAM : warn about this condition; add code to handle? */ - if (catalog[0].Nmeas_off != 0) { - fprintf (stderr, "WARNING: LOAD_MEAS_META mixed with save??\n"); - fprintf (stderr, "WARNING: this should not be allowed to happen!\n"); - } - - /* rewind file pointers and truncate (file is still open) */ - fseek (catfile[0].f, 0, SEEK_SET); - ftruncate (fileno (catfile[0].f), 0); - - /* write table PHU header */ - if (!gfits_fwrite_header (catfile[0].f, &catfile[0].header)) { - fprintf (stderr, "can't write primary header"); - goto failure; - } - - /* this is probably a NOP, do I have to keep it in? */ - gfits_create_matrix (&catfile[0].header, &matrix); - if (!gfits_fwrite_matrix (catfile[0].f, &matrix)) { - fprintf (stderr, "can't write primary matrix"); - goto failure; - } - gfits_free_matrix (&matrix); - - /* write out Measure table (convert to FITS table format) */ - MeasureToFtable (&ftable, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat); - if (!gfits_fwrite_Theader (catfile[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_table (catfile[0].f, &ftable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } + first = catalog[0].Nmeas_disk - catalog[0].Nmeas_off; // first row in memory to write + start = catalog[0].Nmeas_off; // first disk row to write + Nrows = catalog[0].Nmeasure - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first < catalog[0].Nmeasure); + assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off); + + // convert to external table format + if (!MeasureToFtable (&ftable, &catalog[0].measure[first], Nrows, catalog[0].catformat)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Measure table + if (!dvo_catalog_save_subcat (catalog[0].measure_catalog, &ftable, start, Nrows, catalog[0].Nmeas_disk, Nmeas_disk_new)) { + fprintf (stderr, "trouble writing Measure table\n"); + goto failure; + } + gfits_free_header (&header); gfits_free_table (&ftable); - gfits_free_header (&header); - } + } + + /* missing table CANNOT be written unsorted, thus it is always written + out in full */ /*** Missing Table ***/ + if (catalog[0].missing != NULL) { - /* catalog data is stored in separate catalog */ - catfile = catalog[0].missing_catalog; - - /* rewind file pointers and truncate (file is still open) */ - fseek (catfile[0].f, 0, SEEK_SET); - ftruncate (fileno (catfile[0].f), 0); - - /* write table PHU header */ - if (!gfits_fwrite_header (catfile[0].f, &catfile[0].header)) { - fprintf (stderr, "can't write primary header"); - goto failure; - } - - /* this is probably a NOP, do I have to keep it in? */ - gfits_create_matrix (&catfile[0].header, &matrix); - if (!gfits_fwrite_matrix (catfile[0].f, &matrix)) { - fprintf (stderr, "can't write primary matrix"); - goto failure; - } - gfits_free_matrix (&matrix); - - /* write out Missing table (convert to FITS table format) */ - gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing); - if (!gfits_fwrite_Theader (catfile[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_table (catfile[0].f, &ftable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } + if (catalog[0].Nmiss_off != 0) { + fprintf (stderr, "inconsistency: Missing table cannot be written in segments\n"); + goto failure; + } + + // convert to external table format + if (!gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing)) { + fprintf (stderr, "trouble converting format\n"); + goto failure; + } + + // write out Missing table (must write out entire table) + if (!dvo_catalog_save_subcat (catalog[0].missing_catalog, &ftable, 0, catalog[0].Nmissing, catalog[0].Nmissing, catalog[0].Nmissing)) { + fprintf (stderr, "trouble writing Missing Table\n"); + goto failure; + } + gfits_free_header (&header); gfits_free_table (&ftable); - gfits_free_header (&header); } @@ -459,38 +816,24 @@ if (catalog[0].secfilt != NULL) { - /* catalog file data is stored in a separate catalog structure */ - catfile = catalog[0].secfilt_catalog; - - /* rewind file pointers and truncate (file is still open) */ - fseek (catfile[0].f, 0, SEEK_SET); - ftruncate (fileno (catfile[0].f), 0); - - /* write table PHU header */ - if (!gfits_fwrite_header (catfile[0].f, &catfile[0].header)) { - fprintf (stderr, "can't write primary header"); - goto failure; - } - - /* this is probably a NOP, do I have to keep it in? */ - gfits_create_matrix (&catfile[0].header, &matrix); - if (!gfits_fwrite_matrix (catfile[0].f, &matrix)) { - fprintf (stderr, "can't write primary matrix"); - goto failure; - } - gfits_free_matrix (&matrix); - - /* write out SecFilt table (convert to FITS table format) */ - Nitems = catalog[0].Naverage * catalog[0].Nsecfilt; - SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat); - if (!gfits_fwrite_Theader (catfile[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_table (catfile[0].f, &ftable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } + first = catalog[0].Nsecf_disk - catalog[0].Nsecf_off; // first row in memory to write + start = catalog[0].Nsecf_off; // first disk row to write + Nitems = catalog[0].Nsecf_mem; + Nrows = Nitems - first; + + assert (Nrows >= 0); + assert (first >= 0); + assert (first < Nitems); + assert (catalog[0].Nmeas_disk >= catalog[0].Nmeas_off); + + // convert to external table format + SecFiltToFtable (&ftable, &secfilt[first], Nrows, catalog[0].catformat); + + // write out SecFilt table + if (!dvo_catalog_save_subcat (catalog[0].secfilt_catalog, &ftable, start, Nrows, catalog[0].Nsecf_disk, Nsecf_disk_new)) { + fprintf (stderr, "failure writing SecFilt table\n"); + goto failure; + } + gfits_free_header (&header); gfits_free_table (&ftable); - gfits_free_header (&header); } @@ -512,240 +855,4 @@ } -/* update_catalog_split only writes new lines to file. - * if file is empty, call save_catalog_split instead. - * XXX EAM : save_catalog SHOULD do this - */ - -int dvo_catalog_update_split (Catalog *catalog, char VERBOSE) { - - int i, Nx, Ny, Nlines; - int Nitems, Nskip, Nout, Ndisk, Nstart; - Matrix matrix; - Header header; - FTable ftable; - VTable vtable; - Catalog *catfile; - SecFilt *primary, *secfilt; - int j, Nsecfilt, Nallfilt, Ntotal; - - ftable.header = &header; - vtable.header = &header; - - if (catalog[0].Naverage == 0) { - if (VERBOSE) fprintf (stderr, "no stars in catalog, skipping\n"); - return (TRUE); - } - - /** for the appropriate types, pull out the first secfilt and pass to AverageToFtable as primary **/ - if ((catalog[0].catformat == DVO_FORMAT_ELIXIR) || // special case for ELIXIR - (catalog[0].catformat == DVO_FORMAT_LONEOS)) { // special case for LONEOS - if (catalog[0].secfilt == NULL) { - fprintf (stderr, "missing secfilt, cannot build output averages (dvo_catalog_split.c:544)\n"); - exit (1); - } - secfilt = catalog[0].secfilt; - - Nallfilt = catalog[0].Nsecfilt; - Nsecfilt = catalog[0].Nsecfilt - 1; - Ntotal = Nsecfilt * catalog[0].Naverage; - ALLOCATE (primary, SecFilt, catalog[0].Naverage); - ALLOCATE (secfilt, SecFilt, Ntotal); - - for (i = 0; i < catalog[0].Naverage; i++) { - primary[i] = secfilt[i*Nallfilt + 0]; - for (j = 0; j < Nsecfilt; j++) { - secfilt[i*Nsecfilt + j] = catalog[0].secfilt[i*Nallfilt + j + 1]; - } - } - } else { - primary = NULL; - secfilt = catalog[0].secfilt; - Nsecfilt = catalog[0].Nsecfilt; - } - - /* make sure header is consistent with data */ - gfits_modify (&catalog[0].header, "NSTARS", "%d", 1, catalog[0].Naverage); - gfits_modify (&catalog[0].header, "NMEAS", "%d", 1, catalog[0].Nmeasure + catalog[0].Nmeas_off); - gfits_modify (&catalog[0].header, "NMISS", "%d", 1, catalog[0].Nmissing); - gfits_modify (&catalog[0].header, "NSECFILT", "%d", 1, Nsecfilt); - gfits_modify (&catalog[0].header, "EXTEND", "%t", 1, TRUE); - - /* rewind file pointers and truncate (file is still open) */ - fseek (catalog[0].f, 0, SEEK_SET); - - /* write table PHU header - always write this out */ - /* XXX EAM : check if disk file size has changed */ - if (!gfits_fwrite_header (catalog[0].f, &catalog[0].header)) { - fprintf (stderr, "can't write primary header"); - goto failure; - } - - /*** Average Table ***/ - if (catalog[0].average != NULL) { - - /* skip past matrix (already at end of header) */ - Nskip = gfits_data_size (&catalog[0].header); - fseek (catalog[0].f, Nskip, SEEK_CUR); - - /* how many lines to write out? */ - Nout = catalog[0].Naverage - catalog[0].Nave_disk; - - /* write out Average table (convert to FITS table format) */ - AverageToFtable (&ftable, catalog[0].average, catalog[0].Naverage, catalog[0].catformat, primary); - /* convert only output rows to vtable */ - gfits_table_to_vtable (&ftable, &vtable, catalog[0].Nave_disk, Nout); - - if (!gfits_fwrite_Theader (catalog[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_vtable (catalog[0].f, &vtable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } - gfits_free_vtable (&vtable); - gfits_free_table (&ftable); - gfits_free_header (&header); - } - - /*** Measure Table ***/ - if (catalog[0].measure != NULL) { - - catfile = catalog[0].measure_catalog; - - /* skip past PHU header and matrix */ - Nskip = catfile[0].header.size + gfits_data_size (&catfile[0].header); - fseek (catfile[0].f, Nskip, SEEK_SET); - - Ndisk = catalog[0].Nmeas_disk; - Nstart = catalog[0].Nmeas_disk - catalog[0].Nmeas_off; /* where is first new line? */ - Nout = catalog[0].Nmeasure - Nstart; /* how many lines to write out? */ - Nlines = catalog[0].Nmeasure + catalog[0].Nmeas_off; /* how many lines total in file */ - - /* convert to output format FITS table (only rows for output : 0 - Nout) */ - MeasureToFtable (&ftable, &catalog[0].measure[Nstart], Nout, catalog[0].catformat); - - gfits_scan (&header, "NAXIS1", "%d", 1, &Nx); - gfits_scan (&header, "NAXIS2", "%d", 1, &Ny); - - /* convert all output rows to vtable */ - ALLOCATE (vtable.row, int, MAX (1, Nout)); - ALLOCATE (vtable.buffer, char *, MAX (1, Nout)); - for (i = 0; i < Nout; i++) { - ALLOCATE (vtable.buffer[i], char, MAX (1, Nx)); - memcpy (vtable.buffer[i], &ftable.buffer[i*Nx], Nx); - vtable.row[i] = i + Ndisk; - } - - /* modify vtable to represent full disk table */ - gfits_modify (&header, "NAXIS2", "%d", 1, Nlines); - header.Naxis[1] = Nlines; - - vtable.size = gfits_data_size (&header); - vtable.Nrow = Nout; - vtable.pad = vtable.size - Nx*Ny; - - if (!gfits_fwrite_Theader (catfile[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_vtable (catfile[0].f, &vtable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } - gfits_free_vtable (&vtable); - gfits_free_table (&ftable); - gfits_free_header (&header); - } - - /*** Missing Table ***/ - /* missing table CANNOT be written unsorted, thus it is always written - out in full */ - - if (catalog[0].missing != NULL) { - - catfile = catalog[0].missing_catalog; - - /* rewind file pointers and truncate (file is still open) */ - fseek (catfile[0].f, 0, SEEK_SET); - ftruncate (fileno (catfile[0].f), 0); - - /* write table PHU header */ - if (!gfits_fwrite_header (catfile[0].f, &catfile[0].header)) { - fprintf (stderr, "can't write primary header"); - goto failure; - } - - /* this is probably a NOP, do I have to keep it in? */ - gfits_create_matrix (&catfile[0].header, &matrix); - if (!gfits_fwrite_matrix (catfile[0].f, &matrix)) { - fprintf (stderr, "can't write primary matrix"); - goto failure; - } - gfits_free_matrix (&matrix); - - /* write out Missing table (convert to FITS table format) */ - gfits_table_set_Missing (&ftable, catalog[0].missing, catalog[0].Nmissing); - if (!gfits_fwrite_Theader (catfile[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_table (catfile[0].f, &ftable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } - gfits_free_table (&ftable); - gfits_free_header (&header); - } - - /*** Secfilt Table ***/ - if (catalog[0].secfilt != NULL) { - - catfile = catalog[0].secfilt_catalog; - - /* skip past PHU header and matrix */ - Nskip = catfile[0].header.size + gfits_data_size (&catfile[0].header); - fseek (catfile[0].f, Nskip, SEEK_SET); - - /* how many lines to write out? */ - Nout = catalog[0].Nsecfilt * (catalog[0].Naverage - catalog[0].Nave_disk); - Ndisk = catalog[0].Nsecfilt * catalog[0].Nave_disk; - - /* convert to output format FITS table */ - Nitems = catalog[0].Naverage * catalog[0].Nsecfilt; - SecFiltToFtable (&ftable, secfilt, Nitems, catalog[0].catformat); - /* convert only output rows to vtable */ - gfits_table_to_vtable (&ftable, &vtable, Ndisk, Nout); - - if (!gfits_fwrite_Theader (catfile[0].f, &header)) { - fprintf (stderr, "can't write table header"); - goto failure; - } - if (!gfits_fwrite_vtable (catfile[0].f, &vtable)) { - fprintf (stderr, "can't write table data"); - goto failure; - } - gfits_free_vtable (&vtable); - gfits_free_table (&ftable); - gfits_free_header (&header); - } - - /* free temp storage */ - if (primary != NULL) { - free (primary); - free (secfilt); - } - - return (TRUE); - -failure: - /* free temp storage */ - if (primary != NULL) { - free (primary); - free (secfilt); - } - return (FALSE); -} - /* in split mode, extra files are linked to catalog->measure_catalog, etc. Each has a valid filename, f, header. The primary catalog data pointers are set @@ -758,5 +865,16 @@ /* XXX EAM : update is not efficient. MeasureToFtable should only convert the new rows (Nmeas_disk to Nmeasure). the resulting - table represents the end rows of the vtable. we need to define + table represents the end rows of the ftable. we need to define the vtable based on the ftable, but with Ny = Nmeasure */ + + + // * convert to an ftable + // * optionally write the PHU header/matrix + // * advance to the start of the output data block: + // ** Nx * catalog[0].Nmeas_off + // * write out the ftable data block + // * if Nmeas_off + Nmeasure >= Nmeas_disk, update padding + // ** start = Nmeas_off + // ** Nrows = Nmeasure +