Changeset 15728

Timestamp:

Nov 30, 2007, 4:46:27 PM (18 years ago)

Author:

eugene

Message:

working on the dvo_catalog api to support segmented read/write

Location:

branches/eam_branch_20071130/Ohana/src

Files:

• dvosplit/src/dvosplit.c (modified) (4 diffs)
• libdvo/doc/dvo-catalogs.txt (modified) (3 diffs)
• libdvo/src/dvo_catalog.c (modified) (3 diffs)
• libdvo/src/dvo_catalog_mef.c (modified) (6 diffs)
• libdvo/src/dvo_catalog_split.c (modified) (5 diffs)

branches/eam_branch_20071130/Ohana/src/dvosplit/src/dvosplit.c

@@ -8 +8 @@
   // the output catalog needs to inherit the SKY_DEPTH of the input catalog. 
   // if the CATDIR/SkyTable.fits is not found, assume the HST layout (old default)
+
+  // force the output to have SPLIT mode?
 
   // load the sky table for the existing database
@@ -27 +29 @@
     // if (current level == out level) skip: no action is needed
     if (skylist[0].regions[i][0].depth >= OUT_DEPTH) continue;
+    // XXX : we need to copy these files to their new names
 
+    // change outsky.regions[i].depth for these regions
     outlist = SkyListByPatch (outsky, OUT_DEPTH, skylist[0].regions[i]);
 
@@ -42 +46 @@
 
     // skip empty input catalogs
-    if (!incatalog.Nave_disk) {
+    if (!incatalog.Naves_disk) {
       dvo_catalog_unlock (&incatalog);
       dvo_catalog_free (&incatalog);
       continue;
     }
-
-    outcatalog = SkyTableSubdivide (&incatalog, &Noutcatalog);
 
     // split out the average entries:
@@ -67 +69 @@
 
         N = -1;
-        for (n = 0; n < Noutcatalog; n++) {
+        for (n = 0; n < outlist[0].Nregions; n++) {
           if (inR < outregions[n].Rmin) continue;
           if (inR > outregions[n].Rmax) continue;

branches/eam_branch_20071130/Ohana/src/libdvo/doc/dvo-catalogs.txt

@@ -2 +2 @@
 // APIs related to the DVO catalogs:
 
-// XXX change name
-// XXX add SORTED test to load
-// XXX add Nsecfilt test? or add an API?
 load_catalog (Catalog *catalog, int VERBOSE);
-// returns
-//   0: failure to lock catalog
-//   1: success: file is locked and opened
-//   2: success: file is locked but empty
 
 dvo_catalog_lock (Catalog *catalog, int lockmode);
@@ -20 +13 @@
 // layout: RAW, MEF, SPLIT
 // mode:   READ, WRITE
+
+// returns
+//   DVO_CAT_OPEN_FAIL:  failure to lock catalog
+//   DVO_CAT_OPEN_OK:    success: file is locked and opened
+//   DVO_CAT_OPEN_EMPTY: success: file is locked but empty
+
+ the catalog is opened and the data for each of the elements (average,
+ measure, etc) is loaded if requested.  Whether or not the data is
+ loaded, the values of catalog.Naves_disk,Naves_off,Naverage are set
+ to match the segment of data loaded.  this API always loads either
+ the entire data set, or none.  If none, the offset value is set to
+ point at the end of the table.
 
 dvo_catalog_save (Catalog *catalog, int VERBOSE);
@@ -43 +48 @@
 - open and read an existing catalog / error if missing (can be ignored)
 
+dvo_catalog_open (Catalog *catalog, int mode); 
 
-dvo_catalog_open (Catalog *catalog, int mode); 
+dvo_catalog_load_segment_average (Catalog *catalog, int start, int Nrows)
+
+ this function reads in the requested number of rows from the catalog
+ for the Average and SecFilt tables only.  It is possible within the
+ DVO framework to examine the average values in slices.  
+
+dvo_catalog_load_segment_measure (Catalog *catalog, int start, int Nrows)
+
+ this function reads in the requested number of rows from the catalog
+ for the Measure table only.  It is not guaranteed to be possible to
+ examine the measure values in slices if reference is needed to their
+ average values.
+
+dvo_catalog_load_segment_missing (Catalog *catalog, int start, int Nrows)
+
+ this function reads in the requested number of rows from the catalog
+ for the Missing table only.  It is not guaranteed to be possible to
+ examine the missing values in slices if reference is needed to their
+ average values.

branches/eam_branch_20071130/Ohana/src/libdvo/src/dvo_catalog.c

@@ -235 +235 @@
   if (!status) catalog[0].sorted = TRUE;
 
-  // even if the data is sorted on disk, if we only load the MEAS_META, we are
-  // treating it as unsorted (eg, append only)
-  // XXX deprecate this statement?
-  if (catalog[0].catflags & LOAD_MEAS_META) {
-      catalog[0].sorted = FALSE;
-  }
-
   // determine catmode
   if (!gfits_scan (&catalog[0].header, "NAXIS", "%d", 1, &Naxis)) {
@@ -308 +301 @@
 
 int dvo_catalog_update (Catalog *catalog, char VERBOSE) {
+
+  // even if the data is sorted on disk, if we only load the MEAS_META, we are
+  // treating it as unsorted (eg, append only)
+  // XXX deprecate this statement?
+  XXX: 
+  check on this;
+  if (catalog[0].catflags & LOAD_MEAS_META) {
+      catalog[0].sorted = FALSE;
+  }
 
   /* update is only valid for catmode SPLIT */
@@ -398 +400 @@
   style  : raw, mef, split, mysql
 */
+
+int dvo_catalog_load_segment (Catalog *catalog, int VERBOSE, int start, int Nrows) {
+  
+  int Naxis, split, status;
+  char measure[80];
+
+  // catformat determined in dvo_catalog_load_XXX function
+  catalog[0].catformat = DVO_FORMAT_UNDEF;
+
+  switch (catalog[0].catmode) {
+    case DVO_MODE_RAW:
+      if (VERBOSE) fprintf (stderr, "reading catalog (mode DVO_MODE_RAW)\n");
+      fprintf (stderr, "cannot do this in raw mode\n");
+      // dvo_catalog_load_segment_raw (catalog, VERBOSE, start, Nrows);
+      break;
+    case DVO_MODE_MEF:
+      if (VERBOSE) fprintf (stderr, "reading catalog (mode DVO_MODE_MEF)\n");
+      dvo_catalog_load_segment_mef (catalog, VERBOSE, start, Nrows);
+      break;
+    case DVO_MODE_SPLIT:
+      if (VERBOSE) fprintf (stderr, "reading catalog (mode DVO_MODE_SPLIT)\n");
+      dvo_catalog_load_segment_split (catalog, VERBOSE, start, Nrows);
+      break;
+    default:
+      fprintf (stderr, "error getting catalog mode\n");
+      exit (2);
+  }
+  return (TRUE);
+}
+

branches/eam_branch_20071130/Ohana/src/libdvo/src/dvo_catalog_mef.c

@@ -28 +28 @@
   if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0;
 
+  /* save the current number so we can do partial updates */
+  catalog[0].Naves_disk = Naverage;
+  catalog[0].Nmeas_disk = Nmeasure;
+  catalog[0].Nmiss_disk = Nmissing;
+  catalog[0].Nsecf_disk = Naverage * Nsecfilt;
+
   /**  Nsecfilt is unusual: it does not list the number of data items in the table
        instead, the number of items is Nsecfilt * Naverage.  **/
   catalog[0].Nsecfilt = Nsecfilt;
+
+  /* default values, but we will assign these a valid value before we exit (even if empty) */
+  catalog[0].average = NULL;
+  catalog[0].measure = NULL;
+  catalog[0].missing = NULL;
+  catalog[0].secfilt = NULL;
 
   /* validate table mode */
@@ -46 +58 @@
     }
     /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */
-    catalog[0].average = FtableToAverage (&ftable, &catalog[0].Naverage, &catalog[0].catformat, &primary);
+    catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary);
     if (Naverage != catalog[0].Naverage) {
-      fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%d vs %d)\n", Naverage, catalog[0].Naverage);
-    }
+      fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%d vs %d)\n", Naverage, catalog[0].Naves_disk);
+    }
+    catalog[0].Naverage = catalog[0].Naves_disk;
+    catalog[0].Naves_off = 0;
   } else {
     Nbytes = gfits_data_size (&header);
     fseek (f, Nbytes, SEEK_CUR);
+    ALLOCATE (catalog[0].average, Average, 1);
+    catalog[0].Naverage = 0;
+    catalog[0].Naves_off = catalog[0].Naves_disk;
   }
   gfits_free_header (&header);
@@ -72 +89 @@
       fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, catalog[0].Nmeasure);
     }
+    catalog[0].Nmeasure = catalog[0].Nmeas_disk;
+    catalog[0].Nmeas_off = 0;
   } else {
     Nbytes = gfits_data_size (&header);
     fseek (f, Nbytes, SEEK_CUR);
+    ALLOCATE (catalog[0].measure, Measure, 1);
+    catalog[0].Nmeasure = 0;
+    catalog[0].Nmeas_off = catalog[0].Nmeas_disk;
   }
 
@@ -93 +115 @@
       fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%d vs %d)\n", Nmissing, catalog[0].Nmissing);
     }
+    catalog[0].Nmissing = catalog[0].Nmiss_disk;
+    catalog[0].Nmiss_off = 0;
   } else {
     Nbytes = gfits_data_size (&header);
     fseek (f, Nbytes, SEEK_CUR);
+    ALLOCATE (catalog[0].missing, Missing, 1);
+    catalog[0].Nmissing = 0;
+    catalog[0].Nmiss_off = catalog[0].Nmiss_disk;
   }
 
@@ -125 +152 @@
       Ntmpfilt = catalog[0].Nsecfilt;
       Nsecfilt = catalog[0].Nsecfilt + 1;
-      Ntotal = Nsecfilt * catalog[0].Naverage;
+      Ntotal = Nsecfilt * catalog[0].Naves_disk;
       ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal);
-      for (i = 0; i < catalog[0].Naverage; i++) {
+      for (i = 0; i < catalog[0].Naves_disk; i++) {
         catalog[0].secfilt[i*Nsecfilt + 0] = primary[i];
         for (j = 0; j < Ntmpfilt; j++) {
@@ -134 +161 @@
       }         
       catalog[0].Nsecfilt = Nsecfilt;
+      catalog[0].Nsecf_disk = Ntotal;
       free (primary);
     } 
-
+    catalog[0].Nsecf_mem = catalog[0].Nsecf_disk;
+    catalog[0].Nsecf_off = 0;
   } else {
     /* no real need to skip the data array here... */
     Nbytes = gfits_data_size (&header);
     fseek (f, Nbytes, SEEK_CUR);
-    if (primary != NULL) free (primary);
-  }
-
-  /* save the current number so we can do partial updates */
-  catalog[0].Nave_disk  = catalog[0].Naverage;
-  catalog[0].Nmeas_disk = catalog[0].Nmeasure;
-  catalog[0].Nmiss_disk = catalog[0].Nmissing;
+    if (primary != NULL) {
+      free (primary);
+      catalog[0].Nsecfilt ++;
+      catalog[0].Nsecf_disk =  catalog[0].Nsecfilt * catalog[0].Naves_disk;
+    }
+    ALLOCATE (catalog[0].secfilt, SecFilt, 1);
+    catalog[0].Nsecf_mem = 0;
+    catalog[0].Nsecf_off = catalog[0].Nsecf_disk;
+  }
 
   return (TRUE);

branches/eam_branch_20071130/Ohana/src/libdvo/src/dvo_catalog_split.c

@@ -121 +121 @@
     }
     gfits_free_header (&header);
-    catalog[0].Naverage = Naverage;
+    catalog[0].Naverage = catalog[0].Naves_disk;
     catalog[0].Naves_off = 0;
   } else {
     ALLOCATE (catalog[0].average, Average, 1);
     catalog[0].Naverage = 0;
-    catalog[0].Naves_off = Naverage;
+    catalog[0].Naves_off = catalog[0].Naves_disk;
   }
 
@@ -221 +221 @@
       Ntotal = Nsecfilt * catalog[0].Naves_disk;
       ALLOCATE (catalog[0].secfilt, SecFilt, Ntotal);
-      for (i = 0; i < catalog[0].Naverage; i++) {
+      for (i = 0; i < catalog[0].Naves_disk; i++) {
         catalog[0].secfilt[i*Nsecfilt + 0] = primary[i];
         for (j = 0; j < Ntmpfilt; j++) {
@@ -250 +250 @@
 }
 
-// XXX I either need to always read both average and secfilt at the same time, or 
-// do something sloppy to carry around the primary secfilt values...
+// I need to always read both average and secfilt at the same time to correctly manage the
+// primary secfilt values...
+
+// XXX merge these? use the mode to choose which component?
 
 int dvo_catalog_load_segment_split_average (Catalog *catalog, int start, int Nrows) {
@@ -257 +259 @@
   // XXX check the open status of the FILE *f?
 
+  /*** Average (& SecFilt) Table ***/
+  if (catalog[0].catflags & LOAD_AVES) {
+
+    /*** load the Average data ***/
     /* move pointer past header -- must be already read (load_catalog) */
     fseek (catalog[0].f, catalog[0].header.size, SEEK_SET);
@@ -290 +296 @@
     catalog[0].Naves_off = start;
 
-    return (TRUE);
-}
-
-int dvo_catalog_load_segment_split_measure (Catalog *catalog, int start, int Nrows) {
-
-  // XXX check the open status of the FILE *f?
-
-  Catalog *subcat = catalog[0].measure_catalog;
-
-  /* move pointer past header -- must be already read (load_catalog) */
-  fseek (subcat[0].f, subcat[0].header.size, SEEK_SET);
-  /* matrix should be empty : XXX should we drop this step and skip the data? */
-  if (!gfits_fread_matrix (subcat[0].f, &matrix, &subcat[0].header)) {
-    if (VERBOSE) fprintf (stderr, "can't read primary matrix");
-    return (FALSE);
-  }
-  /* read Measure table header */
-  if (!gfits_fread_header (subcat[0].f, &header)) {
-    if (VERBOSE) fprintf (stderr, "can't read table measure header");
-    return (FALSE);
-  }
-  /* read Measure table data : format is irrelevant here */
-  if (!gfits_fread_vtable_range (subcat[0].f, &vtable, start, Nrows)) {
-    if (VERBOSE) fprintf (stderr, "can't read table measure data");
-    return (FALSE);
-  }
-
-  /* convert data format to internal : returns number of row read in Nmeasure */
-  catalog[0].measure = VtableToMeasure (&vtable, &Nmeasure, &catalog[0].catformat);
-  if (Nmeasure != Nrows) {
-    fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, Nrows);
-  }
-  gfits_free_header (&header);
-  gfits_free_matrix (&matrix);
-  catalog[0].Nmeasure = Nmeasure;
-  catalog[0].Nmeas_off = start;
-  return (TRUE);
-}
-
-int dvo_catalog_load_segment_split_missing (Catalog *catalog, int start, int Nrows) {
-
-  // XXX check the open status of the FILE *f?
-
-  Catalog *subcat = catalog[0].missing_catalog;
-
-  /* move pointer past header -- must be already read (load_catalog) */
-  fseek (subcat[0].f, subcat[0].header.size, SEEK_SET);
-  /* matrix should be empty : XXX should we drop this step and skip the data? */
-  if (!gfits_fread_matrix (subcat[0].f, &matrix, &subcat[0].header)) {
-    if (VERBOSE) fprintf (stderr, "can't read primary matrix");
-    return (FALSE);
-  }
-  /* read Missing table header */
-  if (!gfits_fread_header (subcat[0].f, &header)) {
-    if (VERBOSE) fprintf (stderr, "can't read table missing header");
-    return (FALSE);
-  }
-  /* read Missing table data : format is irrelevant here */
-  if (!gfits_fread_vtable_range (subcat[0].f, &vtable, start, Nrows)) {
-    if (VERBOSE) fprintf (stderr, "can't read table missing data");
-    return (FALSE);
-  }
-
-  /* convert data format to internal : returns number of row read in Nmissing */
-  catalog[0].missing = VtableToMissing (&vtable, &Nmissing, &catalog[0].catformat);
-  if (Nmissing != Nrows) {
-    fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%d vs %d)\n", Nmissing, Nrows);
-  }
-  gfits_free_header (&header);
-  gfits_free_matrix (&matrix);
-  catalog[0].Nmissing = Nmissing;
-  catalog[0].Nmiss_off = start;
+    /*** load the secfilt data ***/
+    Catalog *subcat = catalog[0].secfilt_catalog;
+
+    /* move pointer past header -- must be already read (load_catalog) */
+    fseek (subcat[0].f, subcat[0].header.size, SEEK_SET);
+    /* matrix should be empty : XXX should we drop this step and skip the data? */
+    if (!gfits_fread_matrix (subcat[0].f, &matrix, &subcat[0].header)) {
+      if (VERBOSE) fprintf (stderr, "can't read primary matrix");
+      return (FALSE);
+    }
+    /* read Measure table header */
+    if (!gfits_fread_header (subcat[0].f, &header)) {
+      if (VERBOSE) fprintf (stderr, "can't read table measure header");
+      return (FALSE);
+    }
+    /* read Measure table data : format is irrelevant here */
+    if (!gfits_fread_vtable_range (subcat[0].f, &vtable, start, Nrows)) {
+      if (VERBOSE) fprintf (stderr, "can't read table measure data");
+      return (FALSE);
+    }
+
+    Nexpect = Nrows * Nsecfilt;
+    catalog[0].secfilt = VtableToSecFilt (&vtable, &Nitems, &catalog[0].catformat);
+    if (Nitems != Nexpect) {
+      fprintf (stderr, "Warning: mismatch between Nsecfilt items in PHU and Table headers (%d vs %d)\n", Nitems, catalog[0].Nsecf_disk);
+    }
+
+    /* if primary is defined, we were supplied with one additional average magnitude from Average
+       we need to interleave these magnitudes with the secfilt entries just loaded */
+    if (primary != NULL) {
+      int Ntmpfilt, Ntotal, i, j;
+      SecFilt *tmpfilt;
+      tmpfilt  = catalog[0].secfilt;
+      Ntmpfilt = catalog[0].Nsecfilt;
+
+      catalog[0].Nsecfilt ++;
+      catalog[0].Nsecf_disk = catalog[0].Nsecfilt * catalog[0].Naves_disk;
+      Nsecfilt = catalog[0].Nsecfilt;
+
+      // the loaded average rows correspond to the loaded secfilt rows
+      // supplement with the loaded primary secfilt data
+      Nexpect = Nrows * Nsecfilt;
+      ALLOCATE (catalog[0].secfilt, SecFilt, Nexpect);
+      for (i = 0; i < Nrows; i++) {
+        catalog[0].secfilt[i*Nsecfilt + 0] = primary[i];
+        for (j = 0; j < Ntmpfilt; j++) {
+          catalog[0].secfilt[i*Nsecfilt + j + 1] = tmpfilt[i*Ntmpfilt + j];
+        }
+      }         
+      free (primary);
+    } 
+    gfits_free_header (&header);
+    gfits_free_matrix (&matrix);
+    catalog[0].Nsecf_mem = Nexpect;
+    catalog[0].Nsecf_off = start * Nsecfilt;
+
+    // CAREFUL: note that Nsecf_disk and Nsecf_off are set to the values if there were no primary data 
+  }
+
+  // XXX check the open status of the catalog
+  if (catalog[0].catflags & LOAD_MEAS) {
+
+    Catalog *subcat = catalog[0].measure_catalog;
+
+    /* move pointer past header -- must be already read (load_catalog) */
+    fseek (subcat[0].f, subcat[0].header.size, SEEK_SET);
+    /* matrix should be empty : XXX should we drop this step and skip the data? */
+    if (!gfits_fread_matrix (subcat[0].f, &matrix, &subcat[0].header)) {
+      if (VERBOSE) fprintf (stderr, "can't read primary matrix");
+      return (FALSE);
+    }
+    /* read Measure table header */
+    if (!gfits_fread_header (subcat[0].f, &header)) {
+      if (VERBOSE) fprintf (stderr, "can't read table measure header");
+      return (FALSE);
+    }
+    /* read Measure table data : format is irrelevant here */
+    if (!gfits_fread_vtable_range (subcat[0].f, &vtable, start, Nrows)) {
+      if (VERBOSE) fprintf (stderr, "can't read table measure data");
+      return (FALSE);
+    }
+
+    /* convert data format to internal : returns number of row read in Nmeasure */
+    catalog[0].measure = VtableToMeasure (&vtable, &Nmeasure, &catalog[0].catformat);
+    if (Nmeasure != Nrows) {
+      fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, Nrows);
+    }
+    gfits_free_header (&header);
+    gfits_free_matrix (&matrix);
+    catalog[0].Nmeasure = Nmeasure;
+    catalog[0].Nmeas_off = start;
+  }
+
+  // XXX check the open status of the catalog?
+  if (catalog[0].catflags & LOAD_MISS) {
+
+    Catalog *subcat = catalog[0].missing_catalog;
+
+    /* move pointer past header -- must be already read (load_catalog) */
+    fseek (subcat[0].f, subcat[0].header.size, SEEK_SET);
+    /* matrix should be empty : XXX should we drop this step and skip the data? */
+    if (!gfits_fread_matrix (subcat[0].f, &matrix, &subcat[0].header)) {
+      if (VERBOSE) fprintf (stderr, "can't read primary matrix");
+      return (FALSE);
+    }
+    /* read Missing table header */
+    if (!gfits_fread_header (subcat[0].f, &header)) {
+      if (VERBOSE) fprintf (stderr, "can't read table missing header");
+      return (FALSE);
+    }
+    /* read Missing table data : format is irrelevant here */
+    if (!gfits_fread_vtable_range (subcat[0].f, &vtable, start, Nrows)) {
+      if (VERBOSE) fprintf (stderr, "can't read table missing data");
+      return (FALSE);
+    }
+
+    /* convert data format to internal : returns number of row read in Nmissing */
+    catalog[0].missing = VtableToMissing (&vtable, &Nmissing, &catalog[0].catformat);
+    if (Nmissing != Nrows) {
+      fprintf (stderr, "Warning: mismatch between Nmissing in PHU and Table headers (%d vs %d)\n", Nmissing, Nrows);
+    }
+    gfits_free_header (&header);
+    gfits_free_matrix (&matrix);
+    catalog[0].Nmissing = Nmissing;
+    catalog[0].Nmiss_off = start;
+  }
   return (TRUE);
 }