Changeset 15724

Timestamp:

Nov 30, 2007, 9:06:52 AM (18 years ago)

Author:

eugene

Message:

adding support to generalize dvo catalog load segments

Location:

branches/eam_branch_20071130/Ohana/src/libdvo

Files:

• include/dvo.h (modified) (1 diff)
• src/dvo_catalog.c (modified) (2 diffs)
• src/dvo_catalog_split.c (modified) (5 diffs)

branches/eam_branch_20071130/Ohana/src/libdvo/include/dvo.h

@@ -195 +195 @@
   Missing *missing; 
   SecFilt *secfilt;
-  int Naverage, Nmeasure, Nmissing, Nsecfilt;   /* current number of each component */
-  int Nave_disk, Nmeas_disk, Nmiss_disk;        /* number of component on disk */
-  int Nmeas_off;                                /* dist seq of first loaded data value */
-  /* note the different counting for Nsecfilt */
+
+  int Nsecfilt;  /* number of secfilt entries for each average entry */
+  int Naverage,   Nmeasure,   Nmissing,   Nsecf_mem;  /* current number of each component in memory */
+  int Naves_disk, Nmeas_disk, Nmiss_disk, Nsecf_disk; /* current number of each component on disk */
+  int Naves_off,  Nmeas_off,  Nmiss_off,  Nsecf_off;  /* index of first loaded data value */
+
+  /* note the different counting for Nsecfilt:
+     number of secfilt rows on disk is: Nave_disk * Nsecfilt
+     number of secfilt rows in mem  is: Naverage * Nsecfilt
+     *** that is just silly, and bad: convert to using Nsec_mem, Nsec_disk, Nsec_off.
+     *** unless we always require the secfilt and average entries to be loaded sychronously.
+   */
 
   /* pointers to split data files */

branches/eam_branch_20071130/Ohana/src/libdvo/src/dvo_catalog.c

@@ -237 +237 @@
   // even if the data is sorted on disk, if we only load the MEAS_META, we are
   // treating it as unsorted (eg, append only)
+  // XXX deprecate this statement?
   if (catalog[0].catflags & LOAD_MEAS_META) {
       catalog[0].sorted = FALSE;
@@ -276 +277 @@
 
 // write out the data, unlink if empty?
+// XXX consider conditions in which we can allow a save...
 int dvo_catalog_save (Catalog *catalog, char VERBOSE) {
 

branches/eam_branch_20071130/Ohana/src/libdvo/src/dvo_catalog_split.c

@@ -23 +23 @@
   if (!gfits_scan (&catalog[0].header, "NSECFILT", "%d", 1, &Nsecfilt)) Nsecfilt = 0;
 
+  /* save the current number so we can do partial updates */
+  catalog[0].Naves_disk = Naverage;
+  catalog[0].Nmeas_disk = Nmeasure;
+  catalog[0].Nmiss_disk = Nmissing;
+  catalog[0].Nsecf_disk = Naverage * Nsecfilt;
+
+  /* these values are the number of elements loaded into memory */
+  catalog[0].Naverage = 0;
+  catalog[0].Nmeasure = 0;
+  catalog[0].Nmissing = 0;
+  catalog[0].Nsecfilt = 0;
+
+  /* by default, an unloaded catalog is ready to accept more entries after the end of the table */
+  catalog[0].Naves_off = catalog[0].Naves_disk;
+  catalog[0].Nmeas_off = catalog[0].Nmeas_disk;
+  catalog[0].Nmiss_off = catalog[0].Nmiss_disk;
+  catalog[0].Nsecf_off = catalog[0].Nsecf_disk;
+
   /**  Nsecfilt is unusual: it does not list the number of data items in the table
        instead, the number of items is Nsecfilt * Naverage.  **/
-  catalog[0].Nsecfilt = Nsecfilt;
-
-  /* default values */
+  catalog[0].Nsecfilt  = Nsecfilt;
+
+  /* default values, but we will assign these a valid value before we exit (even if empty) */
   catalog[0].average = NULL;
   catalog[0].measure = NULL;
@@ -56 +74 @@
     }
     /* old versions of DVO stored one of the average magnitudes in Average. we save this if needed */
-    catalog[0].average = FtableToAverage (&ftable, &catalog[0].Naverage, &catalog[0].catformat, &primary);
+    catalog[0].average = FtableToAverage (&ftable, &Naverage, &catalog[0].catformat, &primary);
     if (Naverage != catalog[0].Naverage) {
       fprintf (stderr, "Warning: mismatch between Naverage in PHU and Table headers (%d vs %d)\n", Naverage, catalog[0].Naverage);
     }
     gfits_free_header (&header);
-  } 
+  } else {
+    ALLOCATE (catalog[0].average, Average, 1);
+  }
 
   /*** Measure Table ***/
@@ -69 +89 @@
   /* (Full Load) */
   if (catalog[0].catflags & LOAD_MEAS) {
+    /* in split mode, we need to init & open the corresponding measure file */
     ALLOCATE (measure, Catalog, 1);
     dvo_catalog_init (measure, TRUE);
+    catalog[0].measure_catalog = measure;
 
     /* get split filename from main header (paths relative to cpt file) */
@@ -104 +126 @@
     }
     /* convert data format to internal */
-    catalog[0].measure = FtableToMeasure (&ftable, &catalog[0].Nmeasure, &catalog[0].catformat);
+    catalog[0].measure = FtableToMeasure (&ftable, &Nmeasure, &catalog[0].catformat);
     if (Nmeasure != catalog[0].Nmeasure) {
       fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, catalog[0].Nmeasure);
     }
-    catalog[0].Nmeas_off = 0;
     gfits_free_header (&header);
     gfits_free_matrix (&matrix);
-  }
-
-  /* (Meta Load) */
-  if (catalog[0].catflags & LOAD_MEAS_META) {
-    ALLOCATE (measure, Catalog, 1);
-    dvo_catalog_init (measure, TRUE);
-
-    /* get split filename from main header (paths relative to cpt file) */
-    if (!gfits_scan (&catalog[0].header, "MEASURE",  "%s", 1, string)) return (FALSE);
-    ALLOCATE (measure[0].filename, char, strlen(path) + strlen(string) + 2);
-    sprintf (measure[0].filename, "%s/%s", path, string);
-
-    /* lock & open catalog file */
-    if (dvo_catalog_lock (measure, catalog[0].lockmode) != 1) {
-      fprintf (stderr, "cannot access measure file %s\n", measure[0].filename);
-      exit (2);
-    }
-
-    /* read PHU */
-    if (!gfits_load_header (measure[0].f, &measure[0].header)) {
-      if (VERBOSE) fprintf (stderr, "catalog file does not exist: %s\n", measure[0].filename);
-      return (FALSE);
-    }
-
-    /* allocate dummy data for valid realloc */
+  } else {
     ALLOCATE (catalog[0].measure, Measure, 1);
-
-    /* set up the table size information */
-    catalog[0].Nmeasure  = 0;        /* no rows loaded */
-    catalog[0].Nmeas_off = Nmeasure; /* start of new entries */
-  }
-  catalog[0].measure_catalog = measure;
+  }
 
   /*** Missing Table ***/
@@ -272 +264 @@
   catalog[0].secfilt_catalog = secfilt;
 
-  /* save the current number so we can do partial updates */
-  catalog[0].Nave_disk  = Naverage;
-  catalog[0].Nmeas_disk = Nmeasure;
-  catalog[0].Nmiss_disk = Nmissing;
-
   return (TRUE);
 }