Changeset 5234 for trunk/Ohana/src/libohana

Timestamp:

Oct 7, 2005, 6:33:25 AM (21 years ago)

Author:

eugene

Message:

working on the update concept, added split table support

Location:

trunk/Ohana/src/libohana

Files:

• include/dvo.h (modified) (5 diffs)
• src/dvo_catalog.c (modified) (1 diff)
• src/dvo_catalog_split.c (modified) (11 diffs)

trunk/Ohana/src/libohana/include/dvo.h

@@ -24 +24 @@
 
 /* catalog values to be loaded */
-# define LOAD_AVES 0x01
-# define LOAD_MEAS 0x02
-# define LOAD_MISS 0x04
-# define LOAD_SECF 0x08
+# define LOAD_AVES      0x01
+# define LOAD_MEAS      0x02
+# define LOAD_MISS      0x04
+# define LOAD_SECF      0x08 
+# define LOAD_MEAS_META 0x10
 
 /* invalid mag value */
@@ -128 +129 @@
   Missing *missing; 
   SecFilt *secfilt;
-  int Naverage, Nmeasure, Nmissing, Nsecfilt;
+  int Naverage, Nmeasure, Nmissing, Nsecfilt;   /* current number of each component */
+  int Nave_disk, Nmeas_disk, Nmiss_disk;        /* number of component on disk */
+  int Nmeas_off;                                /* dist seq of first loaded data value */
+  /* note the different counting for Nsecfilt */
 
   /* pointers to split data files */
@@ -139 +143 @@
   int catmode;                          /* storage mode (raw, mef, split, mysql) */
   int catformat;                        /* storage format (elixir, panstarrs, etc) */
-  int ordered;                          /* is measure table average-ordered? */
+  int sorted;                           /* is measure table average-sorted? */
   
   /* pointers for data manipulation */
@@ -177 +181 @@
 int load_catalog (Catalog *catalog, char mode, int VERBOSE);
 int save_catalog (Catalog *catalog, char VERBOSE);
+int update_catalog (Catalog *catalog, char VERBOSE);
 
 int FindMosaicForImage (Image *images, int Nimages, int entry);
@@ -291 +296 @@
 int load_catalog_split (Catalog *catalog, char mode, int VERBOSE);
 int save_catalog_split (Catalog *catalog, char VERBOSE);
+int update_catalog_split (Catalog *catalog, char VERBOSE);
 
 /*** DVO image db I/O Functions ***/

trunk/Ohana/src/libohana/src/dvo_catalog.c

@@ -110 +110 @@
       break;
     case DVO_MODE_SPLIT:
-      update_catalog_split (catalog, VERBOSE);
+      /* new file needs to use save_catalog_split */
+      if (catalog[0].Nave_disk == 0) {
+        save_catalog_split (catalog, VERBOSE);
+      } else {
+        update_catalog_split (catalog, VERBOSE);
+      }
       break;
     default:

trunk/Ohana/src/libohana/src/dvo_catalog_split.c

@@ -65 +65 @@
 
   measure = NULL;
+
+  /* (Full Load) */
   if (mode & LOAD_MEAS) {
     ALLOCATE (measure, Catalog, 1);
@@ -104 +106 @@
       fprintf (stderr, "Warning: mismatch between Nmeasure in PHU and Table headers (%d vs %d)\n", Nmeasure, catalog[0].Nmeasure);
     }
-    fits_free_header (&header);
-    fits_free_matrix (&matrix);
+    catalog[0].Nmeas_off = 0;
+    fits_free_header (&header);
+    fits_free_matrix (&matrix);
+  }
+
+  /* (Meta Load) */
+  if (mode & LOAD_MEAS_META) {
+    ALLOCATE (measure, Catalog, 1);
+
+    /* get split filename from main header (paths relative to cpt file) */
+    if (!fits_scan (&catalog[0].header, "MEASURE",  "%s", 1, string)) return (FALSE);
+    ALLOCATE (measure[0].filename, char, strlen(path) + strlen(string) + 2);
+    sprintf (measure[0].filename, "%s/%s", path, string);
+
+    /* lock & open catalog file */
+    if (lock_catalog (measure, catalog[0].lockmode) != 1) {
+      fprintf (stderr, "cannot access measure file %s\n", measure[0].filename);
+      exit (2);
+    }
+
+    /* read PHU */
+    if (!fits_load_header (measure[0].f, &measure[0].header)) {
+      if (VERBOSE) fprintf (stderr, "catalog file does not exist: %s\n", measure[0].filename);
+      return (FALSE);
+    }
+
+    /* set up the table size information */
+    catalog[0].Nmeasure  = 0;        /* no rows loaded */
+    catalog[0].Nmeas_off = Nmeasure; /* start of new entries */
   }
   catalog[0].measure_catalog = measure;
@@ -203 +232 @@
   catalog[0].secfilt_catalog = secfilt;
 
+  /* save the current number so we can do partial updates */
+  catalog[0].Nave_disk  = Naverage;
+  catalog[0].Nmeas_disk = Nmeasure;
+  catalog[0].Nmiss_disk = Nmissing;
+
   return (TRUE);
 }
@@ -275 +309 @@
     measure = catalog[0].measure_catalog;
 
+    /* XXX EAM : warn about this condition; add code to handle? */
+    if (catalog[0].Nmeas_off != 0) {
+      fprintf (stderr, "WARNING: LOAD_MEAS_META mixed with save??\n");
+      fprintf (stderr, "WARNING: this should not be allowed to happen!\n");
+    }
+
     /* rewind file pointers and truncate (file is still open) */
     fseek (measure[0].f, 0, SEEK_SET);
@@ -394 +434 @@
 int update_catalog_split (Catalog *catalog, char VERBOSE) {
 
-  int Nitems;
+  int i, Nx, Ny, Nlines;
+  int Nitems, Nskip, Nout, Ndisk, Nstart;
   Matrix matrix;
   Header header;
   FTable ftable;
+  VTable vtable;
   Catalog *measure, *missing, *secfilt;
 
   ftable.header = &header;
+  vtable.header = &header;
 
   if (catalog[0].Naverage == 0) {
@@ -409 +452 @@
   /* make sure header is consistent with data */
   fits_modify (&catalog[0].header, "NSTARS",   "%d", 1, catalog[0].Naverage);
-  fits_modify (&catalog[0].header, "NMEAS",    "%d", 1, catalog[0].Nmeasure);
+  fits_modify (&catalog[0].header, "NMEAS",    "%d", 1, catalog[0].Nmeasure + catalog[0].Nmeas_off);
   fits_modify (&catalog[0].header, "NMISS",    "%d", 1, catalog[0].Nmissing);
   fits_modify (&catalog[0].header, "NSECFILT", "%d", 1, catalog[0].Nsecfilt);
@@ -424 +467 @@
   }
 
-  /* in split mode, we can save only part of the data */ 
-
   /*** Average Table ***/
 
   if (catalog[0].average != NULL) {
-    ftruncate (fileno (catalog[0].f), catalog[0].header.size);
-
-    /* this is probably a NOP, do I have to keep it in? */
-    fits_create_matrix (&catalog[0].header, &matrix);
-    if (!fits_fwrite_matrix  (catalog[0].f, &matrix)) {
-      fprintf (stderr, "can't write primary matrix");
-      return (FALSE);
-    }
-    fits_free_matrix (&matrix);
+
+    /* skip past matrix (already at end of header) */
+    Nskip = fits_matrix_size (&catalog[0].header);
+    fseek (catalog[0].f, Nskip, SEEK_CUR);
+
+    /* how many lines to write out? */
+    Nout = catalog[0].Naverage - catalog[0].Nave_disk;
 
     /* write out Average table (convert to FITS table format) */
     AverageToFtable (&ftable, catalog[0].average, catalog[0].Naverage, catalog[0].catformat);
+    /* convert only output rows to vtable */
+    fits_table_to_vtable (&ftable, &vtable, catalog[0].Nave_disk, Nout);
+
     if (!fits_fwrite_Theader (catalog[0].f, &header)) {
       fprintf (stderr, "can't write table header");
       return (FALSE);
     }
-    if (!fits_fwrite_table (catalog[0].f, &ftable)) {
-      fprintf (stderr, "can't write table data");
-      return (FALSE);
-    }
+    if (!fits_fwrite_vtable (catalog[0].f, &vtable)) {
+      fprintf (stderr, "can't write table data");
+      return (FALSE);
+    }
+    fits_free_vtable (&vtable);
     fits_free_table (&ftable);
     fits_free_header (&header);
@@ -459 +502 @@
     measure = catalog[0].measure_catalog;
 
-    /* rewind file pointers and truncate (file is still open) */
-    fseek (measure[0].f, 0, SEEK_SET);
-    ftruncate (fileno (measure[0].f), 0);
-
-    /* write table PHU header */
-    if (!fits_fwrite_header  (measure[0].f, &measure[0].header)) {
-      fprintf (stderr, "can't write primary header");
-      return (FALSE);
-    }
-
-    /* this is probably a NOP, do I have to keep it in? */
-    fits_create_matrix (&measure[0].header, &matrix);
-    if (!fits_fwrite_matrix  (measure[0].f, &matrix)) {
-      fprintf (stderr, "can't write primary matrix");
-      return (FALSE);
-    }
-    fits_free_matrix (&matrix);
-
-    /* write out Measure table (convert to FITS table format) */
-    MeasureToFtable (&ftable, catalog[0].measure, catalog[0].Nmeasure, catalog[0].catformat);
+    /* skip past PHU header and matrix */
+    Nskip = measure[0].header.size + fits_matrix_size (&measure[0].header);
+    fseek (measure[0].f, Nskip, SEEK_SET);
+
+    Ndisk  = catalog[0].Nmeas_disk;
+    Nstart = catalog[0].Nmeas_disk - catalog[0].Nmeas_off; /* where is first new line? */
+    Nout   = catalog[0].Nmeasure - Nstart;                   /* how many lines to write out? */
+    Nlines = catalog[0].Nmeasure + catalog[0].Nmeas_off;   /* how many lines total in file */
+
+    /* convert to output format FITS table (only rows for output : 0 - Nout) */
+    MeasureToFtable (&ftable, &catalog[0].measure[Nstart], Nout, catalog[0].catformat);
+
+    fits_scan (&header, "NAXIS1", "%d", 1, &Nx);
+    fits_scan (&header, "NAXIS2", "%d", 1, &Ny);
+
+    /* convert all output rows to vtable */
+    ALLOCATE (vtable.row, int, MAX (1, Nout));
+    ALLOCATE (vtable.buffer, char *, MAX (1, Nout));
+    for (i = 0; i < Nout; i++) {
+      ALLOCATE (vtable.buffer[i], char, MAX (1, Nx));
+      memcpy (vtable.buffer[i], &ftable.buffer[i*Nx], Nx);
+      vtable.row[i] = i + Ndisk;
+    }
+
+    /* modify vtable to represent full disk table */
+    fits_modify (&header, "NAXIS2", "%d", 1, Nlines);
+    header.Naxis[1] = Nlines;
+
+    vtable.size = fits_matrix_size (&header);
+    vtable.Nrow = Nout;
+    vtable.pad = vtable.size - Nx*Ny;
+
     if (!fits_fwrite_Theader (measure[0].f, &header)) {
       fprintf (stderr, "can't write table header");
       return (FALSE);
     }
-    if (!fits_fwrite_table (measure[0].f, &ftable)) {
-      fprintf (stderr, "can't write table data");
-      return (FALSE);
-    }
+    if (!fits_fwrite_vtable (measure[0].f, &vtable)) {
+      fprintf (stderr, "can't write table data");
+      return (FALSE);
+    }
+    fits_free_vtable (&vtable);
     fits_free_table (&ftable);
     fits_free_header (&header);
@@ -492 +548 @@
 
   /*** Missing Table ***/
+  /* missing table CANNOT be written unsorted, thus it is always written 
+     out in full */
 
   if (catalog[0].missing != NULL) {
@@ -535 +593 @@
     secfilt = catalog[0].secfilt_catalog;
 
-    /* rewind file pointers and truncate (file is still open) */
-    fseek (secfilt[0].f, 0, SEEK_SET);
-    ftruncate (fileno (secfilt[0].f), 0);
-
-    /* write table PHU header */
-    if (!fits_fwrite_header  (secfilt[0].f, &secfilt[0].header)) {
-      fprintf (stderr, "can't write primary header");
-      return (FALSE);
-    }
-
-    /* this is probably a NOP, do I have to keep it in? */
-    fits_create_matrix (&secfilt[0].header, &matrix);
-    if (!fits_fwrite_matrix  (secfilt[0].f, &matrix)) {
-      fprintf (stderr, "can't write primary matrix");
-      return (FALSE);
-    }
-    fits_free_matrix (&matrix);
-
-    /* write out SecFilt table (convert to FITS table format) */
+    /* skip past PHU header and matrix */
+    Nskip = secfilt[0].header.size + fits_matrix_size (&secfilt[0].header);
+    fseek (secfilt[0].f, Nskip, SEEK_SET);
+
+    /* how many lines to write out? */
+    Nout  = catalog[0].Nsecfilt * (catalog[0].Naverage - catalog[0].Nave_disk);
+    Ndisk = catalog[0].Nsecfilt * catalog[0].Nave_disk;
+
+    /* convert to output format FITS table */
     Nitems = catalog[0].Naverage * catalog[0].Nsecfilt;
     SecFiltToFtable (&ftable, catalog[0].secfilt, Nitems, catalog[0].catformat);
+    /* convert only output rows to vtable */
+    fits_table_to_vtable (&ftable, &vtable, Ndisk, Nout);
+
     if (!fits_fwrite_Theader (secfilt[0].f, &header)) {
       fprintf (stderr, "can't write table header");
       return (FALSE);
     }
-    if (!fits_fwrite_table (secfilt[0].f, &ftable)) {
-      fprintf (stderr, "can't write table data");
-      return (FALSE);
-    }
+    if (!fits_fwrite_vtable (secfilt[0].f, &vtable)) {
+      fprintf (stderr, "can't write table data");
+      return (FALSE);
+    }
+    fits_free_vtable (&vtable);
     fits_free_table (&ftable);
     fits_free_header (&header);
@@ -578 +630 @@
 
 /* XXX EAM : this file needs work on the error exit conditions and memory leaks, esp under errors */
+
+/* XXX EAM : update is not efficient.  MeasureToFtable should only 
+   convert the new rows (Nmeas_disk to Nmeasure). the resulting
+   table represents the end rows of the vtable.  we need to define
+   the vtable based on the ftable, but with Ny = Nmeasure */  
+