Changeset 15746

Timestamp:

Dec 6, 2007, 3:39:40 PM (18 years ago)

Author:

eugene

Message:

fixing up dvosplit to free memory as needed

Location:

trunk/Ohana/src/dvosplit/src

Files:

• dvosplit.c (modified) (2 diffs)
• split_averages.c (modified) (7 diffs)
• split_measures.c (modified) (4 diffs)

trunk/Ohana/src/dvosplit/src/dvosplit.c

@@ -61 +61 @@
     // split_missings (&incatalog, outlist, outcatalogs, avelinks); 
 
+    free (avelinks[0].outref);
+    free (avelinks[0].outcat);
+
     dvo_catalog_unlock (&incatalog);
 
@@ -72 +75 @@
       outlist[0].regions[j][0].table = TRUE;
     }
-
   }
 
   // save sky table copy
   sprintf (filename, "%s/SkyTable.fits", CATDIR);
-  check_file_access (filename, FALSE, VERBOSE);
+  check_file_access (filename, TRUE, VERBOSE);
   if (!SkyTableSave (sky, filename)) {
     fprintf (stderr, "ERROR: failed to save sky table for %s\n", CATDIR);

trunk/Ohana/src/dvosplit/src/split_averages.c

@@ -1 +1 @@
 # include "dvosplit.h"
 # define NROWS 100000 /* ~10MB per block for measures */
+# define DNOUT 1000
 
 AveLinks *split_averages (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs) {
@@ -6 +7 @@
   double inR, inD;
   int n, block, ave, cat, averef, Nblocks, Ncat, Nout, Nsecfilt;
-  int *outref, *outcat;
+  int *outref, *outcat, *outmem;
   AveLinks *avelinks;
 
   ALLOCATE (outref, int, incatalog[0].Naves_disk);
   ALLOCATE (outcat, int, incatalog[0].Naves_disk);
+  ALLOCATE (outmem, int, outlist[0].Nregions);
 
   Nsecfilt = GetPhotcodeNsecfilt ();
@@ -17 +19 @@
   // contains primary photcodes, which will increase Nsecfilt by one.
   for (cat = 0; cat < outlist[0].Nregions; cat++) {
-    REALLOCATE (outcatalogs[cat].average, Average, NROWS);
-    REALLOCATE (outcatalogs[cat].secfilt, SecFilt, NROWS*(Nsecfilt + 1));
+    outmem[cat] = DNOUT;
+    REALLOCATE (outcatalogs[cat].average, Average, outmem[cat]);
+    REALLOCATE (outcatalogs[cat].secfilt, SecFilt, outmem[cat]*(Nsecfilt + 1));
   }
 
@@ -29 +32 @@
     // read up to NROWS at a time
     dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
+    fprintf (stderr, "splitting %s (averages) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
+
     assert (block*NROWS == incatalog[0].Naves_off);
 
+    // distribute data to the output catalogs
     for (ave = 0; ave < incatalog[0].Naverage; ave++) {
       averef = ave + incatalog[0].Naves_off;
@@ -67 +73 @@
         outcatalogs[Ncat].Nsecf_mem++;
       }
+
+      if (outcatalogs[Ncat].Naverage >= outmem[Ncat]) {
+        outmem[Ncat] += DNOUT;
+        REALLOCATE (outcatalogs[Ncat].average, Average, outmem[Ncat]);
+        REALLOCATE (outcatalogs[Ncat].secfilt, SecFilt, outmem[Ncat]*(Nsecfilt + 1));
+      }
     }
+    dvo_catalog_free_data (incatalog);
 
     // double check the values of Naverage, Nsecf_mem?
@@ -75 +88 @@
       outcatalogs[cat].catflags = LOAD_AVES | LOAD_SECF;
 
-      fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecf_mem, outcatalogs[cat].Nsecf_disk, outcatalogs[cat].Nsecf_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt);
       dvo_catalog_save (&outcatalogs[cat], VERBOSE);
-      fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecf_mem, outcatalogs[cat].Nsecf_disk, outcatalogs[cat].Nsecf_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt);
+      // fprintf (stderr, "secfilt: %d %d %d %d\n", outcatalogs[cat].Nsecf_mem, outcatalogs[cat].Nsecf_disk, outcatalogs[cat].Nsecf_off, outcatalogs[cat].Naverage, outcatalogs[cat].Nsecfilt);
 
-      // XXX I need to advance the pointers and free the current data
+      // advance the pointers and free the current data
       // XXX these should be done within save segment:
       outcatalogs[cat].Naves_disk += outcatalogs[cat].Naverage;
@@ -92 +104 @@
   }
 
+  for (cat = 0; cat < outlist[0].Nregions; cat++) {
+    dvo_catalog_free_data (&outcatalogs[cat]);
+  }
+
+  free (outmem);
+
   ALLOCATE (avelinks, AveLinks, 1);
   avelinks[0].outref = outref;
   avelinks[0].outcat = outcat;
 
+
   return (avelinks);
 }

trunk/Ohana/src/dvosplit/src/split_measures.c

@@ -1 +1 @@
 # include "dvosplit.h"
 # define NROWS 100000 /* ~10MB per row for measures */
+# define DNOUT 1000
 
 int split_measures (Catalog *incatalog, SkyList *outlist, Catalog *outcatalogs, AveLinks *avelinks) {
 
   int block, meas, cat, Nblocks, Ncat, Nout, averef;
-  int *outref, *outcat;
+  int *outref, *outcat, *outmem;
 
   outref = avelinks->outref;
   outcat = avelinks->outcat;
+  ALLOCATE (outmem, int, outlist[0].Nregions);
 
   // allocate enough space for the output buffer
   for (cat = 0; cat < outlist[0].Nregions; cat++) {
-    REALLOCATE (outcatalogs[cat].measure, Measure, NROWS);
+    outmem[cat] = DNOUT;
+    REALLOCATE (outcatalogs[cat].measure, Measure, outmem[cat]);
   }
 
@@ -23 +26 @@
     // read up to NROWS at a time
     dvo_catalog_load_segment (incatalog, VERBOSE, block*NROWS, NROWS);
+    fprintf (stderr, "splitting %s (measures) .. %d of %d\n", incatalog[0].filename, block, Nblocks);
+
+    assert (block*NROWS == incatalog[0].Nmeas_off);
 
     for (meas = 0; meas < incatalog[0].Nmeasure; meas++) {
@@ -33 +39 @@
       outcatalogs[Ncat].measure[Nout].averef = outref[averef];
       outcatalogs[Ncat].Nmeasure++;
+
+      if (outcatalogs[Ncat].Nmeasure >= outmem[Ncat]) {
+        outmem[Ncat] += DNOUT;
+        REALLOCATE (outcatalogs[Ncat].measure, Measure, outmem[Ncat]);
+      }
     }
+    dvo_catalog_free_data (incatalog);
 
     for (cat = 0; cat < outlist[0].Nregions; cat++) {
@@ -45 +57 @@
     }
   }
+
+  for (cat = 0; cat < outlist[0].Nregions; cat++) {
+    dvo_catalog_free_data (&outcatalogs[cat]);
+  }
+
+  free (outmem);
+
   return (TRUE);
 }