Changeset 31524

Timestamp:

May 11, 2011, 10:52:00 AM (15 years ago)

Author:

eugene

Message:

plug some leaks, move CopyMeasure,AverageTiny out of functions (for speed)

Location:

branches/eam_branches/ipp-20110505/Ohana/src/relphot

Files:

• doc/allsky.txt (modified) (1 diff)
• src/MosaicOps.c (modified) (2 diffs)
• src/StarOps.c (modified) (1 diff)
• src/bcatalog.c (modified) (2 diffs)
• src/load_catalogs.c (modified) (2 diffs)
• src/reload_catalogs.c (modified) (3 diffs)
• src/relphot.c (modified) (1 diff)

branches/eam_branches/ipp-20110505/Ohana/src/relphot/doc/allsky.txt

@@ +1 @@
+
+2011.05.10
+
+  Looking into them memory usage of relphot:
+  AverageTiny = 32 bytes (including 8 byte padding)
+  MeasureTiny = 56 bytes (including 8 byte padding)
+  Image Indexes:
+    bin   : Nmeasure * off_t
+    clist : Nmeasure * off_t
+    mlist : Nmeasure * off_t
+  Mosaic Indexes:
+    bin   : Nmeasure * off_t
+    clist : Nmeasure * off_t
+    mlist : Nmeasure * off_t
+  Total:
+    32 bytes * Naverage (values)
+    56 bytes * Nmeasure (values)
+    64 bytes * Nmeasure (indexes)
+
+  In my test run, I have 3.3M average + 37.6M measure:
+    2.1GB values
+    2.2GB indexes
+
+  This matches the memory usage graph.  But, after the indexes have
+  been loaded, the memory footprint grows (slowly) by another 1GB as
+  the processing runs along.
+
+
+using 23153493 of 640663697 stars (192585953 of 289672671 measurements)
+using  3306896 of  48399614 stars ( 37582679 of 193863612 measurements)
+
+
 
 2011.04.12

branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/MosaicOps.c

@@ -485 +485 @@
 
   ALLOCATE (imageOffset, double, Nmosaic);
-
   ALLOCATE ( Slist, int *, Nmosaic); // array of calibrated star indexes on this mosaic
   ALLOCATE (NSlist, int,   Nmosaic); // number of stars on mosaic
@@ -647 +646 @@
   free (starNcount);
 
+  free (seclist);
+  free (NSlist);
+  free (NSLIST);
   for (i = 0; i < Nmosaic; i++){
     free (Slist[i]);
   }
-  free (NSlist);
-  free (NSLIST);
+  free (Slist);
   free (imageOffset);
-  free (seclist);
   return (TRUE);
 }

branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/StarOps.c

@@ -587 +587 @@
   initstats (STATMODE);
   if (VERBOSE) fprintf (stderr, OFF_T_FMT" measures marked poor, "OFF_T_FMT" total\n", Ndel, Nave);
+  free (list);
+  free (dlist);
   free (ilist);
   free (tlist);

branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/bcatalog.c

@@ -5 +5 @@
 int CopyAverageTiny (AverageTiny *averageT, Average *average) {
 
-  averageT[0].R     = average[0].R;
-  averageT[0].D     = average[0].D;
-  averageT[0].flags = average[0].flags;
+  averageT[0].R             = average[0].R;
+  averageT[0].D             = average[0].D;
+  averageT[0].flags         = average[0].flags;
   averageT[0].Nmeasure      = average[0].Nmeasure;
   averageT[0].measureOffset = average[0].measureOffset;
@@ -293 +293 @@
   off_t i;
 
+  AverageTiny *averageT;
+  Average *average;
+
+  MeasureTiny *measureT;
+  Measure *measure;
+
   ALLOCATE (catalog[0].measureT, MeasureTiny, catalog[0].Nmeasure);
   ALLOCATE (catalog[0].averageT, AverageTiny, catalog[0].Naverage);
 
+  average  = catalog[0].average;
+  averageT = catalog[0].averageT;
+
+  measure  = catalog[0].measure;
+  measureT = catalog[0].measureT;
+
   for (i = 0; i < catalog[0].Naverage; i++) {
-    CopyAverageTiny (&catalog[0].averageT[i], &catalog[0].average[i]);
+    // CopyAverageTiny (&catalog[0].averageT[i], &catalog[0].average[i]);
+    averageT[i].R             = average[i].R;
+    averageT[i].D             = average[i].D;
+    averageT[i].flags         = average[i].flags;
+    averageT[i].Nmeasure      = average[i].Nmeasure;
+    averageT[i].measureOffset = average[i].measureOffset;
   }
 
   for (i = 0; i < catalog[0].Nmeasure; i++) {
-    CopyMeasureTiny (&catalog[0].measureT[i], &catalog[0].measure[i]);
+    // CopyMeasureTiny (&catalog[0].measureT[i], &catalog[0].measure[i]);
+    measureT[i].dR       = measure[i].dR;
+    measureT[i].dD       = measure[i].dD;
+    measureT[i].M        = measure[i].M;
+    measureT[i].Mcal     = measure[i].Mcal;
+    measureT[i].dM       = measure[i].dM;
+    measureT[i].airmass  = measure[i].airmass;
+    measureT[i].Xccd     = measure[i].Xccd;
+    measureT[i].Yccd     = measure[i].Yccd;
+    measureT[i].t        = measure[i].t;
+    measureT[i].dt       = measure[i].dt;
+    measureT[i].averef   = measure[i].averef;
+    measureT[i].imageID  = measure[i].imageID;
+    measureT[i].dbFlags  = measure[i].dbFlags;
+    measureT[i].photcode = measure[i].photcode;
   }
 

branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/load_catalogs.c

@@ -3 +3 @@
 Catalog *load_catalogs (SkyList *skylist, int *Ncatalog) {
 
-  int i, Nmeas, Nstar, Nmeas_total, Nstar_total;
+  off_t i, Nmeas, Nstar, Nmeas_total, Nstar_total;
   Catalog *catalog, tcatalog;
 
@@ -50 +50 @@
   }
   if (Nstar < 2) { 
-    fprintf (stderr, "warning: insufficient stars %d\n", Nstar);
+    fprintf (stderr, "warning: insufficient stars "OFF_T_FMT"\n", Nstar);
   }
 
-  fprintf (stderr, "using %d of %d stars (%d of %d measurements)\n", Nstar, Nstar_total, Nmeas, Nmeas_total);
+  fprintf (stderr, "using "OFF_T_FMT" of "OFF_T_FMT" stars ("OFF_T_FMT" of "OFF_T_FMT" measurements)\n", Nstar, Nstar_total, Nmeas, Nmeas_total);
   if (Nstar < 1) Shutdown ("%s", "ERROR: no stars match the minimum requirements; exiting \n");
    

branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/reload_catalogs.c

@@ -18 +18 @@
   double time1 = 0.0;
   double time2 = 0.0;
-  double time3 = 0.0;
+  double time3a = 0.0;
+  double time3b = 0.0;
   double time4 = 0.0;
   double time5 = 0.0;
@@ -57 +58 @@
 
     populate_tiny_values(&catalog);
+    TIMESTAMP(time3a);
 
     initImageBins  (&catalog, 1);
     initMosaicBins (&catalog, 1);
     initGridBins   (&catalog, 1); 
-    TIMESTAMP(time3);
+    TIMESTAMP(time3b);
 
     findImages (&catalog, 1);  // FX
@@ -83 +85 @@
   }
 
-  fprintf (stderr, "time1 %f : find catalog\n", time1);
-  fprintf (stderr, "time2 %f : load catalog\n", time2);
-  fprintf (stderr, "time3 %f : init imbins\n",  time3);
-  fprintf (stderr, "time4 %f : find images\n",  time4);
-  fprintf (stderr, "time5 %f : set Mrel\n",     time5);
-  fprintf (stderr, "time6 %f : save catalog\n", time6);
-  fprintf (stderr, "time7 %f : free catalog\n", time7);
+  fprintf (stderr, "time1  %f : find catalog\n", time1);
+  fprintf (stderr, "time2  %f : load catalog\n", time2);
+  fprintf (stderr, "time3a %f : init imbins\n",  time3a);
+  fprintf (stderr, "time3b %f : init imbins\n",  time3b);
+  fprintf (stderr, "time4  %f : find images\n",  time4);
+  fprintf (stderr, "time5  %f : set Mrel\n",     time5);
+  fprintf (stderr, "time6  %f : save catalog\n", time6);
+  fprintf (stderr, "time7  %f : free catalog\n", time7);
 }

branches/eam_branches/ipp-20110505/Ohana/src/relphot/src/relphot.c

@@ -207 +207 @@
     dvo_image_save (&dbX, VERBOSE);
     dvo_image_unlock (&dbX); 
+    MARKTIME("-- save Image.dat.bck: %f sec\n", dtime);
   }