Changeset 34399 for branches/eam_branches/ipp-20120805/psphot/src/psphotStackReadout.c

Timestamp:

Sep 5, 2012, 4:04:34 PM (14 years ago)

Author:

eugene

Message:

merge changes from trunk

Location:

branches/eam_branches/ipp-20120805/psphot

Files:

• . (modified) (1 prop)
• src (modified) (1 prop)
• src/psphotStackReadout.c (modified) (18 diffs)

branches/eam_branches/ipp-20120805/psphot/src/psphotStackReadout.c

@@ -2 +2 @@
 
 static bool psphotStackLoadWCS(pmConfig *config, const pmFPAview *view, const char *filerule);
+static void logMemStats(const char *heading);
 
 // we have 3 possible real filesets:
@@ -51 +52 @@
     // by the multiple threads, not the total time used by all threads.
     psTimerStart ("psphotReadout");
+
+    logMemStats("Start");
 
     pmModelClassSetLimits(PM_MODEL_LIMITS_LAX); // allow models to have ugly fits (eg, central cusp)
@@ -82 +85 @@
     }
 
-    // Generate the mask and weight images (if not supplied) and set mask bits
+    // Generate the mask and weight images (if not supplied) and set mask bits. 
+    // This also insures that all invalid pixels are masked (this is done for STACK_CNV in psphotStackMatchPSFs)
     if (!psphotSetMaskAndVariance (config, view, STACK_DET)) {
+        return psphotReadoutCleanup (config, view, STACK_SRC);
+    }
+    if (!psphotSetMaskAndVariance (config, view, STACK_OUT)) {
         return psphotReadoutCleanup (config, view, STACK_SRC);
     }
@@ -114 +121 @@
     }
 
+#ifdef MAKE_CHISQ_IMAGE
     // also make the chisq detection image
     if (!psphotStackChisqImage(config, view, STACK_DET, STACK_SRC)) {
@@ -119 +127 @@
         return psphotReadoutCleanup (config, view, STACK_SRC);
     }
+#endif
     if (!strcasecmp (breakPt, "CHISQ")) {
         return psphotReadoutCleanup (config, view, STACK_SRC);
@@ -151 +160 @@
     // psphotDumpTest (config, view, STACK_SRC);
     psMemDump("sourcestats");
+    logMemStats("sourcestats");
 
     // classify sources based on moments, brightness
@@ -196 +206 @@
     // window of size PSF_MOMENTS_RADIUS (same window used to measure the psf-scale moments)
     // but iterates to an appropriately larger size
-    psphotKronIterate(config, view, STACK_SRC);
+        logMemStats("before.kron.1");
+    psphotKronIterate(config, view, STACK_SRC, 1);
+        logMemStats("after.kron.1");
 
     // identify CRs and extended sources
@@ -208 +220 @@
     psphotReplaceAllSources (config, view, STACK_SRC, false); // pass 1 (detections->allSources)
 
+    logMemStats("pass1");
 
     // if we only do one pass, skip to extended source analysis
@@ -299 +312 @@
     }
 
+    logMemStats("prematch");
+
     // generate the objects (objects unify the sources from the different images) NOTE: could
     // this just match the detections for the chisq image, and not bother measuring the source
@@ -331 +346 @@
     // re-measure the kron mags with models subtracted
     // psphotKronMasked(config, view, STACK_SRC);
-    psphotKronIterate(config, view, STACK_SRC);
+    logMemStats("before.kron.2");
+    psphotKronIterate(config, view, STACK_SRC, 2);
+    logMemStats("after.kron.2");
 
     // measure source size for the remaining sources
@@ -338 +355 @@
 
     psMemDump("psfstats");
+
+    // drop matched sources without any useful measurements
+    psphotDropBadMatchedSources (config, view, STACK_SRC, objects);
 
     // measure elliptical apertures, petrosians (objects sorted by S/N)
@@ -354 +374 @@
     }
 
-
     bool radial_apertures = psMetadataLookupBool(NULL, recipe, "RADIAL_APERTURES");
     if (radial_apertures) {
@@ -361 +380 @@
 
         // NOTE: we always do the radial apertures analysis on the convolved image since
-        // those are the ones that are psf matched and the source of STACK_OUT's pixels
+        // those are the ones that are psf matched and are the source of STACK_OUT's pixels
         // XXX: Actually if PSPHOT.STACK.MATCH.PSF.SOURCE were set to RAW this wouldn't be true.
         // but in that case we don't get past the psf matching step because there is no
@@ -419 +438 @@
         // psphotEfficiency wants to have the PSF of the image, but since we are measuring on
         // the convolved images we need to generate PSFs for the DET images
-        if (!psphotChoosePSF (config, view, STACK_DET, false)) { // pass 1
+        if (!psphotChoosePSF (config, view, STACK_DET, false)) {
             psLogMsg ("psphot", 3, "failure to construct a psf model for raw input");
             return psphotReadoutCleanup (config, view, STACK_DET);
@@ -430 +449 @@
     psphotCopyEfficiency (config, view, STACK_OUT, STACK_DET);
 
+    logMemStats("final");
+#if (1)
+    psphotSourceMemory(config, view, STACK_SRC);
+    psphotSourceMemory(config, view, STACK_OUT);
+#endif
+
     // replace failed sources?
     // psphotReplaceUnfitSources (sources);
@@ -440 +465 @@
     psphotSourceFreePixels (config, view, STACK_SRC);
 
+#ifdef MAKE_CHISQ_IMAGE
     // remove chisq image from config->file:PSPHOT.INPUT
     psphotStackRemoveChisqFromInputs(config, STACK_DET);
@@ -445 +471 @@
         psphotStackRemoveChisqFromInputs(config, STACK_SRC);
     }
+#endif
 
     psFree (objects);
@@ -474 +501 @@
     return true;
 }
+
+// read the memory usage data from /proc and log them out
+// This will only work on a system that has /proc (not MacOS for instance) but since this function just
+// tries to open and read from a file it is safe
+// XXX: refine this and move it to psLib
+static void logMemStats(const char *heading) {
+
+    // file containing memory statistics for this process proc. See proc(5)
+    const char* statm_path = "/proc/self/statm";
+
+    FILE *f = fopen(statm_path,"r");
+    if (!f) {
+        psLogMsg ("psphot", PS_LOG_WARN, "failed to open %s", statm_path);
+        return;
+    }
+
+    unsigned long vmSize, resident, share, text, lib, data;
+
+    int nread;
+    nread = fscanf(f,"%ld %ld %ld %ld %ld %ld", &vmSize, &resident, &share, &text, &lib, &data);
+    fclose(f);
+    if (nread != 6) {
+        psLogMsg ("psphot", PS_LOG_WARN, "failed to read 6 items from %s", statm_path);
+        return;
+    }
+  
+    // XXX: assuming 4 KB page size here
+#   define PAGES_TO_MB(_v) (_v * 4.096 / 1024.)
+    psLogMsg ("psphot", PS_LOG_INFO, "Memory usage at %20s: Total VmSize: %8.2f MB   Resident %8.2f MB   Data: %8.2f MB\n",
+        heading, PAGES_TO_MB(vmSize), PAGES_TO_MB(resident), PAGES_TO_MB(data));
+}
+
+
 
 /* here is the process: