Changeset 16989

Timestamp:

Mar 13, 2008, 2:29:14 PM (18 years ago)

Author:

Paul Price

Message:

Fix for outrageous use of memory when combining GPC images --- there are so many chips and cells, each with their own concepts metadata, that the memory is filled with metadataItems. Added a purge of the ones that aren't required. Since the concepts are integral to the system, it's sometimes necessary when iterating to skip cells that have had the concepts purged.

Location:

trunk/ppMerge/src

Files:

• ppMergeCheckInputs.c (modified) (1 diff)
• ppMergeCombine.c (modified) (3 diffs)
• ppMergeMaskAverageConcepts.c (modified) (3 diffs)

trunk/ppMerge/src/ppMergeCheckInputs.c

@@ -106 +106 @@
         }
 
-        data->in->data[i] = pmFPAConstruct(config->camera);
-        pmFPAview *view = pmFPAAddSourceFromHeader(data->in->data[i], header, options->format);
+        pmFPA *fpa = pmFPAConstruct(config->camera);
+        pmFPAview *view = pmFPAAddSourceFromHeader(fpa, header, options->format);
         psFree(view);
+
+        // Cull chips and cells that don't have data
+        // Otherwise the abundance of metadata in the concepts (esp. for GPC) can overload the memory
+        psArray *chips = fpa->chips; // Array of chips in output
+        for (int i = 0; i < chips->n; i++) {
+            pmChip *chip = chips->data[i]; // Chip of interest
+            psArray *cells = chip->cells; // Array of cells
+            int culled = 0;             // Number of culled cells
+            for (int j = 0; j < cells->n; j++) {
+                pmCell *cell = cells->data[j];
+                pmHDU *hdu = pmHDUGetLowest(fpa, chip, cell); // HDU for cell
+                if (!hdu || hdu->blankPHU) {
+                    psFree(cell->concepts);
+                    cell->concepts = NULL;
+                    culled++;
+                }
+            }
+            if (culled == cells->n) {
+                psFree(chip->concepts);
+                chip->concepts = NULL;
+            }
+        }
+        data->in->data[i] = fpa;
+
 
         // Use the first valid input as the basis for the output --- including the header

trunk/ppMerge/src/ppMergeCombine.c

@@ -133 +133 @@
         while ((cell = pmFPAviewNextCell(view, fpa, 1))) {
             cellNum++;
+
+            pmHDU *hdu = pmHDUGetLowest(data->out, chip, cell); // HDU for cell
+            if (!hdu || hdu->blankPHU) {
+                pmCellWrite(cell, data->outFile, config->database, true); // Write header only
+                continue;
+            }
             if (cell->hdu) {
                 // Data will exist soon
@@ -138 +144 @@
                 chip->data_exists = cell->data_exists = true;
             }
-            pmCellWrite(cell, data->outFile, config->database, true); // Write header only
             pmReadout *readout = pmReadoutAlloc(cell); // Output readout of interest
             psArray *stack = psArrayAlloc(filenames->n); // Stack of readouts to combine
@@ -155 +160 @@
 
             // Put version metadata into header
-            pmHDU *hdu = pmHDUFromCell(cell);
             if (hdu && hdu != lastHDU) {
                 if (!hdu->header) {

trunk/ppMerge/src/ppMergeMaskAverageConcepts.c

@@ -18 +18 @@
 // Generate a mask
 bool ppMergeMaskAverageConcepts(ppMergeData *data,  // Data
-                                ppMergeOptions *options, // Options
-                                pmConfig *config    // Configuration
+                                ppMergeOptions *options, // Options
+                                pmConfig *config    // Configuration
     )
 {
@@ -38 +38 @@
         while ((cellOut = pmFPAviewNextCell(view, fpaOut, 1))) {
 
+            pmHDU *hdu = pmHDUGetLowest(fpaOut, chipOut, cellOut);
+            if (!hdu || hdu->blankPHU) {
+                continue;
+            }
+
             // Get list of cells for concepts averaging
-            psList *inCells = psListAlloc(NULL); // List of cells
-            for (int i = 0; i < filenames->n; i++) {
-                if (! filenames->data[i] || strlen(filenames->data[i]) == 0) {
-                    continue;
-                }
-                pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell
-                psListAdd(inCells, PS_LIST_TAIL, cellIn);
-            }
-            if (!pmConceptsAverageCells(cellOut, inCells, NULL, NULL, true)) {
-                psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts.");
-                psFree(inCells);
-                return false;
-            }
-            psFree(inCells);
+            psList *inCells = psListAlloc(NULL); // List of cells
+            for (int i = 0; i < filenames->n; i++) {
+                if (! filenames->data[i] || strlen(filenames->data[i]) == 0) {
+                    continue;
+                }
+                pmCell *cellIn = pmFPAviewThisCell(view, data->in->data[i]); // Input cell
+                psListAdd(inCells, PS_LIST_TAIL, cellIn);
+            }
+            if (!pmConceptsAverageCells(cellOut, inCells, NULL, NULL, true)) {
+                psError(PS_ERR_UNKNOWN, false, "Unable to average cell concepts.");
+                psFree(inCells);
+                return false;
+            }
+            psFree(inCells);
         }
     }
@@ -59 +64 @@
     psList *inFPAs = psListAlloc(NULL); // List of FPAs
     for (int i = 0; i < filenames->n; i++) {
-        if (! filenames->data[i] || strlen(filenames->data[i]) == 0) {
-            continue;
-        }
-        pmFPA *fpaIn = data->in->data[i]; // Input FPA
-        psListAdd(inFPAs, PS_LIST_TAIL, fpaIn);
+        if (! filenames->data[i] || strlen(filenames->data[i]) == 0) {
+            continue;
+        }
+        pmFPA *fpaIn = data->in->data[i]; // Input FPA
+        psListAdd(inFPAs, PS_LIST_TAIL, fpaIn);
     }
 
     if (!pmConceptsAverageFPAs(fpaOut, inFPAs)) {
-        psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts.");
-        psFree(inFPAs);
-        return false;
+        psError(PS_ERR_UNKNOWN, false, "Unable to average FPA concepts.");
+        psFree(inFPAs);
+        return false;
     }
     psFree(inFPAs);