Changeset 10296

Timestamp:

Nov 29, 2006, 1:43:22 PM (19 years ago)

Author:

Paul Price

Message:

Speeding up parsing of huge metadata.

File:

• trunk/ippScripts/scripts/detrend_norm_calc.pl (modified) (4 diffs)

trunk/ippScripts/scripts/detrend_norm_calc.pl

@@ -58 +58 @@
 
 # Get the list of inputs
-my $files;                      # The input files
+my @files;                      # The input files
 {
     my $command = "$dettool -processedimfile -det_id $detId"; # Command to run
@@ -68 +68 @@
     print $stdout . "\n";
     
-    # Parse the output
-    my $metadata = $mdcParser->parse($stdout)
-        or die "unable to parse metadata config doc";
-    $files = parse_md_list($metadata);
+    # Because of the length, need to split into individual metadatas --- it parses SO much quicker!
+    my @whole = split /\n/, $stdout;
+    my @single = ();
+    while ( scalar @whole > 0 ) {
+        my $value = shift @whole;
+        push @single, $value;
+        if ($value =~ /^\s*END\s*$/) {
+            push @single, "\n";
+
+            my $list = parse_md_list( $mdcParser->parse( join( "\n", @single ) ) ) or
+                die "Unable to parse output from dettool.\n";
+            push @files, $list;
+        }
+    }
 }
-
 
 my $norms;                      # MDC with normalisations
@@ -79 +88 @@
 
     my %matrix; # Matrix of statistics as a function of exposures and classes
-    foreach my $file (@$files) {
+    foreach my $file (@files) {
         my $expTag = $file->{'exp_tag'}; # Exposure ID
         my $classId = $file->{'class_id'}; # Class ID
@@ -116 +125 @@
     # It's something that doesn't need normalisation --- just push in a normalisation of 1
     my %classes;                # List of unique classes
-    foreach my $file (@$files) {
+    foreach my $file (@files) {
         my $classId = $file->{'class_id'}; # Class Id
         $classes{$classId} = 1;