Changeset 24210

Timestamp:

May 16, 2009, 2:57:23 PM (17 years ago)

Author:

bills

Message:

edit config dump files to change volume imformation
to match the destination.
I.E. neb://volume/ ===> receiveSource.workdir/

/data/ippxxx.y/ ===> receiveSource.workdir/

File:

• trunk/ippScripts/scripts/receive_file.pl (modified) (9 diffs)

trunk/ippScripts/scripts/receive_file.pl

@@ -162 +162 @@
     # Rather than read it as an mdc and interptet it, we do our substitutions directly
     # This is much faster. Parsing a mdc file for a chip run takes several seconds.
-    # we are very strict on the formatting
-
-    # first line tells us the run type. From this we get the stage
+    # we are very strict about the format of the file
+    #
+    # First comes the data for the Run
+    # Next is the data for each component
+    # The component_id (class_id, skycell_id) must come before any of the paths that we edit
+
+    # The first line tells us the run type. From this we get the stage
     #
     my $line = $lines[0];
@@ -170 +174 @@
     &my_die( "unexpected first line found in $filename: $line\n", $file_id, $PS_EXIT_UNKNOWN_ERROR)
         if !$runType or ($multi ne 'MULTI');
+
     my $stage;
     my $comp_name;
@@ -201 +206 @@
     my $new_workdir_value;
     if ($destdir eq 'none') {
+        # this only appiles to rawExp
         $new_workdir_value = "$workdir";
     } else {
@@ -213 +219 @@
 
         my ($name, $type, $value) = split " ", $line;
-        # we only edit complete lines
+        # only complete lines have things that we need to examine
         if ($name and $type and $value) {
             my $new_value;
+            # we have a new component id, save it and look up the corresponding
+            # component_dir
             if ($name eq $comp_name) {
                 $current_component = $value;
@@ -235 +243 @@
             }
 
+            # if the value changed re-write the line, otherwise just print what we read
             if ($new_value) {
                 $out_line = "   " . $name . "\t\t" . $type . "\t" . $new_value . "\n";
@@ -300 +309 @@
         my $target = "$target_dir/$file"; # Target destination for file
 
+
         $ipprc->file_delete ($target);
 
@@ -308 +318 @@
         }
 
-        system("mv $from $to") == 0 or &my_die( "Unable to move $file into workdir $workdir: $!\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+        if ( $file =~ /.+\.mdc/ ) {
+            # this file is a config dump file edit the paths
+            edit_mdc_file($file_id, $from, $to, $workdir);
+        } else {
+            system("mv $from $to") == 0 or &my_die( "Unable to move $file into workdir $workdir: $!\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+        }
     }
 } else {
@@ -314 +329 @@
 }
 
-if (!$save_temps) {
+if (!$save_temps and -e $filename) {
     unlink $filename or &my_die( "Unable to unlink $filename\n", $file_id, $PS_EXIT_UNKNOWN_ERROR);
 }
@@ -372 +387 @@
 }
 
+# edit a config dump file replacing the "volume" value with the new local value: $workdir
+sub edit_mdc_file
+{
+    my $file_id = shift;
+    my $src = shift;
+    my $dest = shift;
+    my $workdir = shift;
+
+    open my $IN,  "<$src" or &my_die("failed to open $src for input", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+    open my $OUT, ">$dest" or &my_die("failed to open $dest for output", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    # Assumed file structure
+    # stuff
+    # FILES.INPUT metadata
+    # FILES.OUTPUT metadata
+    # more stuff
+    # only the paths in the FILES.* metadata are monkeyed with
+    my $done_editing = 0;
+    my $numFilesMD = 0;
+    foreach my $line (<$IN>) {
+        my $out_line = $line;
+        if (!$done_editing) {
+            my (@words) = split " ", $line;
+            if (scalar @words) {
+                # get rid of any leading blank words
+                while ((scalar @words) and !defined $words[0]) {
+                    shift @words;
+                }
+
+                if ($words[1] and $words[1] eq "METADATA") {
+                    if ( $words[0] =~ /^FILES\..+/ ) {
+                        $numFilesMD++;
+                    }
+                } elsif ($words[0] eq "END") {
+                    # when we get to the end of the second FILES metadata we're done editing
+                    if ($numFilesMD == 2) {
+                        $done_editing = 1;
+                    }
+                } elsif ($numFilesMD and ($words[1] eq "STR"))  {
+                    # we're processing one of the files metadata edit the path
+                    my $key = shift @words;
+                    my $type = shift @words;
+                    my $path = shift @words;
+                    my $extra = join " ", @words;
+
+                    $path = edit_path($file_id, $workdir, $path);
+
+                    $out_line = "\t" . $key ."\t" . "STR" . "\t" . $path;
+                    $out_line .= "\t" . $extra if $extra;
+                    $out_line .= "\n";
+                }
+            }
+        }
+        print $OUT $out_line;
+    }
+
+    close $IN;
+    close $OUT or &my_die("failed to close $dest", $file_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+
+
+# XXX: this should go into a module
+# Replace 'volume portion of path with $workdir/
+# Volume is defined here by
+#   neb://volume/
+#   /xxx/xxxxx/         i.e. /data/ippxxx.y/
+#   file://xxx/xxxxx/   i.e. file://data/ippxxx.y/
+#   path://somepath/
+sub edit_path
+{
+    my $file_id = shift;
+    my $workdir = shift;
+    my $path = shift;
+
+    my $scheme = file_scheme($path);
+    my $tail;
+    if ($scheme) {
+            # strip off scheme://
+        $tail = substr($path, length($scheme) + 3);
+    } elsif (substr($path, 0, 1) eq '/') {
+        $tail = substr($path, 1);
+        $scheme = "";
+    }
+    # remove any leading / that are left
+    while ((substr($tail, 0, 1) eq '/')) {
+        $tail = substr($tail, 1);
+    }
+
+    my @segments;
+    if (($scheme eq 'neb') or ($scheme eq 'path')) {
+        my $volume;
+        ($volume, @segments) = split '/', $tail;
+
+    } elsif (!$scheme or ($scheme eq 'file')) {
+
+        # XXX Here we're assuming the /data/ipp??? structure. This won't be true when data is forwarded
+        # by remote sites. We need a way to configure this
+        my $volume;
+
+        # data/ippxxx/dirs
+        (undef, $volume, @segments) = split '/', $tail;
+    } else {
+        &my_die( "unexpected workdir value: $path\n", $file_id, $PS_EXIT_PROG_ERROR);
+    }
+
+    my $new_path = caturi($workdir, @segments);
+
+    return $new_path;
+}
+
 sub my_die
 {