Changeset 25027 for branches/pap/ippScripts/scripts/dist_advancerun.pl

Timestamp:

Aug 7, 2009, 4:08:25 PM (17 years ago)

Author:

Paul Price

Message:

Merging trunk (r25026) to get up-to-date on old branch.

Location:

branches/pap

Files:

• . (modified) (1 prop)
• ippScripts/scripts/dist_advancerun.pl (modified) (5 diffs)

branches/pap/ippScripts/scripts/dist_advancerun.pl

@@ -16 +16 @@
 use IPC::Cmd 0.36 qw( can_run run );
 use File::Temp qw( tempfile );
-use File::Basename qw( basename );
+use File::Basename qw( dirname);
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
@@ -72 +72 @@
 
 my $tool_cmd;
+my $list_mode;
+my $component_key;
 if ($stage eq "raw") {
     $tool_cmd = "$regtool -exp_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "chip") {
     $tool_cmd = "$chiptool -chip_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "camera") {
     $tool_cmd = "$camtool -cam_id";
+    $list_mode = "-processedexp";
+    $component_key = "";
 } elsif ($stage eq "fake") {
     $tool_cmd = "$faketool -fake_id";
+    $list_mode = "-processedimfile";
+    $component_key = "class_id";
 } elsif ($stage eq "warp") {
     $tool_cmd = "$warptool -warp_id";
+    $list_mode = "-warped";
+    $component_key = "skycell_id";
 } elsif ($stage eq "stack") {
     $tool_cmd = "$stacktool -stack_id";
+    $list_mode = "-sumskyfile";
+    $component_key = "skycell_id";
 } elsif ($stage eq "diff") {
     $tool_cmd = "$difftool -diff_id";
+    $list_mode = "-diffskyfile";
+    $component_key = "skycell_id";
 } else {
     &my_die("Unexpected stage: $stage", $dist_id, $PS_EXIT_CONFIG_ERROR);
 }
 
+$tool_cmd .= " $stage_id";
+
 # XXX should we create a file rule for this?
-my $outfile = "$outdir/dbinfo.$stage.$stage_id.mdc";
-
-{
-    my $command = "$tool_cmd $stage_id -exportrun -outfile $outfile";
-    $command .= " -clean" if defined $clean;
+my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc";
+
+{
+    my $command = "$tool_cmd -exportrun -outfile $dbinfo_file";
+    $command .= " -clean" if ((defined $clean) and ($stage ne "raw"));
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -105 +123 @@
     }
 }
-
-# set distRun.stage = 'full'
-{
-    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
+my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc";
+{
+    my $command = "$tool_cmd $list_mode";
     $command .= " -dbname $dbname" if defined $dbname;
 
@@ -117 +134 @@
         &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
     }
+    if (@$stdout_buf == 0) {
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
+    my $metadata = $mdcParser->parse(join "", @$stdout_buf) or
+        &my_die("Unable to parse metadata config doc", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+    my $components = parse_md_list($metadata) or
+        &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    open MANIFEST, ">$dirinfo" or
+        &my_die("Unable to open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+
+    my $destdir;
+    foreach my $c (@$components) {
+        # take the workdir from the first component
+        if (!$destdir) {
+            my $workdir = $c->{workdir};
+            if ($workdir) {
+                $destdir = stripvolume($workdir, $stage);
+            } elsif ($stage eq 'raw') {
+                $destdir = 'none';
+            } else {
+                &my_die("workdir not found for open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+            }
+            print MANIFEST "destdir METADATA\n";
+            print MANIFEST "\t" , "destdir", "\tSTR\t", $destdir, "\n";
+            print MANIFEST "END\n\n";
+            print MANIFEST "components METADATA\n";
+        }
+        my $component = $c->{$component_key} ? $c->{$component_key} : "exposure";
+        my $path;
+        if ($stage eq 'raw') {
+            $path = $c->{uri};
+        } else {
+            $path = $c->{path_base};
+        }
+        &my_die("unable to find path",  $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path;
+        my $component_dir = find_componentdir($destdir, $path); 
+#        print MANIFEST "$component METADATA\n";
+        print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n";
+    }
+    print MANIFEST "END\n\n";
+    close MANIFEST or 
+        &my_die("Unable to close dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+}
+
+{
+    my $command = "$disttool -updaterun -dist_id $dist_id -set_state full";
+    $command .= " -dbname $dbname" if defined $dbname;
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to perform $command: $error_code", $dist_id, $error_code);
+    }
 }
 
@@ -124 +196 @@
 ### Pau.
 
+sub stripvolume
+{
+    my $path = shift;
+    my $stage = shift;
+    my @segments;
+
+    # workdir isn't what we want for raw stage
+    return "none" if ($stage and ($stage eq 'raw'));
+
+    my $scheme = file_scheme($path);
+    my $tail;
+    if ($scheme) {
+        # strip off scheme://
+        $tail = substr($path, length($scheme) + 3);
+    } elsif (substr($path, 0, 1) eq '/') {
+        $tail = substr($path, 1);
+        $scheme = "";
+    }
+    # remove any leading / that are left
+    while ((substr($tail, 0, 1) eq '/')) {
+        $tail = substr($tail, 1);
+    }
+
+    if (($scheme eq 'neb') or ($scheme eq 'path')) {
+        my $volume;
+        ($volume, @segments) = split '/', $tail;
+
+    } elsif (!$scheme or ($scheme eq 'file')) {
+
+        # XXX Here we're assuming the /data/ipp??? structure. This won't be true when data is forwarded
+        # by remote sites. We need a way to configure this
+        my $volume;
+
+        # data/ippxxx/dirs
+        (undef, $volume, @segments) = split '/', $tail;
+    } else {
+        die( "unexpected workdir value: $path\n");
+    }
+
+    return caturi(@segments);
+}
+
+sub find_componentdir
+{
+    my $destdir = shift;
+    my $path = shift;
+
+    my $result;
+    if ($destdir eq 'none') {
+        $result = stripvolume($path);
+    } else {
+        # find location of destdir in the path
+        my $i = index($path, $destdir);
+
+        $result = substr($path, $i + length($destdir) + 1);
+
+        while (substr($result, 0, 1) eq '/') {
+            $result = substr($result, 1);
+        }
+    }
+    return dirname($result);
+}
 
 sub my_die