Changeset 27840 for branches/simtest_nebulous_branches/ippScripts/scripts/dist_make_fileset.pl

Timestamp:

May 3, 2010, 8:50:52 AM (16 years ago)

Author:

eugene

Message:

updates from trunk

Location:

branches/simtest_nebulous_branches

Files:

• . (modified) (1 prop)
• ippScripts/scripts/dist_make_fileset.pl (modified) (15 diffs)

branches/simtest_nebulous_branches/ippScripts/scripts/dist_make_fileset.pl

@@ -8 +8 @@
 use Sys::Hostname;
 my $host = hostname();
+my $date = `date`;
 print "\n\n";
-print "Starting script $0 on $host\n\n";
+print "Starting script $0 on $host at $date\n\n";
 
 use vars qw( $VERSION );
@@ -39 +40 @@
 
 # Parse the command-line arguments
-my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $prod_id, $product_name, $ds_dbhost, $ds_dbname);
+my ($dist_id, $dist_dir, $target_id, $stage, $stage_id, $dest_id, $product_name, $ds_dbhost, $ds_dbname);
+my ($label, $data_group, $filter);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 
@@ -45 +47 @@
            'dist_id=s'      => \$dist_id,    # distribution run identifier
            'dist_dir=s'     => \$dist_dir,   # directory containing dist run outputs
-           'target_id=s'    => \$target_id,  # 
+           'target_id=s'    => \$target_id,  #
            'stage=s'        => \$stage,      # raw, chip, camera, fake, warp, stack, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, etc.
-           'prod_id=s'      => \$prod_id,    # id for the product
+           'dest_id=s'      => \$dest_id,    # id for the product
            'product_name=s' => \$product_name,  # location of the data store directory for this product
+           'label=s'        => \$label,
+           'data_group=s'   => \$data_group,
+           'filter=s'       => \$filter,
            'ds_dbhost=s'    => \$ds_dbhost,  # database host for the datastore database
            'ds_dbname=s'    => \$ds_dbname,  # database name for the datastore database
@@ -60 +65 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --prod_id --ds_dbhost --ds_dbname",
+pod2usage( -msg => "Required options: --dist_id --dist_dir --target_id --stage --stage_id --data_group --filter --dest_id --ds_dbhost --ds_dbname",
            -exitval => 3) unless
     defined $dist_id and
@@ -67 +72 @@
     defined $stage and
     defined $stage_id and
-    defined $prod_id and
+    defined $data_group and
+    defined $filter and
+    defined $dest_id and
     defined $product_name and
     defined $ds_dbhost and
@@ -79 +86 @@
 my $fs_tag = get_fileset_tag($ipprc, $stage, $stage_id, $dbname);
 
-&my_die("failed to lookup fileset tag", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR) if !$fs_tag;
-
-my $fileset_name = "$fs_tag.$stage.$stage_id.$dist_id.$prod_id";
+&my_die("failed to lookup fileset tag", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR) if !defined $fs_tag;
+
+my $fileset_name = $fs_tag ? "$fs_tag." : "";
+$fileset_name .= "$stage.$stage_id.$dist_id.$dest_id";
 
 print "$fileset_name\n";
@@ -87 +95 @@
 
 # make sure that the database info file for this run exists
-my $dbinfo_file = "dbinfo.$stage.$stage_id.mdc";
-if (! -e "$dist_dir/$dbinfo_file" ) {
-    &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR);
+my $dbinfo_file = "$dist_dir/dbinfo.$stage.$stage_id.mdc";
+if (! -e "$dbinfo_file" ) {
+    &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
 print "dbinfo file $dbinfo_file exists\n" if $verbose;
 
 # make sure that the dirinfo file for this run exists
-my $dirinfo_file = "dirinfo.$stage.$stage_id.mdc";
-if (! -e "$dist_dir/$dirinfo_file" ) {
-    &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $prod_id, $PS_EXIT_UNKNOWN_ERROR);
+my $dirinfo_file = "$dist_dir/dirinfo.$stage.$stage_id.mdc";
+if (! -e "$dirinfo_file" ) {
+    &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
 print "dirinfo file $dirinfo_file exists\n" if $verbose;
@@ -116 +124 @@
     unless ($success) {
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code);
+        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $dest_id, $error_code);
     }
 
     my $metadata = $mdcParser->parse (join "", @$stdout_buf) or
-        &my_die("Unable to parse metadata config doc", $dist_id, $prod_id, $PS_EXIT_PROG_ERROR);
+        &my_die("Unable to parse metadata config doc", $dist_id, $dest_id, $PS_EXIT_PROG_ERROR);
 
     $components = parse_md_list($metadata);
@@ -135 +143 @@
     my $md5sum = $component->{md5sum};
     # name of the file
-    my $name = $component->{name};
+    my $file_name = $component->{name};
     # component id (class_is or skycell_id)
+    my $comp_dir = $component->{outdir};
     my $comp_name = $component->{component};
 
     # XXX: if tarfile is not always the right type we need to add a type to distComponent
-    print $listFile "$name|$size|$md5sum|tgz|$comp_name|\n";
+    print $listFile "$comp_dir/$file_name|$size|$md5sum|tgz|$comp_name|\n";
 }
 
@@ -146 +155 @@
 
 {
-    # XXX: need to chose an appropriate file set type
-    my $command = "$dsreg --add $fileset_name --product $product_name --type notset --list $listFileName";
+    my $command = "$dsreg --add $fileset_name --product $product_name --type IPP-DIST --list $listFileName";
 
     # the data store will refer to the distribution bundle via symlinks back to distRun.outdir
-    $command .= " --datapath $dist_dir --link";
+#    $command .= " --datapath $dist_dir --link";
+    $command .= " --abspath --link";
 
     # set the product specific columns in product list
-    $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $fs_tag";
-
-    $command .= " --dbname $ds_dbname";    # XXX: notyet --dbhost $ds_dbhost
-
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code);
+    my $prod_col_3 = $fs_tag ? $fs_tag : "$stage.$stage_id";
+
+    $command .= " --ps0 $target_id --ps1 $stage --ps2 $stage_id --ps3 $prod_col_3";
+    $command .= " --ps4 $data_group --ps5 $filter";
+
+    $command .= " --dbname $ds_dbname --dbhost $ds_dbhost";
+
+    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+        run(command => $command, verbose => $verbose);
+    unless ($success) {
+        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+         &my_die("Unable to perform $command error_code: $error_code", $dist_id, $dest_id, $error_code);
     }
 }
 
 {
-    my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -name $fileset_name";
+    my $command = "$disttool -addfileset -dist_id $dist_id -dest_id $dest_id -name $fileset_name";
     $command .= " -dbname $dbname" if $dbname;
 
@@ -175 +187 @@
         # We need to have revertfileset check whether the fileset exists and do dsreg -del if it does
         $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $prod_id, $error_code);
+        &my_die("Unable to perform $command error_code: $error_code", $dist_id, $dest_id, $error_code);
     }
 }
@@ -198 +210 @@
     my $dsn = "DBI:mysql:host=$dbserver;database=$dbname";
 
-    my $dbh = DBI->connect($dsn, $dbuser, $dbpassword) 
+    my $dbh = DBI->connect($dsn, $dbuser, $dbpassword)
         or die "Cannot connect to database.\n";
 
@@ -211 +223 @@
     my $dbname = shift;
 
-    if ($stage eq 'stack') {
-        return "stack.$stage_id";
-    } elsif ($stage eq 'diff') {
-        return "diff.$stage_id";
+    if (($stage eq 'stack') or ($stage eq 'diff') or ($stage eq 'SSdiff')) {
+        return "";
     }
 
     #
-    # we are a long ways away from the rawExp in the pipeline. Rather than do some 
+    # we are a long ways away from the rawExp in the pipeline. Rather than do some
     # very long joins in disttool, we look up the exp_name in the database using DBI
     #
     my $dbh = getDBHandle($ipprc, $dbname);
 
-    my $query; 
+    my $query;
 
     if ($stage eq 'raw') {
@@ -248 +258 @@
     my $tag = $ref->{exp_name};
 
-    return $tag;
+    return "$tag";
 }
 
@@ -254 +264 @@
     my $msg = shift;
     my $dist_id = shift;
-    my $prod_id = shift;
+    my $dest_id = shift;
     my $fault = shift;
 
-    # TODO: disttool -adddsfileset -prod_id $prod_id -dist_id $dist_id -fault $fault
+    # TODO: disttool -adddsfileset -dest_id $dest_id -dist_id $dist_id -fault $fault
     print STDERR "$msg\n";
 
-    my $command = "$disttool -addfileset -dist_id $dist_id -prod_id $prod_id -fault $fault";
+    my $command = "$disttool -addfileset -dist_id $dist_id -dest_id $dest_id -fault $fault";
     $command .= " -dbname $dbname" if $dbname;