Changeset 30490

Timestamp:

Feb 3, 2011, 2:55:05 PM (15 years ago)

Author:

bills

Message:

merging in changes to support storing distribution bundles in nebulous files

Location:

tags/ipp-20101215

Files:

• DataStoreServer/scripts/dsreg (modified) (1 diff, 1 prop)
• ippScripts/scripts/dist_advancerun.pl (modified) (8 diffs, 1 prop)
• ippScripts/scripts/dist_bundle.pl (modified) (8 diffs, 1 prop)
• ippScripts/scripts/dist_component.pl (modified) (5 diffs, 1 prop)
• ippScripts/scripts/dist_defineruns.pl (modified) (4 diffs, 1 prop)
• ippScripts/scripts/dist_make_fileset.pl (modified) (3 diffs, 1 prop)
• ippTasks/dist.pro (modified) (6 diffs, 1 prop)
• ippTasks/rcserver.pro (modified) (1 diff, 1 prop)
• ippTasks/survey.pro (modified) (3 diffs, 1 prop)

tags/ipp-20101215/DataStoreServer/scripts/dsreg

@@ -338 +338 @@
                 ## then copy or symlink the files into place
                 foreach my $ref (@files) {
-                    my $src = (defined $abspath ? '' : "$datapath/") . "$ref->{file}";
+                    my $src_path = (defined $abspath ? '' : "$datapath/") . "$ref->{file}";
                     my $filename = fileparse($ref->{file}, ());
                     my $dest = "$fileset_dir/$filename";
 
+                    my $src = $ipprc->file_resolve($src_path);
+                    if (!$src) {
+                        print STDERR "failed to resolve file $src_path\n";
+                        exit $PS_EXIT_SYS_ERROR;
+                    }
                     if ( !-e $src or !-r $src) {
                         print STDERR "source file $src does not exist or is not accessible\n";

tags/ipp-20101215/ippScripts/scripts/dist_advancerun.pl

@@ -23 +23 @@
 use PS::IPP::Config 1.01 qw( :standard );
 
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -29 +28 @@
 
 # Parse the command-line arguments
-my ($dist_id, $stage, $stage_id, $outdir, $clean);
+my ($dist_id, $stage, $stage_id, $outdir, $clean, $camera);
 my ($dbname, $save_temps, $verbose, $no_update, $no_op, $logfile);
 
 GetOptions(
-           'dist_id=s'  => \$dist_id,# Magic destreak run identifier
+           'dist_id=s'      => \$dist_id,# Magic destreak run identifier
+           'camera=s'       => \$camera,
            'stage=s'        => \$stage,      # raw, chip, warp, or diff
            'stage_id=s'     => \$stage_id,   # exp_id, chip_id, warp_id, or diff_id
@@ -53 +53 @@
     defined $stage_id and
     defined $outdir;
+
+my $ipprc = PS::IPP::Config->new($camera); # IPP configuration
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -134 +136 @@
 }
 
-# XXX should we create a file rule for this?
-my $dbinfo_file = "$outdir/dbinfo.$stage.$stage_id.mdc";
-
-{
-    my $command = "$tool_cmd $exportarg -outfile $dbinfo_file";
+
+# work around the fact that $ipprc->file_create does not actually create a file on disk
+# unless the scheme is nebulous
+sub create_file {
+    my $rule = shift;
+    my $path_base = shift;
+    my $ref = shift;
+
+    my $file;
+
+    my $error;
+    $file = $ipprc->prepare_output($rule, $path_base, undef, 1, \$error)
+        or &my_die("Unable to prepare outut for $rule", $dist_id, $PS_EXIT_SYS_ERROR);
+
+    my $scheme = file_scheme($file);
+    $scheme = "" if !$scheme;
+
+    my $resolved;
+    if ($scheme) {
+        $ipprc->file_create($file) 
+            or &my_die("Unable to create $file", $dist_id, $PS_EXIT_SYS_ERROR);
+
+        $resolved = $ipprc->file_resolve($file) 
+            or &my_die("Unable to resolve $file", $dist_id, $PS_EXIT_SYS_ERROR);
+
+        if ($scheme eq 'neb') {
+            &my_die("$resolved not found", $dist_id, $PS_EXIT_SYS_ERROR) unless ($resolved and -e $resolved);
+    }
+    } else {
+        $resolved = $file;
+    }
+    $$ref = $resolved;
+
+    return $file
+}
+
+my $dbinfo_root = "$outdir/dbinfo.$stage.$stage_id";
+my $resolved;
+my $dbinfo_file = create_file("DIST.OUTPUT.DBINFO", $dbinfo_root, \$resolved);
+
+{
+    my $command = "$tool_cmd $exportarg -outfile $resolved";
     $command .= " -clean" if ((defined $clean) and ($stage ne "raw"));
     $command .= " -dbname $dbname" if defined $dbname;
@@ -149 +188 @@
     }
 }
-my $dirinfo = "$outdir/dirinfo.$stage.$stage_id.mdc";
+
+my $dirinfo_root = "$outdir/dirinfo.$stage.$stage_id";
+my $dirinfo = create_file("DIST.OUTPUT.DIRINFO", $dirinfo_root, \$resolved);
+
 {
     my $command = "$tool_cmd $list_mode";
@@ -168 +210 @@
         &my_die("Unable to parse metadata list", $dist_id, $PS_EXIT_UNKNOWN_ERROR);
 
-    open MANIFEST, ">$dirinfo" or
-        &my_die("Unable to open dirinfo file $dirinfo",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
+    open MANIFEST, ">$resolved" or
+        &my_die("Unable to open dirinfo file $resolved",  $dist_id, $PS_EXIT_UNKNOWN_ERROR);
 
     my $destdir;
@@ -197 +239 @@
         &my_die("unable to find path",  $dist_id, $PS_EXIT_UNKNOWN_ERROR) if !$path;
         my $component_dir = find_componentdir($destdir, $path);
-#        print MANIFEST "$component METADATA\n";
         print MANIFEST "\t" , "$component", "\tSTR\t", $component_dir, "\n";
     }
@@ -217 +258 @@
     }
 }
-
 
 exit 0;

tags/ipp-20101215/ippScripts/scripts/dist_bundle.pl

@@ -16 +16 @@
 
 use IPC::Cmd 0.36 qw( can_run run );
-use File::Temp qw( tempfile );
+use File::Temp qw( tempfile tempdir );
 use File::Basename qw( basename );
 use Digest::MD5::File qw( file_md5_hex );
@@ -71 +71 @@
 # Parse the command-line arguments
 my ($camera, $stage, $stage_id, $component, $path_base, $chip_path_base, $alt_path_base, $clean);
-my ($outdir, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix);
+my ($outroot, $run_state, $data_state, $magicked, $no_magic, $poor_quality, $results_file, $prefix);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 
@@ -88 +88 @@
            'magicked'       => \$magicked,   # magicked state for this component
            'alt_path_base=s'=> \$alt_path_base,  # path to alternate inputs
-           'outdir=s'       => \$outdir,     # "directory" for outputs
+           'outroot=s'      => \$outroot,    # outroot
            'prefix=s'       => \$prefix,     # "prefix" to apply to filenames
            'clean'          => \$clean,      # create clean distribution
@@ -99 +99 @@
 
 pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV;
-pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outdir",
+pod2usage( -msg => "Required options: --camera --stage --stage_id --component --path_base --outroot",
            -exitval => 3) unless
     defined $camera and
@@ -106 +106 @@
     defined $component and
     defined $path_base and
-    defined $outdir;
+    defined $outroot;
 
 $ipprc->redirect_output($logfile) if $logfile;
@@ -116 +116 @@
 $ipprc->define_camera($camera);
 
-# create the output directories if it is not a nebulous path and it doesn't exist
-if (index($outdir, "neb://") != 0) {
-    if (! -e $outdir ) {
-        my $code = system "mkdir -p $outdir";
-        &my_die("cannot create output directory $outdir", $component,
-                $code >> 8) if $code;
-    }
-}
+$ipprc->outroot_prepare($outroot);
 
 # Get the list of data products for this component
@@ -138 +131 @@
 # set up directory for temporary files
 
-my $tmpdir  = "$outdir/tmpdir.$component.$$";
-if (-e $tmpdir) {
-    if (-d $tmpdir) {
-        my $rc = system "rm -r $tmpdir";
-        &my_die("cannot rm $tmpdir return code: $rc", $component, $PS_EXIT_UNKNOWN_ERROR) if $rc;
-    } else {
-        unlink $tmpdir or &my_die("cannot delete $tmpdir", $component, $PS_EXIT_UNKNOWN_ERROR);
-    }
-}
-mkdir $tmpdir or &my_die("cannot create temporary directory $tmpdir", $component,
-                       $PS_EXIT_UNKNOWN_ERROR);
+my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR");
+$temproot = "/tmp" if !defined $temproot;
+&my_die("directory for TEMP.DIR $temproot does not exist", $component, $PS_EXIT_CONFIG_ERROR) if ! -e $temproot;
+
+my $tmpdir  = tempdir("$temproot/dist.XXXX", CLEANUP => !$save_temps);
 
 #
@@ -349 +336 @@
 if ($num_files) {
     # create the tarfile
-    # XXX TODO: create a file rule for the tar file name
-    my $tbase = basename($path_base);
-    $tbase .= ".$component" if $component;
-    $file_name = ($prefix ? $prefix : "") . "$tbase.tgz";
-    my $tarfile = "$outdir/$file_name";
+#    my $tbase = basename($path_base);
+#    $tbase .= ".$component" if $component;
+#    $file_name = ($prefix ? $prefix : "") . "$tbase.tgz";
+    my $error;
+    my $output_tarfile;
+    my $rule;
+    if ($stage eq 'chip' or $stage eq 'chip_bg' or $stage eq 'raw') {
+        $rule = "DIST.OUTPUT.CHIP.BUNDLE";
+    } else {
+        $rule = "DIST.OUTPUT.BUNDLE";
+    }
+    $output_tarfile = $ipprc->prepare_output($rule, $outroot, $component, 1, \$error)
+            or &my_die("Failed to prepare output tarfile: $error", $component, $error);
+    $file_name = basename($output_tarfile);
+
+
+    my $scheme = file_scheme($output_tarfile);
+
+    my $tarfile;
+    if ($scheme) {
+        $ipprc->file_create($output_tarfile) 
+            or &my_die("Failed to create $output_tarfile: $error", $component, $error);
+        $tarfile = $ipprc->file_resolve($output_tarfile);
+
+        if ($scheme eq 'neb') {
+            &my_die("output file $output_tarfile not found", $component, $PS_EXIT_SYS_ERROR)
+                unless ($tarfile and -e $tarfile);
+        }
+    } else {
+        # no scheme the filename and the resolved filename are the same
+        $tarfile = $output_tarfile;
+    }
 
     my $command = "tar -C $tmpdir --owner=ipp --group=users -czhf $tarfile .";

tags/ipp-20101215/ippScripts/scripts/dist_component.pl

@@ -21 +21 @@
 use PS::IPP::Metadata::Config;
 use PS::IPP::Metadata::List qw( parse_md_list );
-
 use PS::IPP::Config 1.01 qw( :standard );
-
-my $ipprc = PS::IPP::Config->new(); # IPP configuration
 
 use Getopt::Long qw( GetOptions :config auto_help auto_version gnu_getopt );
@@ -77 +74 @@
     defined $outdir;
 
+my $ipprc = PS::IPP::Config->new($camera); # IPP configuration
 $ipprc->redirect_output($logfile) if $logfile;
+
+my $temproot = metadataLookupStr($ipprc->{_siteConfig}, "TEMP.DIR");
+
+$temproot = "/tmp" if !defined $temproot;
 
 if (($stage eq 'raw') and !$clean and !defined $chip_path_base) {
@@ -85 +87 @@
 my $mdcParser = PS::IPP::Metadata::Config->new; # Parser for metadata config files
 
-my ($rf, $rf_name) = tempfile("/tmp/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
+my ($rf, $rf_name) = tempfile("$temproot/bundleresults.$dist_id.$component.XXXX", UNLINK => !$save_temps);
 close $rf;
+
+my $basename = basename($path_base);
+my $outroot = "$outdir/$basename";
 
 my ($file_name, $bytes, $md5sum);
@@ -92 +97 @@
     my $command = "$dist_make_bundle --camera $camera --stage $stage --stage_id $stage_id";
     $command .= " --results_file $rf_name";
-    $command .= " --component $component --path_base $path_base --outdir $outdir";
+    $command .= " --component $component --path_base $path_base --outroot $outroot";
     $command .= " --chip_path_base $chip_path_base" if $chip_path_base;
     $command .= " --state $run_state" if defined $run_state;
@@ -136 +141 @@
     &my_die("undefined file md5sum from dist_bundle.pl", $dist_id, $component, $outdir, $PS_EXIT_PROG_ERROR) unless defined $md5sum;
 }
+
 
 {

tags/ipp-20101215/ippScripts/scripts/dist_defineruns.pl

@@ -42 +42 @@
 
 # Parse the command-line arguments
-my ($stage, $stage_limit, $dist_root, $no_magic);
+my ($stage, $stage_limit, $dist_root, $workdir, $no_magic);
 my ($dbname, $save_temps, $verbose, $no_update, $logfile);
 my @labels;
@@ -49 +49 @@
            'stage=s'        => \$stage,      # stage to queue
            'label=s'        => \@labels,     # labels
+           'workdir=s'      => \$workdir,     # workdir
            'stage_limit=s'  => \$stage_limit,# maximum number of runs queued for each stage
-           'dist_root=s'    => \$dist_root,  # root of distribution work area
+#           'dist_root=s'    => \$dist_root,  # root of distribution work area
            'no_magic'       => \$no_magic,   # queue runs without requiring magic (for testing only)
            'dbname=s'       => \$dbname,     # Database name
@@ -66 +67 @@
 $ipprc->redirect_output($logfile) if $logfile;
 
-if (!$dist_root) {
-    $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT");
-    &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root;
+if (!$workdir) {
+    print "workdir not supplied will use DISTRIBUTION_ROOT\n";
+    # old method where we set workdir based on a config file
+    if (!$dist_root) {
+        $dist_root = metadataLookupStr($ipprc->{_siteConfig}, "DISTRIBUTION_ROOT");
+        &my_die("failed to find DISTRIBUTION_ROOT in site configuration", $PS_EXIT_CONFIG_ERROR) if !$dist_root;
+    }
+
+    my ($day, $month, $year) = (gmtime)[3,4,5];
+    my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
+
+    $workdir = $dist_root . "/$datestr";
+
+    print "workdir is $workdir\n";
 }
-
-my ($day, $month, $year) = (gmtime)[3,4,5];
-my $datestr = sprintf "%04d%02d%02d", $year+1900, $month + 1, $day;
-
-my $workdir = "$dist_root/$datestr";
-
-print "workdir is $workdir\n";
 
 # if stage is not supplied as an argument, loop over all stages
@@ -90 +95 @@
         my $command = "$disttool -definebyquery -stage $stage -workdir $workdir -label $label";
         $command .= " -no_magic" if $no_magic;
-        $command .= " -dry_run" if $no_update;
+        $command .= " -pretend" if $no_update;
         $command .= " -limit $stage_limit" if $stage_limit;
         $command .= " -set_label $label";

tags/ipp-20101215/ippScripts/scripts/dist_make_fileset.pl

@@ -96 +96 @@
 # make sure that the database info file for this run exists
 my $dbinfo_file = "$dist_dir/dbinfo.$stage.$stage_id.mdc";
-if (! -e "$dbinfo_file" ) {
+if (! $ipprc->file_exists($dbinfo_file) ) {
     &my_die("dbinfo file for dist run $dbinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
@@ -103 +103 @@
 # make sure that the dirinfo file for this run exists
 my $dirinfo_file = "$dist_dir/dirinfo.$stage.$stage_id.mdc";
-if (! -e "$dirinfo_file" ) {
+if (!$ipprc->file_exists($dirinfo_file)) {
     &my_die("dirinfo file for dist run $dirinfo_file not found", $dist_id, $dest_id, $PS_EXIT_UNKNOWN_ERROR);
 }
@@ -277 +277 @@
     $command .= " -dbname $dbname" if $dbname;
 
-    my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
-        run(command => $command, verbose => $verbose);
-    unless ($success) {
-        $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
-        print STDERR "Unable to perform $command error_code: $error_code\n";
+    if (!$no_update) {
+        my ( $success, $error_code, $full_buf, $stdout_buf, $stderr_buf ) =
+            run(command => $command, verbose => $verbose);
+        unless ($success) {
+            $error_code = (($error_code >> 8) or $PS_EXIT_PROG_ERROR);
+            print STDERR "Unable to perform $command error_code: $error_code\n";
+        }
+    } else {
+        print STDERR "skipping $command\n";
     }
     exit $fault;

tags/ipp-20101215/ippTasks/dist.pro

@@ -95 +95 @@
 end
 
+# This is now only used if we are not using a nebulous outroot
 macro get.host.for.component
     if ($0 != 3)
@@ -122 +123 @@
 end
 
+# This is now only used if we are not using a nebulous outroot
 macro set.dist.workdir.by.component
     if ($0 != 5)
@@ -139 +141 @@
     book getword ipphosts distribution count -var count
     if ("$count" == "NULL")
-        echo "ERROR: distributin hosts list is empty"
+        echo "ERROR: distribution hosts list is empty"
         break
     end
@@ -289 +291 @@
     end
 
-
-    set.dist.workdir.by.component $DIST_STAGE_ID $COMPONENT $OUTDIR_TEMPLATE OUTDIR 
-    if ("$OUTDIR" == "NULL")
-        echo ERROR failed to set workdir for $COMPONENT
-        break
-    end
-
     substr $COMPONENT 0 3 COMP_HEAD
     if ("$COMP_HEAD" == "sky")
         set.host.for.skycell $COMPONENT
+        set.workdir.by.skycell $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR
     else 
-        # assume component is a class_id, if not we will default to anyhost
+        # assume component is a class_id, if  it is not we will default to
+        # host "anyhost"
+        # and volume "any"
         set.host.for.camera $CAMERA $COMPONENT
+        set.workdir.by.camera $CAMERA $COMPONENT $OUTDIR_TEMPLATE $default_host OUTDIR
+    end
+
+    if ("$OUTDIR" == "NULL")
+        echo ERROR failed to set workdir for $COMPONENT
+        break
     end
 
     sprintf logfile "%s/dist.%s.%s.log" $OUTDIR $DIST_ID $COMPONENT
-#    stdout $logfile
-#    stderr $logfile
 
     $run = dist_component.pl --dist_id $DIST_ID --camera $CAMERA --stage $DIST_STAGE --stage_id $DIST_STAGE_ID --component $COMPONENT --path_base $PATH_BASE --chip_path_base $CHIP_PATH_BASE --state $STATE --data_state $DATA_STATE $EXTRA_ARGS --outdir $OUTDIR --logfile $logfile
@@ -401 +403 @@
     book getpage distToAdvance 0 -var pageName -key pantaskState INIT
     if ("$pageName" == "NULL") break
+
+    # XXX the sql for disttool -toadvance does not include the camera:
+    # hack around this for now
+    $CAMERA = GPC1
 
     book setword distToAdvance $pageName pantaskState RUN
@@ -414 +420 @@
     end
 
-    host anyhost
-
-    # using $DIST_ID as the "component" works fine here since we only look
-    # at the last two digits. But make sure that there 2 digits
-    $fake_component = $STAGE_ID + 10
-    set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    substr $OUTDIR_TEMPLATE 0 3 SCHEME
+    if ("$SCHEME" == "neb")
+        # XXX This does not work if workdir is not in nebulous
+        # and $default_host == "any" because /data/any does not exist
+        # use "dummy" as component. Result will be 'any'
+        # set.workdir.by.camera $CAMERA dummy $OUTDIR_TEMPLATE $default_host OUTDIR
+        set.workdir.by.camera GPC1 dummy $OUTDIR_TEMPLATE $default_host OUTDIR
+    else 
+        # using $DIST_ID as the "component" works fine here since we only look
+        # at the last two digits. But make sure that there 2 digits
+        $fake_component = $STAGE_ID + 10
+        set.dist.workdir.by.component $fake_component "exposure" $OUTDIR_TEMPLATE OUTDIR 
+    end
+ 
     if ("$OUTDIR" == "NULL")
         echo ERROR failed to set workdir for $DIST_ID
         break
     end
+    host anyhost
 
     sprintf logfile "%s/dist.advance.%s.log" $OUTDIR $DIST_ID
-#    stdout $logfile
-#    stderr $logfile
-
-    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile
+
+    $run = dist_advancerun.pl --dist_id $DIST_ID --stage $STAGE --stage_id $STAGE_ID --camera $CAMERA --outdir $OUTDIR $EXTRA_ARGS --logfile $logfile
     add_standard_args run
 

tags/ipp-20101215/ippTasks/rcserver.pro

@@ -136 +136 @@
 
     sprintf logfile "%s/makefs.%s.%s.log" $DIST_DIR $DIST_ID $DEST_ID
-    stdout $logfile
-    stderr $logfile
 
     book setword rcPendingFS $pageName pantaskState RUN
 
-    $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --data_group $DATA_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME  --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR
+    $run = dist_make_fileset.pl --dist_id $DIST_ID --target_id $TARGET_ID --stage $STAGE --stage_id $STAGE_ID --data_group $DATA_GROUP --filter $FILTER --dest_id $DEST_ID --product_name $PRODUCT_NAME  --ds_dbhost $DS_DBHOST --ds_dbname $DS_DBNAME --dist_dir $DIST_DIR --logfile $logfile
 
     add_standard_args run

tags/ipp-20101215/ippTasks/survey.pro

@@ -287 +287 @@
 # user functions to manipulate dist labels
 macro survey.add.dist
-  if ($0 != 2)
-    echo "USAGE: survey.add.dist (label)"
+  if ($0 != 3)
+    echo "USAGE: survey.add.dist (label) (workdir)"
     break
   end
   book newpage SURVEY_DIST $1
+  book setword SURVEY_DIST $1 WORKDIR $2
   book setword SURVEY_DIST $1 STATE PENDING
 end
@@ -896 +897 @@
     end
 
+    book getword SURVEY_DIST $label WORKDIR -var workdir_base
+    $year = `date +%Y`
+    $month = `date +%m`
+    $day = `date +%d`
+    $workdir = $workdir_base/$label/$year/$month/$day
+
     book setword SURVEY_DIST $label STATE DONE
   
     # note workdir is set by the script based on site.config
-    $run = dist_defineruns.pl --label $label
+    $run = dist_defineruns.pl --label $label --workdir $workdir
 
     if ($DB:n == 0)
@@ -908 +915 @@
     end
     
-#    echo $run
+    if ($VERBOSE > 1 )
+        echo $run
+    end
     command $run
   end