Index: /tags/ipp-20110622/Nebulous/Build.PL =================================================================== --- /tags/ipp-20110622/Nebulous/Build.PL (revision 32475) +++ /tags/ipp-20110622/Nebulous/Build.PL (revision 32476) @@ -116,4 +116,5 @@ bin/neb-mv bin/neb-replicate + bin/neb-repair bin/neb-rm bin/neb-shift Index: /tags/ipp-20110622/Nebulous/MANIFEST =================================================================== --- /tags/ipp-20110622/Nebulous/MANIFEST (revision 32475) +++ /tags/ipp-20110622/Nebulous/MANIFEST (revision 32476) @@ -17,4 +17,5 @@ bin/neb-mv bin/neb-replicate +bin/neb-repair bin/neb-rm bin/neb-shift Index: /tags/ipp-20110622/Nebulous/bin/neb-repair =================================================================== --- /tags/ipp-20110622/Nebulous/bin/neb-repair (revision 32476) +++ /tags/ipp-20110622/Nebulous/bin/neb-repair (revision 32476) @@ -0,0 +1,214 @@ +#!/usr/bin/env perl + +# Copyright (C) 2007-2008 Chris Waters +# + +use strict; +use warnings FATAL => qw( all ); + +use vars qw( $VERSION ); +$VERSION = '0.01'; + +use Nebulous::Client; + +use Getopt::Long qw( GetOptions :config auto_help auto_version ); +use Pod::Usage qw( pod2usage ); + +use Digest::MD5; +use URI; + +my ($validate,$server,$nouris); + +$server = $ENV{'NEB_SERVER'} unless $server; + +$validate = 1; +$nouris = 1; + +GetOptions( + 'server|s=s' => \$server, +) || pod2usage( 2 ); + +my $key = shift; + +pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; +pod2usage( -msg => "Required options: --server", -exitval => 2 ) + unless $server; +pod2usage( -msg => "missing key", exitval => 2 ) + unless defined $key; + +my $neb = Nebulous::Client->new( + proxy => "$server", +); + +die "can't connected to Nebulous Server: $server" + unless defined $neb; + +my $stat = $neb->stat($key); +die "nebulous key: $key not found" unless $stat; +my $instances = $neb->find_instances($key, 'any'); +die "no instances found" unless $instances; + +my $user_copies; +eval { + $user_copies = $neb->getxattr($key, "user.copies"); +}; +unless(defined($user_copies)) { + $user_copies = 2; +} + +my @validation; +my %md5sum_uniq; +my %md5sum_valid; +my $last_f = ''; +my $existing_copies = 0; +my @existance; +my @md5sums; +my @files = map {URI->new($_)->file if $_} @$instances; + +for (my $i = 0; $i <= $#files; $i++) { + if (-e $files[$i]) { + $existance[$i] = 1; + $existing_copies++; + my $fh; + open($fh,$files[$i]); + $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest; + $md5sum_uniq{$md5sums[$i]} = 1; + push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i]; + $last_f = $files[$i]; + close($fh); + } + else { + $existance[$i] = 0; + $md5sums[$i] = 'NON-EXISTANT'; + $md5sum_uniq{$md5sums[$i]} = 1; + push @{ $md5sum_valid{$md5sums[$i]} }, $files[$i]; + } +} + +my $N_mds = scalar(keys(%md5sum_uniq)); +my $N_files = scalar(@files); +if ($N_files < $user_copies) { + print "Insufficient copies ($N_files) of $key, replicating!\n"; + system("neb-replicate $key"); +} +if ($N_mds == 1) { + print "Nebulous key $key is consistent.\n"; +} +else { + my $valid_sum = ''; + my $valid_file = ''; + my $m; + foreach $m (keys %md5sum_valid) { + if (($m ne 'd41d8cd98f00b204e9800998ecf8427e')&&($m ne 'NON-EXISTANT')) { + if ($valid_sum ne '') { + die "Too many acceptable files for key $key\n"; + } + $valid_sum = $m; + $valid_file = ${ $md5sum_valid{$m} }[0]; + } + } + if ($valid_sum eq '') { + die "No acceptable file for key $key\n"; + } + else { + print "Repairing instances\n"; + foreach $m (keys %md5sum_valid) { + if ($m ne $valid_sum) { + foreach my $f (@{ $md5sum_valid{$m} }) { + print "\tcp $valid_file $f\n"; + system("cp $valid_file $f"); + } + } + } + } +} + +if ($N_files > $user_copies) { + my @split = split /\//, $last_f; + my $volume = $split[2]; + print "Culling extra copy: neb-cull --volume $volume $key\n"; + system("neb-cull --volume $volume $key"); +} + + + +__END__ + +=pod + +=head1 NAME + +neb-repair - tool to check and repair instances + +=head1 SYNOPSIS + + neb-repair [--server ] + +=head1 DESCRIPTION + +This program repairs instances and corrects instance counts for a nebulous key. + +=head1 OPTIONS + +=over 4 + +=item * --server|-s + +URL of the Nebulous server to connect to. + +Optional if the appropriate environment variable is set. + +=back + +=head1 ENVIRONMENT + +These environment variables may be used in place of the specified command line +options. All command line option will override the corresponding environment +value. + +=over 4 + +=item * C + +Equivalent to --server|-s + +=back + +=head1 CREDITS + + + +=head1 SUPPORT + +Please contact the author directly via e-mail. + +=head1 AUTHOR + +Chris Waters + +=head1 COPYRIGHT + +Copyright (C) 2007-2008 Chris Waters + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307, USA. + +The full text of the license can be found in the L Pod as supplied +with Perl 5.8.1 and later. + +=head1 SEE ALSO + +L, L, L, L, L, +L, C + +=cut Index: /tags/ipp-20110622/Nebulous/bin/neb-shift =================================================================== --- /tags/ipp-20110622/Nebulous/bin/neb-shift (revision 32475) +++ /tags/ipp-20110622/Nebulous/bin/neb-shift (revision 32476) @@ -29,7 +29,7 @@ pod2usage( -msg => "Unknown option: @ARGV", -exitval => 2 ) if @ARGV; -pod2usage( -msg => "Required options: --server ", +pod2usage( -msg => "Required options: --server --volume ", -exitval => 2 ) - unless defined $server and defined $src and defined $abandon_vol; + unless defined $server and defined $src and defined $abandon_vol and defined $volume; my $neb = Nebulous::Client->new( @@ -45,12 +45,14 @@ } +my $all_uris = $neb->find_instances($src); + my $status; -if ($volume) { - $status = $neb->replicate( $src, $volume); +$status = $neb->replicate( $src, $volume); +die "Replicate phase failed for $src ($abandon_vol) ($volume)" unless $status; + +$uris = $neb->find_instances($src, "~$volume"); +unless (defined $uris) { + die "Replicate phase failed for $src ($abandon_vol) ($volume)"; } -else { - $status = $neb->replicate($src); -} -die "Replicate phase failed for $src ($abandon_vol) ($volume)" unless $status; $status = $neb->cull($src,$abandon_vol); Index: /tags/ipp-20110622/Nebulous/bin/neb-stat =================================================================== --- /tags/ipp-20110622/Nebulous/bin/neb-stat (revision 32475) +++ /tags/ipp-20110622/Nebulous/bin/neb-stat (revision 32476) @@ -16,5 +16,8 @@ use Pod::Usage qw( pod2usage ); -my ($server); +use Digest::MD5; +use URI; + +my ($validate,$server,$nouris); $server = $ENV{'NEB_SERVER'} unless $server; @@ -22,4 +25,6 @@ GetOptions( 'server|s=s' => \$server, + 'validate|v' => \$validate, + 'files' => \$nouris, ) || pod2usage( 2 ); @@ -44,16 +49,84 @@ die "no instances found" unless $instances; +my $user_copies; +eval { + $user_copies = $neb->getxattr($key, "user.copies"); +}; +unless(defined($user_copies)) { + $user_copies = 1; +} + +my @validation; +my %md5sum_uniq; +my $existing_copies = 0; +if (defined $validate) { + my @existance; + my @md5sums; + my @files = map {URI->new($_)->file if $_} @$instances; + + for (my $i = 0; $i <= $#files; $i++) { + if (-e $files[$i]) { + $existance[$i] = 1; + $existing_copies++; + my $fh; + open($fh,$files[$i]); + $md5sums[$i] = Digest::MD5->new->addfile($fh)->hexdigest; + $md5sum_uniq{$md5sums[$i]} = 1; + close($fh); + } + else { + $existance[$i] = 0; + $md5sums[$i] = 'NON-EXISTANT'; + $md5sum_uniq{$md5sums[$i]} = 1; + } + if (defined($nouris)) { + $validation[$i] = sprintf("% 3d %32s %s", + $existance[$i], + $md5sums[$i], + $files[$i]); + } + else { + $validation[$i] = sprintf("% 3d %32s %s", + $existance[$i], + $md5sums[$i], + ${ $instances }[$i]); + } + } +} + no warnings qw(uninitialized); -print - "object id: ", @$stat[0], "\n", - "key: ", @$stat[1], "\n", - "read lock: ", @$stat[2], "\n", - "write lock: ", @$stat[3], "\n", - "epoch: ", @$stat[4], "\n", - "mtime: ", @$stat[5], "\n", - "available instances: ", @$stat[6], "\n", - "total instances: ", @$stat[7], "\n", - "instance location:\n", " " x 4, - join("\n" . " " x 4, @$instances), "\n"; +if (defined $validate) { + print + "object id: ", @$stat[0], "\n", + "key: ", @$stat[1], "\n", + "read lock: ", @$stat[2], "\n", + "write lock: ", @$stat[3], "\n", + "epoch: ", @$stat[4], "\n", + "mtime: ", @$stat[5], "\n", + "md5sum count: ", scalar(keys %md5sum_uniq), "\n", + "requested instances: ", $user_copies, "\n", + "available instances: ", @$stat[6], "\n", + "existing instances: ", $existing_copies, "\n", + "total instances: ", @$stat[7], "\n", + + "instance location:\n", " " x 4, + join("\n" . " " x 4, @validation), "\n"; +} +else { + print + "object id: ", @$stat[0], "\n", + "key: ", @$stat[1], "\n", + "read lock: ", @$stat[2], "\n", + "write lock: ", @$stat[3], "\n", + "epoch: ", @$stat[4], "\n", + "mtime: ", @$stat[5], "\n", + "available instances: ", @$stat[6], "\n", + "requested instances: ", $user_copies, "\n", + "total instances: ", @$stat[7], "\n", + + "instance location:\n", " " x 4, + join("\n" . " " x 4, @$instances), "\n"; +} + __END__ @@ -67,5 +140,5 @@ =head1 SYNOPSIS - neb-stat [--server ] + neb-stat [--server ] [--validate] =head1 DESCRIPTION @@ -82,4 +155,12 @@ Optional if the appropriate environment variable is set. + +=item * --validate|-v + +Optional - prints out additional information including md5sum. + +=item * --files + +Optional - prints out disk files instead of uris during validate stage. =back