IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 17072


Ignore:
Timestamp:
Mar 20, 2008, 11:13:28 AM (18 years ago)
Author:
jhoblitt
Message:

server/client split

Location:
trunk/Nebulous-Server
Files:
4 added
19 edited

Legend:

Unmodified
Added
Removed

  • trunk/Nebulous-Server/Build.PL

    r16565 r17072  
    22# See perldoc Module::Build for details of how this works
    33
    4 my $build_pkg   = eval { require Apache::TestMB }
    5                 ? 'Apache::TestMB'
    6                 : 'Module::Build';
    7 
    8 my $class = $build_pkg->subclass(code => <<'EOF');
    9 use Cwd;
    10 
    11 my $pkg_dir = "./nebclient";
    12 
    13 sub ACTION_code {
    14     my $self = shift;
    15 
    16     $self->SUPER::ACTION_code(@_);
    17 
    18     my $old_pwd = getcwd();
    19     chdir $pkg_dir;
    20 
    21     unless (-e "configure") {
    22         system("./autogen.sh") == 0 or die "install failed: $?";
    23     }
    24 
    25     chdir $old_pwd;
    26 }
    27 
    28 sub ACTION_build {
    29     my $self = shift;
    30 
    31     $self->SUPER::ACTION_build(@_);
    32 
    33     my $old_pwd = getcwd();
    34     chdir $pkg_dir;
    35 
    36     unless (-e "Makefile") {
    37         system("sh ./configure") == 0 or die "build failed: $?";
    38     }
    39 
    40     system("make") == 0 or die "build failed: $?";
    41 
    42     chdir $old_pwd;
    43 }
    44 
    45 # Do not attempt to install nebclient.  It is bundled in this package for
    46 # testing only.
    47 #
    48 # sub ACTION_install {
    49 #     my $self = shift;
    50 #
    51 #     $self->SUPER::ACTION_install(@_);
    52 #
    53 #     my $old_pwd = getcwd();
    54 #     chdir $pkg_dir;
    55 #
    56 #     system("make install") == 0 or die "install failed: $?";
    57 #
    58 #     chdir $old_pwd;
    59 # }
    60 
    61 sub ACTION_clean {
    62     my $self = shift;
    63 
    64     $self->SUPER::ACTION_clean(@_);
    65 
    66     my $old_pwd = getcwd();
    67     chdir $pkg_dir;
    68 
    69     system("make clean") == 0 or die "install failed: $?";
    70 
    71     chdir $old_pwd;
    72 }
    73 EOF
    74 
    75 $class->new(
     4Module::Build->new(
    765    module_name         => 'Nebulous',
    776    dist_version_from   => 'lib/Nebulous/Server.pm',
    … …  
    809    create_makefile_pl  => 'passthrough',
    8110    requires            => {
     11        'Apache2::Const'        => 0,
    8212        'Class::Accessor::Fast' => 0,
    8313        'Config::YAML'          => '1.42',
    … …  
    8919        'File::Spec'            => 0,
    9020        'File::Spec::Functions' => 0,
     21        'File::Temp'            => 0,
    9122        'Log::Log4perl'         => '0.48',
    9223        'Net::Server::Daemonize'=> '0.05',
    … …  
    9728    },
    9829    build_requires      => {
    99         'Apache2::SOAP'         => 0,
    100         'Apache::DBI'           => '1.05',
    101         'Apache::Test'          => '1.27',  # bundles Apache::TestMB
    102         'Apache::TestMB'        => 0,
    103         'File::Temp'            => 0,
    10430        'Test::More'            => '0.49',
    10531        'Test::URI'             => '1.06',
    … …  
    10733    recommends          => {
    10834        'Test::Distribution'    => '1.22',
     35        'Apache2::SOAP'         => 0,
     36        'Apache::DBI'           => '1.05',
    10937    },
    11038    script_files        => [qw(
    11139        bin/neb-addvol
    112         bin/neb-cat
    113         bin/neb-cp
    114         bin/neb-cull
    115         bin/neb-df
    11640        bin/neb-fsck
    11741        bin/neb-initdb
    118         bin/neb-locate
    119         bin/neb-ls
    120         bin/neb-mv
    121         bin/neb-replicate
    122         bin/neb-rm
    123         bin/neb-stat
    124         bin/neb-touch
    12542        bin/nebdiskd
    12643    )],

  • trunk/Nebulous-Server/MANIFEST

    r16264 r17072  
    77README
    88Todo
    9 autogen.sh
    109bin/neb-addvol
    11 bin/neb-cat
    12 bin/neb-cp
    13 bin/neb-cull
    14 bin/neb-df
    1510bin/neb-fsck
    1611bin/neb-initdb
    17 bin/neb-locate
    18 bin/neb-ls
    19 bin/neb-mv
    20 bin/neb-replicate
    21 bin/neb-rm
    22 bin/neb-stat
    23 bin/neb-touch
    2412bin/nebdiskd
    2513docs/c_api.h
    … …  
    3018docs/tmp.txt
    3119examples/Makefile
     20examples/uri_test.pl
    3221examples/nebexample.c
    3322examples/nebexample.pl
    34 lib/Nebulous/Client.pm
    35 lib/Nebulous/Client.pod
    36 lib/Nebulous/Client/HTTP.pm
    37 lib/Nebulous/Client/Log.pm
    38 lib/Nebulous/Client/QuickStart.pod
    39 lib/Nebulous/Keys.pod
     23lib/Test/Nebulous.pm
     24lib/Nebulous/Keys.pm
    4025lib/Nebulous/Server.pm
    4126lib/Nebulous/Server.pod
    … …  
    4530lib/Nebulous/Server/SOAP.pm
    4631lib/Nebulous/Server/SQL.pm
    47 lib/Nebulous/Util.pm
    48 nebclient/Doxyfile.in
    49 nebclient/Makefile.am
    50 nebclient/Makefile.in
    51 nebclient/aclocal.m4
    52 nebclient/autogen.sh
    53 nebclient/compile
    54 nebclient/config.guess
    55 nebclient/config.h.in
    56 nebclient/config.sub
    57 nebclient/configure
    58 nebclient/configure.ac
    59 nebclient/depcomp
    60 nebclient/install-sh
    61 nebclient/ltmain.sh
    62 nebclient/missing
    63 nebclient/nebclient.pc.in
    64 nebclient/nebulous.wsdl
    65 nebclient/src/Makefile.am
    66 nebclient/src/Makefile.in
    67 nebclient/src/SOAP.nsmap
    68 nebclient/src/nebclient.c
    69 nebclient/src/nebclient.h
    70 nebclient/src/soapC.c
    71 nebclient/src/soapClient.c
    72 nebclient/src/soapH.h
    73 nebclient/src/soapStub.h
    74 nebclient/src/stdsoap2.c
    75 nebclient/src/stdsoap2.h
    76 nebclient/src/xmalloc.c
    77 nebclient/src/xmalloc.h
    78 nebclient/tests/tap/INSTALL
    79 nebclient/tests/tap/Makefile.am
    80 nebclient/tests/tap/Makefile.in
    81 nebclient/tests/tap/README
    82 nebclient/tests/tap/aclocal.m4
    83 nebclient/tests/tap/compile
    84 nebclient/tests/tap/configure
    85 nebclient/tests/tap/configure.in
    86 nebclient/tests/tap/src/
    87 nebclient/tests/tap/src/Makefile.am
    88 nebclient/tests/tap/src/Makefile.in
    89 nebclient/tests/tap/src/config.h.in
    90 nebclient/tests/tap/src/tap.c
    91 nebclient/tests/tap/src/tap.h
    92 nebclient/tests/Makefile.am
    93 nebclient/tests/Makefile.in
    94 nebclient/tests/tests.c
    9532scripts/bench_test.pl
    9633scripts/nebulous.cgi
    … …  
    11148t/13_server_rename_object.t
    11249t/14_server_xattr.t
    113 t/50_client_new.t
    114 t/51_client_create.t
    115 t/51_client_open_create.t
    116 t/52_client_replicate.t
    117 t/53_client_cull.t
    118 t/54_client_lock.t
    119 t/55_client_unlock.t
    120 t/56_client_find_instances.t
    121 t/57_client_find.t
    122 t/58_client_open.t
    123 t/59_client_delete.t
    124 t/60_client_copy.t
    125 t/61_client_move.t
    126 t/62_client_delete_instance.t
    127 t/63_client_stat.t
    128 t/64_client_find_objects.t
    129 t/75_parse_neb_key.t
    130 t/90_nebclient.t
    131 t/TEST.PL
    132 t/Test/Nebulous.pm
    133 t/conf/extra.conf.in
    134 t/conf/startup.pl.in

  • trunk/Nebulous-Server/bin/neb-fsck

    r16264 r17072  
    33# Copyright (C) 2005-2008  Joshua Hoblitt
    44#
    5 # $Id: neb-fsck,v 1.1 2008-01-30 00:19:34 jhoblitt Exp $
     5# $Id: neb-fsck,v 1.2 2008-03-20 21:12:06 jhoblitt Exp $
    66
    77use strict;
    … …  
    5959my $sql = Nebulous::Server::SQL->new();
    6060
     61print "Pass 1: Look for objects with inaccessable instances\n";
     62
    6163# so_id, ext_id, instances, available_instances, need_recovery, recoverable
    6264my $query = $dbh->prepare( $sql->find_objects_with_unavailable_instances );
    … …  
    7072
    7173foreach my $obj (@rows) {
     74#    use Data::Dumper;
     75#    print Dumper($obj);
    7276    my $so_id = $obj->{so_id};
    7377    my $key = $obj->{ext_id};
    7478    my $has = $obj->{instances};
    75     my $available = $obj->{instances_available};
    76     my $need = $obj->{need_recovery};
     79    my $available = $obj->{available_instances} || 0;
     80    my $need_recovery = $obj->{need_recovery};
    7781    my $recoverable = $obj->{recoverable};
     82    # if the copies xattr is unset and the object has 2 or more instances, we
     83    # will assume a target of 2 copies
     84    my $copies = $obj->{copies} || 2;
    7885
    7986    # objects with only a single instance that are offline are unrecoverable so
    8087    # we don't need to handle that special case
    81     next unless $need;
     88    next unless $need_recovery;
    8289    unless ($recoverable) {
    8390        warn "so_id: $so_id key: \'$key\' ",
    … …  
    8693    }
    8794
    88     $available ||= 0;
    89     next unless $available < 2;
    90 
    91     warn "so_id: $so_id key: \'$key\' ",
    92             " has only 1 instance avaiable -- replicating";
    93 
    94     $neb->replicate($obj->{ext_id})
    95         or warn "so_id: $so_id key: \'$key\' failed to replicate";
     95    my $need = $copies - $available;
     96    $need = 0 if $need < 0;
     97
     98    next unless $need;
     99
     100    for (; $need > 0; $need--) {
     101        warn "so_id: $so_id key: \'$key\' ",
     102            "has $available available instances, target $copies -- replicating";
     103        $neb->replicate($obj->{ext_id})
     104            or warn "so_id: $so_id key: \'$key\' replication failed";
     105
     106        $available++;
     107    }
    96108}
     109
     110print "Pass 2: Check volumes for files without an instance\n";
     111
     112# get a list of volumes and their paths
     113# get a list of all instances on a volume
     114# get a list of all files under the volumes path
     115# compare the two lists
    97116
    98117__END__

  • trunk/Nebulous-Server/lib/Nebulous/Server.pm

    r16566 r17072  
    11# Copyright (c) 2004  Joshua Hoblitt
    22#
    3 # $Id: Server.pm,v 1.54 2008-02-21 23:21:09 jhoblitt Exp $
     3# $Id: Server.pm,v 1.55 2008-03-20 21:10:57 jhoblitt Exp $
    44
    55package Nebulous::Server;
    … …  
    2121use Nebulous::Server::Log;
    2222use Nebulous::Server::SQL;
    23 use Nebulous::Util qw( :standard );
     23use Nebulous::Keys qw( parse_neb_key );
    2424use Params::Validate qw( validate_pos SCALAR SCALARREF );
    2525use URI::file;

  • trunk/Nebulous-Server/lib/Nebulous/Server/Config.pm

    r10628 r17072  
    11# Copyright (C) 2005  Joshua Hoblitt
    22#
    3 # $Id: Config.pm,v 1.2 2006-12-12 00:06:27 jhoblitt Exp $
     3# $Id: Config.pm,v 1.3 2008-03-20 21:10:57 jhoblitt Exp $
    44
    55package Nebulous::Server::Config;
    … …  
    1212use base qw( Class::Accessor::Fast );
    1313
    14 use Nebulous::Util qw( %LEVELS );
     14use Log::Log4perl qw( :levels );
    1515use Params::Validate qw( validate SCALAR );
     16
     17our %LEVELS = (
     18    off     => $OFF,
     19    fatal   => $FATAL,
     20    error   => $ERROR,
     21    warn    => $WARN,
     22    info    => $INFO,
     23    debug   => $DEBUG,
     24    all     => $ALL,
     25);
    1626
    1727my $new_validate = {

  • trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm

    r16282 r17072  
    11# Copyright (c) 2004  Joshua Hoblitt
    22#
    3 # $Id: SQL.pm,v 1.48 2008-02-02 02:10:52 jhoblitt Exp $
     3# $Id: SQL.pm,v 1.49 2008-03-20 21:10:57 jhoblitt Exp $
    44
    55package Nebulous::Server::SQL;
    … …  
    246246    find_objects_with_unavailable_instances => qq{
    247247        SELECT
    248             so_id,
     248            storage_object.so_id,
    249249            ext_id,
    250250            count(ins_id) as instances,
    251251            count(mountedvol.vol_id) as available_instances,
    252252            count(mountedvol.vol_id) < count(ins_id) as need_recovery,
    253             count(mountedvol.vol_id) > 0 as recoverable
     253            count(mountedvol.vol_id) > 0 as recoverable,
     254            storage_object_xattr.value as copies
    254255        FROM storage_object
    255256        JOIN storage_object_attr
    256257            USING(so_id)
    257         JOIN storage_object_xattr
     258        JOIN instance
     259            USING(so_id)
     260        LEFT JOIN storage_object_xattr
    258261            ON storage_object.so_id = storage_object_xattr.so_id
    259262            AND storage_object_xattr.name = 'copies'
    260         JOIN instance
    261             USING(so_id)
    262263        LEFT JOIN mountedvol
    263264            USING(vol_id)

  • trunk/Nebulous-Server/t/01_load.t

    r4440 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 01_load.t,v 1.7 2005-06-30 02:35:06 jhoblitt Exp $
     5# $Id: 01_load.t,v 1.8 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1010use lib qw( ./t ./lib );
    1111
    12 use Test::More tests => 8;
     12use Test::More tests => 4;
    1313
    14 BEGIN { use_ok( 'Nebulous::Client' ); }
    15 BEGIN { use_ok( 'Nebulous::Client::Log' ); }
    16 BEGIN { use_ok( 'Nebulous::Client::HTTP' ); }
    17 BEGIN { use_ok( 'Nebulous::Util' ); }
    1814BEGIN { use_ok( 'Nebulous::Server' ); }
    1915BEGIN { use_ok( 'Nebulous::Server::SQL' ); }

  • trunk/Nebulous-Server/t/03_server_create_object.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 03_server_create_object.t,v 1.24 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 03_server_create_object.t,v 1.25 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1414use File::ExtAttr qw( getfattr );
    1515use Nebulous::Server;
    16 use Nebulous::Util qw( :standard );
    1716use Test::Nebulous;
    1817use Test::URI;

  • trunk/Nebulous-Server/t/04_server_replicate_object.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 04_server_replicate_object.t,v 1.11 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 04_server_replicate_object.t,v 1.12 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1414use File::ExtAttr qw( getfattr );
    1515use Nebulous::Server;
    16 use Nebulous::Util qw( :standard );
    1716use Test::Nebulous;
    1817use Test::URI;

  • trunk/Nebulous-Server/t/05_server_lock_object.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 05_server_lock_object.t,v 1.9 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 05_server_lock_object.t,v 1.10 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::Nebulous;
    1716

  • trunk/Nebulous-Server/t/06_server_unlock_object.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 06_server_unlock_object.t,v 1.9 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 06_server_unlock_object.t,v 1.10 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::Nebulous;
    1716

  • trunk/Nebulous-Server/t/07_server_find_instances.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 07_server_find_instances.t,v 1.9 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 07_server_find_instances.t,v 1.10 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::URI;
    1716use Test::Nebulous;

  • trunk/Nebulous-Server/t/08_server_delete_instance.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 08_server_delete_instance.t,v 1.9 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 08_server_delete_instance.t,v 1.10 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::Nebulous;
    1716

  • trunk/Nebulous-Server/t/09_server_stat_object.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 09_server_stat_object.t,v 1.12 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 09_server_stat_object.t,v 1.13 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::Nebulous;
    1716

  • trunk/Nebulous-Server/t/10_server_is_valid_volume_name.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 10_server_is_valid_volume_name.t,v 1.4 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 10_server_is_valid_volume_name.t,v 1.5 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::Nebulous;
    1716

  • trunk/Nebulous-Server/t/11_server_is_valid_object_key.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 11_server_is_valid_object_key.t,v 1.2 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 11_server_is_valid_object_key.t,v 1.3 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::Nebulous;
    1716

  • trunk/Nebulous-Server/t/12_server_find_objects.t

    r16281 r17072  
    33# Copryight (C) 2004-2005  Joshua Hoblitt
    44#
    5 # $Id: 12_server_find_objects.t,v 1.2 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 12_server_find_objects.t,v 1.3 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::Nebulous;
    1716

  • trunk/Nebulous-Server/t/13_server_rename_object.t

    r16281 r17072  
    33# Copryight (C) 2007  Joshua Hoblitt
    44#
    5 # $Id: 13_server_rename_object.t,v 1.3 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 13_server_rename_object.t,v 1.4 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::Nebulous;
    1716

  • trunk/Nebulous-Server/t/14_server_xattr.t

    r16281 r17072  
    33# Copryight (C) 2007  Joshua Hoblitt
    44#
    5 # $Id: 14_server_xattr.t,v 1.3 2008-02-02 01:51:29 jhoblitt Exp $
     5# $Id: 14_server_xattr.t,v 1.4 2008-03-20 21:10:14 jhoblitt Exp $
    66
    77use strict;
    … …  
    1313
    1414use Nebulous::Server;
    15 use Nebulous::Util qw( :standard );
    1615use Test::Nebulous;
    1716
Note: See TracChangeset for help on using the changeset viewer.