Changeset 13180

Timestamp:

May 2, 2007, 5:21:28 PM (19 years ago)

Author:

jhoblitt

Message:

overhaul Nebulous::Client->cull()
add the number of instances to ->stat_object()

Location:

trunk

Files:

• Nebulous-Server/lib/Nebulous/Server.pm (modified) (2 diffs)
• Nebulous-Server/lib/Nebulous/Server/SQL.pm (modified) (3 diffs)
• Nebulous-Server/t/09_server_stat_object.t (modified) (6 diffs)
• Nebulous/lib/Nebulous/Client.pm (modified) (4 diffs)
• Nebulous/lib/Nebulous/Server.pm (modified) (2 diffs)
• Nebulous/lib/Nebulous/Server/SQL.pm (modified) (3 diffs)
• Nebulous/t/09_server_stat_object.t (modified) (6 diffs)
• Nebulous/t/53_client_cull.t (modified) (5 diffs)

trunk/Nebulous-Server/lib/Nebulous/Server.pm

@@ -1 +1 @@
 # Copyright (c) 2004  Joshua Hoblitt
 #
-# $Id: Server.pm,v 1.32 2007-05-03 01:44:23 jhoblitt Exp $
+# $Id: Server.pm,v 1.33 2007-05-03 03:21:28 jhoblitt Exp $
 
 package Nebulous::Server;
@@ -1021 +1021 @@
     my ( $key ) = validate_pos( @_,
         {
-            type => SCALAR,
-        },
-    );
-
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug( "entered - @_" );
+            type        => SCALAR,
+            callbacks   => {
+                'is valid object key' => sub { $self->_is_valid_object_key($_[0]) },
+            },
+        },
+    );
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  =$self->db;
+
+    $log->debug("entered - @_");
 
     my $stat;
     eval {
-        my $query = $db->prepare_cached( $sql->get_object );
-        my $rows = $query->execute( $key );
-
-        unless ( $rows == 1 ) {
-            $log->logdie( "no storage object found" );
+        my $query = $db->prepare_cached( $sql->stat_object );
+        my $rows = $query->execute($key);
+
+        unless ($rows == 1) {
+            $log->logdie("no storage object found");
         }
 
         $stat = $query->fetchrow_arrayref;
         $query->finish;
-
-        $db->commit;
     };
     if ( $@ ) {
         $db->rollback;
-        $log->logdie( "database error: $@" );
-    }
-
-    $log->debug( "leaving" );
+        $log->logdie("database error: $@");
+    }
+
+    $log->debug("leaving");
 
     return $stat;

trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm

@@ -1 +1 @@
 # Copyright (c) 2004  Joshua Hoblitt
 #
-# $Id: SQL.pm,v 1.35 2007-05-03 01:49:55 jhoblitt Exp $
+# $Id: SQL.pm,v 1.36 2007-05-03 03:21:28 jhoblitt Exp $
 
 package Nebulous::Server::SQL;
@@ -53 +53 @@
     get_object          => qq{
         SELECT
-            storage_object.so_id,
+            so_id,
             ext_id,
             read_lock,
@@ -63 +63 @@
         USING (so_id)
         WHERE ext_id = ?
+    },
+    stat_object          => qq{
+        SELECT
+            storage_object.so_id,
+            storage_object.ext_id,
+            read_lock,
+            write_lock,
+            storage_object_attr.epoch,
+            storage_object_attr.mtime,
+            COUNT(instance.so_id) as instances
+        FROM storage_object
+        JOIN storage_object_attr
+            USING (so_id)
+        JOIN instance
+            USING (so_id)
+        WHERE ext_id = ?
+        GROUP BY storage_object.so_id
     },
     # Note: this sets an update lock

trunk/Nebulous-Server/t/09_server_stat_object.t

@@ -3 +3 @@
 # Copryight (C) 2004-2005  Joshua Hoblitt
 #
-# $Id: 09_server_stat_object.t,v 1.10 2007-05-01 02:52:04 jhoblitt Exp $
+# $Id: 09_server_stat_object.t,v 1.11 2007-05-03 03:21:28 jhoblitt Exp $
 
 use strict;
 use warnings FATAL => qw( all );
 
-use Test::More tests => 11;
+use Test::More tests => 12;
 
 use lib qw( ./t ./lib );
@@ -23 +23 @@
 
 Test::Nebulous->setup;
-
 {
-    my $uri = $neb->create_object("foo");
+    $neb->create_object("foo");
 
     my $info = $neb->stat_object("foo");
 
-    is(scalar @$info, 6, "number of columns");
+    is(scalar @$info, 7, "number of columns");
 }
 
@@ -35 +34 @@
 
 {
-    my $uri = $neb->create_object("foo", "node01");
+    $neb->create_object("foo", "node01");
 
     my $info = $neb->stat_object("foo");
 
-    is(scalar @$info, 6,                       "number of columns");
+    is(scalar @$info, 7,                       "number of columns");
     is(@$info[0], 1,                           "so_id");
     is(@$info[1], "foo",                       "ext_id");
@@ -46 +45 @@
     like(@$info[4], qr/....-..-.. ..:..:../,   "epoch");
     like(@$info[5], qr/....-..-.. ..:..:../,   "mtime");
+    is(@$info[6], 1,                           "instances");
 }
 
@@ -53 +53 @@
     $neb->stat_object("foo");
 };
-like($@, qr/no storage object found/, "object does not exist");
+like($@, qr/is valid object key/, "object does not exist");
 
 Test::Nebulous->setup;
@@ -65 +65 @@
 
 eval {
+    $neb->create_object("foo");
     $neb->stat_object("foo", 2);
 };

trunk/Nebulous/lib/Nebulous/Client.pm

@@ -1 +1 @@
 # Copyright (c) 2004  Joshua Hoblitt
 #
-# $Id: Client.pm,v 1.29 2007-05-01 02:52:04 jhoblitt Exp $
+# $Id: Client.pm,v 1.30 2007-05-03 03:21:28 jhoblitt Exp $
 
 package Nebulous::Client;
@@ -227 +227 @@
     my $self = shift;
 
-    my ( $key, @params ) = validate_pos( @_,
+    my ($key, $vol_name) = validate_pos(@_,
         {
             type => SCALAR,
@@ -240 +240 @@
     $log->debug( "entered - @_" );
 
-    # need some way to determine which is the best instance to remove
-    my $locations = $self->find_instances( $key, @params );
-    unless ( $locations ) {
+    my $locations;
+    # if vol_name is specified we need to stat the file to find out how many
+    # instances there are on all volumes.  Otherwise we wouldn't know if it's
+    # safe to remove the last instance on the specified volume.
+    # XXX We need some way to determine which is the best instance to remove
+    if (defined $vol_name) {
+        my @stats = $self->stat($key);
+        my $instances = $stats[6];
+        if (defined $instances and $instances < 2) {
+            $log->debug("can not cull - not enough instances");
+            $log->debug("leaving");
+
+            return;
+        }
+
+        $locations = $self->find_instances($key, $vol_name);
+        unless (scalar @{ $locations }) {
             $log->debug( "no instances" );
             $log->debug( "leaving" );
 
-            return undef;
-    }
-
-    if ( scalar @{ $locations } < 2 ) {
-        $log->debug( "can not cull - not enough instances" );
-        $log->debug( "leaving" );
-
-        return undef;
-    }
-
-    my $uri = $self->delete_instance( $locations->[0] );
-
-    $log->debug( "leaving" );
+            return;
+        }
+    } else {
+        $locations = $self->find_instances($key);
+        unless ($locations) {
+            $log->debug("no instances");
+            $log->debug("leaving");
+
+            return;
+        }
+
+        if (scalar @{ $locations } < 2) {
+            $log->debug("can not cull - not enough instances");
+            $log->debug("leaving");
+
+            return;
+        }
+    }
+
+    my $uri = $self->delete_instance( @$locations[0] );
+
+    $log->debug("leaving");
 
     return $uri;
@@ -638 +661 @@
     my $self = shift;
 
-    my ( $uri ) = validate_pos( @_,
+    my ($uri) = validate_pos(@_,
         {
             type => SCALAR,

trunk/Nebulous/lib/Nebulous/Server.pm

@@ -1 +1 @@
 # Copyright (c) 2004  Joshua Hoblitt
 #
-# $Id: Server.pm,v 1.32 2007-05-03 01:44:23 jhoblitt Exp $
+# $Id: Server.pm,v 1.33 2007-05-03 03:21:28 jhoblitt Exp $
 
 package Nebulous::Server;
@@ -1021 +1021 @@
     my ( $key ) = validate_pos( @_,
         {
-            type => SCALAR,
-        },
-    );
-
-    my $log = $self->log;
-    my $sql = $self->sql;
-    my $db  =$self->db;
-
-    $log->debug( "entered - @_" );
+            type        => SCALAR,
+            callbacks   => {
+                'is valid object key' => sub { $self->_is_valid_object_key($_[0]) },
+            },
+        },
+    );
+
+    my $log = $self->log;
+    my $sql = $self->sql;
+    my $db  =$self->db;
+
+    $log->debug("entered - @_");
 
     my $stat;
     eval {
-        my $query = $db->prepare_cached( $sql->get_object );
-        my $rows = $query->execute( $key );
-
-        unless ( $rows == 1 ) {
-            $log->logdie( "no storage object found" );
+        my $query = $db->prepare_cached( $sql->stat_object );
+        my $rows = $query->execute($key);
+
+        unless ($rows == 1) {
+            $log->logdie("no storage object found");
         }
 
         $stat = $query->fetchrow_arrayref;
         $query->finish;
-
-        $db->commit;
     };
     if ( $@ ) {
         $db->rollback;
-        $log->logdie( "database error: $@" );
-    }
-
-    $log->debug( "leaving" );
+        $log->logdie("database error: $@");
+    }
+
+    $log->debug("leaving");
 
     return $stat;

trunk/Nebulous/lib/Nebulous/Server/SQL.pm

@@ -1 +1 @@
 # Copyright (c) 2004  Joshua Hoblitt
 #
-# $Id: SQL.pm,v 1.35 2007-05-03 01:49:55 jhoblitt Exp $
+# $Id: SQL.pm,v 1.36 2007-05-03 03:21:28 jhoblitt Exp $
 
 package Nebulous::Server::SQL;
@@ -53 +53 @@
     get_object          => qq{
         SELECT
-            storage_object.so_id,
+            so_id,
             ext_id,
             read_lock,
@@ -63 +63 @@
         USING (so_id)
         WHERE ext_id = ?
+    },
+    stat_object          => qq{
+        SELECT
+            storage_object.so_id,
+            storage_object.ext_id,
+            read_lock,
+            write_lock,
+            storage_object_attr.epoch,
+            storage_object_attr.mtime,
+            COUNT(instance.so_id) as instances
+        FROM storage_object
+        JOIN storage_object_attr
+            USING (so_id)
+        JOIN instance
+            USING (so_id)
+        WHERE ext_id = ?
+        GROUP BY storage_object.so_id
     },
     # Note: this sets an update lock

trunk/Nebulous/t/09_server_stat_object.t

@@ -3 +3 @@
 # Copryight (C) 2004-2005  Joshua Hoblitt
 #
-# $Id: 09_server_stat_object.t,v 1.10 2007-05-01 02:52:04 jhoblitt Exp $
+# $Id: 09_server_stat_object.t,v 1.11 2007-05-03 03:21:28 jhoblitt Exp $
 
 use strict;
 use warnings FATAL => qw( all );
 
-use Test::More tests => 11;
+use Test::More tests => 12;
 
 use lib qw( ./t ./lib );
@@ -23 +23 @@
 
 Test::Nebulous->setup;
-
 {
-    my $uri = $neb->create_object("foo");
+    $neb->create_object("foo");
 
     my $info = $neb->stat_object("foo");
 
-    is(scalar @$info, 6, "number of columns");
+    is(scalar @$info, 7, "number of columns");
 }
 
@@ -35 +34 @@
 
 {
-    my $uri = $neb->create_object("foo", "node01");
+    $neb->create_object("foo", "node01");
 
     my $info = $neb->stat_object("foo");
 
-    is(scalar @$info, 6,                       "number of columns");
+    is(scalar @$info, 7,                       "number of columns");
     is(@$info[0], 1,                           "so_id");
     is(@$info[1], "foo",                       "ext_id");
@@ -46 +45 @@
     like(@$info[4], qr/....-..-.. ..:..:../,   "epoch");
     like(@$info[5], qr/....-..-.. ..:..:../,   "mtime");
+    is(@$info[6], 1,                           "instances");
 }
 
@@ -53 +53 @@
     $neb->stat_object("foo");
 };
-like($@, qr/no storage object found/, "object does not exist");
+like($@, qr/is valid object key/, "object does not exist");
 
 Test::Nebulous->setup;
@@ -65 +65 @@
 
 eval {
+    $neb->create_object("foo");
     $neb->stat_object("foo", 2);
 };

trunk/Nebulous/t/53_client_cull.t

@@ -3 +3 @@
 # Copryight (C) 2004-2005  Joshua Hoblitt
 #
-# $Id: 53_client_cull.t,v 1.4 2007-05-03 01:53:20 jhoblitt Exp $
+# $Id: 53_client_cull.t,v 1.5 2007-05-03 03:21:28 jhoblitt Exp $
 
 use strict;
@@ -14 +14 @@
 use lib qw( ./t ./lib );
 
-use Nebulous::Client trace => 'debug';
+use Nebulous::Client;
 use Nebulous::Util qw( :standard );
 use Test::Nebulous;
@@ -27 +27 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->create( "foo" );
-    $neb->replicate( "foo" );
+    $neb->create("foo");
+    $neb->replicate("foo");
 
-    my $uri = $neb->cull( "foo" );
+    my $uri = $neb->cull("foo");
 
-    ok( $uri, "good cull" );
-    ok( ! -e _get_file_path( $uri ), "file doesn't exist" );
+    ok($uri, "good cull");
+    ok(! -e _get_file_path($uri), "file doesn't exist");
 }
 
@@ -39 +39 @@
 
 {
-
     # key, $volume
     my $neb = Nebulous::Client->new(
         proxy => "http://$hostport/nebulous",
     );
-    $neb->create( "foo" );
-    $neb->replicate( "foo", "node01" );
+    $neb->create("foo");
+    $neb->replicate("foo", "node01");
 
-    my $uri = $neb->cull( "foo", "node01" );
+    my $uri = $neb->cull("foo", "node01");
 
-    ok( $uri, "good cull" );
-    ok( ! -e _get_file_path( $uri ), "file exists" );
+    ok($uri, "good cull");
+    ok(! -e _get_file_path($uri), "file exists");
 }
 
@@ -60 +59 @@
         proxy => "http://$hostport/nebulous",
     );
-    $neb->create( "foo" );
+    $neb->create("foo");
 
-    $neb->replicate( "foo", "node01" );
-    $neb->replicate( "foo", "node02" );
+    $neb->replicate("foo", "node01");
+    $neb->replicate("foo", "node02");
 
-    my $uri1 = $neb->cull( "foo" );
-    my $uri2 = $neb->cull( "foo" );
+    my $uri1 = $neb->cull("foo");
+    my $uri2 = $neb->cull("foo");
 
-    ok( $uri1, "good cull" );
-    ok( $uri2, "good cull" );
-    ok( ! -e _get_file_path( $uri1 ), "file exists" );
-    ok( ! -e _get_file_path( $uri2 ), "file exists" );
+    ok($uri1, "good cull");
+    ok($uri2, "good cull");
+    ok(! -e _get_file_path($uri1), "file exists");
+    ok(! -e _get_file_path($uri2), "file exists");
 }