Changeset 26292

Timestamp:

Nov 30, 2009, 1:54:49 PM (16 years ago)

Author:

watersc1

Message:

Nebulous: Implemented a number of bug fixes and modifications to

ensure no loss of data.

• Each volume can now be associated into a cabinet, defining their physical arrangement.
• File creation: new files without a specified volume are created on a random volume from a subset of the N least full.
• Replication: file replication uses the same volume selection method, with the constraints that a copy not be automatically placed on the same volume as another, and that the first copy not be housed within the same volume.
• Cull: instead of removing the first available instance, first attempt to remove an instance on the same volume as another, then attempt to remove an instance in the same cabinet as another, and then remove the first only if those fail.
• Corrected bug that prevented the deletion of an object if it had more than one unavailable instance.
• Fixed nebdiskd to synchronize between volume->mountedvol tables and to properly remove volumes that were no longer accessible.

Location:

trunk

Files:

• Nebulous-Server/Build.PL (modified) (1 diff)
• Nebulous-Server/MANIFEST (modified) (1 diff)
• Nebulous-Server/bin/neb-cabadd (added)
• Nebulous-Server/bin/neb-voladm (modified) (9 diffs)
• Nebulous-Server/bin/nebdiskd (modified) (4 diffs)
• Nebulous-Server/lib/Nebulous/Server.pm (modified) (10 diffs)
• Nebulous-Server/lib/Nebulous/Server/SQL.pm (modified) (10 diffs)
• Nebulous/lib/Nebulous/Client.pm (modified) (3 diffs)

trunk/Nebulous-Server/Build.PL

@@ -46 +46 @@
         bin/neb-fsck
         bin/neb-initdb
+        bin/neb-cabadd
         bin/neb-voladd
         bin/neb-voladm

trunk/Nebulous-Server/MANIFEST

@@ -8 +8 @@
 bin/neb-initdb
 bin/neb-voladd
+bin/neb-cabadd
+bin/neb-voladm
 bin/nebdiskd
 docs/database_setup.txt

trunk/Nebulous-Server/bin/neb-voladm

@@ -23 +23 @@
     $allocate,
     $available,
+    $mounted,
     $db,
     $dbhost,
@@ -31 +32 @@
     $vname,
     $xattr,
+    $cab_id,
 );
 
@@ -41 +43 @@
     'allocate=i'        => \$allocate,
     'available=i'       => \$available,
+    'mounted'           => \$mounted,
     'db|d=s'            => \$db,
     'debug'             => \$debug,
@@ -48 +51 @@
     'vhost=s'           => \$vhost,
     'vname|n=s'         => \$vname,
+    'cab_id|c=i'        => \$cab_id,
     'xattr=i'           => \$xattr,
 ) || pod2usage( 2 );
@@ -56 +60 @@
 
 # are we listing the volumes or updating?
-if (defined $allocate or defined $available or defined $xattr) {
-    pod2usage( -msg => "Options: --allocate, --available, and --xattr Require options --vhost or --vname", -exitval => 2 )
+if (defined $allocate or defined $available or defined $xattr or defined $cab_id) {
+    pod2usage( -msg => "Options: --allocate, --available, --xattr, and --cab_id Require options --vhost or --vname", -exitval => 2 )
         unless defined $vhost or defined $vname;
     pod2usage( -msg => "Options: --allocate, --available, and --xattr Must be 0 or 1", -exitval => 2)
@@ -84 +88 @@
 $set{'v.available'} = $available if defined $available;
 $set{'v.xattr'} = $xattr if defined $xattr;
+$set{'v.cab_id'} = $cab_id if defined $cab_id;
 
 if (%set) {
@@ -97 +102 @@
         die $@;
     }
+    if ($mounted) {
+        eval {
+            my ($q, @bind) = sql_interp("UPDATE mountedvol AS v SET", \%set, "WHERE", \%constraint);
+            warn "$q\n" if $debug;
+            my $query = $dbh->prepare($q);
+            $query->execute(@bind);
+            $dbh->commit;
+        };
+        if ($@) {
+            $dbh->rollback;
+            die $@;
+        }
+    }
 } 
 
@@ -132 +150 @@
     neb-voladm [--vname <volume name>] [--vhost <volume host>]
     [--allocate <0|1>] [--available <0|1>] [--xattr <0|1>]
+    [--cab_id <cab_id>]
     [--host <database host> ] [--db <database name>]
     [--user <database username>] [--pass <database password>]
@@ -164 +183 @@
 
 Enables/Disables these flags on the storage volume.
+
+Requires either the C<--vname> or c<--vhost> options.
+
+=item * --cab_id <cab_id>
+
+Sets the cabinet id for this storage volume.
 
 Requires either the C<--vname> or c<--vhost> options.

trunk/Nebulous-Server/bin/nebdiskd

@@ -177 +177 @@
 
     eval {
-        my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT *, ?, ? FROM volume WHERE mountpoint = ?");
+        my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, ?, ? FROM volume WHERE mountpoint = ?");
+#       my $d_query = $dbh->prepare_cached("UPDATE mountedvol SET allocate = ?, available = ? WHERE mountpoint = ?");
         my $d_query = $dbh->prepare_cached("DELETE FROM mountedvol WHERE mountpoint = ?");
 
@@ -185 +186 @@
             # there may be multiple vol_ids per mountpoint but we only need to
             # check each mointpont once
-            my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume WHERE available = 1");
+            my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume");
             $query->execute;
             while (my $row = $query->fetchrow_hashref) {
@@ -200 +201 @@
             # automounter and it fails to mount
             my $tries = 0;
+            my $valid_mountpoint = 1;
             TEST: eval {
                 $tries++;
                 unless (is_mountpoint($mountpoint)) {
-                    $log->warn("$mnt is not a valid mountpoint");
+                    $log->warn("$mountpoint is not a valid mountpoint ($tries)");
+                    $valid_mountpoint = 0;
                 }
             };
-            if ($@) {
+            if (!$valid_mountpoint) {
                 # try is_mountpoint() again if $retry > 1
                 if ($tries < $retry) {
@@ -212 +215 @@
                     goto TEST; 
                 }
-                $log->warn($@);
+                $log->warn("Removing $mountpoint from the mountedvol table ($tries > $retry)");
+ #               $log->warn($@);
+
                 $d_query->execute($mountpoint);
                 next;

trunk/Nebulous-Server/lib/Nebulous/Server.pm

@@ -35 +35 @@
 use constant NFS_RETRY_WAIT     => 1;
 use constant TRANS_RETRY_WAIT   => 1;
+
+# This is the umask hack.
+umask(0002);
+
+# This determines how many entries from the list of volumes sorted by free space are randomized.
+my $topfew_count = 3;
 
 # transaction restart/retry regex
@@ -905 +911 @@
             }
 
-            # In MySQL you can't select from a table your deleting rows from so
+            # In MySQL you can't select from a table you're deleting rows from so
             # we first have to get a list of instances to be removed, and then
-            # removed them.
+            # remove them.
             my $rows_found;
             {
                 my $query = $db->prepare_cached( $sql->find_dead_instances_by_so_id );
                 $rows_found = $query->execute( $so_id );
-                foreach my $row ($query->fetchrow_hashref) {
+                my $i = 0;
+                while (my $row = $query->fetchrow_hashref) {
                     # remove dead instances
+#                   $log->warn("copied: $rows_copied found: $rows_found removed: $rows_removed");
                     my $query = $db->prepare_cached( $sql->delete_instance_by_ins_id);
                     $rows_removed += $query->execute( $row->{ins_id} );
@@ -919 +927 @@
                 $query->finish;
             }
-
+#           $log->warn("copied: $rows_copied found: $rows_found removed: $rows_removed");
             # sanity check
             die("instances inaccessible ($rows_copied) != instances removed ($rows_removed)")
@@ -1692 +1700 @@
 }
 
+sub find_instances_for_cull
+{
+    my $self = shift;
+
+    my $log = $self->log;
+    $log->debug("entered - @_");
+
+    my ($key, $vol_name) = validate_pos(@_,
+        {
+            type        => SCALAR,
+            callbacks   => {
+                'is valid object key' => sub { $self->_is_valid_object_key($_[0]) },
+            },
+        },
+        {
+            type        => SCALAR|UNDEF,
+#            callbacks   => {
+#                # check that the volume name requested is valid
+#                'is valid volume name' => sub {
+#                    return 1 if not defined $_[0];
+#                    $self->_is_valid_volume_name($_[0])
+#                },
+#            },
+            optional    => 1,
+        },
+    );
+
+    my $sql = $self->sql;
+
+#    unless ($key) {
+#        $log->warn("key was undefined after validate_pos(), trying again...");
+#        return $self->find_instances(@_);
+#    }
+
+    # vol_name overrides the key implied volume
+    eval {
+        $key = parse_neb_key($key, $vol_name);
+    };
+    $log->logdie("$@") if $@;
+    $vol_name = $key->volume;
+
+    my $db  = $self->db($key);
+
+    # the key's volume can't be validiated on input for this method so we have
+    # to check it after parsing the key
+    if (defined $vol_name
+        and not $self->_is_valid_volume_name($key, $key->volume)) {
+        if ($key->hard_volume) {
+            $log->logdie("$vol_name is not a valid volume name");
+        } else {
+            $log->warn( "$vol_name is not a known volume name" );
+            $vol_name = undef;
+        }
+    }
+
+    my @locations;
+    eval {
+        my $query;
+        if ($vol_name) {
+            $query = $db->prepare_cached( $sql->get_object_instances_by_vol_name );
+            # ext_id, name, available
+            my $rows = $query->execute($key->path, $vol_name, 1);
+            unless ($rows > 0) {
+                $query->finish;
+                die("no instances on storage volume or volume is not avaiable for key: $key volume: $vol_name");
+            }
+        } else {
+            $query = $db->prepare_cached( $sql->get_object_instances );
+            # ext_id, available
+            my $rows = $query->execute($key->path, 1);
+            unless ($rows > 0) {
+                $query->finish;
+                die("no instances available for key: $key");
+            }
+        }
+        
+        while (my $row = $query->fetchrow_hashref) {
+#           my $instance_hash  = { uri => $row->{ 'uri' }, 
+#                                  vol_id => $row->{ 'vol_id' }, 
+#                                  cab_id => $row->{ 'cab_id'} };
+            my $instance = $row->{ 'uri' };
+            push @locations, $row if $instance;
+        }
+    };
+    if ($@) {
+        $db->rollback;
+        # handle soft volumes
+        if (defined $vol_name and not defined $key->hard_volume) {
+            $log->debug("retrying with 'any' volume");
+            return $self->find_instances($key->path, 'any');
+        }
+        $log->logdie("database error: $@");
+    }
+
+    # XXX remove this?
+    $log->logdie("no instances found") unless (scalar @locations);
+
+    $log->debug("found: @locations");
+
+    $log->debug("leaving");
+
+    return \@locations;
+}
+
 
 sub delete_instance
@@ -1984 +2096 @@
 
     my ($key, $name, $hard_volume) = @_;
-
+    
+#    $log->warn("_g_s_v: key:>$key< name:>$name< hard_vol:>$hard_volume<");
     my $sql = $self->sql;
     my $db  = $self->db($key);
@@ -1996 +2109 @@
             # %free, name, avaiable, allocate
             $rows = $query->execute(0.95, $name, 1, 1);
-            # XXX destinguish between non-existant and unaviable
+            # XXX destinguish between non-existant and unavailable
             unless ($rows > 0) {
                 $query->finish;
@@ -2016 +2129 @@
             $query = $db->prepare_cached( $sql->get_storage_volume );
             # %free, avaiable, allocate
-            $rows = $query->execute(0.95, 1, 1);
+            $rows = $query->execute(0.95, 1, 1, $topfew_count);
             # there has to be atleast one storage volume
             unless ($rows > 0) {
@@ -2062 +2175 @@
     my $db  = $self->db($key);
 
-    my ($vol_id, $vol_host, $vol_path, $xattr);
+    my ($vol_id, $vol_host, $vol_path, $xattr, $forbidden_cabinet);
     eval {
         my $rows;
-        my $query = $db->prepare_cached( $sql->get_replication_volume_for_ext_id );
+        
+        my $query = $db->prepare_cached( $sql->get_cabinets_for_ext_id );
+        $rows = $query->execute($key->path);
+        unless ($rows > 0) {
+            $query->finish;
+            die("Requested key $key does not exist");
+        }
+        if ($rows == 1) {
+            ($forbidden_cabinet) = $query->fetchrow_array;
+            unless (defined($forbidden_cabinet)) {
+                $forbidden_cabinet = 0;
+            }
+            $query->finish;
+        }
+        else {
+            $forbidden_cabinet = 0;
+            $query->finish;
+        }
+            
+
+        $query = $db->prepare_cached( $sql->get_replication_volume_for_ext_id );
         # ext_id, %free, avaiable, allocate
-        $rows = $query->execute($key->path, 0.95, 1, 1);
+        $rows = $query->execute($key->path, 0.95, 1, 1, $forbidden_cabinet, $topfew_count);
         # XXX destinguish between non-existant and unaviable
         unless ($rows > 0) {
@@ -2175 +2308 @@
 
     # handle "any" volume
-    if ($vol_name eq 'any') {
+    if (($vol_name eq 'any')||($vol_name eq 'any.0')) {
         $log->debug( "found volume name $vol_name" );
         $log->debug( "leaving" );
@@ -2200 +2333 @@
     $log->debug( "leaving" );
 
-    return;
+    return 0;
 }
 

trunk/Nebulous-Server/lib/Nebulous/Server/SQL.pm

@@ -237 +237 @@
             storage_object.so_id,
             uri,
-            available
+            mountedvol.available,
+            vol_id,
+            cab_id
         FROM storage_object
         JOIN instance
@@ -243 +245 @@
         JOIN mountedvol
             USING(vol_id)
-        WHERE ext_id = ?
-            AND available = ?
+        JOIN volume
+            USING(vol_id)
+        WHERE ext_id = ?
+            AND mountedvol.available = ?
     },
     get_object_instances_by_vol_name => qq{
         SELECT
             storage_object.so_id,
-            uri
+            uri,
+            vol_id,
+            cab_id
         FROM storage_object
         JOIN instance
@@ -255 +261 @@
         JOIN mountedvol
             USING(vol_id)
-        WHERE ext_id = ?
-            AND name = ?
-            AND available = ?
-    },
+        JOIN volume
+            USING(vol_id)
+        WHERE ext_id = ?
+            AND mountedvol.name = ?
+            AND mountedvol.available = ?
+    },
+    # volume handler
     get_storage_volume_by_name   => qq{
         SELECT
@@ -275 +284 @@
         LIMIT 1
     },
-    get_replication_volume_for_ext_id   => qq{
+    # volume handler
+    get_cabinets_for_ext_id            => qq{
+        SELECT DISTINCT 
+            cab_id 
+        FROM instance 
+        JOIN volume ON instance.vol_id = volume.vol_id 
+        JOIN storage_object USING(so_id) 
+        WHERE ext_id = ?
+    },
+    get_replication_volume_for_ext_id    => qq{
+        SELECT * FROM (
         SELECT
             m.vol_id,
-            host,
-            path,
-            xattr,
+            m.host,
+            m.path,
+            m.xattr,
             total - used as free
         FROM mountedvol AS m
-        LEFT JOIN instance AS i
+        JOIN volume AS v USING(vol_id)
+        LEFT JOIN (
+                   SELECT
+                       instance.vol_id,
+                       so_id
+                   FROM instance
+                   JOIN volume 
+                   ON instance.vol_id = volume.vol_id
+                  ) AS i
             ON m.vol_id = i.vol_id
             AND i.so_id = (
@@ -290 +317 @@
                 WHERE ext_id = ?
             )
-        WHERE
-            i.vol_id IS NULL
-            AND used / total < ? 
-            AND available = ? 
-            AND allocate = ? 
-            ORDER BY RAND()
-            LIMIT 1
-    },
+         WHERE
+             i.vol_id IS NULL
+             AND used / total < ?
+             AND m.available = ?
+             AND m.allocate = ?
+             AND ( (v.cab_id IS NULL) ||
+                   (v.cab_id != ?) )
+         ORDER BY free DESC
+         LIMIT ?) as topfew
+         ORDER BY RAND()
+         LIMIT 1
+    },
+    # volume handler
     get_storage_volume          => qq{
+        SELECT * from (
         SELECT
             vol_id,
@@ -311 +344 @@
             AND allocate = ?
         ORDER BY free DESC
+        LIMIT ?) as topfew
+        ORDER BY RAND()
         LIMIT 1
     },
+    new_cabinet         => qq{
+        INSERT INTO cabinet (name, location, cab_id)
+        VALUES (?, ?, NULL)
+    },
     new_volume          => qq{
-        INSERT INTO volume (name, host, path, allocate, available, xattr, mountpoint)
-        VALUES (?, ?, ?, TRUE, TRUE, FALSE, ?)
+        INSERT INTO volume (name, host, path, allocate, available, xattr, mountpoint, cab_id)
+        VALUES (?, ?, ?, TRUE, TRUE, FALSE, ?, NULL)
     },
     get_volume_by_name => qq{
@@ -331 +370 @@
             v.available,
             v.xattr,
-            mountedvol.vol_id IS NOT NULL as mounted
+            mountedvol.vol_id IS NOT NULL as mounted,
+            v.cab_id
         FROM volume AS v
         LEFT JOIN mountedvol
@@ -614 +654 @@
 ###
 
+CREATE TABLE cabinet (
+    cab_id INT NOT NULL AUTO_INCREMENT,
+    name VARCHAR(255) NOT NULL,
+    location VARCHAR(255),
+    PRIMARY KEY(cab_id),
+    UNIQUE KEY(name),
+    KEY (location)
+) ENGINE=innodb DEFAULT CHARSET=latin1;
+
+###
+
 CREATE TABLE volume (
     vol_id INT NOT NULL AUTO_INCREMENT,
@@ -623 +674 @@
     xattr BOOLEAN DEFAULT FALSE,
     mountpoint VARCHAR(255) NOT NULL,
+    cab_id INT,
     PRIMARY KEY(vol_id),
     UNIQUE KEY(name),
@@ -629 +681 @@
     KEY(allocate),
     KEY(available),
+    FOREIGN KEY(cab_id) REFERENCES cabinet(cab_id),
     KEY(mountpoint(255))
 ) ENGINE=innodb DEFAULT CHARSET=latin1;

trunk/Nebulous/lib/Nebulous/Client.pm

@@ -315 +315 @@
     my $locations;
     if (defined $vol_name) {
-        $locations = $self->find_instances($key, $vol_name);
+        $locations = $self->find_instances_for_cull($key, $vol_name);
     } else {
-        $locations = $self->find_instances($key);
+        $locations = $self->find_instances_for_cull($key);
     }
 
@@ -326 +326 @@
         return;
     }
-
-    my $uri = $self->delete_instance($key, @$locations[0]);
+    
+    # Calculate which is the best instance to remove. First, see if we have two copies
+    # on one volume, if so, delete the first of those copies. 
+    my $delete_index = -1;
+
+    for (my $i = 0 ; $i <= $#{ $locations }; $i++) {
+        for (my $j = $i + 1; $j <= $#{ $locations }; $j++) {
+            if (@$locations[$i]->{vol_id} eq @$locations[$j]->{vol_id}) {
+                $delete_index = $i;
+            }
+        }
+    }
+    # If we don't have two copies on one volume, see if we have two copies on a single 
+    # cabinet. If so, delete the first of those copies.
+    if ($delete_index == -1) {
+        for (my $i = 0 ; $i <= $#{ $locations }; $i++) {
+            for (my $j = $i + 1; $j <= $#{ $locations }; $j++) {
+                if (@$locations[$i]->{cab_id} eq @$locations[$j]->{cab_id}) {
+                    $delete_index = $i;
+                }
+            }
+        }
+    }
+    # Fail-safe. We didn't have any duplicates (the instances are "well-mixed"), so 
+    # delete the first.
+    if ($delete_index == -1) {
+        $delete_index = 0;
+    }    
+    my $uri = $self->delete_instance($key, @$locations[$delete_index]->{uri});
 
     eval {
-        _nuke_file(_get_file_path(@$locations[0]));
+        _nuke_file(_get_file_path(@$locations[$delete_index]->{uri}));
     };
     if ($@) {
@@ -758 +785 @@
 }
 
+sub find_instances_for_cull
+{
+    my $self = shift;
+
+    my ( $key, @params ) = validate_pos( @_,
+        {
+            type        => SCALAR,
+        },
+        {
+            #volume
+            type        => SCALAR|UNDEF,
+            optional    => 1,
+        },
+    );
+
+    $log->debug( "entered - @_" );
+
+    my $response = $self->{ 'server' }->find_instances_for_cull( $key, @params );
+    if ( $response->fault ) {
+        $self->set_err($response->faultstring);
+        # check to see if this failure is because $key doesn't exist
+        if ($response->faultstring =~ /is valid object key/) {
+            $log->debug( "leaving" );
+            return;
+        }
+        # key is valid but no instances are on the specified volume
+        if ($response->faultstring =~ /no instances on storage volume/) {
+            $log->debug( "leaving" );
+            return;
+        }
+
+        $log->logdie("unhandled fault - ", $self->err);
+    }
+
+    my $uris = $response->result;
+
+    $log->debug( "server found: @$uris" );
+
+    $log->debug( "leaving" );
+
+    return $uris;
+}
+
 
 sub find