IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 23755


Ignore:
Timestamp:
Apr 8, 2009, 3:15:29 PM (17 years ago)
Author:
jhoblitt
Message:

create a pseduo directory structure on key creation

Location:
branches/neb_distrib_20081210/Nebulous-Server
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • branches/neb_distrib_20081210/Nebulous-Server/Build.PL

    r23708 r23755  
    3434        'Test::More'            => '0.49',
    3535        'Test::URI'             => '1.06',
     36        'Test::DBUnit'          => '0.20',
    3637    },
    3738    recommends          => {

  • branches/neb_distrib_20081210/Nebulous-Server/Changes

    r23727 r23755  
    1212    - disallow Nebulous::Server->rename_object() when it would cause the db
    1313      hash of a key to change
     14    - create a pseduo directory structure on key creation
    1415     
    15160.16

  • branches/neb_distrib_20081210/Nebulous-Server/lib/Nebulous/Server.pm

    r23727 r23755  
    1515use DBI;
    1616use Digest::SHA1 qw( sha1_hex );
    17 use File::Basename qw( dirname );
     17use File::Basename qw( basename dirname fileparse );
    1818use File::ExtAttr qw( setfattr );
    1919use File::Path;
    … …  
    210210        = $self->_get_storage_volume($key, $vol_name, $key->soft_volume);
    211211
     212    my $parent_id = $self->_resolve_dir_parent_id($key);
     213
    212214    my $uri;
    213215TRANS: while (1) {
    … …  
    216218                # create storage_object
    217219                my $query = $db->prepare_cached( $sql->new_object );
    218                 $query->execute('NULL', $key->path);
     220                $query->execute('NULL', $key->path, basename($key->path), $parent_id);
    219221            }
    220222
    … …  
    294296
    295297
     298sub _resolve_dir_parent_id
     299{
     300    my $self = shift;
     301
     302    my ($key) = validate_pos(@_,
     303        {
     304            isa => 'Nebulous::Key',
     305        },
     306    );
     307
     308    my $log = $self->log;
     309    my $sql = $self->sql;
     310    my $db  = $self->db($key);
     311
     312    # resolve parent directory
     313    my @dirs;
     314
     315    # File::Spec->splitpath was causing ->splitdir to always an extra dir
     316    # named "" because of a trailing /
     317    @dirs = File::Spec->splitdir(dirname($key->path));
     318    # dirname returns "." if there is no dir component to the path, we have
     319    # to filter this out
     320    @dirs = grep(!/^\.$/, @dirs);
     321
     322    # start at the root dir; '/' == 1
     323    my $parent_id = 1;
     324TRANS: while (1) {
     325        eval {
     326            foreach my $dir (@dirs) {
     327                my $dir_id;
     328                {
     329                    my $query = $db->prepare_cached($sql->get_directory);
     330                    $query->execute($parent_id, $dir);
     331                    if ($query->rows) {
     332                        $dir_id = $query->fetchrow_hashref->{'dir_id'};
     333                    }
     334                    $query->finish;
     335                }
     336
     337                # if we found a dir_id, a row for this directory already exists
     338                if (defined $dir_id) {
     339                    $parent_id = $dir_id;
     340                    # note that you can't exit an eval {} with next
     341                    next;
     342                }
     343
     344                {
     345                    # dir doesn't exist, create it
     346                    my $query = $db->prepare_cached($sql->new_directory);
     347                    $query->execute($dir, $parent_id);
     348                }
     349
     350                # get the dir_id of the new directory entry
     351                {
     352                    my $query = $db->prepare_cached($sql->last_insert_id);
     353                    $query->execute();
     354
     355                    # the new dir_id will be the parent_id of the next
     356                    # descendent directory
     357                    ($parent_id) = $query->fetchrow_array;
     358                    $query->finish;
     359                }
     360                $log->logdie("failed to get LAST_INSERT_ID()")
     361                    unless $parent_id;
     362
     363                $db->commit;
     364            }
     365        };
     366        if ($@) {
     367            $db->rollback;
     368            $log->debug("rollback");
     369            if ($@ =~ /Deadlock found/) {
     370                $log->warn("database deadlock retrying transaction: $@");
     371                redo TRANS;
     372            }
     373            $log->logdie("error: $@");
     374        }
     375        last;
     376    }
     377
     378    return $parent_id;
     379}
     380
     381
    296382sub rename_object
    297383{
    … …  
    314400    );
    315401
    316     # XXX this may require database migration!
    317 
    318402    # ignore volumes
    319403    $key    = parse_neb_key($key);
    … …  
    326410    $log->debug("entered - @_");
    327411
     412    # XXX this may require database migration in the future
    328413    unless ($self->_db_index_for_key($key)
    329414         == $self->_db_index_for_key($newkey)) {
    … …  
    336421            my $query = $db->prepare_cached($sql->rename_object);
    337422            # this SQL statment takes the new key name as the first param
    338             my $rows = $query->execute($newkey->path, $key->path);
     423            my $rows = $query->execute($newkey->path, basename($newkey->path), $key->path);
    339424
    340425            # if we affected more then one row something very bad has happened.
    … …  
    412497              my $query = $db->prepare_cached($sql->rename_object);
    413498              # this SQL statment takes the new key name as the first param
    414               my $rows = $query->execute($key1->path . ".swap", $key1->path);
     499              my $rows = $query->execute($key1->path . ".swap", basename($key1->path) . ".swap", $key1->path);
    415500
    416501              # if we affected more then one row something very bad has happened.
    … …  
    425510              my $query = $db->prepare_cached($sql->rename_object);
    426511              # this SQL statment takes the new key name as the first param
    427               my $rows = $query->execute($key1->path, $key2->path);
     512              my $rows = $query->execute($key1->path, basename($key1->path), $key2->path);
    428513
    429514              # if we affected more then one row something very bad has happened.
    … …  
    438523              my $query = $db->prepare_cached($sql->rename_object);
    439524              # this SQL statment takes the new key name as the first param
    440               my $rows = $query->execute($key2->path, $key1->path . ".swap");
     525              my $rows = $query->execute($key2->path, basename($key2->path), $key1->path . ".swap");
    441526
    442527              # if we affected more then one row something very bad has happened.

  • branches/neb_distrib_20081210/Nebulous-Server/lib/Nebulous/Server/SQL.pm

    r23708 r23755  
    88use warnings FATAL => qw( all );
    99
    10 our $VERSION = '0.03';
     10our $VERSION = '0.04';
    1111
    1212use base qw( Class::Accessor::Fast );
    … …  
    2626    new_object          => qq{
    2727        INSERT INTO storage_object
    28         (so_id, ext_id, type)
    29         VALUES (?, ?, 'REG_FILE')
     28        (so_id, ext_id, ext_id_basename, type, dir_id)
     29        VALUES (?, ?, ?, 'REG_FILE', ?)
    3030    },
    3131    new_object_attr  => qq{
    … …  
    6060        USING (so_id)
    6161        WHERE ext_id = ?
     62    },
     63    get_directory       => qq{
     64        SELECT
     65            dir_id
     66        FROM directory
     67        WHERE parent_id = ?
     68            AND dirname = ?
     69    },
     70    new_directory       => qq{
     71        INSERT INTO directory
     72        (dirname, parent_id)
     73        VALUES (?, ?)
    6274    },
    6375    check_object_name => qq{
    … …  
    304316    rename_object => qq{
    305317        UPDATE storage_object
    306         SET ext_id = ?
     318        SET ext_id = ?, ext_id_basename = ?
    307319        WHERE ext_id = ?
    308320    },
    … …  
    410422DROP TABLE IF EXISTS log;
    411423DROP TABLE IF EXISTS mountedvol;
     424DROP TABLE IF EXISTS directory;
    412425DROP PROCEDURE IF EXISTS getmountedvol;
    413426SET FOREIGN_KEY_CHECKS=1
    … …  
    431444
    432445__DATA__
     446CREATE TABLE directory (
     447    dir_id BIGINT NOT NULL AUTO_INCREMENT,
     448    dirname CHAR(255) NOT NULL,
     449    parent_id BIGINT NOT NULL,
     450    FOREIGN KEY(parent_id) REFERENCES directory(dir_id),
     451    PRIMARY KEY(dir_id),
     452    KEY(parent_id)
     453) ENGINE=innodb DEFAULT CHARSET=latin1;
     454
     455###
     456
     457INSERT INTO directory (dir_id, dirname, parent_id) VALUES (1, '/', 1);
     458
     459###
     460
    433461CREATE TABLE storage_object (
    434462    so_id BIGINT NOT NULL AUTO_INCREMENT,
    435463    ext_id VARCHAR(255) NOT NULL UNIQUE,
     464    ext_id_basename VARCHAR(255) NOT NULL,
     465    dir_id BIGINT NOT NULL,
     466    FOREIGN KEY(dir_id) REFERENCES directory(dir_id),
    436467    type enum('REG_FILE'),
    437468    PRIMARY KEY(so_id),

  • branches/neb_distrib_20081210/Nebulous-Server/t/03_server_create_object.t

    r20900 r23755  
    88use warnings FATAL => qw( all );
    99
    10 use Test::More tests => 89;
     10use Test::More tests => 99;
    1111
    1212use lib qw( ./t ./lib );
    1313
     14use File::Basename qw( basename );
    1415use File::ExtAttr qw( getfattr );
    1516use Nebulous::Server;
    … …  
    334335}
    335336
     337# test for properly row creation in the directories table
     338use Test::DBUnit dsn => $NEB_DB, username => $NEB_USER, password => $NEB_PASS;
     339
     340Test::Nebulous->setup;
     341
     342{
     343    my $key = "foo";
     344    $neb->create_object($key);
     345
     346    expected_dataset_ok(
     347        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     348        storage_object  => [so_id => 1, ext_id => $key, dir_id => 1],
     349    );
     350}
     351
     352Test::Nebulous->setup;
     353
     354{
     355    my $key = "a/foo";
     356    $neb->create_object($key);
     357
     358    expected_dataset_ok(
     359        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     360        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
     361        storage_object  => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 2],
     362    );
     363
     364}
     365
     366Test::Nebulous->setup;
     367
     368{
     369    my $key = "a/b/foo";
     370    $neb->create_object($key);
     371
     372    expected_dataset_ok(
     373        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     374        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
     375        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
     376        storage_object  => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 3],
     377    );
     378}
     379
     380Test::Nebulous->setup;
     381
     382{
     383    my $key = "a/b/c/foo";
     384    $neb->create_object($key);
     385
     386    expected_dataset_ok(
     387        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     388        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
     389        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
     390        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
     391        storage_object  => [so_id => 1, ext_id => $key, ext_id_basename => basename($key), dir_id => 4],
     392    );
     393}
     394
     395Test::Nebulous->setup;
     396
     397{
     398    my $key1 = "a/b/c/foo";
     399    my $key2 = "foo";
     400    $neb->create_object($key1);
     401    $neb->create_object($key2);
     402
     403    expected_dataset_ok(
     404        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     405        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 1],
     406        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
     407        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
     408        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
     409        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
     410    );
     411}
     412
     413Test::Nebulous->setup;
     414
     415{
     416    my $key1 = "a/b/c/foo";
     417    my $key2 = "a/foo";
     418    $neb->create_object($key1);
     419    $neb->create_object($key2);
     420
     421    expected_dataset_ok(
     422        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     423        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
     424        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 2],
     425        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
     426        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
     427        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
     428    );
     429}
     430
     431Test::Nebulous->setup;
     432
     433{
     434    my $key1 = "a/b/c/foo";
     435    my $key2 = "d/foo";
     436    $neb->create_object($key1);
     437    $neb->create_object($key2);
     438
     439    expected_dataset_ok(
     440        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     441        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
     442        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
     443        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
     444        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
     445        directory       => [dir_id => 5, dirname => 'd', parent_id => 1],
     446        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5],
     447    );
     448}
     449
     450Test::Nebulous->setup;
     451
     452{
     453    my $key1 = "a/b/c/foo";
     454    my $key2 = "a/d/foo";
     455    $neb->create_object($key1);
     456    $neb->create_object($key2);
     457
     458    expected_dataset_ok(
     459        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     460        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
     461        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
     462        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
     463        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
     464        directory       => [dir_id => 5, dirname => 'd', parent_id => 2],
     465        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5],
     466    );
     467}
     468
     469Test::Nebulous->setup;
     470
     471{
     472    my $key1 = "a/b/c/foo";
     473    my $key2 = "d/a/foo";
     474    $neb->create_object($key1);
     475    $neb->create_object($key2);
     476
     477    expected_dataset_ok(
     478        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     479        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
     480        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
     481        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
     482        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
     483        directory       => [dir_id => 5, dirname => 'd', parent_id => 1],
     484        directory       => [dir_id => 6, dirname => 'a', parent_id => 5],
     485        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 6],
     486    );
     487}
     488
     489Test::Nebulous->setup;
     490
     491{
     492    my $key1 = "a/b/c/foo";
     493    my $key2 = "a/b/c/d/foo";
     494    $neb->create_object($key1);
     495    $neb->create_object($key2);
     496
     497    expected_dataset_ok(
     498        directory       => [dir_id => 1, dirname => '/', parent_id => 1],
     499        directory       => [dir_id => 2, dirname => 'a', parent_id => 1],
     500        directory       => [dir_id => 3, dirname => 'b', parent_id => 2],
     501        directory       => [dir_id => 4, dirname => 'c', parent_id => 3],
     502        storage_object  => [so_id => 1, ext_id => $key1, ext_id_basename => basename($key1), dir_id => 4],
     503        directory       => [dir_id => 5, dirname => 'd', parent_id => 4],
     504        storage_object  => [so_id => 2, ext_id => $key2, ext_id_basename => basename($key2), dir_id => 5],
     505    );
     506}
     507
    336508Test::Nebulous->setup;
    337509
Note: See TracChangeset for help on using the changeset viewer.