Changeset 27103

Timestamp:

Feb 26, 2010, 11:01:34 AM (16 years ago)

Author:

watersc1

Message:

Increase the allowed disk consumption fraction from 0.95 to 0.98.

Rework the error/warning reporting of nebdiskd so that it isn't so
annoying about saying the same things over and over again. This also
disables the "Mailer" log interface.

Location:

trunk/Nebulous-Server

Files:

• bin/nebdiskd (modified) (5 diffs)
• lib/Nebulous/Server.pm (modified) (7 diffs)

trunk/Nebulous-Server/bin/nebdiskd

@@ -69 +69 @@
 # stuff to controll failures.
 my %host_failure_counts = ();
+my %host_removed = ();
 my $failure_limit = 5;
 
@@ -96 +97 @@
 # start up logging
 my $conf = '
-    log4perl.category.nebdiskd = WARN, Screen, SERVERLOGFILE, Mailer
+    log4perl.category.nebdiskd = WARN, Screen, SERVERLOGFILE
 
     log4perl.appender.Screen        = Log::Log4perl::Appender::Screen
@@ -113 +114 @@
     log4perl.filter.MatchWarn.AcceptOnMatch = off
 
-    log4perl.appender.Mailer         = Log::Dispatch::Email::MailSend
-    log4perl.appender.Mailer.to      = ps-ipp-ops@ifa.hawaii.edu
-    log4perl.appender.Mailer.subject = nebdiskd alert
-    log4perl.appender.Mailer.buffered = 0
-    log4perl.appender.Mailer.Filter= MatchWarn
-    log4perl.appender.Mailer.layout = Log::Log4perl::Layout::PatternLayout
-    log4perl.appender.Mailer.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
-
     log4perl.appender.Limiter              = Log::Log4perl::Appender::Limit
     log4perl.appender.Limiter.appender     = Mailer
     log4perl.appender.Limiter.block_period = 300
 ';
+# I've removed this bit.
+#     log4perl.appender.Mailer         = Log::Dispatch::Email::MailSend
+#     log4perl.appender.Mailer.to      = ps-ipp-ops@ifa.hawaii.edu
+#     log4perl.appender.Mailer.subject = nebdiskd alert
+#     log4perl.appender.Mailer.buffered = 0
+#     log4perl.appender.Mailer.Filter= MatchWarn
+#     log4perl.appender.Mailer.layout = Log::Log4perl::Layout::PatternLayout
+#     log4perl.appender.Mailer.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
+
+
 Log::Log4perl::init(\$conf);
 my $log = Log::Log4perl::get_logger("nebdiskd");
@@ -215 +218 @@
                 $tries++;
                 unless (is_mountpoint($mountpoint)) {
-                    $log->warn("$mountpoint is not a valid mountpoint ($tries)");
+                    unless(defined($host_removed{$mountpoint})) {
+                        $log->warn("$mountpoint is not a valid mountpoint ($tries $host_failure_counts{$mountpoint})");
+                    }
                     $valid_mountpoint = 0;
                 }
@@ -226 +231 @@
                 }
                 $host_failure_counts{$mountpoint}++;
-
-                if ($host_failure_counts{$mountpoint} > $failure_limit) {
-                    $log->warn("Removing $mountpoint from the mountedvol table ($host_failure_counts{$mountpoint} > $failure_limit)");
-                    $d_query->execute($mountpoint);
+                
+                if (!(defined($host_removed{$mountpoint})) || !($host_removed{$mountpoint})) {
+                    if (($host_failure_counts{$mountpoint} > $failure_limit)) {
+                        $host_removed{$mountpoint} = 1;
+                        $log->warn("Removing $mountpoint from the mountedvol table ($host_failure_counts{$mountpoint} > $failure_limit) No further warnings unless state changes.");
+                        $d_query->execute($mountpoint);
+                    }
+                    else {
+                        $log->warn("Mountpoint $mountpoint has had $host_failure_counts{$mountpoint} failures. Will remove after $failure_limit");
+                    }
                 }
-                else {
-                    $log->warn("Mountpoint $mountpoint has had $host_failure_counts{$mountpoint} failures. Will remove after $failure_limit");
-                }
-                next;
-            }
+                next;
+                
+            }
             if ($host_failure_counts{$mountpoint} != 0) {
+                $host_removed{$mountpoint} = 0;
                 $log->warn("Mountpoint $mountpoint failures cleared ($host_failure_counts{$mountpoint})");
                 $host_failure_counts{$mountpoint} = 0;

trunk/Nebulous-Server/lib/Nebulous/Server.pm

@@ -41 +41 @@
 # This determines how many entries from the list of volumes sorted by free space are randomized.
 my $topfew_count = 15;
-
+my $max_used_space = 0.98
 # transaction restart/retry regex
 my $trans_regex = qr/Deadlock Found|Lock wait timeout exceeded|try restarting transaction|Can't connect to MySQL server/i;
@@ -1606 +1606 @@
     $log->debug("entered - @_");
 
-    my ($key, $vol_name) = validate_pos(@_,
+    my ($key, $vol_name, $find_invalid) = validate_pos(@_,
         {
             type        => SCALAR,
@@ -1624 +1624 @@
             optional    => 1,
         },
+        {
+            # find_invalid
+            type        => SCALAR|UNDEF,
+            optional    => 1,
+        }, 
     );
 
@@ -1667 +1672 @@
         } else {
             $query = $db->prepare_cached( $sql->get_object_instances );
+            my $rows;
             # ext_id, available
-            my $rows = $query->execute($key->path, 1);
+            if (defined($find_invalid)) {
+                $rows = $query->execute($key->path, 0);
+            }
+            else {
+                $rows = $query->execute($key->path, 1);
+            }
             unless ($rows > 0) {
                 $query->finish;
@@ -2108 +2119 @@
             $query = $db->prepare_cached( $sql->get_storage_volume_by_name );
             # %free, name, avaiable, allocate
-            $rows = $query->execute(0.95, $name, 1, 1);
+            $rows = $query->execute($max_used_space, $name, 1, 1);
             # XXX destinguish between non-existant and unavailable
             unless ($rows > 0) {
@@ -2129 +2140 @@
             $query = $db->prepare_cached( $sql->get_storage_volume );
             # %free, avaiable, allocate
-            $rows = $query->execute(0.95, 1, 1, $topfew_count);
+            $rows = $query->execute($max_used_space, 1, 1, $topfew_count);
 #           $log->warn("Storage_volume: $rows $topfew_count");
             # there has to be atleast one storage volume
@@ -2201 +2212 @@
         $query = $db->prepare_cached( $sql->get_replication_volume_for_ext_id );
         # ext_id, %free, avaiable, allocate
-        $rows = $query->execute($key->path, 0.95, 1, 1, $forbidden_cabinet, $topfew_count);
+        $rows = $query->execute($key->path, $max_used_space, 1, 1, $forbidden_cabinet, $topfew_count);
         # XXX destinguish between non-existant and unaviable
         unless ($rows > 0) {