Index: branches/simmosaic_branches/Nebulous-Server/bin/nebdiskd
===================================================================
--- branches/simmosaic_branches/Nebulous-Server/bin/nebdiskd	(revision 24860)
+++ branches/simmosaic_branches/Nebulous-Server/bin/nebdiskd	(revision 27839)
@@ -67,4 +67,11 @@
 $retry      ||= $c->get_retry   || 1;
 
+# stuff to controll failures.
+my %host_failure_counts = ();
+my %host_removed = ();
+my $failure_limit = 5;
+
+
+
 #my $mounts = $c->get_mounts;
 my $poll_interval = $c->get_poll_interval || 5;
@@ -90,5 +97,5 @@
 # start up logging
 my $conf = '
-    log4perl.category.nebdiskd = WARN, Screen, SERVERLOGFILE, Mailer
+    log4perl.category.nebdiskd = WARN, Screen, SERVERLOGFILE
 
     log4perl.appender.Screen        = Log::Log4perl::Appender::Screen
@@ -107,16 +114,18 @@
     log4perl.filter.MatchWarn.AcceptOnMatch = off
 
-    log4perl.appender.Mailer         = Log::Dispatch::Email::MailSend
-    log4perl.appender.Mailer.to      = ps-ipp-ops@ifa.hawaii.edu
-    log4perl.appender.Mailer.subject = nebdiskd alert
-    log4perl.appender.Mailer.buffered = 0
-    log4perl.appender.Mailer.Filter= MatchWarn
-    log4perl.appender.Mailer.layout = Log::Log4perl::Layout::PatternLayout
-    log4perl.appender.Mailer.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
-
     log4perl.appender.Limiter              = Log::Log4perl::Appender::Limit
     log4perl.appender.Limiter.appender     = Mailer
     log4perl.appender.Limiter.block_period = 300
 ';
+# I've removed this bit.
+#     log4perl.appender.Mailer         = Log::Dispatch::Email::MailSend
+#     log4perl.appender.Mailer.to      = ps-ipp-ops@ifa.hawaii.edu
+#     log4perl.appender.Mailer.subject = nebdiskd alert
+#     log4perl.appender.Mailer.buffered = 0
+#     log4perl.appender.Mailer.Filter= MatchWarn
+#     log4perl.appender.Mailer.layout = Log::Log4perl::Layout::PatternLayout
+#     log4perl.appender.Mailer.layout.ConversionPattern = %d{yyyy-MM-dd HH:mm:ss} | %H | %p | %M - %m%n
+
+
 Log::Log4perl::init(\$conf);
 my $log = Log::Log4perl::get_logger("nebdiskd");
@@ -133,4 +142,5 @@
 $SIG{INT}  = $SIG{TERM};
 $SIG{HUP}  = sub { $c = read_rcfile($rcfile) }; 
+
 
 while (1) {
@@ -177,5 +187,6 @@
 
     eval {
-        my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT *, ?, ? FROM volume WHERE mountpoint = ?");
+        my $r_query = $dbh->prepare_cached("REPLACE INTO mountedvol SELECT vol_id,name,host,path,allocate,available,xattr,mountpoint, ?, ? FROM volume WHERE mountpoint = ?");
+#	my $d_query = $dbh->prepare_cached("UPDATE mountedvol SET allocate = ?, available = ? WHERE mountpoint = ?");
         my $d_query = $dbh->prepare_cached("DELETE FROM mountedvol WHERE mountpoint = ?");
 
@@ -185,5 +196,5 @@
             # there may be multiple vol_ids per mountpoint but we only need to
             # check each mointpont once
-            my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume WHERE available = 1");
+            my $query = $dbh->prepare_cached("SELECT DISTINCT mountpoint FROM volume");
             $query->execute;
             while (my $row = $query->fetchrow_hashref) {
@@ -200,11 +211,18 @@
             # automounter and it fails to mount
             my $tries = 0;
+	    my $valid_mountpoint = 1;
+	    unless (exists($host_failure_counts{$mountpoint})) {
+		$host_failure_counts{$mountpoint} = 0;
+	    }
             TEST: eval {
                 $tries++;
                 unless (is_mountpoint($mountpoint)) {
-                    $log->warn("$mnt is not a valid mountpoint");
+		    unless(defined($host_removed{$mountpoint})) {
+			$log->warn("$mountpoint is not a valid mountpoint ($tries $host_failure_counts{$mountpoint})");
+		    }
+		    $valid_mountpoint = 0;
                 }
             };
-            if ($@) {
+            if (!$valid_mountpoint) {
                 # try is_mountpoint() again if $retry > 1
                 if ($tries < $retry) {
@@ -212,8 +230,24 @@
                     goto TEST; 
                 }
-                $log->warn($@);
-                $d_query->execute($mountpoint);
-                next;
-            }
+		$host_failure_counts{$mountpoint}++;
+		
+		if (!(defined($host_removed{$mountpoint})) || !($host_removed{$mountpoint})) {
+		    if (($host_failure_counts{$mountpoint} > $failure_limit)) {
+			$host_removed{$mountpoint} = 1;
+			$log->warn("Removing $mountpoint from the mountedvol table ($host_failure_counts{$mountpoint} > $failure_limit) No further warnings unless state changes.");
+			$d_query->execute($mountpoint);
+		    }
+		    else {
+			$log->warn("Mountpoint $mountpoint has had $host_failure_counts{$mountpoint} failures. Will remove after $failure_limit");
+		    }
+		}
+		next;
+		
+	    }
+	    if ($host_failure_counts{$mountpoint} != 0) {
+		$host_removed{$mountpoint} = 0;
+		$log->warn("Mountpoint $mountpoint failures cleared ($host_failure_counts{$mountpoint})");
+		$host_failure_counts{$mountpoint} = 0;
+	    }
 
             # fetch stats on the mounted device.  this has to be done AFTER