Changeset 28158

Timestamp:

May 28, 2010, 11:51:46 AM (16 years ago)

Author:

eugene

Message:

redirect pcontrol stderr to log file; add bits to harvest zombies that may accumulate

Location:

trunk/Ohana/src/opihi

Files:

• include/pcontrol.h (modified) (1 diff)
• pcontrol/CheckIdleHost.c (modified) (6 diffs)
• pcontrol/CheckSystem.c (modified) (1 diff)
• pcontrol/StopHosts.c (modified) (4 diffs)
• pcontrol/pcontrol.c.in (modified) (2 diffs)
• pcontrol/test/machines.sh (modified) (2 diffs)

trunk/Ohana/src/opihi/include/pcontrol.h

@@ -273 +273 @@
 int    StopHostResponse (Host *host);
 int    HarvestHost (int pid);
+int    AddZombie(int pid);
+int    DelZombies();
+int    CheckZombies();
 
 /*** JobOps.c ***/

trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c

@@ -8 +8 @@
 static float MAX_WANTHOST_WAIT = 10.0;
 static float MAX_CONNECT_TIME = 36000.0;
-static FILE *logfile = NULL;
 
   /* if this host has been connected for too long, disconnect (will automatically reconnect) */
@@ -35 +34 @@
   struct timeval now;
   float dtime;
-
-  if (logfile == NULL) { 
-    logfile = fopen ("pcontrol.log", "w");
-  }
 
   ASSERT (host, "host not set");
@@ -69 +64 @@
     host[0].job = (struct Job *) job;
 
-    // if (logfile) fprintf (logfile, "start needhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    // gprint (GP_ERR, "start needhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     AddMachineJob (host, job);
 
@@ -90 +85 @@
     host[0].job = (struct Job *) job;
 
-    // if (logfile) fprintf (logfile, "start wanthost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    // gprint (GP_ERR, "start wanthost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     AddMachineJob (host, job);
 
@@ -111 +106 @@
     host[0].job = (struct Job *) job;
 
-    // if (logfile) fprintf (logfile, "start  anyhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    // gprint (GP_ERR, "start  anyhost %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     AddMachineJob (host, job);
 
@@ -133 +128 @@
     if (!CheckMachineJobs (host, job)) continue;
 
-    if (logfile) fprintf (logfile, "start wanthost(2) %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
+    gprint (GP_ERR, "start wanthost(2) %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);
     AddMachineJob (host, job);
 

trunk/Ohana/src/opihi/pcontrol/CheckSystem.c

@@ -107 +107 @@
       TestCheckPoint ();
       Nhostchecks += CheckDownHosts(0.100); /* launch the host */
+      TestCheckPoint ();
+      CheckZombies(); /* launch the host */
       TestCheckPoint ();
     }

trunk/Ohana/src/opihi/pcontrol/StopHosts.c

@@ -1 +1 @@
 # include "pcontrol.h"
+
+// we attempt to harvest the 'down' hosts in HarvestHost.  However, sometimes the
+// child is busy and does not exit in the timeout period.  we need to keep a list and
+// try again occasionally to free up the needed resources
+static int NUNHARVESTED = 0;
+static int Nunharvested = 0;
+static int *unharvested = NULL;
 
 void DownHost (Host *host) {
@@ -140 +147 @@
       switch (errno) {
         case ECHILD:
-          gprint (GP_ERR, "unknown PID, not a child proc\n");
+          gprint (GP_ERR, "HarvestHost: unknown PID (%d), not a child proc\n", pid);
           gprint (GP_ERR, "did process already exit?  programming error?\n");
           break;
@@ -152 +159 @@
       
     case 0:
-      gprint (GP_ERR, "HarvestHost: child with connection to remote host failed to exit: may be hung");
+      gprint (GP_ERR, "HarvestHost: child with connection to remote host failed to exit: may be hung\n");
+      AddZombie(pid);
       break;
 
@@ -174 +182 @@
   return (TRUE);
 }
+
+int AddZombie(int pid) {
+
+  if (unharvested == NULL) {
+    NUNHARVESTED = 128;
+    ALLOCATE (unharvested, int, NUNHARVESTED);
+    memset (unharvested, 0, NUNHARVESTED*sizeof(int));
+  }
+  unharvested[Nunharvested] = pid;
+
+  Nunharvested ++;
+  if (Nunharvested >= NUNHARVESTED) {
+    NUNHARVESTED += 128;
+    REALLOCATE (unharvested, int, NUNHARVESTED);
+    memset (&unharvested[Nunharvested], 0, (NUNHARVESTED - Nunharvested)*sizeof(int));
+  }
+  return TRUE;
+}
+
+int DelZombies() {
+
+  int i, j;
+
+  if (!unharvested) return FALSE;
+  if (!Nunharvested) return FALSE;
+  if (!NUNHARVESTED) return FALSE;
+
+  int *newlist = NULL;
+
+  ALLOCATE (newlist, int, NUNHARVESTED);
+  memset (newlist, 0, NUNHARVESTED*sizeof(int));
+
+  j = 0;
+  for (i = 0; i < NUNHARVESTED; i++) {
+    if (!unharvested[i]) continue;
+    newlist[j] = unharvested[i];
+    j++;
+  }
+  free (unharvested);
+  unharvested = newlist;
+  Nunharvested = j;
+  return TRUE;
+}
+
+int CheckZombies() {
+
+  int pid, i, result, waitstatus;
+
+  if (!unharvested) return FALSE;
+  if (!Nunharvested) return FALSE;
+  if (!NUNHARVESTED) return FALSE;
+
+  for (i = 0; i < Nunharvested; i++) {
+    if (!unharvested[i]) continue;
+    pid = unharvested[i];
+    result = waitpid (pid, &waitstatus, WNOHANG);
+    switch (result) {
+      case -1:  /* error with waitpid */
+        switch (errno) {
+          case ECHILD:
+            gprint (GP_ERR, "CheckZombies: unknown PID (%d), not a child proc\n", pid);
+            gprint (GP_ERR, "did process already exit?  programming error?\n");
+            break;
+          case EINTR:
+          case EINVAL:
+          default:
+            perror ("unexpected error");
+            ABORT ("CheckZombies impossible condition");
+        }
+        break;
+      
+      case 0:
+        if (VerboseMode()) gprint (GP_ERR, "CheckZombies: still waiting on %d\n", pid);
+        break;
+
+      default:
+        if (result != pid) {
+          gprint (GP_ERR, "waitpid error: mis-matched PID (%d vs %d).  programming error\n", result, pid);
+          ABORT ("CheckZombies impossible condition");
+        }
+        
+        if (WIFEXITED(waitstatus)) {
+          if (VerboseMode()) gprint (GP_ERR, "child exited with status %d\n", WEXITSTATUS(waitstatus));
+        }
+        if (WIFSIGNALED(waitstatus)) {
+          if (VerboseMode()) gprint (GP_ERR, "child crashed with status %d\n", WTERMSIG(waitstatus));
+        }
+        if (WIFSTOPPED(waitstatus)) {
+          ABORT ("waitpid returns 'stopped': programming error\n");
+        }
+        unharvested[i] = 0;
+        break;
+    }
+  }
+  DelZombies();
+  return (TRUE);
+}

trunk/Ohana/src/opihi/pcontrol/pcontrol.c.in

@@ -1 +1 @@
 # include "pcontrol.h"
+
+# define STDERR_FILE "pcontrol.log"
 
 # define opihi_name "PCONTROL"
@@ -24 +26 @@
   signal (SIGTSTP, gotsignal);
   signal (SIGTTIN, gotsignal);
+
+  // stdin / stdout are used for communication with pantasks.
+  // redirect stderr so various error messages are saved
+  stderr = freopen (STDERR_FILE, "a", stderr);
+  if (!stderr) {
+      fprintf (stdout, "failed to open %s for error output\n", STDERR_FILE);
+      exit (1);
+  }
 
   rl_readline_name = opihi_name;

trunk/Ohana/src/opihi/pcontrol/test/machines.sh

@@ -6 +6 @@
   host add pikake
   host add pikake
-
   host add ipp022
   host add ipp022
@@ -26 +25 @@
 end
 
+macro load.hosts.zombie
+  parameters connect = 2.0
+
+  host add pikake
+  host add pikake
+  host add ipp022
+  host add ipp022
+  host add ipp022
+  host add ipp022
+
+  machines
+end
+
+macro load.jobs.zombie
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+  job sleep 10
+end
+