Changeset 21379

Timestamp:

Feb 6, 2009, 10:37:19 AM (17 years ago)

Author:

eugene

Message:

report pending / process time with status; cycle over all hosts for each of NEED, WANT, ANY, OLDWANT; wait up to 10 sec for WANT before giving up

Location:

trunk/Ohana/src/opihi

Files:

• include/pcontrol.h (modified) (3 diffs)
• pcontrol/CheckIdleHost.c (modified) (5 diffs)
• pcontrol/CheckSystem.c (modified) (4 diffs)
• pcontrol/JobOps.c (modified) (1 diff)
• pcontrol/check.c (modified) (4 diffs)
• pcontrol/status.c (modified) (2 diffs)
• pcontrol/test/hosttargets.sh (added)

trunk/Ohana/src/opihi/include/pcontrol.h

@@ -25 +25 @@
   PCONTROL_JOB_NEEDHOST,
 } JobMode;
+
+/** job mode check stages **/
+typedef enum {
+  PCONTROL_JOB_STAGE_ANYHOST,
+  PCONTROL_JOB_STAGE_WANTHOST,
+  PCONTROL_JOB_STAGE_NEEDHOST,
+  PCONTROL_JOB_STAGE_OLDWANT,
+} JobCheckStage;
 
 /** job thread options values **/
@@ -180 +188 @@
 int   CheckDoneHosts (float delay);
 int   CheckDownHosts (float delay);
-int   CheckIdleHosts (float delay);
+int   CheckIdleHosts (float delay, int Stage);
 int   CheckLiveHosts (float delay);
 int   SetRunSystem (int state);
@@ -202 +210 @@
 
 int StartHost (Host *host);
-int CheckIdleHost (Host *host);
+int CheckIdleHost (Host *host, int Stage);
 int CheckDoneJob (Job *job, Host *host);
 int GetJobOutput (char *command, Host *host, JobOutput *output);

trunk/Ohana/src/opihi/pcontrol/CheckIdleHost.c

@@ -4 +4 @@
 
 /* the supplied host is not on a stack: it cannot be taken by the other thread */
-int CheckIdleHost (Host *host) {
+int CheckIdleHost (Host *host, int Stage) {
 
   int i;
   Stack *stack;
   Job *job;
+  struct timeval now;
+  float dtime;
 
   if (logfile == NULL) { 
@@ -28 +30 @@
   
   /* look for first NEEDHOST matching this host */
-  for (i = 0; i < stack[0].Nobject; i++) {
+  for (i = 0; (Stage == PCONTROL_JOB_STAGE_NEEDHOST) && (i < stack[0].Nobject); i++) {
     job = (Job *) stack[0].object[i];
     if (job[0].mode != PCONTROL_JOB_NEEDHOST) continue;
@@ -48 +50 @@
 
   /* no NEEDHOST entry, look for first WANTHOST matching this host */
-  for (i = 0; i < stack[0].Nobject; i++) {
+  for (i = 0; (Stage == PCONTROL_JOB_STAGE_WANTHOST) && (i < stack[0].Nobject); i++) {
     job = (Job *) stack[0].object[i];
     if (job[0].mode != PCONTROL_JOB_WANTHOST) continue;
@@ -68 +70 @@
 
   /* no WANTHOST entry, look for first ANYHOST matching this host */
-  for (i = 0; i < stack[0].Nobject; i++) {
+  for (i = 0; (Stage == PCONTROL_JOB_STAGE_ANYHOST) && (i < stack[0].Nobject); i++) {
     job = (Job *) stack[0].object[i];
     if (job[0].mode != PCONTROL_JOB_ANYHOST) continue;
@@ -86 +88 @@
 
   /* no ANYHOST entry, look for first WANTHOST with old time */
-  /* XXX perhaps I should add this to the conditions for ANYHOST instead of
-     running a separate loop?  ie, WANTHOST && time > X == ANYHOST */
-  for (i = 0; i < stack[0].Nobject; i++) {
+  for (i = 0; (Stage == PCONTROL_JOB_STAGE_OLDWANT) && (i < stack[0].Nobject); i++) {
     job = (Job *) stack[0].object[i];
     if (job[0].mode != PCONTROL_JOB_WANTHOST) continue;
-    // XXX test the job age and skip if too young
+
+    // allow WANT jobs to wait up to 10.0 sec for the host to be free before giving up
+    gettimeofday (&now, (void *) NULL);
+    dtime = DTIME (now, job[0].start);
+    if (dtime > 10.0) continue;
 
     if (logfile) fprintf (logfile, "start wanthost(2) %s (job host %s) : %s\n", host[0].hostname, job[0].hostname, job[0].argv[0]);

trunk/Ohana/src/opihi/pcontrol/CheckSystem.c

@@ -22 +22 @@
 
   /* we want to give each block a maximum allowed time */
-  CheckIdleHosts(0.020); /* submit a new job */
+  CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_NEEDHOST); /* submit a new job */
+  CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_WANTHOST); /* submit a new job */
+  CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_ANYHOST);  /* submit a new job */
+  CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_OLDWANT);  /* submit a new job */
 
   CheckBusyJobs(0.020);  /* get job status */
@@ -109 +112 @@
     if (RunLevel == PCONTROL_RUN_ALL) {
       // we want to give each block a maximum allowed time
-      Nhostchecks += CheckIdleHosts(0.020); /* submit a new job (PCLIENT) */
+      Nhostchecks += CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_NEEDHOST); /* submit a new job (PCLIENT) */
+      Nhostchecks += CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_WANTHOST); /* submit a new job (PCLIENT) */
+      Nhostchecks += CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_ANYHOST); /* submit a new job (PCLIENT) */
+      Nhostchecks += CheckIdleHosts(0.015, PCONTROL_JOB_STAGE_OLDWANT); /* submit a new job (PCLIENT) */
       TestCheckPoint ();
     }
@@ -367 +373 @@
 }
 
-int CheckIdleHosts (float MaxDelay) {
+// if we have any IDLE hosts, check if there are jobs to be launched 
+// for each pass, we only check one type of job: stage = NEED, WANT, ANY, OLDWANT
+int CheckIdleHosts (float MaxDelay, int Stage) {
 
   struct timeval start, stop;
@@ -389 +397 @@
     host = PullStackByLocation (stack, STACK_TOP);
     if (host == NULL) break;
-    CheckIdleHost (host);
-    gettimeofday (&stop, (void *) NULL);
-    dtime = DTIME (stop, start);
-  }
+    CheckIdleHost (host, Stage);
+    gettimeofday (&stop, (void *) NULL);
+    dtime = DTIME (stop, start);
+  }
+
   if (DEBUG) gprint (GP_ERR, "checked %d hosts\n", i);
   return (i);

trunk/Ohana/src/opihi/pcontrol/JobOps.c

@@ -231 +231 @@
   PutJob (job, PCONTROL_JOB_PENDING, STACK_BOTTOM);
 
+  // until the job is launched, we use 'start' to time how long the job is waiting on the queue
+  gettimeofday (&job[0].start, (void *) NULL);
+
   if (VerboseMode()) gprint (GP_ERR, "added new job\n");
   return (JobID);

trunk/Ohana/src/opihi/pcontrol/check.c

@@ -3 +3 @@
 int check (int argc, char **argv) {
 
+  int N, Save;
   int JobID, HostID;
 
@@ -8 +9 @@
   Job *job = NULL;
   Host *host = NULL;
+
+  Save = FALSE;
+  if ((N = get_argument (argc, argv, "-save"))) {
+    remove_argument (N, &argc, argv);
+    Save = TRUE;
+  }
 
   if (argc != 3) {
@@ -39 +46 @@
         gprint (GP_LOG, "HOSTNAME NONE\n");
     }
+
+    if (Save) {
+        set_str_variable ("JOB_STATUS", GetJobStackName(job[0].stack));
+        set_int_variable ("JOB_EXITST", job[0].exit_status);
+        set_int_variable ("JOB_STDOUT_SIZE", job[0].stdout.size);
+        set_int_variable ("JOB_STDERR_SIZE", job[0].stderr.size);
+        set_variable ("JOB_DTIME", job[0].dtime);
+        set_str_variable ("JOB_HOSTNAME", job[0].hostname);
+        if (job[0].realhost) {
+            set_str_variable ("JOB_REALHOST", job[0].realhost);
+        } else {
+            set_str_variable ("JOB_REALHOST", "NONE");
+        }
+    }
+
     PushStack (stack, STACK_BOTTOM, job, job[0].JobID, job[0].argv[0]);
     return (TRUE);
@@ -53 +75 @@
     }
     gprint (GP_LOG, "host %s\n", GetHostStackName(host[0].stack));
+
+    if (Save) {
+        set_str_variable ("HOST_STATE", GetHostStackName(host[0].stack));
+    }
+
     PushStack (stack, STACK_BOTTOM, host, host[0].HostID, host[0].hostname);
     return (TRUE);

trunk/Ohana/src/opihi/pcontrol/status.c

@@ -17 +17 @@
   Stack *stack;
   Job *job;
+  struct timeval now;
+  float dtime;
 
   stack = GetJobStack (Nstack);
@@ -34 +36 @@
     }
     gprint (GP_LOG, "%7s  ", GetJobStackName (job[0].state));
+
+    switch (job[0].state) {
+        // for active jobs or pending jobs, print time since start (or create in the case of pending)
+      case PCONTROL_JOB_PENDING:
+      case PCONTROL_JOB_BUSY:
+      case PCONTROL_JOB_RESP:
+      case PCONTROL_JOB_HUNG:
+        gettimeofday (&now, (void *) NULL);
+        dtime = DTIME (now, job[0].start);
+        gprint (GP_LOG, "%8.2f ", dtime);
+        break;
+
+        // for active jobs or pending jobs, print time since start (or create in the case of pending)
+      case PCONTROL_JOB_DONE:
+      case PCONTROL_JOB_KILL:
+      case PCONTROL_JOB_EXIT:
+      case PCONTROL_JOB_CRASH:
+      default:
+        dtime = DTIME (job[0].stop, job[0].start);
+        gprint (GP_LOG, "%8.2f ", dtime);
+        break;
+    }
+
     for (j = 0; j < job[0].argc; j++) {
       gprint (GP_LOG, "%s ", job[0].argv[j]);