IPP Software Navigation Tools IPP Links Communication Pan-STARRS Links

Changeset 29268


Ignore:
Timestamp:
Sep 29, 2010, 3:33:18 PM (16 years ago)
Author:
rhenders
Message:

Added ability to stop and run pantasks servers

Location:
trunk/ippMonitor
Files:
1 added
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/ippMonitor/Makefile.in

    r29031 r29268  
    1616$(DESTBIN)/czartool_getServerStatus.pl \
    1717$(DESTBIN)/czartool_revert.pl \
     18$(DESTBIN)/czartool_serverstoprun.pl \
    1819$(DESTBIN)/build_histogram.dvo \
    1920$(DESTBIN)/helpers.dvo \

  • trunk/ippMonitor/raw/czartool_labels.php

    r29260 r29268  
    2525
    2626menu($myMenu, 'Czartool on '.$lastUpdateTime, 'ipp.css', $ID['link'], $ID['proj']);
    27 
    28 $pass = $ID['pass'];
    29 $proj = $ID['proj'];
    30 $menu = $ID['menu'];
    31 
    32 $selectedStage = $_GET[stage];
    33 $selectedLabel = $_GET[label];
    34 $selectedServer = $_GET[server];
    35 $selectedRevertStage = $_GET[revertstage];
    36 $selectedRevertMode = $_GET[revertmode];
    37 $plotType = $_GET[plottype];
    38 
    39 if ($selectedLabel == "") { $selectedLabel = "all_stdscience_labels"; }
    40 if ($selectedStage == "") { $selectedStage = "all_stages"; }
    41 if ($plotType == "") { $plotType = "linear"; }
    42 
    43 $nsStatus = getNightlyScienceStatus($czardb);
    44 $plotTypeLink = ($plotType == "linear") ? "log" : "linear";
    45 $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
    46 echo "<p  align=\"center\"> Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses). <br>Documentation can be found <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\">here</a> and cluster load monitored <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&c=IPP%2520Production\">here</a><br>Current nightly science status: $nsStatus<br>Use <a href=\"$link\"> $plotTypeLink</a> plots </p>";
    47 
    48 
    49 // deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
    50 if ($selectedRevertStage != "" && $selectedRevertMode != "") {
    51 
    52     exec("czartool_revert.pl -t $selectedRevertStage -o $selectedRevertMode", $response, $status);
    53     $currentRevertMode;
    54     if ($response[0] == "off") $currentRevertMode = 0;
    55     else if ($response[0] == "on") $currentRevertMode = 1;
    56 
    57     setRevertStatus($czardb, $selectedRevertStage, $currentRevertMode);
    58 }
    59 $debug = 0;
    60 
    61 $stdsLabels = getLabels($czardb, "stdscience");
    62 $distLabels = getLabels($czardb, "distribution");
    63 $pubLabels = getLabels($czardb, "publishing");
    64 $updateLabels = getLabels($czardb, "update");
    65 
    66 if ($debug) {
    67     echo "prog returned $status, and output:<br>";
    68     for ($i = 0; $i < count($distLabels); $i++) {
    69         echo "$distLabels[$i]<br>";
    70     }
    71 }
    7227
    7328$states=array("full","new","drop","wait");
    … …  
    8641        "summitcopy");
    8742
     43$pass = $ID['pass'];
     44$proj = $ID['proj'];
     45$menu = $ID['menu'];
     46
     47$selectedStage = $_GET[stage];
     48$selectedLabel = $_GET[label];
     49$selectedServer = $_GET[server];
     50$selectedRevertStage = $_GET[revertstage];
     51$selectedRevertMode = $_GET[revertmode];
     52$plotType = $_GET[plottype];
     53$serverCmd = $_GET[servercmd];
     54$allServerCmd = $_GET[allservercmd];
     55
     56if ($selectedLabel == "") { $selectedLabel = "all_stdscience_labels"; }
     57if ($selectedStage == "") { $selectedStage = "all_stages"; }
     58if ($plotType == "") { $plotType = "linear"; }
     59
     60$nsStatus = getNightlyScienceStatus($czardb);
     61$plotTypeLink = ($plotType == "linear") ? "log" : "linear";
     62$link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=" . $selectedStage . "&revertstage=" . $stage . "&plottype=$plotTypeLink";
     63echo "<p  align=\"center\"> Current status of the IPP as of $lastUpdateTime (faults are shown in parentheses). <br>Documentation can be found <a href=\"http://svn.pan-starrs.ifa.hawaii.edu/trac/ipp/wiki/Processing\">here</a> and cluster load monitored <a href=\"http://ganglia.pan-starrs.ifa.hawaii.edu/?r=hour&s=descending&c=IPP%2520Production\">here</a><br>Current nightly science status: $nsStatus<br>Use <a href=\"$link\"> $plotTypeLink</a> plots </p>";
     64
     65
     66// deal with reverts: turn on or off if requested and pass current revert state for this stage onto labels table later
     67if ($selectedRevertStage && $selectedRevertMode) turnRevertsOnOff($czardb, $selectedRevertStage, $selectedRevertMode);
     68
     69// tell selected server to stop or run
     70if ($selectedServer && $serverCmd) serverStopRun($czardb, $selectedServer,  $serverCmd);
     71
     72// tell all servers to stop or run
     73if ($allServerCmd) {
     74
     75    foreach ($servers as &$server) serverStopRun($czardb, $server,  $allServerCmd);
     76}
     77
     78
     79$debug = 0;
     80
     81$stdsLabels = getLabels($czardb, "stdscience");
     82$distLabels = getLabels($czardb, "distribution");
     83$pubLabels = getLabels($czardb, "publishing");
     84$updateLabels = getLabels($czardb, "update");
     85
     86if ($debug) {
     87    echo "prog returned $status, and output:<br>";
     88    for ($i = 0; $i < count($distLabels); $i++) {
     89        echo "$distLabels[$i]<br>";
     90    }
     91}
     92
     93
    8894// set up the form
    8995echo "<form action=\"czartool_labels.php\" method=\"POST\">\n";
    … …  
    95101
    96102echo "<table>\n";
    97   echo "<tr>\n";
    98     echo "<td>\n";
    99       echo "<img src=\"czartool_getplot.php?type=t&label=$selectedLabel&stage=$selectedStage&plottype=$plotType\"><br>";
    100     echo "</td>\n";
    101 
    102     echo "<td> \n";
    103       createLabelsTable($pass, $proj, $czardb, "stdscience", $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $plotType);
    104     echo "</td>\n";
    105   echo "</tr>\n";
    106 
    107   echo "<tr>\n";
    108     echo "<td>\n";
    109       echo "<img src=\"czartool_getplot.php?type=h&label=$selectedLabel&stage=$selectedStage&plottype=linear\"><br>";
    110     echo "</td>\n";
    111 
    112     echo "<td> \n";
    113       createLabelsTable($pass, $proj, $czardb, "update", $updateLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $plotType);
    114     echo "</td>\n";
    115   echo "</tr>\n";
    116 
    117   echo "<tr>\n";
    118     echo "<table>\n";
    119       echo "<tr valign=top>\n";
    120         echo "<td> \n";
    121           echo "<img src=\"czartool_getplot.php?type=s\"><br>";
    122         echo "</td>\n";
    123         echo "<td>\n";
    124           createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage, $plotType);
    125         echo "</td>\n";
    126 
    127         echo "<td> \n";
    128           $today = date("Y-m-d");
    129           showSummitData($gpc1db, $today);
    130         echo "</td>\n";
    131       echo "</tr>\n";
    132     echo "</table>\n";
    133 
    134     echo "<table>\n";
    135       echo "<tr valign=top>\n";
    136         echo "<td> \n";
    137         echo "</td>\n";
    138         echo "<td>\n";
    139       if ($selectedServer) showServerStatus($selectedServer);
    140         echo "</td>\n";
    141       echo "</tr>\n";
    142     echo "</table>\n";
    143   echo "</tr>\n";
     103echo "<tr>\n";
     104echo "<td>\n";
     105echo "<img src=\"czartool_getplot.php?type=t&label=$selectedLabel&stage=$selectedStage&plottype=$plotType\"><br>";
     106echo "</td>\n";
     107
     108echo "<td> \n";
     109createLabelsTable($pass, $proj, $czardb, "stdscience", $stdsLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $plotType);
     110echo "</td>\n";
     111echo "</tr>\n";
     112
     113echo "<tr>\n";
     114echo "<td>\n";
     115echo "<img src=\"czartool_getplot.php?type=h&label=$selectedLabel&stage=$selectedStage&plottype=linear\"><br>";
     116echo "</td>\n";
     117
     118echo "<td> \n";
     119createLabelsTable($pass, $proj, $czardb, "update", $updateLabels, $distLabels, $pubLabels, $stages, $states, "new", $selectedLabel, $selectedStage, $plotType);
     120echo "</td>\n";
     121echo "</tr>\n";
     122
     123echo "<tr>\n";
     124echo "<table>\n";
     125echo "<tr valign=top>\n";
     126echo "<td> \n";
     127echo "<img src=\"czartool_getplot.php?type=s\"><br>";
     128echo "</td>\n";
     129echo "<td>\n";
     130createServersTable($pass, $proj,$czardb, $servers, $selectedLabel, $selectedStage, $plotType);
     131echo "</td>\n";
     132
     133echo "<td> \n";
     134$today = date("Y-m-d");
     135showSummitData($gpc1db, $today);
     136echo "</td>\n";
     137echo "</tr>\n";
     138echo "</table>\n";
     139
     140echo "<table>\n";
     141echo "<tr valign=top>\n";
     142echo "<td> \n";
     143echo "</td>\n";
     144echo "<td>\n";
     145if ($selectedServer && !$serverCmd) showServerStatus($selectedServer);
     146echo "</td>\n";
     147echo "</tr>\n";
     148echo "</table>\n";
     149echo "</tr>\n";
    144150echo "</table>\n";
    145151
    … …  
    258264    write_header_cell($class, "Label (in order of priority)");
    259265    foreach ($stages as &$stage) {
    260        
     266
    261267        if ($stage == $selectedStage) $link = "";
    262268        else $link = "czartool_labels.php?pass=" . $pass . "&proj=" . $proj . "&label=" . $selectedLabel . "&stage=".$stage."&plottype=".$plotType;
    … …  
    385391
    386392    return $row[0];
     393}
     394
     395###########################################################################
     396#
     397#  Turns reverts on or off for a given task
     398#
     399###########################################################################
     400function turnRevertsOnOff($db, $stage, $mode) {
     401
     402    exec("czartool_revert.pl -t $stage -o $mode", $response, $status);
     403    $currentRevertMode;
     404    if ($response[0] == "off") $currentRevertMode = 0;
     405    else if ($response[0] == "on") $currentRevertMode = 1;
     406
     407    setRevertStatus($db, $stage, $currentRevertMode);
    387408}
    388409
    … …  
    449470    write_header_cell($class, "Server");
    450471    write_header_cell($class, "Alive?");
    451     write_header_cell($class, "Scheduler running?");
     472    $link = "czartool_labels.php?pass=".$pass."&proj=".$proj."&label=".$selectedLabel."&stage=".$selectedStage."&plottype=".$plotType."&allservercmd=stop";
     473    write_table_cell($class, '%s', $link, "Stop all");
     474    $link = "czartool_labels.php?pass=".$pass."&proj=".$proj."&label=".$selectedLabel."&stage=".$selectedStage."&plottype=".$plotType."&allservercmd=run";
     475    write_table_cell($class, '%s', $link, "Run all");
    452476    echo "</tr>\n";
    453477
    454478    foreach ($servers as &$server) {
    455479
     480        getServerStatus($db, $server, $alive, $running);
     481
     482        echo "<tr><td></td>\n";
    456483        $link = "czartool_labels.php?pass=".$pass."&proj=".$proj."&server=".$server."&label=".$selectedLabel."&stage=".$selectedStage."&plottype=".$plotType;
    457         //      $link = "";
    458 
    459         getServerStatus($db, $server, $alive, $running);
    460 
    461         echo "<tr><td></td>\n";
    462484        write_table_cell($class, '%s', $link, $server);
    463485        write_table_cell($class, '%s', "", $alive ? "yes" : "NO");
    464         write_table_cell($class, '%s', "", $running ? "yes" : "NO");
     486
     487        if ($alive) {
     488
     489            $link = "czartool_labels.php?pass=".$pass."&proj=".$proj."&label=".$selectedLabel."&stage=".$selectedStage."&plottype=".$plotType."&server=".$server."&servercmd=";
     490            if ($running)  {
     491
     492                $link = $link . "stop";
     493                write_table_cell($class, '%s', $link, "stop");
     494                write_table_cell($class, '%s', "", "running");
     495            }
     496            else {
     497
     498                $link = $link . "run";
     499                write_table_cell($class, '%s', "", "stopped");
     500                write_table_cell($class, '%s', $link, "run");
     501            }
     502        }
     503        else {
     504       
     505            write_table_cell($class, '%s', "", "");
     506            write_table_cell($class, '%s', "", "");
     507        }
    465508        echo "</tr>\n";
    466509    }
    … …  
    468511    echo "</table>\n";
    469512}
     513
     514###########################################################################
     515#
     516#  Commands a server to stop or run
     517#
     518###########################################################################
     519function serverStopRun($db, $server, $cmd) {
     520
     521    exec("czartool_serverstoprun.pl -s $server -c $cmd", $response, $status);
     522    $alive = 0;
     523    $running = 0;
     524    if ($response[0] == "running") {$alive = 1; $running = 1;}
     525    else if ($response[0] == "stopped") {$alive = 1; $running = 0;}
     526    else if ($response[0] == "dead") {$alive = 0; $running = 0;}
     527    setServerStatus($db, $server, $alive, $running);
     528    print "$cmd $server $alive, $running<br>";
     529}
     530
     531###########################################################################
     532#
     533# Sets server status
     534#
     535###########################################################################
     536function setServerStatus($db, $server, $alive, $running) {
     537
     538    $sql = "INSERT INTO servers (server, alive, running) VALUES ('$server', $alive, $running)";
     539
     540    if ($debug) {echo "$sql<br>";}
     541
     542    $qry = $db->query($sql);
     543    if (dberror($qry)) {echo "<b>error with $sql </b><br>\n";}
     544}
     545
    470546
    471547###########################################################################
Note: See TracChangeset for help on using the changeset viewer.