#!/bin/bash -f

#
# This script is supposed to be run as ipp user (on ipp004)
#
# for each computer of the cluster
#   get the result of the command 'top -d 3 -b -n 5 -i'
#   i.e.:
#   - show active processes (-i)
#   - 5 times (-n 5)
#   - run in batch mode (-b)
#   - every 3 seconds (-d 3)
#
# If nothing is changed in this script, data can be analyzed using
# analyze_logs.pl
#

#
# TODO: 
# - make it parametered?
# - add an option for ssh timeout
# - add it to crontab
# - split hosts by pantasks server?
#

# The hosts list on which we're sshing
HOSTS="ippdb00 ippdb01 ippdb02 ippdb03\
           ippc01 ippc02 ippc03 ippc04 ippc05 ippc06 ippc07 ippc08 ippc09\
    ippc10 ippc11 ippc12 ippc13 ippc14 ippc15 ippc16 ippc17 ippc18 ippc19\
    ippc20 ippc21 ippc22 ippc23 ippc24 ippc25 ippc26 ippc27 ippc28 ippc29\
    ippc30 ippc31 ippc32 ippc33 ippc34 ippc35 ippc36 ippc37 ippc38 ippc39\
    ippc40 ippc41 ippc42 ippc43 ippc44 ippc45 ippc46 ippc47 ippc48 ippc49\
    ippc50 ippc51 ippc52 ippc53 ippc54 ippc55 ippc56 ippc57 ippc58 ippc59\
    ippc60 ippc61 ippc62 ippc63\
                                ipp004 ipp005 ipp006 ipp007 ipp008 ipp009\
    ipp010 ipp011 ipp012 ipp013 ipp014 ipp015 ipp016 ipp017 ipp018 ipp019\
    ipp020 ipp021        ipp023 ipp024 ipp025 ipp026 ipp027 ipp028 ipp029\
    ipp030 ipp031 ipp032 ipp033 ipp034 ipp035 ipp036 ipp037 ipp038 ipp039\
    ipp040 ipp041 ipp042 ipp043 ipp044 ipp045 ipp046 ipp047 ipp048 ipp049\
    ipp050 ipp051 ipp052 ipp053 ipp054 ipp055 ipp056 ipp057 ipp058 ipp059\
    ipp060 ipp061 ipp062 ipp063 ipp064 ipp065 ipp066\
    stare00 stare01 stare02 stare03 stare04"

# Basename of the files 
LOGFILE_BASE="/home/panstarrs/ipp/.tmp/log_"

# Run
for computer in $HOSTS; do
    LOGFILE=$LOGFILE_BASE$computer
    # Delete existing logfile
    echo "" > $LOGFILE
    echo "########### $computer ###########"
    echo "########### $computer ###########" >> $LOGFILE
#    /usr/bin/ssh $computer 'setenv TERM xterm; top -d 3 -b -n 5 -i -c' >> $LOGFILE 2> /dev/null &
    /usr/bin/ssh $computer 'setenv TERM xterm; setenv COLUMNS 1000; top -d 3 -b -n 5 -c -i' >> $LOGFILE 2> /dev/null &
done

#
# We should wait 15 to 20 seconds to get the results. The perl script uses any input file though.
#
