#!/bin/ksh
##             
## SCRIPT NAME: rmtcmds4sar_syscalls_sortOFallhosts
##             
## Where:  in   $FEDIR/scripts   where   $FEDIR=/apps/nns_com/fea
##             
############################################################################
## PURPOSE: To show the TOP SYSCALLS (reads,writes,chars/sec) on SGI hosts
##          --- 'A SORTED OVERVIEW OF ALL HOSTS ON THE NETWORK'.
##
##      Uses a 'sar -c' command on all hosts, like 'sar -c 2 1' ---
##      two seconds of data collection, and one output interval (sample).
##
##      *************************************************************
##      ** Helps find 'forgotten' processes, in a tight processing **
##      ** generating lots of file I/O, from network clients.      **
##      *************************************************************
##
##         (These processes can have a WORSE IMPACT on the network than
##          'forgotten' processes, in a tight processing loop,
##          gobbling LOTS OF CPU CYCLES, on network clients ---
##          when those CPU-gobblers ARE NOT generating network I/O.)
##
##       This script puts report output in a file and displays it.
##
##       The script puts the hostname beside each 'syscalls' line
##       --- and it sorts the resulting list, for all hosts, 
##       by syscalls (and/or several other 'sar -c' columns ---
##       so the TOP HOSTS ON THE NETWORK 
##                 (in terms of syscall-read-write-chars/sec activity)
##       POP UP TO THE TOP OF THE LIST.
##
############################################################################
## CALLED BY: nethosts_tools -> nethosts_tools.chestdef
##                                                        in $FEDIR/scripts
##
############################################################################
## CALL FORMAT: 
##      /apps/nns_com/fea/scripts/rmtcmds4sar_syscalls_sortOFallhosts
##             
##
## See  nethosts_tools.chestdef in $FEDIR/scripts
## for a call initiated within a 'winterm'. 
##
############################################################################
## MAINTENANCE HISTORY:
## Written by: B.Montandon  O06 18Apr2000 Based on
##                                   'rmtcmds4netstat_colls_sortOFallhosts'
##                                                      in $FEDIR/scripts
## Updated by: B.Montandon  O06 19Apr2000 Add 'sysreads/sec' and 'syswrites/sec'
##                                        sorts to the report.
############################################################################

if test "$FEDIR" = ""
then
    FEDIR="/apps/nns_com/fea"
fi


##############################################################################
## GENERATE THE NAME OF THE OUTPUT-REPORT FILE.
##############################################################################

. $FEDIR/scripts/set_localoutlist
OUTLIST=${OUTLIST}_sar_syscalls
OUTLIST_PRESORT=${OUTLIST}_presort

  rm -f $OUTLIST
  rm -f $OUTLIST_PRESORT

##############################################################################
## GENERATE THE HOSTLIST from NIS (ypcat hosts).
##############################################################################

## . /apps/nns_com/fea/scripts/alarm_sethostlist

## . /apps/ideas/cron/set_hostlist

  HOSTLIST=`ypcat hosts | grep -v "^#"  | grep iaw | awk '{print $3}' | sort`

# HOSTLIST=$HOSTLIST" engvis00 engprd00 sgia sgib"

## FOR TESTING:
#   HOSTLIST="iaw005 iaw007 iaw021 iaw030 iaw141"

#############################################################################
## PREPARE A HEADER FOR THE REPORT.
#############################################################################
## SAMPLE 'sar -c' OUTPUT:
## $ sar -c 2 1
##
## IRIX iaw030 6.5 04151556 IP22    04/18/00
##
## 17:04:31 scall/s sread/s swrit/s  fork/s  exec/s rchar/s wchar/s
## 17:04:33     122       8       9    0.00    0.00     572     570
#############################################################################

SAMPLE_DURATION=2
SAMPLES_TAKEN=1

COLHEAD0="  1        2         3        4         5       6     7        8            9"
COLHEAD1="HOST   END-TIME  SYSCALLs  SYSREADs SYSWRITEs FORKs EXECs    READ         WRITE"
COLHEAD2="NAME   OF SAMPLE     /sec      /sec      /sec  /sec  /sec    chars/sec    chars/sec"
COLHEAD3="------ --------- -------- --------- --------- ----- ----- ------------ ------------"

echo "\
************************* `date '+%Y %b %d  %T%p'` ******************************
NNS SGI NETWORK
HOSTS SORTED BY 'SYSCALL' ACTIVITY (reads,writes,chars/sec)
--- BUSIEST HOSTS AT THE TOP.
                                   SEVERAL DIFFERENT SORTS.

    (Data was collected for $SAMPLE_DURATION seconds on each host.
     See comments at bottom of this report.)
" > $OUTLIST

##############################################################################
## IF WE WERE GOING TO EXECUTE A COMMAND/SCRIPT ON EACH HOST,
## WE COULD ASSURE THE USER HAD A .rhosts FILE.
##############################################################################
#  echo  "+ $USER" > $HOME/.rhosts
    . $FEDIR/scripts/mak_rhosts


##############################################################################
## LOOP THRU HOSTS -- TO EXECUTE THE 'sar-awk' COMMAND PIPE, with 'rsh'.
##############################################################################
##   Use 'ping' to check the accessibility of the host before issuing
##   'rsh'.  'ping' returns faster on an inaccessible host than 'rsh'.
##############################################################################
##   CATCH THE OUTPUT FROM A SINGLE PING WITH A SMALL 4-BYTE PACKET.
##############################################################################

for HOST in $HOSTLIST
do
   echo  "\nPinging $HOST to check availability."

   ## FOR TESTING:
   # set -x

   ###########################################################################
   ## PING THE HOST BEFORE TRYING 'rsh' --- and CATCH THE OUTPUT
   ## FROM A SINGLE PING WITH A SMALL 4-BYTE PACKET --- to use to
   ## determine whether to do 'rsh' on the host.
   ###########################################################################

   PINGOUT=`/usr/etc/ping -s 4 -c 1 $HOST`

   ## FOR TESTING:
   #   echo  "$PINGOUT
   #
   # "

   ## FOR TESTING:
   # set -

   PINGCHECK=`echo $PINGOUT | grep '100.0% packet loss'`

   if test "$PINGCHECK" = ""
   then

#######################################################
## Get IP address of host.  
## (better way? with 'netstat -in'? 'ifconfig'?)
#######################################################
##     NOT USED AT THIS TIME.
#######################################################
##
##     HOSTNAME=$HOST
##     HOSTNAME_ARP=`/usr/etc/arp $HOSTNAME`
##     HOSTNAME2=`echo "$HOSTNAME_ARP" |sed "s|-- no entry||"`
## 
##     BOOTTIME=`who -b`
## #    DATETIME=`date`
##     DATETIME=`date '+%Y %b %d  %T%p'` 
#######################################################


############################################################################
## Using 'rsh $HOST',
## get the 'sar -c' output and reformat it.
############################################################################
## SAMPLE 'sar -c' OUTPUT:
##
## $ sar -c 2 1
##
## IRIX iaw030 6.5 04151556 IP22    04/18/00
##
## 17:04:31 scall/s sread/s swrit/s  fork/s  exec/s rchar/s wchar/s
## 17:04:33     122       8       9    0.00    0.00     572     570
##
## $1            $2      $3      $4      $5      $6      $7      $8
############################################################################
## Reformat the 'sar -c 2 1' output.
## Put this in an environment variable, WINMSG0.
## Could use a work file if necessary.
############################################################################

## FOR TESTING:
#  set -x

# WINMSG0=`rsh $HOST sar -c $SAMPLE_DURATION $SAMPLES_TAKEN | nawk 'BEGIN {

WINMSG0=`rsh $HOST sar -c $SAMPLE_DURATION $SAMPLES_TAKEN | awk 'BEGIN {
}
     NR == 1 {next}
     NR == 2 {next}
     NR == 3 {next}
     NR == 4 {next}

     {printf ("%-8s %9s %9s %9s %5s %5s %12s %12s \n", \
                $1, $2, $3, $4, $5, $6,  $7,  $8 )}

' `


################################################################# DELETE?
#     RSH_RETCODE=$?
#
#      if test $RSH_RETCODE = 1
#      then
# 
#          echo "COMMAND WAS NOT RUN ON $HOST. 
# IT APPEARS THAT 'rsh' FAILED FOR USERID $USER.\n" >> $OUTLIST
#
#      else
############################################################################


############################################################################
## Output those syscall-info lines.
## Add hostname at the front of each line.
############################################################################

echo "$WINMSG0" | sed "s|^|$HOST |"  >> $OUTLIST_PRESORT


###################################################################  DELETE?
#     fi
#     ## END OF if test $RSH_RETCODE = 1
############################################################################

   else 
   ## ELSE OF if test "$PINGCHECK" = ""

      echo  "
NO OUTPUT from $HOST. NOT PING-ABLE."

## >> $OUTLIST

   fi
   ## END OF if test "$PINGCHECK" = ""

done
## END OF for HOST in $HOSTLIST


## FOR TESTING:
#   xpg $OUTLIST_PRESORT


########################################################################
## Sort $OUTLIST_PRESORT by SYSCALLS/SEC column and add to $OUTLIST.
########################################################################

echo "
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
HOST-SORT #1

SORTED BY ** SYSCALLS/SEC **  (column 3)
___________________________________________________________________________________

$COLHEAD0
$COLHEAD1
$COLHEAD2
$COLHEAD3
" >> $OUTLIST

# sort  +2nr -3 $OUTLIST_PRESORT >> $OUTLIST

sort +2nr -3 $OUTLIST_PRESORT >> $OUTLIST

## FOR TESTING:
#   xpg $OUTLIST


########################################################################
## Sort $OUTLIST_PRESORT by SYSREADS/SEC column and add to $OUTLIST.
########################################################################

echo "
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
HOST-SORT #2

SORTED BY ** SYSREADS/SEC **  (column 4)
___________________________________________________________________________________

$COLHEAD0
$COLHEAD1
$COLHEAD2
$COLHEAD3
" >> $OUTLIST

# sort  +3nr -4 $OUTLIST_PRESORT >> $OUTLIST

sort +3nr -4 $OUTLIST_PRESORT >> $OUTLIST

## FOR TESTING:
#   xpg $OUTLIST


########################################################################
## Sort $OUTLIST_PRESORT by SYSWRITES/SEC column and add to $OUTLIST.
########################################################################

echo "
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
HOST-SORT #3

SORTED BY ** SYSWRITES/SEC **  (column 5)
___________________________________________________________________________________

$COLHEAD0
$COLHEAD1
$COLHEAD2
$COLHEAD3
" >> $OUTLIST

# sort  +4nr -5 $OUTLIST_PRESORT >> $OUTLIST

sort +4nr -5 $OUTLIST_PRESORT >> $OUTLIST

## FOR TESTING:
#   xpg $OUTLIST


########################################################################
## Sort $OUTLIST_PRESORT by READ-CHARS/SEC column and add to $OUTLIST.
########################################################################

echo "
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
HOST-SORT #4

SORTED BY ** READ-CHARS/SEC **  (column 8)
___________________________________________________________________________________

$COLHEAD0
$COLHEAD1
$COLHEAD2
$COLHEAD3
" >> $OUTLIST

# sort  +7nr -8 $OUTLIST_PRESORT >> $OUTLIST

sort +7nr -8 $OUTLIST_PRESORT >> $OUTLIST

## FOR TESTING:
#   xpg $OUTLIST


########################################################################
## Sort $OUTLIST_PRESORT by WRITE-CHARS/SEC column and add to $OUTLIST.
########################################################################

echo "
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
HOST-SORT #5

SORTED BY ** WRITE-CHARS/SEC **  (column 9)
___________________________________________________________________________________

$COLHEAD0
$COLHEAD1
$COLHEAD2
$COLHEAD3
" >> $OUTLIST

# sort  +8nr -9 $OUTLIST_PRESORT >> $OUTLIST

sort +8nr -9 $OUTLIST_PRESORT >> $OUTLIST

## FOR TESTING:
#   xpg $OUTLIST



########################################################################
## Add TRAILER to report.
########################################################################

echo "
************************* `date '+%Y %b %d  %T%p'` ***************************

The report above shows currently ping-able SGI 'network' hosts ---

with their 'SYSCALL' activity --- sorted several different ways:

    - by syscalls/sec
    - by sysreads/sec
    - by syswrites/sec
    - by read-chars/sec
    - by write-chars/sec

The 'busiest' hosts, 'at the moment', appear at the top of each sort.


*****************************************************************************
This report can be used to find 'forgotten' processes that are in a tight
processing loop on a host, generating lots of packets on the network.
*****************************************************************************

The short sampling period of $SAMPLE_DURATION seconds on each host ---
and the fact that applications (especially interactive applications)
will generate system calls in 'bursts' --- means that this picture
could change quite a bit from one 'snapshot' to the next.

However, these 'snapshots' can still prove to be quite valuable in
revealing hosts (& applications & users) in intense processing situations.

--------

This report was generated from SGI 'network' hosts  ---  by userid $USER ---

with the candidate hostnames provided via
NIS = Network Information Service, i.e. by the 'ypcat hosts' command.

Hence, the hosts are limited to 'network' hosts on which $USER can login.

--------

The report was assembled by the script

                $0

The script uses a sequence of commands  ('sar -c $SAMPLE_DURATION $SAMPLES_TAKEN','awk','sed')
on each host, followed by several uses of the 'sort' command,
applied to the collected output from all hosts ---

so the HOSTNAMES WITH THE HIGHEST SYSCALL-ACTIVITY ON THE NETWORK
POP UP TO THE TOP OF EACH SORT LIST.

--------
IMPLEMENTATION METHODS:

The ' `basename $0` ' script is intended to be
run periodically (preferably almost daily) by SGI network
Administrators --- Application (CAD/FEA) or System (root) Administrators.

It could be run at a 'quiet' time --- like noon-hour or evening/night.

The script could be implemented as
    - a command alias, via an Administrator's .profile file;
    - a desktop icon, via the 'Find, File QuickFind' tool drawers;
    - a drawer in the SGI toolchest, via an Administrator's .auxchestrc file.

Or the script could be accessed via a drawer in a command 
toolchest-utility,  like 'nethosts_tools'.

This report can be generated via nnsFEAmenu option 'u n ?'
(Utilities, Net-vu, ?).  

...........................................................................
" >> $OUTLIST


#####################################################################
## SHOW THE REPORT.
#####################################################################

## FOR TESTING:
#    echo "
#    DISPLAY: $DISPLAY"
#
#    set -x

#####################################################################
## $FEDIR/scripts/shofil Does not work in an 'xwsh' from toolchest.
## Apparently,
## because of '&' batch invocation of shofil.tk within this script.
#####################################################################
# $FEDIR/scripts/shofil $OUTLIST
#####################################################################

SHOFILENAME=$OUTLIST
export SHOFILENAME

XLPHP_FORMAT="AV"
export XLPHP_FORMAT

## $FEDIR/tkGUIs/shofil.tk &   ## DOES NOT SHOW UP in an 'xwsh'.
$FEDIR/tkGUIs/shofil.tk