#!/bin/ksh ## ## SCRIPT NAME: rmtcmds4netstat_deltapkts_sortOFallhosts ## ## Where: in $FEDIR/scripts where $FEDIR=/apps/nns_com/fea ## ############################################################################ ## PURPOSE: To show the TOP Ethernet PACKET I/O on SGI hosts --- ## 'A SORTED OVERVIEW OF ALL HOSTS ON THE NETWORK' --- ## 'TOP' being in terms of Number of Out-Packets over a sample ## interval of a few seconds. ## ## ******************************************************************* ## ** Helps find 'forgotten' processes, in a tight processing loop, ** ## ** generating lots of file I/O, from/to network clients. ** ## ******************************************************************* ## ## (These processes can have a WORSE IMPACT ON THE NETWORK than ## 'forgotten' processes, in a tight processing loop, ## gobbling LOTS OF CPU CYCLES, on network clients --- when those ## CPU-gobblers that are NOT GENERATING NETWORK I/O.) ## ## This script puts report output in a file and displays it. ## ## The script puts the hostname beside each netstat delta-I/O ## line --- and it sorts the resulting list, for all hosts, ## by delta-I/O --- so the BUSIEST-delta-I/O HOSTS ON THE NETWORK ## POP UP TO THE TOP OF THE LIST. ## ############################################################################ ## CALLED BY: nethosts_tools -> nethosts_tools.chestdef ## in $FEDIR/scripts ## ############################################################################ ## CALL FORMAT: ## /apps/nns_com/fea/scripts/rmtcmds4netstat_deltapkts_sortOFallhosts ## ## ## See nethosts_tools.chestdef in $FEDIR/scripts ## for a call initiated within a 'winterm'. ## ############################################################################ ## METHOD: ## Checks 'delta' packet activity by interface (ec0) on a specified host ## using the 'netstat -i' command. Gets a couple of 'netstat -i' ## reports and differences them. ## Ref: p.184 of 'System Performance Tuning', Mike Loukides, ## O'Reilly & Associates, 1992 ## ############################################################################ ## MAINTENANCE HISTORY: ## Written by: B.Montandon O06 19Apr2000 Based on ## 'rmtcmds4netstat_colls_sortOFallhosts' ## and 'alarm_netidelta_1host' ## in $FEDIR/scripts ## Updated by: B.Montandon O06 19Apr2000 ## ############################################################################ if test "$FEDIR" = "" then FEDIR="/apps/nns_com/fea" fi ############################################################################## ## GENERATE THE NAME OF THE OUTPUT-REPORT FILE. ############################################################################## . $FEDIR/scripts/set_localoutlist OUTLIST=${OUTLIST}_netstat_deltapkts OUTLIST_PRESORT=${OUTLIST}_presort rm -f $OUTLIST rm -f $OUTLIST_PRESORT ############################################################################## ## GENERATE THE HOSTLIST from NIS (ypcat hosts). ############################################################################## ## . /apps/nns_com/fea/scripts/alarm_sethostlist ## . /apps/ideas/cron/set_hostlist HOSTLIST=`ypcat hosts | grep -v "^#" | grep iaw | awk '{print $3}' | sort` # HOSTLIST=$HOSTLIST" engvis00 engprd00 sgia sgib" ## FOR TESTING: # HOSTLIST="iaw005 iaw007 iaw021 iaw030 iaw141" ############################################################################# ## PREPARE A HEADER FOR THE REPORT. ############################################################################# IFACE1="ec0" IFACE2="ef0" # SLEEPSECS=5 SLEEPSECS=3 ############################### ## NOT USED. ## See 'alarm_netidelta_1host' ############################### ## LIMPERCENT=5 # LIMPERCENT=10 # ## LIMRATE=100 # LIMRATE=500 ### FOR TESTING: ## LIMRATE=1 ############################### echo "\ ************************* `date '+%Y %b %d %T%p'` ****************************** NNS SGI NETWORK HOSTS AND THEIR ** ETHERNET PACKET I/O RATES ** at Interface ' $IFACE1 ' or ' $IFACE2 ' SORTED BY PACKET-I/O RATE --- HIGHEST RATES AT THE TOP. (Sample interval at each host is $SLEEPSECS seconds. See comments at bottom of this list.) " > $OUTLIST COLHEAD0=" 1 2 3 4 5 6 7 8" COLHEAD1=" Network" COLHEAD2=" Inter-" COLHEAD3="HOST face In-packets In-errs Out-packets Out-errs Collisions %Collisions" COLHEAD4="NAME Name /sec /sec /sec /sec /sec (% of Opkts)" COLHEAD5="-------- ----- ------------ -------- ------------ -------- ------------ ------------" ############################################################################## ## TO EXECUTE A COMMAND/SCRIPT ON EACH HOST, ## WE ASSURE THE USER HAD A .rhosts FILE. ############################################################################## # echo "+ $USER" > $HOME/.rhosts . $FEDIR/scripts/mak_rhosts ############################################################################## ## LOOP THRU HOSTS -- TO EXECUTE THE 'ps-nawk-sort' COMMAND PIPE, with 'rsh'. ############################################################################## ## Use 'ping' to check the accessibility of the host before issuing ## 'rsh'. 'ping' returns faster on an inaccessible host than 'rsh'. ############################################################################## ## CATCH THE OUTPUT FROM A SINGLE PING WITH A SMALL 4-BYTE PACKET. ############################################################################## for HOST in $HOSTLIST do echo "\nPinging $HOST to check availability." ## FOR TESTING: # set -x ########################################################################### ## PING THE HOST BEFORE TRYING 'rsh' --- and CATCH THE OUTPUT ## FROM A SINGLE PING WITH A SMALL 4-BYTE PACKET --- to use to ## determine whether to do 'rsh' on the host. ########################################################################### PINGOUT=`/usr/etc/ping -s 4 -c 1 $HOST` ## FOR TESTING: # echo "$PINGOUT # # " ## FOR TESTING: # set - PINGCHECK=`echo $PINGOUT | grep '100.0% packet loss'` if test "$PINGCHECK" = "" then ####################################################### ## Get IP address of host. ## (better way? with 'netstat -in'? 'ifconfig'?) ####################################################### ## NOT USED AT THIS TIME. ####################################################### ## ## HOSTNAME=$HOST ## HOSTNAME_ARP=`/usr/etc/arp $HOSTNAME` ## HOSTNAME2=`echo "$HOSTNAME_ARP" |sed "s|-- no entry||"` ## ## BOOTTIME=`who -b` ## # DATETIME=`date` ## DATETIME=`date '+%Y %b %d %T%p'` ####################################################### ############################################################################## ## Using 'rsh $HOST', ## get the '/usr/etc/netstat -i' output, TWICE --- and ## calculate delta-Ipkts & delta-Opkts. ############################################################################## ## Output of '/usr/etc/netstat -i': ## ## Cols: ## ## Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll ## ec0 1500 150.10.2 iaw030.cae.nns~ 69668 0 26274 0 229 ## lo0 8304 loopback localhost 5097 0 5097 0 0 ## ## $1 $2 $3 $4 $5 $6 $7 $8 $9 ## ## Importance to this list: ## ## Name Mtu Network Address Ipkts Ierrs Opkts Oerrs Coll ## * * ** * ** ** ## ## Ref. O'Reilly Assoc. 'System Performance Tuning', p.181-184 & 187-194 ## See awk-based script to diff the netstat output, p.184. ############################################################################## ################################################################################# ## We capture two 'netstat -i' outputs -- about 5 secs apart. ## Put this output in environment variables: ## IPKTS1, IERRS1, OPKTS1, OERRS1, COLLS1 ## IPKTS2, IERRS2, OPKTS2, OERRS2, COLLS2 ################################################################################ ################################################################################ ## First 'netstat -i' snapshot. ################################################################################ SNAP1=`rsh $HOST /usr/etc/netstat -i` RSH_RETCODE=$? ## FOR TESTING: # echo "RSH_RETCODE: $RSH_RETCODE SNAP1: $SNAP1" # if test $RSH_RETCODE = 1 if test "$SNAP1" = "" then ############################################################################# ## Handle the case when 'rsh' fails. ############################################################################# echo "$HOST" >> $OUTLIST_PRESORT else ## FOR TESTING: # set -x IFACE=`echo "$SNAP1" | egrep "$IFACE1|$IFACE2" | awk '{print $1}'` IPKTS1=`echo "$SNAP1" | grep $IFACE | awk '{print $5}'` IERRS1=`echo "$SNAP1" | grep $IFACE | awk '{print $6}'` OPKTS1=`echo "$SNAP1" | grep $IFACE | awk '{print $7}'` OERRS1=`echo "$SNAP1" | grep $IFACE | awk '{print $8}'` COLLS1=`echo "$SNAP1" | grep $IFACE | awk '{print $9}'` sleep $SLEEPSECS ############################################################################# ## Second 'netstat -i' snapshot. ############################################################################# SNAP1=`rsh $HOST /usr/etc/netstat -i` IPKTS2=`echo "$SNAP1" | grep $IFACE | awk '{print $5}'` IERRS2=`echo "$SNAP1" | grep $IFACE | awk '{print $6}'` OPKTS2=`echo "$SNAP1" | grep $IFACE | awk '{print $7}'` OERRS2=`echo "$SNAP1" | grep $IFACE | awk '{print $8}'` COLLS2=`echo "$SNAP1" | grep $IFACE | awk '{print $9}'` ############################################################################# ## Get the differences and add the results line to $OUTLIST_PRESORT. ## Add hostname at the front of each line. ############################################################################# ## FOR TESTING: # set -x IPKTS=`expr $IPKTS2 - $IPKTS1` IERRS=`expr $IERRS2 - $IERRS1` OPKTS=`expr $OPKTS2 - $OPKTS1` OERRS=`expr $OERRS2 - $OERRS1` COLLS=`expr $COLLS2 - $COLLS1` IRATE=`expr $IPKTS / $SLEEPSECS` ORATE=`expr $OPKTS / $SLEEPSECS` IERATE=`expr $IERRS / $SLEEPSECS` OERATE=`expr $OERRS / $SLEEPSECS` COLLRATE=`expr $COLLS / $SLEEPSECS` COLLPCTO=`expr 100 \* $COLLS / $OPKTS` ## COLLPCTIO=`expr 100 \* $COLLS / \( $OPKTS + $OPKTS \)` ############################################################ ## COULD APPLY A THRESHOLD CRITERION FOR PRINTOUT ## OF I/O RATES FOR A HOST: ############################################################ ## ## if test \( $IPKTS -gt $LIMRATE -o $OPKTS -gt $LIMRATE -o \ ## $COLLPCTO -gt $LIMPERCENT \) ## then ############################################################ ## FOR TESTING: # set -x echo "$HOST $IFACE $IRATE $IERATE $ORATE $OERATE $COLLRATE $COLLPCTO" | \ awk '{printf ("%-8s %-5s %12d %8d %12d %8d %12d %7d \n", \ $1, $2, $3, $4, $5, $6, $7, $8 ) }' >> $OUTLIST_PRESORT ## FOR TESTING: # set - ############################################################ ## fi ## ## END OF if test \( $IPKTS -gt ... -o $OPKTS -gt ... ############################################################ fi ## END OF if test "$SNAP1" = "" [$RSH_RETCODE = 1] else ## ELSE OF if test "$PINGCHECK" = "" echo " NO OUTPUT from $HOST. NOT PING-ABLE." ## >> $OUTLIST fi ## END OF if test "$PINGCHECK" = "" done ## END OF for HOST in $HOSTLIST ######################################################################## ## Sort $OUTLIST_PRESORT by $ORATE column and add to $OUTLIST. ######################################################################## echo "\ _____________________________________________________________________________________ SORT BY **OUTPUT** PACKET-I/O RATE --- HIGHEST RATES AT THE TOP. (column 5) _____________________________________________________________________________________ $COLHEAD0 $COLHEAD1 $COLHEAD2 $COLHEAD3 $COLHEAD4 $COLHEAD5 " >> $OUTLIST ## FOR TESTING: # xpg $OUTLIST_PRESORT # sort +4nr -5 $OUTLIST_PRESORT >> $OUTLIST sort +4nr -5 +2nr -3 $OUTLIST_PRESORT >> $OUTLIST ## FOR TESTING: # xpg $OUTLIST ######################################################################## ## Sort $OUTLIST_PRESORT by $IRATE column and add to $OUTLIST. ######################################################################## echo "\ _____________________________________________________________________________________ SORT BY **INPUT** PACKET-I/O RATE --- HIGHEST RATES AT THE TOP. (column 3) _____________________________________________________________________________________ $COLHEAD0 $COLHEAD1 $COLHEAD2 $COLHEAD3 $COLHEAD4 $COLHEAD5 " >> $OUTLIST ## FOR TESTING: # xpg $OUTLIST_PRESORT # sort +2nr -3 $OUTLIST_PRESORT >> $OUTLIST sort +2nr -3 +4nr -5 $OUTLIST_PRESORT >> $OUTLIST ## FOR TESTING: # xpg $OUTLIST ######################################################################## ## Add TRAILER to report. ######################################################################## echo " ************************* `date '+%Y %b %d %T%p'` *************************** The report above shows currently ping-able SGI 'network' hosts --- with their 'Network Interface' activity --- sorted in a couple of ways: - by Out-Packets/sec - by In-Packets/sec The 'busiest' hosts, 'at the moment', appear at the top of each sort. ***************************************************************************** This report can be used to find 'forgotten' processes that are in a tight processing loop on a host, generating lots of packets on the network. ***************************************************************************** The Out-packet (& In-packet & Collisions) data is accumulated by a sample taken over $SLEEPSECS seconds on each host. The short sampling period of $SLEEPSECS seconds on each host --- and the fact that applications (especially interactive applications) will generate network I/O in 'bursts' --- means that this picture could change quite a bit from one 'snapshot' to the next. However, these 'snapshots' can still prove to be quite valuable in revealing hosts (& applications & users) in intense processing situations. The snapshot gives a relatively quick overview of 'network health' --- AND it provides other hosts with which to compare, in one comprehensive list. -------- This report was generated from SGI 'network' hosts --- by userid $USER --- with the candidate hostnames provided via NIS = Network Information Service, i.e. by the 'ypcat hosts' command. Hence, the hosts are limited to 'network' hosts on which $USER can login. -------- The report was assembled by the script $0 The script uses a sequence of commands ('/usr/etc/netstat -i','grep','awk') on each host, followed by a 'sort', by Out-Packets-Rate, applied to the collected output from all hosts --- so the HOSTS WITH THE HIGHEST OUT-PACKETS-RATE ON THE NETWORK ( at the NIC = Network Interface Card ' $IFACE1 ' or ' $IFACE2 ' ) POP UP TO THE TOP OF THE LIST. -------- IMPLEMENTATION METHODS: The ' `basename $0` ' script is intended to be run periodically (preferably almost daily) by SGI network Administrators --- Application (CAD/FEA) or System (root) Administrators. It could be run at a 'quiet' time --- like noon-hour or evening/night --- to detect 'forgotten' processes. The script could be implemented as - a command alias, via an Administrator's .profile file; - a desktop icon, via the 'Find, File QuickFind' tool drawers; - a drawer in the SGI toolchest, via an Administrator's .auxchestrc file. Or the script could be accessed via a drawer in a command toolchest-utility, like 'nethosts_tools'. This report can be generated via nnsFEAmenu option 'u n ?' (Utilities, Net-vu, ?). ........................................................................... " >> $OUTLIST ##################################################################### ## SHOW THE REPORT. ##################################################################### ## FOR TESTING: # echo " # DISPLAY: $DISPLAY" # # set -x ##################################################################### ## $FEDIR/scripts/shofil Does not work in an 'xwsh' from toolchest. ## Apparently, ## because of '&' batch invocation of shofil.tk within this script. ##################################################################### # $FEDIR/scripts/shofil $OUTLIST ##################################################################### SHOFILENAME=$OUTLIST export SHOFILENAME XLPHP_FORMAT="AV" export XLPHP_FORMAT ## $FEDIR/tkGUIs/shofil.tk & ## DOES NOT SHOW UP in an 'xwsh'. $FEDIR/tkGUIs/shofil.tk