#!/bin/ksh # # Script: alarm_netpdelta_1host # # Where: in $FEDIR/scripts where $FEDIR=/apps/nns_com/fea # # # Purpose: To check the host that this script runs on for 'exception'/'alarm' # conditions -- with respect to 'netstat -s', i.e. packet rates at # different TCP/IP protocol levels -- IP, UDP, TCP: # like packets in/out greater than 1,000 per sec. # # Checks 'delta' packet activity by protocol level (ip,tcp,udp) # on a specified host # using the 'netstat -s' command. Gets a couple of 'netstat -s' # reports and differences them. # Similar to a technique for Ethernet-level collisions # based on the 'netstat -i' command. # Ref: p.184 of 'System Performance Tuning', Mike Loukides, # O'Reilly & Associates, 1992 # # Opens an alarm-msg-window on the host specified in $1. # # Could send alarm messages/mail to I-DEAS Admin hosts/people # -- like iaw030/bmo01, iaw070/rgb07, iaw025/wcs05. # Also could append the alarm messages to file $2 (name passed to this # script). # # # Called by: The $FEDIR/scripts/alarm_scan script via # $FEDIR/scripts/feamain -> feautils.menu -> netscan.menu -> # netalarm.menu # OR # The $FEDIR/scripts/alarm_exec1host.menu script via # $FEDIR/scripts/feamain -> feautils.menu -> netscan.menu -> # netalarm.menu # OR # Manually # # Technique: 'alarm_scan' repeatedly does # # rsh iaw### script where ### is 000, 001, 002, ..., 100, ... # # where 'script' is this script # "$FEDIR/scripts/alarm_netpdelta_1host $HOST4MSG" # # Written by: B. Montandon 5May97 to look for clients generating heavy I/O # on the 'sgia' file server -- or clients # clobbering themselves or their sub-net neighbors. # Updated by: B. Montandon 5May97 ############################################################################ # Set DISPLAY location for the following xwsh-winmsg display. ############################################################################ if test ! "$1" = "" then DISPLAY=${1}:0 export DISPLAY fi ############################################################################ # Set a work file (& list file, unused at this time). But could use an in-memory # variable instead of the work file. ############################################################################ # if test -w /local/scratch # then # OUTWORK=/local/scratch/topcpu_temp.all # # OUTLIST=/local/scratch/topcpu_temp.lis # else # OUTWORK=$HOME/topcpu_temp.all # # OUTLIST=$HOME/topcpu_temp.lis # fi ############################################################################ # Prepare alarm message heading. ############################################################################ HOSTNAME=`hostname` HOSTNAME3=`/usr/etc/arp $HOSTNAME` HOSTNAME2=`echo "$HOSTNAME3" |sed "s|-- no entry||"` # echo "*** $HOSTNAME2 ***" >> $1 LIMRATE=100 # For test: # LIMRATE=1 SLEEPSECS=5 DATETIME=`date` WINMSG=`echo "\ ****************** $DATETIME ********************* THIS SGI WORKSTATION HAS EXPERIENCED AN IP/UDP/TCP PACKET IN/OUT RATE GREATER THAN ${LIMRATE} per ${SLEEPSECS} secs. *** HOST: $HOSTNAME2 *** For real-time detail, use '/usr/etc/netstat -C' on this host. IP-in IP-out UDP-in UDP-out TCP-in TCP-out -------- -------- -------- -------- -------- -------- "` ############################################################################## ###### See SAMPLE 'netstat -s' OUTPUT at bottom of this script. ########### ############################################################################## ################################################################################# # Difference two 'netstat -s' outputs -- about 5 or 10 secs apart. # Put this output in environment variables: # IP_IN1, IP_IN2, UDP_IN1, UDP_IN2, TCP_IN1, TCP_IN2 # IP_OUT1, IP_OUT2, UDP_OUT1, UDP_OUT2, TCP_OUT1, TCP_OUT2 ################################################################################ ################################################################################# # First 'netstat -s' snapshot. ################################################################################ SNAP1=`/usr/etc/netstat -s` # set -x IP_IN1=`echo "$SNAP1" | grep 'total packets received' | awk '{print $1}'` IP_OUT1=`echo "$SNAP1" | grep 'packets sent from this host' | awk '{print $1}'` # Some Other IP-in stats to consider: # 'fragments received' # 'packets for this host' # 'packets forwarded' # and about 10 others # Some Other IP-out stats to consider: # 'output packets discarded due to no route' # 'fragments created' UDP_IN1=`echo "$SNAP1" | grep 'total datagrams received' | awk '{print $1}'` UDP_OUT1=`echo "$SNAP1" | grep 'datagrams output' | awk '{print $1}'` # Some Other UDP-in stats to consider: # 'broadcast/multicast datagrams dropped due to no socket' # and 5 others # 'datagrams delivered' # Some Other UDP-out stats to consider: # NONE? TCP_IN1=`echo "$SNAP1" | grep 'packets received' | awk '{print $1}' | tail -1` TCP_OUT1=`echo "$SNAP1" | grep 'packets sent' | awk '{print $1}' | tail -1` # Some Other TCP-in stats to consider: # 'data packets (##...## bytes)' # 'control packets' # and 5 others # Some Other TCP-out stats to consider: # 'acks (for ##...## bytes)' # 'packets (##...## bytes) received in-sequence' # 'out-of-order packets (##...## bytes)' # and about 14 others sleep $SLEEPSECS ################################################################################# # Second 'netstat -s' snapshot. ################################################################################ SNAP1=`/usr/etc/netstat -s` IP_IN2=`echo "$SNAP1" | grep 'total packets received' | awk '{print $1}'` IP_OUT2=`echo "$SNAP1" | grep 'packets sent from this host' | awk '{print $1}'` UDP_IN2=`echo "$SNAP1" | grep 'total datagrams received' | awk '{print $1}'` UDP_OUT2=`echo "$SNAP1" | grep 'datagrams output' | awk '{print $1}'` TCP_IN2=`echo "$SNAP1" | grep 'packets received' | awk '{print $1}' | tail -1` TCP_OUT2=`echo "$SNAP1" | grep 'packets sent' | awk '{print $1}' | tail -1` ################################################################################# # Get the differences and attach the results line to WINMSG2. ################################################################################ # set -x IP_IN=`expr $IP_IN2 - $IP_IN1` IP_OUT=`expr $IP_OUT2 - $IP_OUT1` UDP_IN=`expr $UDP_IN2 - $UDP_IN1` UDP_OUT=`expr $UDP_OUT2 - $UDP_OUT1` TCP_IN=`expr $TCP_IN2 - $TCP_IN1` TCP_OUT=`expr $TCP_OUT2 - $TCP_OUT1` WINMSG2="" if test \( $IP_IN -gt $LIMRATE -o $IP_OUT -gt $LIMRATE -o \ $UDP_IN -gt $LIMRATE -o $UDP_OUT -gt $LIMRATE -o \ $TCP_IN -gt $LIMRATE -o $TCP_OUT -gt $LIMRATE \) then WINMSG2=`echo "$IP_IN $IP_OUT $UDP_IN $UDP_OUT $TCP_IN $TCP_OUT" | \ awk '{printf ("\n %8d %8d %8d %8d %8d %8d \n", \ $1, $2, $3, $4, $5, $6 ) }'` fi ############################################################################ # Display the alarm-msg display -- if WINMSG2 is not empty. ############################################################################ if test ! "$WINMSG2" = "" then WINMSG=${WINMSG}$WINMSG2 export WINMSG MSGLINES=`echo "$WINMSG" | wc -l` MSGLINES=`expr $MSGLINES + 5` ROWLOC=`date +%S` ROWLOC=`expr 10 \* $ROWLOC` # set -x # xwsh -bg gray78 -fg black -hold -autofork \ xwsh -bg red2 -fg snow1 -autofork \ -title "PROTOCOLS_$HOSTNAME" -name "PROTOCOLS_$HOSTNAME" \ -fn -*-screen-medium-r-normal--15-*-*-*-m-80-iso8859-1 \ -geometry 95x${MSGLINES}+090+${ROWLOC} -e /apps/nns_com/fea/scripts/winmsg fi # set - exit ############################################################################## ###### SAMPLE 'netstat -s' OUTPUT ########################################## ############################################################################## ## Output of '/usr/etc/netstat -s': ## ## ip: ## 6742 total packets received ## 0 bad header checksums ## 0 with size smaller than minimum ## 0 with data size < data length ## 0 with header length < data size ## 0 with data length < header length ## 0 with bad options ## 353 fragments received ## 0 fragments dropped (dup or out of space) ## 0 fragments dropped after timeout ## 6462 packets for this host ## 0 packets recvd for unknown/unsupported protocol ## 0 packets forwarded (forwarding enabled) ## 0 packets not forwardable ## 0 redirects sent ## 6351 packets sent from this host ## 0 output packets dropped due to no bufs, etc. ## 15 output packets discarded due to no route ## 1 datagram fragmented ## 5 fragments created ## 0 datagrams that can't be fragmented ## icmp: ## 1 call to icmp_error ## 0 errors not generated 'cuz old message was icmp ## Output histogram: ## destination unreachable : 1 ## 0 messages with bad code fields ## 0 messages < minimum length ## 0 bad checksums ## 0 messages with bad length ## Input histogram: ## destination unreachable : 1 ## time stamp reply : 4 ## 0 message responses generated ## igmp: ## 0 messages received ## 0 messages received with too few bytes ## 0 messages received with bad checksum ## 0 membership queries received ## 0 membership queries received with invalid field(s) ## 0 membership reports received ## 0 membership reports received with invalid field(s) ## 0 membership reports received for groups to which we belong ## 0 membership reports sent ## tcp: ## 874 packets sent ## 80 data packets (4330 bytes) ## 0 data packets (0 bytes) retransmitted ## 433 ack-only packets (22 delayed) ## 0 URG only packets ## 0 window probe packets ## 29 window update packets ## 332 control packets ## 916 packets received ## 651 pcb cache misses ## 386 acks (for 4637 bytes) ## 2 ack predictions ok ## 203 duplicate acks ## 0 acks for unsent data ## 255 packets (113362 bytes) received in-sequence ## 94 in-sequence predictions ok ## 0 completely duplicate packets (0 bytes) ## 0 packets with some dup. data (0 bytes duped) ## 102 out-of-order packets (0 bytes) ## 0 packets (0 bytes) of data after window ## 0 window probes ## 0 window update packets ## 1 packet received after close ## 0 discarded for bad checksums ## 0 discarded for bad header offset fields ## 0 discarded because packet too short ## 0 discarded because of old timestamp ## 128 connection requests ## 103 connection accepts ## 208 connections established (including accepts) ## 228 connections closed (including 0 drops) ## 23 embryonic connections dropped ## 491 segments updated rtt (of 514 attempts) ## 0 retransmit timeouts ## 0 connections dropped by rexmit timeout ## 0 persist timeouts ## 0 keepalive timeouts ## 0 keepalive probes sent ## 0 connections dropped by keepalive ## udp: ## 5548 total datagrams received ## 0 with incomplete header ## 0 with bad data length field ## 0 with bad checksum ## 1 datagram dropped due to no socket ## 17 broadcast/multicast datagrams dropped due to no socket ## 0 datagrams dropped due to full socket buffers ## 5530 datagrams delivered ## 5476 datagrams output ## ##