#!/bin/ksh ## ## SCRIPT NAME: select_subdir_siz_recs_Nlevels_bygui ## ## Where: in $FEDIR/scripts where $FEDIR=/apps/nns_com/fea ## ############################################################################## ## ## PURPOSE: Creates a SIZE-SORTED * /data * OR * /apps * ## SUB-DIRECTORY-SIZES LISTING --- ## by extracting lines from a very large weekly-created file like ## ## ${REPORTDIR}/spac_data_dirs_siz.lis ## or ${REPORTDIR}/spac_apps_dirs_siz.lis ## ## where REPORTDIR="/usr/people/ideasadm/cleandir" ## ## The extract is done in either of two ways ## 1) specifying a number of DIRECTORY LEVELS to select&show ## (1,2,3,4 or 0=all) ## 2) specifying a DIR-STRING to search for. Examples: ## /data/subs, /cae/, /cae, /apps/ideas, /apps/ideas/, ## /subs, /ideas/, /shared, /datamgmt, flowmaster, etc. ## OR 3) a combination of the two, DIR-LEVS & DIR-STRING. ## ## NOTE: For a given number of directory levels, this utility shows ## subdirectories with EXACTLY that number of directory levels. ## ## An older utility script, select_subdir_siz_recs_bygui, ## shows subdirectories to NO MORE THAN that number of dir-levels. ## ## SOME EXAMPLES: ## ## Example1: On ${REPORTDIR}/spac_data_dirs_siz.lis, ## DIRLEVS = 4 AND DIR-string = /data/subs/cae/ gives a summary ## of sizes of subdirectories like /data/subs/cae/USERID --- ## only directories that contain the string /data/subs/cae/ ## AND that contain exactly 4 forward-slash (/) characters. ## ## Example2: On ${REPORTDIR}/spac_apps_dirs_siz.lis, ## DIRLEVS = 3 AND DIR-string = gives a summary ## of sizes of subdirectories of form /engprod/apps/xxxx, ## since /apps is actually a link to /engprod/apps on the SGI server. ## ## Example3: On ${REPORTDIR}/spac_data_dirs_siz.lis, ## DIRLEVS = 0 AND DIR-string = /data/subs/ideas/team/ gives a ## summary of sizes of subdirectories of /data/subs/ideas/team, ## including master, projects, datamgmt#, and shared# subdirectories ## and ANY OTHER sub-directories of /data/subs/ideas/team/ --- ## including directories at lower levels, if there are any at ## lower levels. ## ############################################################################## ## SOURCE OF ALL-SUBDIRECTORIES REPORT FILES: ## The /data & /apps subdir report files (well over 100 pages) ## are created by the ideasadm crontab script ## ## creat_datadirsiz_lists.scr in /apps/ideas/cron ## ## which runs on SGI server 'engprd00'. ## ## See 'crontab -l' on engprd00, as ideasadm. ## See /apps/ideas/cron/READMEcrontab_ideas ## and /ideas/cron/CRONTAB_IDEASADM_engprd00 ## and /apps/ideas/cron/zREADMEcrontab_ideas.OLD97jul14 ## ############################################################################## ## SOME HISTORY OF ALL-SUBDIRECTORIES REPORT FILES: ## ## An older version of 'creat_datadirsiz_lists.scr' created ## summary reports for a "restricted" set of I-DEAS directories like ## /data//ideas/team ## and /data//cae ## where = subs, carrier, nuclear, other -- somewhat ## like Examples 1 & 3 above, but for fewer/fixed 's. ## ## This was inadequate to get a view of which directories were ## eating up most of /data -- as installations like cvn76, cvn77, ## ssn21, etc. were added. ## ## A copy of the old version is in ## /apps/ideas/cron/creat_datadirsiz_lists.OLD99may21 ############################################################################## ## ## CALLED BY: 'spacetools' in $FEDIR/scripts, ## actually 'spacetools.chestdef' in $FEDIR/scripts. ## ## The site SGI toolchest-drawer path to this script, as the user sees it: ## ## SGI Toolchest -> ## Handy Tools -> ## Space Tools -> ## /data, /apps SUBDIRS SIZE-QUERY Utility (weekly 'du' snapshot) -> ## Option to show Subdirectories with EXACTLY N levels ## ## with the other option drawer being ## Option to show Subdirectories to NO MORE THAN N levels ##---------- ## ## An SGI user could also call 'spacetools' or this particular script via ## ## 1) an alias in $HOME/.profile, like ## alias st="/apps/nns_com/fea/scripts/spacetools' ## ## 2) a desktop icon setup via 'Find, File QuickFind' in SGI toolchest ## ## 3) an entry in $HOME/.auxchestrc, like ## "SpaceTools" f.checkexec.sh "/apps/nns_com/fea/scripts/spacetools" ## ## ############################################################################## ## MAINTENANCE HISTORY: ## Written by: Blaise Montandon 05nov2004 Based on script ## 'select_subdir_siz_recs_bygui' ## in $FEDIR/scripts. ## Updated by: Blaise Montandon 09nov2004 Add SUBDIR_UTIL_USAGE_INFO var, to ## set Usage-Info text in the Tk GUI. ## ############################################################################## ##################################################################### ## SET UTILITY-SCRIPT DIR VAR(s), if not set already. ##################################################################### if test "$FEDIR" = "" then FEDIR="/apps/nns_com/fea" fi ######################################################################## ## SET TITLE FOR GUI WINDOW. ######################################################################## WINTITLE="\ SubDir Size List, Extract -- show subdirs with EXACTLY N levels" export WINTITLE ######################################################################## ## COULD ALSO INITIALIZE GUI QUERY PARMS, ## different from the defaults built into the GUI. ######################################################################## # DIRRPT="/usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis" # DIRRPT="/usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis" # DIRLEV=5 # SEARCHSTR="/data/genesis" # export DIRRPT DIRLEV SEARCHSTR ## FOR TESTING: # DIRRPT="/local/scratch/$USER/TEST_data_subdir_sizes.lis" # export DIRRPT DIRLEV_SCALE_LABEL="2) EXACT Dir Levels (depth, N):" export DIRLEV_SCALE_LABEL SUBDIR_UTIL_USAGE_INFO="\ The Directory search string is optional. But if DirLevs=0 (consider ALL dir levels), a search string should be specified -- otherwise, you get the entire directory report --- which is huge ... many 1,000's of directories. In fact, THIS EXTRACT TAKES ABOUT A MINUTE. You can paste an example 'directory Search-String', from below, to the Search-String entry field above. _______________________________________________________________________________ EXAMPLES FOR /data QUERIES from report files spac_data_dirs_siz.lis or spac_data_dirs_siz.lis_prev or spac_data_dirs_siz.lis_prev2: DirLevs SearchString Shows sizes of sub-directories like ------- ----------------- --------------------------------------------- 2 NONE /data/ 3 /cae /data/*/cae 4 /cae/ /data/*/cae/USERID 6 /ideas/team/datamgmt /data/*/ideas/team/datamgmt* 6 /ideas/team/shared /data/*/ideas/team/shared* 4 /data/subs/cae /data/subs/cae/USERID 0 /data/subs/ideas/team /data/subs/ideas/team/datamgmt* AND /data/subs/ideas/team/shared* _______________________________________________________________________________ EXAMPLES FOR /apps QUERIES from report files spac_apps_dirs_siz.lis or spac_apps_dirs_siz.lisprev or spac_apps_dirs_siz.lisprev2: DirLevs SearchString Shows sizes of sub-directories like ------- ----------------- --------------------------------------------- 3 NONE /engprod/apps/ 0 /ideas_ms6/ all subdirs of /engprod/apps/ideas_ms6/ 4 /apps/ideas/ top subdirs of /engprod/apps/ideas/ 0 /apps/msc/ all subdirs of /engprod/apps/msc/ 0 /apps/ABAQUS- all subdirs of form /engprod/apps/ABAQUS-* 0 /apps/CF all subdirs of form /engprod/apps/CFDesign* or /engprod/apps/CFdesign* 0 /apps/flowmaster all subdirs of form /engprod/apps/flowmaster* 0 /apps/aw all subdirs of form /engprod/apps/aw* 0 /apps/eai all subdirs of /engprod/apps/eai 0 /apps/envision all subdirs of form /engprod/apps/envision* 0 /apps/matlab all subdirs of form /engprod/apps/matlab* " export SUBDIR_UTIL_USAGE_INFO ######################################################################## ## PROMPT FOR QUERY PARMS -- cron-SUBDIRS-SIZE-LISTING, DIR-LEVELS, ## and (optional) DIR-SEARCH-STRING. ######################################################################## ## NOTE: We activated the while-loop --- and when the TkGUI script, ## 'enter_subdir_queries.tk', ## is shown a 2nd, 3rd, ... time, it is updated to use the ## DIRRPT, DIRLEVS, SEARCHSTR env vars ## to re-initialize the GUI to current (last used) settings. ######################################################################## ## while true while : do TEMP=`$FEDIR/tkGUIs/enter_subdir_queries.tk` eval "$TEMP" ####################################################################### ## TEMP contains a string like ## DIRRPT=""; DIRLEVS=""; SEARCHSTR="" ####################################################################### ## FOR TESTING: # echo "$TEMP" if test "$DIRRPT" = "" then exit fi ##################################################################### ## CHECK THAT THE REPORT FILE ENTERED EXISTS. ##################################################################### if test ! -f $DIRRPT then CONFIRM_TEXT="\ Specified Directory-Report Filename: $DIRRPT not found or does not exist. A search of an appropriate subdir-sizes list file, like spac_data_dirs_siz.lis or spac_apps_dirs_siz.lis will take on the order of 20 SECS TO A MINUTE. Exiting. " export CONFIRM_TEXT CONFIRM_GEOM="+080+080" CONFIRM_MINSIZE="350 150" CONFIRM_TITLE="$WINTITLE" export CONFIRM_TITLE CONFIRM_MINSIZE CONFIRM_GEOM CONFIRM_COLOR="#3DADFF" export CONFIRM_COLOR CONFIRM_FONT="-adobe-helvetica-bold-r-normal--*-100-*-*-p-*-*-*" export CONFIRM_FONT JUNK=`$FEDIR/tkGUIs/confirm.tk DISMISS` & ## exit export DIRRPT DIRLEVS SEARCHSTR continue fi ##################################################################### ## At this point, 'eval $TEMP' should have set the three vars --- ## DIRRPT DIRLEVS SEARCHSTR. Example: ## ## DIRRPT="/usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis"; ## DIRLEVS="3"; ## SEARCHSTR="" ##################################################################### ## Make sure that we ## EXIT if DIRLEVS & SEARCHSTR are null (user cancelled query). ##################################################################### if test \( "$DIRLEVS" = "" -a "$SEARCHSTR" = "" \) then exit fi ##################################################################### ## NOTE: At this point the Report file var, DIRRPT, should be set. ## Typical 'ideasadm' cron job output names in May99 are ## /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis ## /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis_prev ## /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis_prev2 ## and ## /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis ## /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis_prev ## /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis_prev2 ##################################################################### ##################################################################### ## PREP TEXT OUTPUT LIST FILE -- and a temp file. ##################################################################### . $FEDIR/scripts/set_localoutlist OUTLIST2=${OUTLIST}_temp ##################################################################### ## PREP HEADER FOR LIST FILE. Cases considered: ## 1) SEARCHSTR not-blank & DIRLEVS = 0 (ALL) ## 2) SEARCHSTR not-blank & DIRLEVS > 0 ## 3) SEARCHSTR blank & DIRLEVS > 0 ## (show ALL subdirs of DIRRPT with EXACTLY $DIRLEVS subdir levs) ## 4) SEARCHSTR blank & DIRLEVS = 0 ## (shows ALL subdirs of the DIRRPT, with formatting of Kbytes to Gig) ## ## Also add 1st rec of DIRRPT list file (create-date) to the Extract ## Report Header. ##################################################################### SIZE_UNITS="Gigabytes" if test \( ! "$SEARCHSTR" = "" -a "$DIRLEVS" = "0" \) then SEARCH_CRIT="\ ****************************************************** Dir-Levs: $DIRLEVS Search-string: $SEARCHSTR ****************************************************** (I.e. into this report, EXTRACT *ALL* sub-directory size records that CONTAIN THE SEARCH STRING.)" fi if test \( ! "$SEARCHSTR" = "" -a ! "$DIRLEVS" = "0" \) then SEARCH_CRIT="\ ****************************************************** Dir-Levs: $DIRLEVS Search-string: $SEARCHSTR ****************************************************** (I.e. into this report, EXTRACT sub-directory size records that contain the search string and **EXACTLY** $DIRLEVS directory levels.)" fi if test \( "$SEARCHSTR" = "" -a "$DIRLEVS" = "0" \) then SEARCH_CRIT="\ ****************************************************** Dir-Levs: $DIRLEVS Search-string: $SEARCHSTR ****************************************************** I.e. Show *ALL* sub-directory size recs in this report. No extract. This is essentially the same as simply looking at the original 'cron' report --- but with reformatting sizes in Kilobytes to sizes in $SIZE_UNITS. Report is PROBABLY HUGE. DO NOT PRINT." fi if test \( "$SEARCHSTR" = "" -a ! "$DIRLEVS" = "0" \) then SEARCH_CRIT="\ ****************************************************** Dir-Levs: $DIRLEVS Search-string: $SEARCHSTR ****************************************************** (I.e. into this report, EXTRACT *ALL* sub-directory size records for directories with *EXACTLY* $DIRLEVS directory levels.)" fi DATELINE=`head -1 $DIRRPT` echo "\ ................... `date '+%Y %b %d %a %T%p %Z'` ...................... DISK USAGE of SUB-DIRECTORIES. (SIZE-SORTED) AN EXTRACT FROM 'cron' REPORT file $DIRRPT which was generated mid-week. **** First line of 'cron' report file --- *DATE* that cron job was run: **** $DATELINE **************** SEARCH CRITERION FOR THIS report-EXTRACT: **************** $SEARCH_CRIT Disk usage Subdirectory (Gigabytes) name ------------- ------------ TerGig.MegKil | | | | " > $OUTLIST ###################################################################### ## Run 'awk' to select recs for report -- using $DIRLEVS and $SEARCHSTR. ###################################################################### ## IF DIRLEVS NOT = 0, use an awk program that counts the slashes (/) ## in the directory name of each record of the $DIRRPT file and ## prints the records for which the count EQUALS $DIRLEVS. ## ## We 'consider' the non-comment, non-blank recs of the $DIRRPT file ## that match the $DIRLEVS request. ##################################################################### ## ELSE (IF DIRLEVS = 0), 'consider' ALL the non-comment, non-blank recs ## of the $DIRRPT file. ##################################################################### ## In either case, right-justify the subdir size-numbers in the 1st col, ## using awk and format the sizes in Gigabytes instead of Kbytes. ##################################################################### if test "$SEARCHSTR" = "NONE" then SEARCHSTR="" fi if test ! "$DIRLEVS" = "0" then ###################################################################### ## Use an awk that prints $DIRRPT lines whose subdirectory names ## have *EXACTLY* the number of slashes indicated by $DIRLEVS. ###################################################################### ## Use ## grep -v "^#" $DIRRPT | grep -v "^$" ## to remove the comment lines and null lines from the input list. ###################################################################### ## We might have to use code fragments like these: ## ## if test "$SEARCHSTR" = "" ## then ## ... ## else ## ... ## fi ## ## awk -v SEL_STR="$SEARCHSTR" -v DIR_LEVEL2MATCH="$DIRLEVS" \ ###################################################################### grep -v "^#" $DIRRPT | grep -v "^$" | grep "$SEARCHSTR" | \ awk -v SEL_STR="$SEARCHSTR" -v DIR_LEVEL2MATCH="$DIRLEVS" \ '{ ## if ( $0 ~ SEL_STR ) { IXfirstslash = index($0,"/") FULLDIRNAME = substr($0,IXfirstslash) ## FOR TESTING: # printf (" Current Record: %s \n", $0 ) # printf (" FULLDIRNAME : %s \n", $FULLDIRNAME ) LEFTOVER = FULLDIRNAME DIRLEV_CNT = 0 while ( LEFTOVER != "" ) { IXslash = index(LEFTOVER,"/") if (IXslash == 0) {break} DIRLEV_CNT = DIRLEV_CNT + 1 LEFTOVER = substr(LEFTOVER,IXslash + 1) } ## END OF while ( LEFTOVER != "" ) if ( DIRLEV_CNT == DIR_LEVEL2MATCH ) { printf ("%13.6f %s \n", $1/1000000, $2 ) } ## END OF if ( DIRLEV_CNT == DIR_LEVEL2MATCH ) ## } ## ## END OF if ( $0 ~ SEL_STR ) }' >> $OUTLIST else ###################################################################### ## Consider *ALL* the dir-report-lines for the output. ## I.e. if "$DIRLEVS" = "0". ###################################################################### if test "$SEARCHSTR" = "" then grep -v "^#" $DIRRPT | grep -v "^$" | \ awk \ '{printf ("%13.6f %s \n", $1/1000000, $2 )}' \ >> $OUTLIST else grep -v "^#" $DIRRPT | grep -v "^$" | \ awk -v SEL_STR="$SEARCHSTR" \ '{ if ( $0 ~ SEL_STR ) {printf ("%13.6f %s \n", $1/1000000, $2 )}}' \ >> $OUTLIST fi fi ## END OF if test ! "$DIRLEVS" = "0" ################################################# ## ADD TRAILER TO THE REPORT. ################################################# echo " | | | | TerGig.MegKil ------------- ------------ Disk usage Subdirectory (Gigabytes) name ................... `date '+%Y %b %d %a %T%p %Z'` ...................... ADDITIONAL INFO ABOUT THIS REPORT EXTRACT: This report is SORTED BY SUB-DIRECTORY SIZE --- LARGEST SUB-DIRECTORIES AT THE TOP. This *EXTRACT* was obtained from a list of 1,000's of subdirectory sizes in file $DIRRPT The 'cron' job that built that report file essentially did the command du -k | sort +0 -1 -nr on the server where the /data & /apps directories 'reside'. The cron script uses = /data and = /apps. IT TAKES MANY MINUTES FOR THAT CRON JOB TO RUN --- so these 'du' commands are not ones that you would want to do manually at the time of an out-of-disk-space emergency. To reduce load on the system and, mainly, to get relatively quick response, THIS REPORT IS EXTRACTED FROM one of these 'ideasadm' weekly CRON JOB REPORT LISTINGs done on the server. This extract normally runs in less than a minute. This relatively fast query time is especially helpful when quite a few different types of extracts/listings must be generated. ------------------------------------- HOW TO GENERATE THIS REPORT *EXTRACT*: This EXTRACT was built by the script $0 which can be run via a site SGI 'spacetools' toolchest. A toolchest-drawer path to this utility: SGI Toolchest -> Handy Tools -> Space Tools -> /data, /apps SUBDIRS SIZE-QUERY Utility (weekly 'du' snapshot) -> Option to show Subdirectories with EXACTLY N levels ---- Another drawer (and utility script), namely Option to show Subdirectories to NO MORE THAN N levels provides a slightly different type of extract. It is not as easy to read, having various directory levels inter-mixed in the list --- but it shows sizes of PARENT directories. ------------------------------------- ALTERNATE WAYS TO INVOKE THIS UTILITY: You can see other ways of initiating this subdirectory-sizes extract utility --- i.e. other ways of initiating the script $0 --- by using the 'Help' button at the bottom of the 'SubDir Size List, Extract' GUI. That Help option shows the file $FEDIR/helps/subdir_queries.hlp In particular, that help points out that a user could call 'spacetools' or this particular script via 1) an alias in $HOME/.profile, like alias st='$FEDIR/scripts/spacetools' 2) a desktop icon setup via 'Find, File QuickFind' in SGI toolchest 3) an entry in $HOME/.auxchestrc, like \"SpaceTools\" f.checkexec.sh \"$FEDIR/scripts/spacetools\" --------------------- THE WHOLE BALL OF WAX (the complete 'raw' listings): The complete sub-directory size listings of /data and /apps can be seen with the site 'xpg' utility (or with 'nedit -read'). xpg /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis xpg /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis_prev xpg /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis_prev2 xpg /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis xpg /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis_prev xpg /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis_prev2 Warning: There are tens of 1000's of lines (sub-directories) in these listings. These file sizes are about 2 to 4 Megabytes. You should not print these files, but you can use the 'xpg' 'Show All Matches' button to get extracts from those huge lists. If the extracts are sufficiently small, you may want to print those extracts using the 'Print' button on the 'xpg' utility. ........................................................................... " >> $OUTLIST ##################################################################### ## SHOW THE SUBDIRECTORY-SIZES EXTRACT. ##################################################################### ## FOR TESTING: # set -x ## A METHOD OF USING 'shofil.tk': # export SHOFILENAME="$OUTLIST" # $FEDIR/tkGUIs/shofil.tk & $FEDIR/scripts/shofil $OUTLIST ## FOR TESTING: # set - ##################################################################### ## SET ENTRY FIELDS (QUERY PARMS) FOR NEXT GUI PROMPT. ##################################################################### # DIRRPT= # DIRLEVS= # SEARCHSTR= export DIRRPT DIRLEVS SEARCHSTR ##################################################################### ## END OF PROMPTING LOOP, for cron-SUBDIRS-SIZE-LISTING, DIR-LEVELS, ## and (optional) DIR-SEARCH-STRING. ##################################################################### done ## END OF 'while :' (which is equivalent to 'while true')