#!/bin/ksh ## ## SCRIPT NAME: select_subdir_siz_recs_bygui ## ## Where: in $FEDIR/scripts where $FEDIR=/apps/nns_com/fea ## ############################################################################## ## ## PURPOSE: Creates a SIZE-SORTED * /data * OR * /apps * ## SUB-DIRECTORY-SIZES LISTING --- ## by extracting lines from a very large weekly-created file like ## ## ${REPORTDIR}/spac_data_dirs_siz.lis ## or ${REPORTDIR}/spac_apps_dirs_siz.lis ## ## where REPORTDIR="/usr/people/ideasadm/cleandir" ## ## The extract is done in either of two ways ## 1) specifying a number of DIRECTORY LEVELS to select&show ## (1,2,3,4,... or 0=all) ## 2) specifying a DIR-STRING to search for. Examples: ## /data/subs, /cae/, /cae, /apps/ideas, /apps/ideas/, ## /subs, /ideas/, /shared, /datamgmt, flowmaster, etc. ## OR 3) a combination of the two, DIR-LEVS & DIR-STRING. ## ## NOTE: For a given number of directory levels, N, this utility shows ## subdirectories to NO MORE THAN that number of directory levels ## --- i.e. this utility shows parent directories, down to the ## sub-directories consisting of N levels (N slashes in the full name). ## ## A newer utility script, select_subdir_siz_recs_Nlevels_bygui, ## shows subdirectories with EXACTLY N directory-levels. ## ## SOME EXAMPLES: ## ## Example1: For input list ${REPORTDIR}/spac_data_dirs_siz.lis, ## DIRLEVS = 4 AND DIR-string = /data/subs/cae/ gives a summary ## of sizes of subdirectories like /data/subs/cae/USERID --- ## including parent directories like /data and /data/subs and ## /data/subs/cae. ## ## Example2: On ${REPORTDIR}/spac_apps_dirs_siz.lis, ## DIRLEVS = 3 AND DIR-string = gives a summary ## of sizes of subdirectories of form /engprod/apps/xxxx --- ## including parent directories like /engprod/apps. ## ## Note: Since /apps is actually a link to /engprod/apps, ## on the SGI server, we get directories prefixed by ## /engprod/apps, rather than simply prefixed by /apps. ## ## Example3: For input list ${REPORTDIR}/spac_data_dirs_siz.lis, ## DIRLEVS = 0 AND DIR-string = /data/subs/ideas/team/ gives a ## summary of sizes of ALL subdirectories of /data/subs/ideas/team, ## including master, projects, datamgmt#, and shared# subdirectories ## and ANY other sub-directories of /data/subs/ideas/team/ --- ## including directories at lower levels, if there are any at ## lower levels. ## ############################################################################## ## SOURCE OF ALL-SUBDIRECTORIES REPORT FILES: ## The /data & /apps subdir report files (well over 100 pages) ## are created by the ideasadm crontab script ## ## creat_datadirsiz_lists.scr in /apps/ideas/cron ## ## --- which runs on the SGI file server, 'engprd00'. ## ## See 'crontab -l' on engprd00, as ideasadm. ## See /apps/ideas/cron/READMEcrontab_ideasapps ## and /ideas/cron/CRONTAB_IDEASADM_engprd00 ## and /apps/ideas/cron/zREADMEcrontab_ideas.OLD97jul14 ## ############################################################################## ## SOME HISTORY OF ALL-SUBDIRECTORIES REPORT FILES: ## ## An older version of 'creat_datadirsiz_lists.scr' created ## summary reports for a "restricted" set of I-DEAS directories like ## /data//ideas/team ## and /data//cae ## where = subs, carrier, nuclear, other -- somewhat ## like Examples 1 & 3 above, but for fewer, fixed 's. ## ## This was inadequate to get a view of which directories were ## eating up most of /data -- as installations like cvn76, cvn77, ## ssn21, etc. were added. ## ## A copy of the old version is in ## /apps/ideas/cron/creat_datadirsiz_lists.OLD99may21 ############################################################################## ## ## CALLED BY: 'spacetools' in $FEDIR/scripts, ## actually 'spacetools.chestdef' in $FEDIR/scripts. ## ## The site SGI toolchest-drawer path to this script, as the user sees it: ## ## SGI Toolchest -> ## Handy Tools -> ## Space Tools -> ## /data, /apps SUBDIRS SIZE-QUERY Utility (weekly 'du' snapshot) -> ## Option to show Subdirectories to NO MORE THAN N levels ## ## with the other option drawer being ## Option to show Subdirectories with EXACTLY N levels ##---------- ## ## An SGI user could also call 'spacetools' or this particular script via ## ## 1) an alias in $HOME/.profile, like ## alias st="/apps/nns_com/fea/scripts/spacetools' ## ## 2) a desktop icon setup via 'Find, File QuickFind' in SGI toolchest ## ## 3) an entry in $HOME/.auxchestrc, like ## "SpaceTools" f.checkexec.sh "/apps/nns_com/fea/scripts/spacetools" ## ## ############################################################################## ## MAINTENANCE HISTORY: ## Written by: B.Montandon O06 2Jun1999 based on text-prompting version ## $FEDIR/scripts/select_subdir_siz_recs -- ## which was based on scripts like ## $FEDIR/scripts/get_ideasfe_usage ## $FEDIR/scripts/prtplt_lookup_Qinfo ## and /apps/ideas/cron/creat_datadirsiz_lists ## ## Updated by: B.Montandon O06 3Jun1999 Change to accept DIRRPT as a full ## report filename, from an entry field ## in the GUI -- rather than a report ## indicator (/data or /apps) from ## radio-buttons in the GUI. ## ## Updated by: B.Montandon O06 21Jun1999 Add top (date) line of DIRRPT ## to the extracted report. ## Updated by: B.Montandon O06 13Sep2000 Chg size col from Kbytes to ## Megabtyes. Move desc. of report ## to bottom of report. ## Updated by: B.Montandon O06 14Sep2000 Improve speed by doing select in ## awk rather than separate grep step. ## Move the build of report- ## header (timestamp) before the awk. ## Updated by: B.Montandon O06 14Sep2000 Add check if DIRRPT file exists, ## to catch user keying errors. ## Updated by: B.Montandon O06 19Sep2000 Implement 'while' loop for ## repeat prompts by the GUI. ## Updated by: Blaise Montandon 05nov2004 Chg output units from Megabytes ## to Gigabytes. Also update some ## text in the report trailer and ## make some changed based on the new ## script 'select_subdir_siz_recs_Nlevels_bygui'. ## E.g. chg WINTITLE & add DIRLEV_SCALE_LABEL. ## ## Updated by: Blaise Montandon 09nov2004 Add SUBDIR_UTIL_USAGE_INFO var, to ## set Usage-Info text in the Tk GUI. ## ############################################################################## ##################################################################### ## SET UTILITY-SCRIPT DIR VAR(s), if not set already. ##################################################################### if test "$FEDIR" = "" then FEDIR="/apps/nns_com/fea" fi ######################################################################## ## SET TITLE FOR GUI WINDOW(S). ######################################################################## WINTITLE="\ SubDir Size List, Extract --- show subdirs to NO MORE THAN N levels" export WINTITLE ######################################################################## ## COULD ALSO INITIALIZE GUI QUERY PARMS, ## different from the defaults built into the GUI. ######################################################################## # DIRRPT="/usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis" # DIRRPT="/usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis" # DIRLEV=5 # SEARCHSTR="/data/genesis" # export DIRRPT DIRLEV SEARCHSTR ## FOR TESTING: # DIRRPT="/local/scratch/$USER/TEST_data_subdir_sizes.lis" # export DIRRPT DIRLEV_SCALE_LABEL="2) MAX Dir Levels (max depth, N):" export DIRLEV_SCALE_LABEL SUBDIR_UTIL_USAGE_INFO="\ The Directory search string is optional. But if DirLevs=0 (consider ALL dir levels), a search string should be specified -- otherwise, you get the entire directory report --- which is huge ... many 1,000's of directories. In fact, THIS EXTRACT TAKES ABOUT A MINUTE. You can paste an example 'directory Search-String', from below, to the Search-String entry field above. _______________________________________________________________________________ EXAMPLES FOR /data QUERIES from report files spac_data_dirs_siz.lis or spac_data_dirs_siz.lis_prev or spac_data_dirs_siz.lis_prev2: DirLevs SearchString Shows sizes of sub-directories like ------- ----------------- --------------------------------------------- 2 NONE /data/ 3 /cae /data/*/cae 4 /cae/ /data/*/cae/USERID 6 /ideas/team/datamgmt /data/*/ideas/team/datamgmt* 6 /ideas/team/shared /data/*/ideas/team/shared* 4 /data/subs/cae /data/subs/cae/USERID 0 /data/subs/ideas/team /data/subs/ideas/team/datamgmt* AND /data/subs/ideas/team/shared* _______________________________________________________________________________ EXAMPLES FOR /apps QUERIES from report files spac_apps_dirs_siz.lis or spac_apps_dirs_siz.lisprev or spac_apps_dirs_siz.lisprev2: DirLevs SearchString Shows sizes of sub-directories like ------- ----------------- --------------------------------------------- 3 NONE /engprod/apps/ 0 /ideas_ms6/ all subdirs of /engprod/apps/ideas_ms6/ 4 /apps/ideas/ top subdirs of /engprod/apps/ideas/ 0 /apps/msc/ all subdirs of /engprod/apps/msc/ 0 /apps/ABAQUS- all subdirs of form /engprod/apps/ABAQUS-* 0 /apps/CF all subdirs of form /engprod/apps/CFDesign* or /engprod/apps/CFdesign* 0 /apps/flowmaster all subdirs of form /engprod/apps/flowmaster* 0 /apps/aw all subdirs of form /engprod/apps/aw* 0 /apps/eai all subdirs of /engprod/apps/eai 0 /apps/envision all subdirs of form /engprod/apps/envision* 0 /apps/matlab all subdirs of form /engprod/apps/matlab* " export SUBDIR_UTIL_USAGE_INFO ######################################################################## ## PROMPT FOR QUERY PARMS -- cron-SUBDIRS-SIZE-LISTING, DIR-LEVELS, ## and (optional) DIR-SEARCH-STRING. ######################################################################## ## NOTE: We activated the while-loop --- and when the TkGUIscript, ## 'enter_subdir_queries.tk', ## is shown a 2nd, 3rd, ... time, it is updated to use the ## DIRRPT, DIRLEVS, SEARCHSTR env vars ## to re-initialize the GUI to current (last used) settings. ######################################################################## ## while true while : do TEMP=`$FEDIR/tkGUIs/enter_subdir_queries.tk` eval "$TEMP" ####################################################################### ## TEMP contains a string like ## DIRRPT=""; DIRLEVS=""; SEARCHSTR="" ####################################################################### ## For testing: # echo "$TEMP" if test "$DIRRPT" = "" then exit fi ##################################################################### ## CHECK THAT THE REPORT FILE ENTERED EXISTS. ##################################################################### if test ! -f "$DIRRPT" then CONFIRM_TEXT="\ Specified Directory-Report Filename: $DIRRPT not found or does not exist. REMINDER: A search-extract of an appropriate subdir-sizes list file, like spac_data_dirs_siz.lis or spac_apps_dirs_siz.lis file will take on the order of 20 SECS TO A MINUTE. Exiting. " export CONFIRM_TEXT CONFIRM_GEOM="+080+080" CONFIRM_MINSIZE="350 150" CONFIRM_TITLE="$WINTITLE" export CONFIRM_TITLE CONFIRM_MINSIZE CONFIRM_GEOM CONFIRM_COLOR="#3DADFF" export CONFIRM_COLOR CONFIRM_FONT="-adobe-helvetica-bold-r-normal--*-100-*-*-p-*-*-*" export CONFIRM_FONT JUNK=`$FEDIR/tkGUIs/confirm.tk DISMISS` & ## exit export DIRRPT DIRLEVS SEARCHSTR continue fi ##################################################################### ## At this point, 'eval $TEMP' should have set the three vars --- ## DIRRPT DIRLEVS SEARCHSTR. Example: ## ## DIRRPT="/usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis"; ## DIRLEVS="3"; ## SEARCHSTR="" ##################################################################### ## EXIT if DIRLEVS & SEARCHSTR are null (user cancelled query) ##################################################################### if test \( "$DIRLEVS" = "" -a "$SEARCHSTR" = "" \) then exit fi ##################################################################### ## NOTE: At this point the Report file var, DIRRPT, should be set. ## Typical 'ideasadm' cron job output names in May99 are ## /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis ## /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis_prev ## /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis_prev2 ## and ## /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis ## /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis_prev ## /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis_prev2 ##################################################################### ##################################################################### ## PREP TEXT OUTPUT LIST FILE -- and a temp file. ##################################################################### . $FEDIR/scripts/set_localoutlist OUTLIST2=${OUTLIST}_temp ##################################################################### ## PREP HEADER FOR LIST FILE. Cases considered: ## 1) SEARCHSTR not-blank & DIRLEVS = 0 (ALL) ## 2) SEARCHSTR not-blank & DIRLEVS > 0 ## 3) SEARCHSTR blank & DIRLEVS > 0 ## (shows ALL dirs of DIRRPT with UP TO $DIRLEVS of subdir levs) ## 4) SEARCHSTR blank & DIRLEVS = 0 ## (shows ALL subdirs of the DIRRPT, with formatting of Kbytes to Gig) ## ## Also add 1st rec of DIRRPT list file (create-date) to the Extract ## Report Header. ##################################################################### SIZE_UNITS="Gigabytes" if test \( ! "$SEARCHSTR" = "" -a "$DIRLEVS" = "0" \) then SEARCH_CRIT="\ ****************************************************** Dir-Levs: $DIRLEVS Search-string: $SEARCHSTR ****************************************************** (I.e. into this report, EXTRACT *ALL* sub-directory size records that CONTAIN THE SEARCH STRING.)" fi if test \( ! "$SEARCHSTR" = "" -a ! "$DIRLEVS" = "0" \) then SEARCH_CRIT="\ ****************************************************** Dir-Levs: $DIRLEVS Search-string: $SEARCHSTR ****************************************************** (I.e. into this report, EXTRACT sub-directory size records whose sub-directory name CONTAINS THE SEARCH STRING AND whose sub-directory name contains NO MORE THAN $DIRLEVS directory levels.)" fi if test \( "$SEARCHSTR" = "" -a "$DIRLEVS" = "0" \) then SEARCH_CRIT="\ ****************************************************** Dir-Levs: $DIRLEVS Search-string: $SEARCHSTR ****************************************************** I.e. Show *ALL* sub-directory size recs in this report. No extract. This is essentially the same as simply looking at the original 'cron' report --- but with reformatting of sizes in Kilobytes to sizes in $SIZE_UNITS. Report is PROBABLY HUGE. DO NOT PRINT." fi if test \( "$SEARCHSTR" = "" -a ! "$DIRLEVS" = "0" \) then SEARCH_CRIT="\ ****************************************************** Dir-Levs: $DIRLEVS Search-string: $SEARCHSTR ****************************************************** (I.e. into this report, EXTRACT all sub-directory size records whose sub-directory name contains NO MORE THAN $DIRLEVS directory levels.)" fi DATELINE=`head -1 $DIRRPT` echo "\ ................... `date '+%Y %b %d %a %T%p %Z'` ...................... DISK USAGE of SUB-DIRECTORIES. (SIZE-SORTED) AN EXTRACT FROM 'cron' REPORT file $DIRRPT which was generated mid-week. **** First line of 'cron' report file --- *DATE* that cron job was run: **** $DATELINE **************** SEARCH CRITERION FOR THIS report-EXTRACT: **************** $SEARCH_CRIT Disk usage Subdirectory (Gigabytes) name ------------- ------------ TerGig.MegKil | | | | " > $OUTLIST ###################################################################### ## Run 'awk' to select recs for report -- using DIRLEVS and SEARCHSTR. ###################################################################### ## IF DIRLEVS NOT = 0, Set a DIRMASK according to # of DIRLEVS ## requested -- like DIRMASK=/.*/.*/ (Three slashes) when DIRLEVS=2 ## -- and 'consider' the non-comment, non-blank recs of the $DIRRPT file ## that match that DIRMASK. ##################################################################### ## ELSE (IF DIRLEVS = 0), 'consider' ALL the non-comment, non-blank recs ## of the $DIRRPT file. ##################################################################### ## In either case, right-justify the subdir size-numbers in the 1st col, ## using awk and format the sizes in Gigabytes instead of Kbytes. ##################################################################### if test "$SEARCHSTR" = "NONE" then SEARCHSTR="" fi if test ! "$DIRLEVS" = "0" then ###################################################################### ## Build $DIRMASK to use in 'grep -v' that removes dir-report-lines ## that have *MORE THAN* the number of slashes indicated by $DIRLEVS. ###################################################################### ## For example: ' grep -v "/.*/.*/" ' will throw away all dirnames ## with 3 or more slashes --- leaving names like /xxx and /xxx/yyy. ###################################################################### DIRMASK="/" COUNT=0 while test $COUNT -lt $DIRLEVS do DIRMASK=${DIRMASK}".*/" COUNT=`expr $COUNT + 1` done if test "$SEARCHSTR" = "" then grep -v "^#" $DIRRPT | grep -v "^$" | grep -v "$DIRMASK" | \ awk \ '{printf ("%13.6f %s \n", $1/1000000, $2 )}' \ >> $OUTLIST else grep -v "^#" $DIRRPT | grep -v "^$" | grep -v "$DIRMASK" | \ awk -v SEL_STR="$SEARCHSTR" \ '{ if ( $0 ~ SEL_STR ) {printf ("%13.6f %s \n", $1/1000000, $2 )}}' \ >> $OUTLIST fi ###################################################################### ## *OLD* - BEFORE CONVERSION OF KBYTES TO GIG AND BEFORE USE OF SEL_STR. # awk '{printf ("%11s %s \n", $1, $2 )}' > $OUTLIST2 ###################################################################### else ###################################################################### ## Consider *ALL* the dir-report-lines for the output. ## I.e. no 'grep -v' using $DIRMASK. ###################################################################### if test "$SEARCHSTR" = "" then grep -v "^#" $DIRRPT | grep -v "^$" | \ awk \ '{printf ("%13.6f %s \n", $1/1000000, $2 )}' \ >> $OUTLIST else grep -v "^#" $DIRRPT | grep -v "^$" | \ awk -v SEL_STR="$SEARCHSTR" \ '{ if ( $0 ~ SEL_STR ) {printf ("%13.6f %s \n", $1/1000000, $2 )}}' \ >> $OUTLIST fi ###################################################################### ## OLD - BEFORE CONVERSION OF KBYTES TO GIG AND BEFORE USE OF SEL_STR. # awk '{printf ("%11s %s \n", $1, $2 )}' > $OUTLIST2 ###################################################################### fi ##################################################################### ## NOT NEEDED ANY MORE. PUT SELECT-FUNCTION IN THE 'awk' above. ##################################################################### # ##################################################################### # ## IF SEARCHSTR NOT = blank, extract search string recs # ## from OUTLIST2, which contains a specified number of DIRLEVS recs. # ##################################################################### # # if test ! "$SEARCHSTR" = "" # then # ## Remove any blanks in the search-string (esp. trailing blanks) # ## -- used before 'string-trim' of DIRRPT was put in the GUI. # # SEARCHSTR=`echo "$SEARCHSTR" | sed 's| ||g'` # # mv $OUTLIST2 $OUTLIST # grep "$SEARCHSTR" $OUTLIST > $OUTLIST2 # fi # # ################################################# # ## ADD THE REPORT SUBDIR SIZE RECS (in OUTLIST2) # ## TO THE HEADER (in OUTLIST). # ################################################# # # cat $OUTLIST2 >> $OUTLIST # # rm $OUTLIST2 ##################################################################### ################################################# ## ADD TRAILER TO THE REPORT. ################################################# echo " | | | | TerGig.MegKil ------------- ------------ Disk usage Subdirectory (Gigabytes) name ................... `date '+%Y %b %d %a %T%p %Z'` ...................... ADDITIONAL INFO ABOUT THIS REPORT EXTRACT: This report is SORTED BY SUB-DIRECTORY SIZE --- LARGEST SUB-DIRECTORIES AT THE TOP. This *EXTRACT* was obtained from a list of 1,000's of subdirectory sizes in file $DIRRPT The 'cron' job that built that report file essentially did the command du -k | sort +0 -1 -nr on the server where the /data & /apps directories 'reside'. The cron script uses = /data and = /apps. IT TAKES MANY MINUTES FOR THAT CRON JOB TO RUN --- so these 'du' commands are not ones that you would want to do manually at the time of an out-of-disk-space emergency. To reduce load on the system and, mainly, to get relatively quick response, THIS REPORT IS EXTRACTED FROM one of these 'ideasadm' weekly CRON JOB REPORT LISTINGs done on the server. This extract normally runs in less than a minute. This relatively fast query time is especially helpful when quite a few different types of extracts/listings must be generated. ------------------------------------- HOW TO GENERATE THIS REPORT *EXTRACT*: This EXTRACT was built by the script $0 which can be run via a site SGI 'spacetools' toolchest. SGI Toolchest -> Handy Tools -> Space Tools -> /data, /apps SUBDIRS SIZE-QUERY Utility (weekly 'du' snapshot) -> Option to show Subdirectories to NO MORE THAN N levels ---- Another drawer (and utility script), namely Option to show Subdirectories with EXACTLY N dir-levels provides a slightly different type of extract. It is easier to read, not having various directory levels inter-mixed in the list --- but this utility shows sizes of PARENT directories of the N-level sub-directories. ------------------------------------- ALTERNATE WAYS TO INVOKE THIS UTILITY: You can see other ways of initiating this directory-sizes extract utility --- i.e. other ways of initiating the script $0 --- by using the 'Help' button at the bottom of the 'SubDir Size List, Extract' GUI. That Help option shows the file $FEDIR/helps/subdir_queries.hlp In particular, that help points out that a user could call 'spacetools' or this particular script via 1) an alias in $HOME/.profile, like alias st='$FEDIR/scripts/spacetools' 2) a desktop icon setup via 'Find, File QuickFind' in SGI toolchest 3) an entry in $HOME/.auxchestrc, like \"SpaceTools\" f.checkexec.sh \"$FEDIR/scripts/spacetools\" --------------------- THE WHOLE BALL OF WAX (the complete 'raw' listings): The complete sub-directory size listings of /data and /apps can be seen with the site 'xpg' utility (or with 'nedit -read'). xpg /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis xpg /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis_prev xpg /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis_prev2 xpg /usr/people/ideasadm/cleandir/spac_apps_dirs_siz.lis xpg /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis_prev xpg /usr/people/ideasadm/cleandir/spac_data_dirs_siz.lis_prev2 Warning: There are tens of 1000's of lines (sub-directories) in these listings. These file sizes are about 2 to 4 Megabytes. You should not print these files, but you can use the 'xpg' 'Show All Matches' button to get extracts from those huge lists. If the extracts are sufficiently small, you may want to print those extracts using the 'Print' button on the 'xpg' utility. ........................................................................... " >> $OUTLIST ##################################################################### ## SHOW INFO FOUND ##################################################################### ## FOR TESTING: # set -x ## A METHOD OF USING 'shofil.tk': # export SHOFILENAME="$OUTLIST" # $FEDIR/tkGUIs/shofil.tk & $FEDIR/scripts/shofil $OUTLIST ## FOR TESTING: # set - ##################################################################### ## SET ENTRY FIELDS (QUERY PARMS) FOR NEXT GUI PROMPT. ##################################################################### # DIRRPT= # DIRLEVS= # SEARCHSTR= export DIRRPT DIRLEVS SEARCHSTR ##################################################################### ## END OF PROMPTING LOOP, for cron-SUBDIRS-SIZE-LISTING, DIR-LEVELS, ## and (optional) DIR-SEARCH-STRING. ##################################################################### done ## END OF 'while :' (which is equivalent to 'while true')