Best Practices
#!/bin/bash # Set shebang .. interpreter (sh/python/bash) #set -x # debug flag, shows all output and variables set -e # exit when an error occurs, Dont use this when sourcing a scriptor using bash ./scriptnaam gebruikt. set -u # exit when a variable isn't defined set -o pipefail # exit when a pipe command fails # Add comments to your scripts! PIPEFAIL Example grep string /nonexistenddir/file | sort # Does NOT give an error, sort works! So echo $? gives 0 When set -o pipefail is set, above example will print 1 or 2
Bash options example
#!/bin/bash
usage ()
{
echo >&2 "usage: $0 <list-of-options>"
}
main ()
{
[ $# -lt 1 ] && usage
INSTALL_DIR=`dirname $0`
for i in $@; do
echo "$INSTALL_DIR $i"
done
}
main $@
Execute output from a script.
Sometimes i write scripts which print the command I want to have executed. Most of the times to check the generated commandline.
simple example
ls | grep 2000 | while read ; do echo rm -f "$REPLY" ;done
Above only prints the lines
rm -f <filename with 2000>
rm -f <other filename with 2000>
Instead of removing the echo command, you can add a | bash , to have it executed.
ls | grep 2000 | while read ; do echo rm -f "$REPLY" ;done | bash
Or even shorter ( Use !! for previous command )
!! | bash
History
Search with CTRL-R in your command history, and use !<number> to execute this command again.
touch file1 file2 file3 file4 chmod 664 !* # will chmod only the files
When you used typed
systemctl stop httpd
and want to start again use replace
^stop^start
This will take previous command containing stop and places stop with start and executes this.
Toggle history on/off with “set -o history”, depending on your setup you can use “<space>command to be executed”
History log original username per date when sudo is being used
(creates .history.20230103.fash in /root/ )
HISTFILE=~/.history.$(date +%y%b%d).$(who am i | awk '{ print $1 }')
Skip first 2 lines and join 2 lines
#skip 2 lines tail -n +3 #join 2 lines sed 'N;s/\n/,/' #3rd line from a file sed "3q;d" /etc/hosts #join on line line using comma's paste -sd "," -
Find Tricks
#Remove empty directories find ??? -empty -type d -delete #Find multiple extentions find archieven/ \( -name "*.png" -o -name "*.xml" \) -print0 to handle filenames with spaces
Remove huge directory structures FAST
mkdir /tmp/empty ; rsync -a --delete /tmp/empty /path/blah
Check program installed and in path
which zenity >/dev/null 2>/tmp/err || ( echo "zenity not found, please install" ; exit 1 )
Change directory to location script for relative path usage
cd $(dirname $0) ls relativesubdir/
IFS (Internal Field Separator)
IFS=$' ' # internal field separator, strings split here for f in $(cat /etc/hosts) ; do echo $f ;done #outputs 127.0.0.1 # localhost IFS=$'\n\t' # internal field separator, split end of line for f in $(cat /etc/hosts) ; do echo $f ;done # Outputs 127.0.0.1 localhost
Difference for and while read example
echo "This will print every word" > text # Places text in file echo "second line" >> text # Append text for f in $(cat text) ; do echo $f ;done # for example This will print every word second line cat text | while read f ; do echo $f ; done # Read example This will print every word second line
Correct way to make a tempfile
tempfile=$(mktemp -d -t /tmp/log.$$) # Temp file, with unique name $$ is the process number Usage in script : ls > $tempfile Cleanup: rm -f "/tmp/$tempfile" # Remove
Direcory empty test
[ "$(ls -A /tmp)" ] && echo "Not empty" || echo "Empty" # test directory Empty/filled test -d /tmp/1 && rmdir /tmp/1 # Removes a directory when it exists, will give an error when NOT empty
Using Expand
On directories archive30_tmp till archive35_tmp setting recursive chmod 2775
find achief{30..35}_tmp -type d -exec chmod 2775 {} \;
or
mkdir tmp{1..3} # will create tmp1 tmp2 tmp3
echo pr{ut,utser}s # Outputs "pruts prutsers"
Test root user
if [ $USER == "root" ] ; then … ; fi # execute only when root if $USER is empty, this will give an error. if [ w$USER == "wroot" ] works qouting $USER also, but qouting a number using less/greater test could be problematic
Mount test
grep -qs /media /proc/mounts && echo "/media is mounted"
Date tricks
datum=$(date +%Y%m%d) # datum is yyyymmdd 20230103 today=$(date +%F) tomorrow=$(date --date="next day" +%F) p3=$(date --date="$p2" +%F) dater=$p3 #now=$(date +%s --date="1 days ago") now=$(date -d $(date --date="1 days ago" +%F) +%s) p3epoch=$(date --date="$p3" +%s) dater=$(date -d "$p3 1 year" +%F) date -d @<UNIX timestamp> # Timestamp to date
Size test in directory
if [ "$(df /tmp |grep -v Available | awk {' print $4 }')" -lt 1000000 ] ; then echo "not enough free in /tmp" ;fi
Parallel Tasks
4 parallel jobs find jpg -type f -name \*.jpg -print0 | xargs -0 -n1 -P4 ./convert.sh
Cluster ssh trick
## clusterssh trick
Only needed to do stuff on server having a certain directory
clusterssh storageservers # ssh to 24 storage servers
sudo su -
cd /bricks/*/backup2 # change directory to this if exists
cd /alternatedir/brick0*/backup2 # change to this alternate directory if it exists
# directories not found? then you are still in /root
pwd | grep testdir || logout # no testdir in found subdirs? (there are non in /root so you will be logged out)
id | grep myuser && logout # if mortal user? then logout again .. you will be disconnected from servers not containing the backup2 directories
Do your work as root
Forgot to sudo?
systemctl restart httpd (wont work as user) sudo !! (this will do "sudo systemctl restart httpd")
Top 5 homedir users
du -hsx /home/* | sort -rh | head -5
Removing a huge file takes a long time .. lets truncate it
: > hugefile rm hugefile
Check memory banks using dmidecode
sudo dmidecode| grep -i -B1 "Form Factor" (B1 means BEFORE 1 line, A is AFTER)
Size: 8192 MB
Form Factor: DIMM
--
Size: 8192 MB
Form Factor: DIMM
--
Size: No Module Installed
Form Factor: DIMM
--
sudo dmidecode -t memory | grep -i size
Size: 4096 MB
--
sudo lshw -short -C memory
H/W path Device Class Description
==========================================================
/0/0 memory 96KiB BIOS
/0/1000 memory 4GiB System Memory
/0/1000/0 memory 4GiB DIMM RAM
Script log replay
script --timing=/tmp/time.txt /tmp/script.log
start your script
scriptreplay -t /tmp/time.txt /tmp/script.log
replays
Deleted file takes up disk space (not being released)
lsof httpd 12209 root 6w REG 253,3 5233639424 2097234 /var/log/httpd/access_log-20200711 (deleted) cd /proc/12209/fd find /proc/12209/fd -ls | grep '(deleted)' 121613145 0 l-wx------ 1 root root 64 Jul 10 12:04 /proc/12209/fd/6 -> /var/log/httpd/access_log-20200711\ (deleted) : > "/proc/12209/fd/6" # Truncate
Keyword in line grep
cat file | grep -o 'skip_reason.*' # till end cat file | grep -o 'skip_reason.*tillhere'
Remove space filename
mv *\ * a rm *\ *
Upper to lower case
tr '[:upper:]' '[:lower:]'
Whole directories to lowercase
#!/bin/bash
#print usage
if [ -z $1 ];then
echo "Usage :$(basename $0) parent-directory"
exit 1
fi
#process all subdirectories and files in parent directory
all="$(find $1 -depth)"
for name in ${all}; do
#set new name in lower case for files and directories
new_name="$(dirname "${name}")/$(basename "${name}" | tr '[A-Z]' '[a-z]')"
#check if new name already exists
if [ "${name}" != "${new_name}" ]; then
[ ! -e "${new_name}" ] && mv -T "${name}" "${new_name}"; echo "${name} was renamed to ${new_name}" || echo "${name} wasn't renamed!"
fi
done
echo
echo
#list directories and file new names in lowercase
echo "Directories and files with new names in lowercase letters"
find $(echo $1 | tr 'A-Z' 'a-z') -depth
exit 0
Sort by lenght
awk '{ print length(), $0 | "sort -n" }'
Order / Delimiter AWK
echo 8.168.192.in-addr.arpa. | awk -F "." '/1/ { print $3,$2,$1 }'
From – Till using AWK
cat file | awk '/Title1/,/-----/'
Replace Tabs
sed -e 's/\t/ /g'
Sed Case Insensitive
sed s/a/b/gI
And using lynx std in
cat index.html | lynx --dump -stdin
Random sleep
sleep .$[ ( $RANDOM % 4 ) + 1 ]s #remove . for whole seconds instead of 10th of seconds
Copy pasting a lot from terminal , use xclip to copy to clipboard
xclip -sel c < bigsourcefile.c
Cache dir in memory for high IO processes
#!/bin/bash [ ! -d /ramcache ] && mkdir -p /ramcache mount -t tmpfs -o size=2G,nr_inodes=5k,mode=777 tmpfs /ramcache
Have a dual boot machine? And want to remotely start the other OS?
reboot2windows.sh
WINDOWS_TITLE=`grep -i "^menuentry 'Windows" /boot/grub/grub.cfg|head -n 1|cut -d"'" -f2` sudo grub-reboot "$WINDOWS_TITLE" sudo reboot
Move files in subdirectory per create hour
for f in $(seq -w 0 23) ; do find . -type f -print0 | xargs -0 -r stat -c "%y %N" | grep -E " $f:" | cut -f2 -d\' | while read ; do mkdir -p $f ; mv "$REPLY" $f/ ; done ;d
Exif time adjust
#Movies and images exiftool '-*CreateDate-=5' '-*ModifyDate-=5' '-DateTimeOriginal-=5' dir/ exiftool '-FileCreateDate<CreateDate' '-FileModifyDate<CreateDate' dir/
