SHELL

範例與常用技巧

檔頭常用宣告
# quickly syntax
set -euo pipefail

# let script exit if a command fails
set -o errexit 
# OR
set -e

# let script exit if an unsed variable is used
set -o nounset
# OR
set -u

# This setting prevents errors in a pipeline from being masked.
set -o pipefail

# for Debug
set -x

# setting IFS
IFS=$'\n\t'

How set -o pipefail works

$ grep some-string /non/existent/file | sort
grep: /non/existent/file: No such file or directory
$ echo $?
0

$ set -o pipefail
$ grep some-string /non/existent/file | sort
grep: /non/existent/file: No such file or directory
$ echo $?
2

How the IFS  works

#!/bin/bash
names=(
  "Aaron Maxwell"
  "Wayne Gretzky"
  "David Beckham"
  "Anderson da Silva"
)

echo "With default IFS value..."
for name in ${names[@]}; do
  echo "$name"
done

echo ""
echo "With strict-mode IFS value..."
IFS=$'\n\t'
for name in ${names[@]}; do
  echo "$name"
done

############### Output #############
With default IFS value...
Aaron
Maxwell
Wayne
Gretzky
David
Beckham
Anderson
da
Silva

With strict-mode IFS value...
Aaron Maxwell
Wayne Gretzky
David Beckham
Anderson da Silva
檔案的目錄位置
Script 檔案名稱
$ echo $0
./test.sh

$ echo `basename $0`
test.sh 
輸出多行的文字訊息
cat <<EOF
Welcome .....
	
Here are the messages that you want to show up
		
EOF
所有輸出訊息導入一個檔案

Sample #1

#!/bin/sh
LOG="my.log"
(
....
) 2>&1 | tee -a $LOG 

不適用在內容裡有 python 指令的訊息輸入,輸入等待的畫面會無法顯示。

Sample #2

temp=$(mktemp)
exec &> ${temp}

echo "All outputs will be saved into the file ${temp}."

Sample #3

#!/bin/bash
set -eu 
exec 3>&1 4>&2
trap 'exec 2>&4 1>&3' 0 1 2 3
exec 1>/path/to/script.log 2>&1

# rest of the script below
dnf -y in foo bar
# firewall rules goes here 
....
...
快速修改大量檔案的副檔名
## *.old -> *.new
for fname in $(ls *.old);do echo "mv $fname ->"; echo $(echo $fname |sed 's/.old/.new/');mv $fname $(echo $fname | sed 's/.old/.new/');done
快速新增或列出多個指定目錄
mkdir {AAA,BBB,CCC}
快速備份設定檔
cp my.cfg{,.bak}
計算檔案總數量

用 find

# 計算目前目錄底下所有檔案數
find . -type f | wc -l 

# 計算第一層所有子目錄的檔案數並排序清單
find . -maxdepth 1 -type d -print0 | xargs -0 -I {} sh -c 'echo $(find {} -type f | wc -l) {}' | sort -n
NOTE: 第一層子目錄名稱不可包含空格 

用 rsync

rsync --stats --dry-run -ax /usr /tmp

Number of files: 326,373 (reg: 211,698, dir: 24,284, link: 90,391)
Number of created files: 326,373 (reg: 211,698, dir: 24,284, link: 90,391)
Number of deleted files: 0
Number of regular files transferred: 211,698
Total file size: 7,180,685,730 bytes
Total transferred file size: 7,178,524,818 bytes

NOTE: 這指令實際不會作檔案複製,/tmp 只是一個假目錄,回傳結果是 reg: 211698 就是檔案總數

用 tree

tree /mydir -a | tail -n 1

5 directories, 56 files

NOTE: 當目錄底下有包含 symbolic link 也會被計算
執行外部 SHELL 或指令

1. 使用 pipe line

echo"md5sum $X > $X.sum "| bash

2. 使用 eval

get_arch="uname -p"
if [ "`eval "$get_arch"`" = "i686" ]; then
....
fi 
提示字元的路徑名稱太長

加上這變數

PROMPT_DIRTRIM=2
處理 CSV 檔
#!/bin/bash
INPUT=data.cvs
OLDIFS=$IFS
IFS=','
[ ! -f $INPUT ] && { echo "$INPUT file not found"; exit 99; }
while read flname dob ssn tel status
do
	echo "Name : $flname"
	echo "DOB : $dob"
	echo "SSN : $ssn"
	echo "Telephone : $tel"
	echo "Status : $status"
done < $INPUT
IFS=$OLDIFS

CSV and JSON

# JSON to CSV
cat df.json | jq -r '.[]| join(",")'
cat bingbot.json | jq -r '.prefixes[] | {cidr: .ipv4Prefix, comment: "BingBot"} | join(",")' > bingbot.csv
JSON 檔
建立暫存檔
tmpfile1=$(mktemp)
tmpfile2="/tmp/$(basename $0).$$.tmp"
Get my public IP
curl ifconfig.me
curl ifconfig.me/ip
curl ifconfig.co
curl checkip.amazonaws.com
curl icanhazip.com
curl ipecho.net/plain

dig +short myip.opendns.com @resolver1.opendns.com
dig TXT +short o-o.myaddr.l.google.com @ns1.google.com
dig TXT +short o-o.myaddr.l.google.com @ns1.google.com | awk -F'"' '{print $2}'
主機 IP
# On Linux
hostname -I
hostip=$(/sbin/ip a | awk '/eth[012]:|ens192:|bond0:/,/^$/' | grep -E "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | head -1 | awk -F " " '{print $2}' | cut -d"/" -f1)

# On AIX
hostip=$( ifconfig -a | grep inet | awk '{print $2}' | head -1 )
hostip=$( ifconfig -a | grep inet | awk '{print $2}' | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p' | head -1 )
找出最大的目錄或檔案
# Way 1
du -a /var | sort -n -r | head -n 10

# Way 2
cd /path/to/some/where
du -hsx * | sort -rh | head -10
du -hsx -- * | sort -rh | head -10

# Way 3
find /path/to/dir/ -printf '%s %p\n'| sort -nr | head -10
find . -printf '%s %p\n'| sort -nr | head -10
## Skip directories and only display files
find /path/to/search/ -type f -printf '%s %p\n'| sort -nr | head -10

# Create a shell alias
## shell alias ##  
alias ducks='du -cks * | sort -rn | head'
## deal with special files names ##
alias ducks='du -cks -- * | sort -rn | head'
檔名移除空白字元
# With tr
for f in *; do mv "$f" `echo $f | tr ' ' '_'`; done

# With find
find . -type f -name "* *.xml" -exec bash -c 'mv "$0" "${0// /_}"' {} \;
輸入密碼
echo -n "MySQL username: " ; read username
echo -n "MySQL password: " ; stty -echo ; read password ; stty echo ; echo
shuf: 隨機排序
curl -s https://www.imdb.com/list/ls020046354/export | cut -d ',' -f 6  | shuf
tree: 顯示專案目錄的檔案樹
tree --dirsfirst --filelimit 10 --sort=name

# Display size of files
tree -s

# Display permissions of files
tree -p

# Display directory only
tree -d

# Display till a certain level/depth
tree -L 1

# List only those files that match pattern given
tree -P *screenshot*
sort : 資料排序
sort -t ',' -k5,5 -k1,1 -k9,9 -k3,3 -k11,11 my.csv
timeout : 自動停止執行
timeout 10 tail -f /var/log/httpd/access.log
timeout 5m ping 8.8.8.8
timeout 300 tcpdump -n -w data.pcap

# Sending specific signal
# To get a list of all available signals, use the command kill -l .
timeout -s SIGKILL ping 8.8.8.8
variables : 變數
變數 說明
 $0  腳本檔名
 $1  第1個參數
 $2  第2個參數
 ${10}  第10個參數 #10
 $#  參數的個數
 $*  顯示所有參數 (作為一個字串)
 $@  顯示所有參數 (每個為一個獨立字串)
 ${$*}  傳遞到腳本中的參數的個數
 ${$@}  傳遞到腳本中的參數的個數
 $?  腳本結束後的傳回值
 $$  腳本的程序 ID
 $_  前個命令的最後一個參數
 $!  最後一個執行程序的 ID
u=${1:-root}  如果 $1 未指定,就賦予值 root
u=${USER:-foo}  如果 $USER 未指定,就賦予值 foo
len=${#var}  計算 $var 字串的長度

如果 $2 未指定或空值,輸出錯誤訊息

${varName?Error varName is not defined}
${varName:?Error varName is not defined or is empty}

建立目錄並且切換至目錄

mkdir my-dir && cd $_
cut: 切割文字
# AAA = BBB, 取出 BBB
cut -d= -f2

# 111 2222 33 444444 555, 取出 33 以後的所有內容
cut -d ' ' -f3-

# 檢視超長的文字
head -n 1 data.csv | wc  # 計算行的字元數
head data.csv | cut -c -30  # 列出前幾行的前 30 個字元, 如果要顯示後 30 個字元,可以用 30-
tr: 置換
# aaa bbb ccc
# 換成
# aaa
# bbb
# ccc
echo "aaa bbb ccc" | tr " " "\n"
xargs

接入(Pipe) 不支援 stdin 指令的替代方法

printf: 格式化輸出
printf "%-40s ..................%s\n" "Disable the service $1" "[$2]"

Disable the service apmd                 ..................[OK]
Disable the service bluetooth            ..................[OK]
Disable the service hidd                 ..................[OK]
Disable the service cups                 ..................[OK]
Disable the service firstboot            ..................[OK]
Disable the service readahead_early      ..................[OK]

printf "%40s ..................%s\n" "Disable the service $1" "[$2]"

                Disable the service apmd ..................[OK]
           Disable the service bluetooth ..................[OK]
                Disable the service hidd ..................[OK]
                Disable the service cups ..................[OK]
           Disable the service firstboot ..................[OK]
     Disable the service readahead_early ..................[OK] 

數字四捨五入

printf "%.0f" 9.46666667 # output: 10

printf "%.2f" 9.46666667 # output: 9.47
印出一個長符號
printf -- '-%.0s' {1..80}
printf -- '=%.0s' {1..80}
ping: 掃描一個 IP 範圍
{ for p in {1..254}; do ping -c1 -w1 10.22.9.$p & done } | grep "64 bytes"
nice: Reduce CPU and Disk load of backup scripts
# Reduce the I/O priority of the /usr/local/bin/backup.sh script so that it does not interfere with other processes
# The -n parameter must be between 0 and 7, where lower numbers mean higher priority
/usr/bin/ionice -c2 -n7 /usr/local/bin/backup.sh

# To reduce the CPU priority, use the command nice
# The -n parameter can range from -20 to 19, where lower numbers mean higher priority
/usr/bin/nice -n 19 /usr/local/bin/backup.sh

# Nice and ionice can also be combined, to run a script at low I/O and CPU priority
/usr/bin/nice -n 19 /usr/bin/ionice -c2 -n7 /usr/local/bin/backup.sh
Hex to ASCII
# hex = 54657374696e672031203220330
# ascii = Testing 1 2 3

# xxd
echo 54657374696e672031203220330 | xxd -r -p && echo ''

# printf
printf '\x54\x65\x73\x74\x69\x6e\x67\x20\x31\x20\x32\x20\x33\x0' && echo ''

# sed
echo -n 54657374696e67203120322033 | sed 's/\([0-9A-F]\{2\}\)/\\\\\\x\1/gI' | xargs printf && echo ''
隨機密碼生成
genpasswd() { 
	local l=$1
       	[ "$l" == "" ] && l=16
      	tr -dc A-Za-z0-9_ < /dev/urandom | head -c ${l} | xargs 
}
tr -dc A-Za-z0-9_ < /dev/urandom | head -c 16 | xargs

# Generate more than one
tr -dc A-Za-z0-9_ < /dev/urandom | fold -16 | head -5

#
echo FooBar$RANDOM | md5sum | base64 | cut -c 1-12
ls: 進階技巧
# Find the biggest zip file
ls -lSrh ~/Downloads/*.zip
stat: 檔案屬性
❯ stat --printf='Name: %n\nPermissions: %a\n' my.log
Name: my.log
Permissions: 777

❯ stat --format="%F" my.log
regular file

# Symlink
❯ stat ~/bin/FoxitReader
  File: /home/alang/bin/FoxitReader -> /home/alang/opt/foxitsoftware/foxitreader/FoxitReader.sh
  Size: 56        	Blocks: 0          IO Block: 4096   symbolic link
Device: 10302h/66306d	Inode: 787474      Links: 1
Access: (0777/lrwxrwxrwx)  Uid: ( 1000/   alang)   Gid: ( 1000/   alang)
Access: 2023-07-16 10:26:13.412193581 +0800
Modify: 2023-02-26 12:13:50.234374171 +0800
Change: 2023-02-26 12:13:50.234374171 +0800
 Birth: 2023-02-26 12:13:50.234374171 +0800

❯ stat -L ~/bin/FoxitReader
  File: /home/alang/bin/FoxitReader
  Size: 120       	Blocks: 8          IO Block: 4096   regular file
Device: 10302h/66306d	Inode: 1457222     Links: 1
Access: (0755/-rwxr-xr-x)  Uid: ( 1000/   alang)   Gid: ( 1000/   alang)
Access: 2023-07-02 14:34:09.957428285 +0800
Modify: 2023-02-26 12:13:50.246374250 +0800
Change: 2023-02-26 12:13:50.246374250 +0800
 Birth: 2023-02-26 12:13:48.530362986 +0800

grep

教學網站

文字過濾
# 使用 . (period) 表示一個任意字元
> grep dev.sda /etc/fstab
/dev/sda3       /               reiserfs        noatime,ro 1 1
/dev/sda1       /boot           reiserfs        noauto,noatime,notail 1 2
/dev/sda2       swap            swap            sw 0 0
#/dev/sda4      /mnt/extra      reiserfs        noatime,rw 1 1

# 使用 [ ]
> grep dev.sda[12] /etc/fstab
/dev/sda1       /boot           reiserfs        noauto,noatime,notail 1 2
/dev/sda2       swap            swap            sw 0 0 

# 使用 [^12] 表示非1,2字元
> grep dev.sda[^12] /etc/fstab
/dev/sda3       /               reiserfs        noatime,ro 1 1
#/dev/sda4      /mnt/extra      reiserfs        noatime,rw 1 1

# 使用正規表示
> grep '^#' /etc/fstab

# /etc/fstab: static file system information.
#

> grep '^#.*\.$' /etc/fstab
# /etc/fstab: static file system information.
#

> grep 'foo$' filename

# 列出有符合的關鍵字
> echo "AAA BBB ccc ddd" | grep -wE  -o "(cc|BBB|ddd)"

# 搜尋與關鍵字大小寫一致的行
> grep -w "boo" myfile.txt
> grep "\<boo\>" myfile.txt

# 搜尋包含特殊字元 *** 的關鍵字
> grep '\*\*\*' myfile.txt

# 搜尋 email 地址
> grep -E -o "\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6}\b" /path/to/data

# 電話號碼, 1-XXX-ZZZ-YYYY, XXX-ZZZ-YYYY
> grep -E '(1-)?[[:digit:]]{3}-[[:digit:]]{3}-[[:digit:]]{4}' sample.txt

# 搜尋 IP 位址
> egrep '\b(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?\.){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' /etc/hosts
> grep -E -o "(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" /var/log/auth.log

# 更多應用語法
> grep '[vV][iI][Vv][Ee][kK]' filename
> grep '[vV]ivek' filename
> grep -w '[vV]ivek[0-9]' filename
> grep 'foo[0-9][0-9]' filename
> grep '[A-Za-z]' filename
> grep [wn] filename
搜尋關鍵字

> 搜尋多個關鍵字(使用正規運算式)

ls /var/log/ | grep -E "http|secure"

ls /var/log/ | grep "http\|secure" 

grep -Ev "^(#|;)" example.txt

> 在許多檔案內尋找特定字串內容

應用:程式碼編寫及除錯

cd /var/www/html
grep -r "[搜尋關鍵字]" *

grep -Ril "specific_text" /path/to/dir 

> 搜尋特定字串的文字段落內容

應用:檢查系統的硬體裝置,以及類似 AIX 的 grep -p

見附檔: grepp.awk

> 搜尋字串並顯示前或後幾行的內容

# Before n lines
grep -B1 "^SQL" runstats.log

# After n lines
grep -A1 "^SQL" runstats.log

# Without separator '--'
grep -B1 --no-group-separator "^SQL" runstats.log
檢查設定檔的參數
 CONFIG_CHECK=`grep "^# SparkleShare$" /etc/ssh/sshd_config`
  if ! [ "$CONFIG_CHECK" = "# SparkleShare" ]; then
    echo "" >> /etc/ssh/sshd_config
    echo "# SparkleShare" >> /etc/ssh/sshd_config
    echo "Match User storage" >> /etc/ssh/sshd_config
    echo "    PasswordAuthentication no" >> /etc/ssh/sshd_config
  fi
排除 grep 自己,使用 ps 時
↪  ps -ef | grep 'plank'                                 西元2022年04月24日 (週日) 09時30分51秒 CST
alang       2575    2166  0 09:22 ?        00:00:02 plank
alang       4831    3916  0 09:30 pts/0    00:00:00 grep --color=auto plank     <=========

↪  ps -ef | grep '[p]lank'                               西元2022年04月24日 (週日) 09時30分55秒 CST
alang       2575    2166  0 09:22 ?        00:00:02 plank
外部變數
# 匯出系統密碼檔
while read line; do grep -w "^${line%%:*}" /etc/shadow; done <passwd.bak >shadow.bak

 

Cheat Sheet

grep-cheatsheet.png

test

字串或文字的比對
Str1 = str2 | 當str1與str2相同時, 傳回True
Str1 != str2| 當str1與str2不同時, 傳回True
Str1 < Str2 
Str1 <= Str2
Str1 > Str2
Str1 >= Str2
Str     | 當str不是空字符時, 傳回True
-n str  | 當str的長度大於0時, 傳回True
-z str  | 當str的長度是0時, 傳回True
數字(整數) 比對
Int1 -eq int2 |當int1等於int2時, 傳回True
Int1 -ge int2 |當int1大於/等於int2時, 傳回True
Int1 -le int2 |當int1小於/等於int2時, 傳回True
Int1 -gt int2 |當int1大於int2時, 傳回True
Int1 -ne int2 |當int1不等於int2時, 傳回True
Int1 -lt int2 |當int1小於 int2時, 傳回True 
檔案的比對
-e file   | 檔案是否存在
-d file   | 當file是一個目錄時, 傳回 True
-f file   | 當file是一個普通檔案時, 傳回 True
-r file   | 當file是一個可讀檔案時, 傳回 True
-s file   | 當file檔案內容長度大於0時, 傳回 True; 空白內容傳回 False
-w file   | 當file是一個可寫檔案時, 傳回 True
-x file   | 當file是一個可執行檔案時, 傳回 True

File operators list

Operator Returns
-a FILE True if file exists.
-b FILE True if file is block special.
-c FILE True if file is character special.
-d FILE True if file is a directory.
-e FILE True if file exists.
-f FILE True if file exists and is a regular file.
-g FILE True if file is set-group-id.
-h FILE True if file is a symbolic link.
-L FILE True if file is a symbolic link.
-k FILE True if file has its `sticky' bit set.
-p FILE True if file is a named pipe.
-r FILE True if file is readable by you.
-s FILE True if file exists and is not empty.
-S FILE True if file is a socket.
-t FD True if FD is opened on a terminal.
-u FILE True if the file is set-user-id.
-w FILE True if the file is writable by you.
-x FILE True if the file is executable by you.
-O FILE True if the file is effectively owned by you.
-G FILE True if the file is effectively owned by your group.
-N FILE True if the file has been modified since it was last read.
! EXPR Logical not.
EXPR1 && EXPR2 Perform the and operation.
EXPR1 || EXPR2 Perform the or operation.

布林變數應用
# Let us Declare Two Boolean Variables
# Set this one to true
jobstatus=true
# Check it 
if [ "$jobstatus" = true ] ; then
	echo 'Okay :)'
else
	echo 'Noop :('
fi
# Double bracket format syntax to test Boolean variables in bash
bool=false
if [[ "$bool" = true ]] ; then
	echo 'Done.'
else
	echo 'Failed.'
fi
更多範例
# 如果前個指令執行有錯誤時,會執行兩個程序
[ ! $? -eq 0 ] && echo "Abort the process!! you can try it agian after you made change." && exit 1

# 或者
[ ! $? -eq 0 ] && {
 echo "Abort the process!! you can try it agian after you made change."
 exit 1
} 

#
if [[ $RETURN_CODE != 0 ]]; then
        echo "Zulip first start database initi failed in \"initialize-database\" exit code $RETURN_CODE. Exiting."
        exit $RETURN_CODE
fi

#
if [[ $TIMEOUT -eq 0 ]]; then
            echo "Could not connect to database server. Exiting."
            unset PGPASSWORD
            exit 1
fi
#
if (($? > 0)); then
            echo "$SECRET_KEY = $SECRET_VAR" >> "$DATA_DIR/zulip-secrets.conf"
            echo "Secret added for \"$SECRET_KEY\"."
fi
#
if test "x$newbranch" = x; then
    newbranch=`git branch -a | grep "*" | cut -d ' ' -f2`
fi

# Check router home directory.
[ -d "$PROD_HOME" ] || {
	echo "Router home directory ($PROD_HOME) not found"
	exit 1
}

# AND
autoBackupConfiguration() {
    if ([ "$AUTO_BACKUP_ENABLED" != "True" ] && [ "$AUTO_BACKUP_ENABLED" != "true" ]); then
        rm -f /etc/cron.d/autobackup
        echo "Auto backup is disabled. Continuing."
        return 0
    fi
}

# OR
if [ "$MANUAL_CONFIGURATION" = "False" ] || [ "$MANUAL_CONFIGURATION" = "false" ]; then
        databaseConfiguration
        secretsConfiguration
        authenticationBackends
        zulipConfiguration
fi

# Die if $f1 or $f2 is missing 
if [ ! -f "$f1"  ] ||  [ ! -f "$f2" ]
then
	echo "Required files are missing."
else
	echo "Let us build SFTP jail."
fi
# And
if [[ $age -ge 18 ]] && [[ $nat -eq "Indian" ]];then
    echo "You can vote!!!"
else
    echo "You can not vote"

fi

# multiple AND
if [ -e "$DATA_DIR/.initiated" ] && ([ "$FORCE_FIRST_START_INIT" != "True" ] && [ "$FORCE_FIRST_START_INIT" != "true" ]); then
    echo "First Start Init not needed. Continuing."
    return 0
fi


# one-liner
 [[ -z "$var" ]] && echo "NULL" || echo "NOT NULL"
# if age is greater than or equal to 18 then the program will print "Adult" or it will print "Minor".
[[ $age -ge 18 ]] && echo "Adult" || echo "Minor"

# Contain a substring
if [[ $var = *pattern1* ]]; then
    echo "Do something"
fi

[[ $1 != *cyberciti.biz/faq/* ]] && { printf "Error: Specify faq url (e.g., http://www.cyberciti.biz/faq/url-1-2-3/)\n"; exit 2; }

if [[ $fullstring == *"$substr"* ]];

# Regex
if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then

head="win00000:hascom_command.ksh : hostname:wintstc, monitor port#:17911"
if [[ $head =~ ^win00000.*$ ]]; then

# Check the the number of the version
[ "$(echo "$TMUX_VERSION >= 2.4" | bc)" = 1 ] || echo "The version $TMUX_VERSION is outdated" 

awk

教學網站

搜尋字串

awk '/^this/{print $0}'         #與 sed -n '/^this/p' 相同

搜尋取代

# 移除分號
awk { gsub(/\;/, "") }

移除重複資料的行

我們經常使用 sort 與 uniq 指令,從檔案中找出並移除重複項目。不過如果你不希望你的原始檔被排序或更動,這時正是 awk 派上用場的時候,我們可以用 awk 截取不重複記錄並儲存在新的檔案中

awk '!x[$0]++' filewithdupes > newfile

搜尋含 disabled 的行,並列出第 1, 3 欄的內容

awk '/disabled/{print $1, $3}'

取值做計算

awk '{print "up " $1 /60 " minutes"}' /proc/uptime

df -lP -text4 |awk '{sum += $4} END {printf "%d GiB\n", sum/1048576}'
df -lP -text4 |awk '{sum += $4} END {printf "%d GiB\n", sum/2**20}'

加上判斷式

df -k |grep "/dev/" | awk '($2 > 0 && ((1 - $3/$2)  > 0.9) ) {print $0 }'

awk -F" " '{print ($7 != "A")?$0"***":$0}' myfile

搜尋每行的第9欄,如果不是 0x00000000 時就顯示該行訊息

cat info.out | awk '($9 != "0x00000000") {print}'

列出 uid >= 500 且 <= 10000 的行

export UGIDLIMIT=500
awk -v LIMIT=$UGIDLIMIT -F: '($3>=LIMIT) && ($3<=10000)' /etc/passwd

解決長整數顯示問題

$ awk  'BEGIN {print 12345678901234567890}'
1.23457e+19

方法一
$ awk  'BEGIN {printf("%d\n", 12345678901234567890)}'
12345678901234567168

方法二
$ awk  'BEGIN {OFMT="%.0f"; print 12345678901234567890}'
12345678901234567168 

列出 uid=0 的帳號

awk -F: '($3 == "0") {print}' /etc/passwd

列出最後一個欄位的值

ls -ltd */ | awk -F ' ' '{print $NF}'

列出長度大於 64 的行

awk 'length > 64'

格式化輸出

awk '{ printf("1-minute: %s\n5-minute: %s\n15-minute: \
 %s\n",$1,$2,$3); }' /proc/loadavg

計算目錄的檔案大小

foldersize() {
    if [ -d $1 ]; then
        ls -alRF $1/ | grep '^-' | awk 'BEGIN {tot=0} { tot=tot+$5 } END { print tot }'
    else
        echo "$1: folder does not exist"
    fi
    }

計算單字總數(以符號 "空格" 作為單字的識別)

awk '{total=total+NF}; END {print total+0}'

搜尋特定字串的文字段落內容

# lspci -v | awk '/ATI/,/^$/'
01:03.0 VGA compatible controller: ATI Technologies Inc Rage XL (rev 27) (prog-if 00 [VGA])
        Subsystem: Compaq Computer Corporation: Unknown device 001e
        Flags: bus master, stepping, medium devsel, latency 64
        Memory at fc000000 (32-bit, non-prefetchable) [size=16M]
        I/O ports at 3000 [size=256]
        Memory at fbff0000 (32-bit, non-prefetchable) [size=4K]
        Capabilities: [5c] Power Management version 2

批次 Kill 名稱包含有 /plugins/mactrack 的程式

ps -ef | grep "/plugins/mactrack" | awk '{system("kill " $2);}'

使用兩個不同的區隔符號: 空格 + =

# 顯示回應時間
ping 8.8.8.8 | awk -F[\ =] '{print $10}'

取出每十行的資料(第 10, 20, 30, ...)

awk '!(NR % 10)' file

自動清理舊檔案,保留最近一個檔案

# Sort nmon files by time, delete a file far from the current time, always keep only one nmon file:
ls -t ~/*.nmon |awk '/\.nmon/ {if (NR >1){system ("rm " $1)}}'

CSV 指定欄位值

# 統計第 13 欄 APPNAME 每個值的計數
cat ./ISO27001/db2/fdctest_validate.csv |  awk -F, '{a[$13]++} END {for (k in a) print k, a[k]}'

# 變更欄位 3 的值 
awk '{ $3 = toupper(substr($3,1,1)) substr($3,2) } $3' FS=, OFS=, file

# 欄位 3 變更成大寫
awk '$3 { print toupper($0); }' file

清除空白行

awk NF test.txt
範例: 段落文字的解析

script: aud2csv.sh

Raw Data:

timestamp=2023-01-08-23.13.02.322992;
  category=CHECKING;
  audit event=CHECKING_OBJECT;
  event correlator=107;
  event status=0;
  database=RPTDB;
  userid=winmfg;
  authid=WINMFG;
  application id=10.8.25.30.64020.230108151301;
  application name=EXCEL.EXE;
  package schema=NULLID;
  package name=SYSSH200;
  package section=4;
  object schema=ISTRPT;
  object name=FHOPEHS;
  object type=TABLE;
  access approval reason=OBJECT;
  access attempted=SELECT;
  local transaction id=0x00000001b6550792;
  global transaction id=0x0000000000000000000000000000000000000000;
  instance name=istrpt;
  hostname=BSMDB_B;

函式說明:

# 宣告: 紀錄的間隔符號為 空行, 欄位間隔符號為 =
# 一個段落文字為一筆紀錄,每一行以 = 為間隔區別不同欄位
BEGIN {
    FS="=";
    RS="";
}

# 過濾條件: 欄位總數是 nfp2 的值的資料 
# 此實例包含了有不一致欄位數的紀錄,所以必須先做過濾
NF==nfp2 {

}

# 移除 分號字元
{ gsub(/\;/, "") }

# TIMESTAMP = 欄位 2
# CATEGORY = 欄位 4
# f1 = 欄位 1
# f2 = 欄位 3
{
    TIMESTAMP=2; CATEGORY=4; AUDIT_EVENT=6; EVENT_CORRELATOR=8; EVENT_STATUS=10; DATABASE=12; USERID=14; AUTHID=16; APPLICATION_ID=18; APPLICATION_NAME=20; PACKAGE_SCHEMA=22; PACKAGE_NAME=24; PACKAGE_SECTION=26; OBJECT_SCHEMA=28; OBJECT_NAME=30; OBJECT_TYPE=32; ACCESS_APPROVAL_REASON=34; ACCESS_ATTEMPTED=36; LOCAL_TRANSACTION_ID=38; GLOBAL_TRANSACTION_ID=40; INSTANCE_NAME=42; HOSTNAME=44;  
    
    f1=1; f2=3; f3=5; f4=7; f5=9; f6=11; f7=13; f8=15; f9=17; f10=19; f11=21; f12=23; f13=25; f14=27; f15=29; f16=31; f17=33; f18=35; f19=37; f20=39; f21=41; f22=43 
}

# 印出 CSV 的 Header 行
if (! headline)
    {
        headline = sprintf( "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s", $f1, $f2, $f3, $f4, $f5, $f6, $f7, $f8, $f9, $f10, $f11, $f12, $f13, $f14, $f15, $f16, $f17, $f18, $f19, $f20, $f21, $f22 );
        print headline;
    }

# 印出 CSV 的資料
dataline = sprintf("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s", $TIMESTAMP, $CATEGORY, $AUDIT_EVENT, $EVENT_CORRELATOR, $EVENT_STATUS, $DATABASE, $USERID, $AUTHID, $APPLICATION_ID, $APPLICATION_NAME, $PACKAGE_SCHEMA, $PACKAGE_NAME, $PACKAGE_SECTION, $OBJECT_SCHEMA, $OBJECT_NAME, $OBJECT_TYPE, $ACCESS_APPROVAL_REASON, $ACCESS_ATTEMPTED, $LOCAL_TRANSACTION_ID, $GLOBAL_TRANSACTION_ID, $INSTANCE_NAME, $HOSTNAME );
print dataline;
範例: 與 bash 整合
# System Memory Section
mem_raw=$(free -b | awk '/Mem:/ {print $2, $3, $4, $7}')
read -r -a mem_array <<< "$mem_raw"

mem_total_bytes=${mem_array[0]}
mem_used_bytes=${mem_array[1]}
mem_free_bytes=${mem_array[2]}
mem_avail_bytes=${mem_array[3]}

mem_total_gb=$(awk "BEGIN {printf \"%.2f\", $mem_total_bytes / 1024 / 1024 / 1024}")
mem_used_gb=$(awk "BEGIN {printf \"%.2f\", $mem_used_bytes / 1024 / 1024 / 1024}")
mem_free_gb=$(awk "BEGIN {printf \"%.2f\", $mem_free_bytes / 1024 / 1024 / 1024}")
mem_avail_gb=$(awk "BEGIN {printf \"%.2f\", $mem_avail_bytes / 1024 / 1024 / 1024}")

avail_mem_percentage=$(awk "BEGIN {printf \"%.2f\", 100 * $mem_avail_bytes / $mem_total_bytes}")
is_low_mem=$(awk "BEGIN {print ($avail_mem_percentage < 10)}")

if (( is_low_mem )); then
  mem_info="Total: ${mem_total_gb} GB, Used: ${mem_used_gb} GB, Free: ${mem_free_gb} GB, Available: ${mem_avail_gb} GB \033[0;31m(Warning!: ${avail_mem_percentage}%%)\033[0m"
else
  mem_info="Total: ${mem_total_gb} GB, Used: ${mem_used_gb} GB, Free: ${mem_free_gb} GB, Available: ${mem_avail_gb} GB"
fi

 

Cheatsheet

awk_cheatsheet.png

AWK.jpg

sed

教學網站

分隔符號

除了常見的 / ,也能使用 @ , | , %

Tip: 有時遇到要搜尋某些特殊符號,可能會需要更換不同的分隔符號。

For s command

# 將 /usr/bin 加上 local 成為 /usr/bin/local
sed 's@/usr/bin@&/local@g' path.txt

Other command

# 開頭要加一個反斜線
echo $PATH | sed -n '\@/usr/local/sbin@p'
常用語法範例
# 移除註解, 空白行
sed '/^#\|^$\| *#/d' my.conf
# 只移除註解文字,但不刪除整行
sed 's/#.*//' file.txt

# 搜尋取代: 每一行第 1 個 Linux 換成 Unix
sed 's/Linux/Unix/' linuxteck.txt
# 搜尋取代: 每一行第 1 個 Linux 換成 Unix (不分大小寫)
sed 's/Linux/Unix/i' linuxteck.txt

# 搜尋取代: 每一行第 2 個 Linux 換成 Unix
sed 's/Linux/Unix/2' linuxteck.txt
# 搜尋取代: 第二行第 1 個 Linux 換成 Unix
sed '2 s/Linux/Unix/' linuxteck.txt
# 搜尋取代: 全文的 Linux 換成 Unix
sed 's/Linux/Unix/g' linuxteck.txt

# 列印: 行號
sed -n p linuxteck.txt
# 列印: 第 2-4 行
sed -n '2,4p' linuxteck.txt
# 列印: 移除第 2-4 行
sed '2,4d' linuxteck.txt
# 列印: 第 2-3, 5-6 行
sed -n -e '2,3p' -e '5,6p' linuxteck.txt
# 列印: 包含 operating 的行
sed -n /operating/p linuxteck.txt

# 每一行之間加一個空白行
sed G linuxteck.txt
# 在每個註解行下方加一個空行
sed '/^#/G' file.txt
列出搜尋內容
# 列出第 100 行以下的所有文字
# 若要刪除,可將 p 換成 d。
sed -n '100,$p'  my.txt

# 列出第 3 行以上的所有文字
sed '3q' my.txt

# 列出第 130 行
sed -n '130{p;q}' my.txt

# 列出第 100 - 130 行文字
sed -n '100,130p' my.txt
sed -n '100,130p;130q' my.txt

# 列出關鍵字 all 以下所有行
sed -n '/all/,$p' my.txt

# 列出關鍵字 start 以上的所有文字
sed '/start/q' file.txt

# 搜尋關鍵字,並列出這個段落的所有文字
sed -n '/keyword/,/^$/p'

# 列出關鍵字 Top 與 Bottom 之間的所有文字
sed -n '/Top/,/Bottom/p'

# 搜尋字串的那一行
sed -n "/PATTERN/p" my.txt
sed -n "/PATN1\|PATN2\|PATN3/p" my.txt
刪除搜尋內容
# 刪除行首為# 的行
sed '/^#/d' 

# 刪除行首為 # 或空白的行
sed '/^#\|^$/d'
sed '/^#/d;/^$/d' # for AIX

# 刪除多個關鍵字的行, ADMIN_ERP_ROLE, RPT01,...
sed '/ADMIN_ERP_ROLE/d;/RPT01/d'

# 刪除空白行
sed '/^$/d'
sed '/^\s*$/d'

# 刪除第 42 行文字
sed 42d my.txt

# 刪除空格
sed 's/[[:space:]]//g' mywords
sed 's/^[ \t]*//;s/[ \t]*$//' mywords
sed 's/^\s*\|\s*$//g' mywords

# 刪除所有行尾的空格字元
sed 's/[[:space:]]*$//'

# 刪除行尾的 ^M (CR 字元)
sed 's/^M//g'      NOTE: to get ^M type CTRL+V followed by CTRL+M
sed 's/\r$//g'

# 刪除 iXXXX 的帳號, XXXX 是數字
sed '/^i[0-9]*/d' mypasswd

# 刪除最後一行
sed '$d' my.txt
搜尋後取代/插入文字
# 在第 2 行的上方,插入文字 xxx
sed '2i xxx' my.txt

# 在第 2 行的下方,插入文字 yyy
sed '2a yyy' my.txt

# 在第二行的行尾,插入文字 ****
sed '3 s/$/ ****/'

# 在搜尋的行下方新增字串 newstring
sed '/patterm/a\newstring' my.txt

# 在所有文字的行首加上 #
sed 's/^/# /' my.txt

# 在所有文字的行尾加上 End
sed 's/$/ End/' my.txt

# 將每一行裡是兩個以上空白字元, 都換成一個 comma 符號
sed "s/  \+/,/g"

# 快速取代字串
sed 's/old/new/' mywords
sed 's|old|new|' mywords

# 搜尋關鍵字 'astlogdir =>' 的這一行,取代行字串為 'astlogdir => /mnt/usb/asterisk_log'
# .*$ 表示行尾前的所有字串
sed -i 's/astlogdir =>.*$/astlogdir => \/mnt\/usb\/asterisk_log/g'

# 搜尋 'none',並且在該行行首加上 '#'
sed 's/none/#&/g' mywords 

# 在字串  'daemon' 的下方插入另一個檔案 3.txt 的內容
sed '/daemon/r 3.txt' mywords 
sed '/INCLUDE/r foo.h' sample.c

# 特殊字元 / \ & 前加上跳脫符號
sed 's/[\/&]/\\&/g' mywords

# 搜尋文字 'daemon',將符合的內容寫到檔案 3.txt
sed -n '/daemon/w 3.txt' 1.txt 

# 將符合文字 'root' 內容的下一行,插入特定的文字 'test test'
[root@linux-3 ~]# sed ‘/root/a test test’ 1.txt
root:x:0:0:root:/root:/bin/bash
test test
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin 

# 將符合文字 'daemon' 內容的上一行,插入特定的文字 'test test'
[root@linux-3 ~]# sed '/^daemon/i test test' 1.txt
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
test test
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin

# 搜尋所有文字是 'ext3' 的,用 [ ] 框起來
sed 's/ext3\+/[&]/g' my.txt

# 搜尋文字並取代部份的內容
sed -i 's/^\(User\|Group\).*/\1 asterisk/' /etc/httpd/conf/httpd.conf

# 搜尋開頭是 revoke 的每一行尾,加上分號. & 表示符合搜尋的內容
sed 's/^revoke.*$/&\;/' revoke_winieop.sql

# 搜尋每一行, 並格式為 <<<STRING>>>. "^.*$" 表示每一行, & 表示符合搜尋的內容
sed 's@^.*$@<<<&>>>@g' path.txt

# 將 /usr/bin 加上 local 成為 /usr/bin/local
sed 's@/usr/bin@&/local@g' path.txt

備份 *.sh 檔案且移除第一行 #!/bin/bash

sed -i.bak '1i #!/bin/bash' edit.sh

執行多個規則

sed -e '/^#/d' -e '/^$/d' 

找出關鍵字的右側字串內容

$ cat cookie.txt
xxxx  FALSE   /       FALSE   0       PHPSESSID       dbbr5nsmib9tgm0h97sbq8ovd0

$ cat cookie.txt | sed 's/^.*PHPSESSID[ \t]*//' 
dbbr5nsmib9tgm0h97sbq8ovd0

找出行的左側內容
列出 10.14.25.196 與 GA016E38 (Hex IP)

$ cat input.txt
 
 GA160223.MAE3.219594040120
10.14.25.196.49611.200203080358
GA016E38.O5FA.259D81215343
10.4.1.29.41266.200114031620
GA12640A.M6EE.20EDC3093010

$ sed 's/^\(.*\)\.[A-Z0-9]*\.[A-Z0-9]*/\1/' input.txt

GA160223
10.14.25.196
GA016E38
10.4.1.29
GA12640A

對 CSV 檔案的內容,將所有分隔符號 comma 置換成 @@,但必須排除有包含 comma 的文字敘述,這些文字內的 comma 前個字元會有個空白

sed 's/\(,\)\([^ ]\)/@@\2/g' orig.csv

s 搜尋

\( \) \( \) 用括弧區分兩個字元

[^ ] 非空白的字元

\2 第二個字元,這裡要配合括弧的用法

g 作全文置換,若沒有此參數,預設只會置換第一個符合的字元

搜尋 A.AA B.BB C.CC 3個數值,並以指定的格式輸出

sed 's/^\([0-9]\+\.[0-9]\+\) \([0-9]\+\.[0-9]\+\) \([0-9]\+\.[0-9]\+\).*$/1-minute: \1\n5-minute: \2\n15-minute: \3/g' /proc/loadavg

^\([0-9]\+\.[0-9]\+\) \([0-9]\+\.[0-9]\+\) \([0-9]\+\.[0-9]\+\).*$ 搜尋語法

1-minute: \1\n5-minute: \2\n15-minute: \3 輸出語法

搜尋/驗證 IP 位址

sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p' my.txt

取代所有的 mailbox = xxx 為 mailbox = xxx@context

sed 's/^mailbox = [0-9]*/&@context/g' users.conf

搜尋特定區段的 AllowOverride None

只取代這個區段所包含的關鍵字
<Directory "/var/www/html">
...
<Directory>

sed -i ':a;N;$!ba;s/AllowOverride None/AllowOverride All/2' /etc/httpd/conf/httpd.conf

擷取 { } 所包含的所有文字

<br/> <b>Notice</b>: Undefined variable: sn in <b>/var/www/raida/service/fix.php</b> on line <b>259</b><br/> {"server":"RAIDA17","sn":"","status":"success","message":"Fixed: Unit's AN was changed to the PAN. Update your AN to the new PAN.","time":"2017-04-08 06:08:25"}

echo $( cat http_get.txt ) | sed  's/.* \({.*}\)$/\1/'

擷取 [ ] 內的文字

echo "[1070059:1,1070060:1,1070039:1]" | sed  's/^\[\(.*\)\]$/\1/'
排除法搜尋
# 移除所有行尾結束字元不是雙引號的
sed '/\"$/!d'

# 移除所有行開頭不是 "2021-09-26-23.13.02.097806" 這樣格式的內容
sed '/^\"[0-9]\+\-[0-9]\+\-[0-9]\+\-[0-9]\+\.*/!d'

# 排除 PATTERN1 + 搜尋 PATTERN2 + 取代 WORDS
sed '/PATTERN1/!s/PATTERN2/WORDS/g'

# 除第5行以外的所有文字,搜尋 foo 並以 bar 取代
sed '5!/s/foo/bar/' file.txt

# 排除 start 到 end 之間段落的所有文字
sed -rn '/start/,/end/ !p' file.txt
段落搜尋
# 列出關鍵字 Top 與 Bottom 之間段落的所有文字
sed -n '/Top/,/Bottom/p'

# 在 start 與 end 之間段落,搜尋井字開頭的文字,並刪除;井字前的文字仍會保留
sed -E '/start/,/end/ s/#.*//' file.txt
進階應用範例

將數字 123456 格式化成 123,456

echo "123456" | sed 's/\(^\|[^0-9.]\)\([0-9]\+\)\([0-9]\{3\}\)/\1\2,\3/g'

(W)elcome (T)o (T)he (G)eek (S)tuff

echo "Welcome To The Geek Stuff" | sed 's/\(\b[A-Z]\)/\(\1\)/g'

Get the list of usernames in /etc/passwd

sed 's/\([^:]*\).*/\1/' /etc/passwd

變數資料

sed -n '\@/usr/local/bin@p' <<< $PATH

將群組清單以行方式列出

Tip: ( ) - + 等符號需要加上跳脫字元

groups

# domain users@winfoundry.com vdi_fc_it_std01@winfoundry.com pab_zwin_all2@winfoundry.com mis@winfoundry.com prtg_mfgs-r@winfoundry.com 1-3_vpn_it@winfoundry.com

groups | sed 's/\([a-z0-9_ \-]\+@winfoundry.com \)/\1\n/g'

#domain users@winfoundry.com
#quota@winfoundry.com
#it-w@winfoundry.com
#iso 14001-r@winfoundry.com
#mis@winfoundry.com
#it@winfoundry.com
#cqa_spc-r_old@winfoundry.com
#fab_image-r@winfoundry.com
#hp4155b@winfoundry.com
#1-3_vpn_it@winfoundry.com

Markdown 內容

設定 "第 X 章" 為標題 Level 5 格式

# Search: **第 一 章 總則**
# Replace: ##### 第 一 章 總則

sed 's/^.*\(第\ .*\ 章 .*\)\*\*.*$/##### \1/' markdown/勞動基準法.md

# Serch: **法規名稱:**性別平等工作法
# Replace: #### 法規名稱:性別平等工作法

sed -i 's/^\*\*\(法規名稱:\)\*\*\(.*\)$/#### \1\2/' markdown/勞動基準法.md

設定 "第 X 條" 為標題 Level 6 格式

# Search: **第 1 條**
# Replace: ###### 第 1 條

sed 's/^.*\(第\ .*\ 條\).*$/###### \1/' markdown/勞動基準法.md

修正不正確的斷行

1
要派單位不得於派遣事業單位與派遣勞工簽訂勞動契約前,有面試該派遣勞工或其他指定特定派遣勞工之行為。

2
要派單位違反前項規定,且已受領派遣勞工勞務者,派遣勞工得於要派單位提供勞務之日起九十日內,以書面向要派單位提出訂定勞動契約之意思表示。

修正後

1 要派單位不得於派遣事業單位與派遣勞工簽訂勞動契約前,有面試該派遣勞工或其他指定特定派遣勞工之行為。

2 要派單位違反前項規定,且已受領派遣勞工勞務者,派遣勞工得於要派單位提供勞務之日起九十日內,以書面向要派單位提出訂定勞動契約之意思表示。
sed 'N;s/^\([123456789]\)\n/\1 /' markdown/勞動基準法.md

Sample Scripts

線上範例
Auto-rar.sh

解壓縮多個分割檔案 (*.part0XX.rar)

#!/bin/bash

echo "-> Started: "`date +%m/%d/%y\ %H:%M\ %Z`
echo "-> As:"`whoami`

i=1
ARCHIVES="$@"
for f in $ARCHIVES; do
   PARTCHECK=$(expr "$f" : ".*\([Pp][Aa][Rr][Tt][0-9]\+\.[Rr][Aa][Rr]\)")
   RARCHECK=$(expr "$f" : "\(.*\.[Rr][Aa][Rr]\)")
   echo "-> Processing($i of $#): $f"

   if [ "$PARTCHECK" ] && [ -f $f ]; then
      echo "-> Extracting Multipart Archive"
      unrar x $f
      if [ $? -eq 0 ]; then
           FILES=$(expr "$f" : "\(.*[Pp][Aa][Rr][Tt]\).*")
           echo "-> Extraction Successful"
           echo "-> Removing $FILES*.rar"
           rm $FILES*.rar
      else
           echo "-> **Extraction Failed"
           exit 1
      fi
   else
      if [ "$RARCHECK" ] && [ -f $f ]; then
         echo "-> Extracting Single Archive"
         unrar x $f
         if [ $? -eq 0 ]; then
             echo "-> Extraction Successful"
             echo "-> Removing $f"
             rm $f
         else
             echo "-> **Extraction Failed"
             exit 1
         fi
      fi
   fi
      echo ""
      i=$(( i+1 ))
done
Domaincheck.sh
#!/bin/bash

#Specify all the domains you want to check
DOMAINS="google.com dribbble.com facebook.com youtube.com"

current_epoch=`date '+%s'`
for dm in $DOMAINS
do
  expiry_date=`whois $dm | egrep -i "Expiration Date:|Expires on"| head -1 | awk '{print $NF}'`
  echo -n " $dm - Expires on $expiry_date "
  expiry_epoch=`date --date="$expiry_date" '+%s'`
  epoch_diff=`expr $expiry_epoch - $current_epoch`
  days=`expr $epoch_diff / 86400`
  echo " $days days remaining. "
done
Check my.cnf and file permissions
# Check for .my.cnf
if [ ! -f ~/.my.cnf ]; then
  echo "It seems MySQL credentials are missing."
  echo "Please create a ~/.my.cnf file with the following structure:"
  echo ""
  echo "[client]"
  echo "user=your_username"
  echo "password=your_password"
  echo ""
  echo "Ensure the file permissions are secure: chmod 600 ~/.my.cnf"
  exit 1
else
  # Check file permissions
  file_perm=$(stat -c "%a" ~/.my.cnf)
  if [ "$file_perm" -ne 600 ]; then
    echo "Error: ~/.my.cnf permissions are $file_perm. Please set them to 600 using 'chmod 600 ~/.my.cnf'."
    exit 1
  fi
  echo "Credentials found in ~/.my.cnf, but the command still failed."
  echo "MySQL admin output:"
  echo "$output"
  echo "Please verify your credentials or server status."
  exit 1
fi



Regex 正規表示式

教學網站

測試 Regex 語法 (awk)
# echo "Testing regex using awk" | awk '/regex/{print $0}'
Testing regex using awk

# echo "\ is a special character" | awk '/\\/{print $0}'
\ is a special character

# echo "likegeeks website" | awk '/^likegeeks/{print $0}'
likegeeks website
基本搜尋
# 搜尋字串區分大小寫
echo "This is a test" | sed -n '/test/p'
echo "This is a test" | awk '/test/{print $0}'

# 可以包含空白與數字
echo "This is a test 2 again" | awk '/test 2/{print $0}' 
Operators
Description
[abc]
Match any single character from from the listed characters
[a-z]
Match any single character from the range of characters
[^abc]
Match any single character not among listed characters
[^a-z]
Match any single character not among listed range of characters
.
Match any single character except a newline
\
Turn off (escape) the special meaning of a metacharacter
^
Match the beginning of a line.
$
Match the end of a line.
*
Match zero or more instances of the preceding character or regex.
?
Match zero or one instance of the preceding character or regex.
+
Match one or more instances of the preceding character or regex.
{n,m} Match a range of occurrences (at least n, no more than m) of preceding character of regex.
|
Match the character or expression to the left or right of the vertical bar.
數字字元
特殊字元

字元:.*[]^${}\+?|()

$ cat myfile
There is 10$ on my pocket

awk '/\$/{print $0}' myfile

echo "\ is a special character" | awk '/\\/{print $0}' 

echo "This ^ is a test" | sed -n '/s ^/p'
搜尋字首與字尾
# 搜尋字首
$ echo "likegeeks website" | awk '/^likegeeks/{print $0}'

# 搜尋字尾
$ echo "This is a test" | awk '/test$/{print $0}' 

# 搜尋相同字元行的文字
$ awk '/^this is a test$/{print $0}' myfile

# 不顯示空白行
$ awk '!/^$/{print $0}' myfile
搜尋字串

用 dot

# cat myfile
this is a test
This is another test
And this is one more
start with this

# awk '/.st/{print $0}' myfile
this is a test
This is another test

用 [ ]

// 包含 oth 與 ith
$ awk '/[oi]th/{print $0}' myfile
This is another test
start with this

// 不包含 oth 與 ith
$ awk '/[^oi]th/{print $0}' myfile
And this is one more
start with this                  <== 有兩個 th

使用範圍

$ awk '/[e-p]st/{print $0}' myfile
this is a test
This is another test

$ echo "123" | awk '/[0-9][0-9][0-9]/'

$ awk '/[a-fm-z]st/{print $0}' myfile
this is a test
This is another test 

管理與操作

基本操作

查出目前使用哪種 SHELL

ps -p $$
echo $0
echo $SHELL 

執行 SHELL 的方式

sh file
. file
source file

執行前一個指令的參數

ls -l /home/alang/test.py
vi !$
或
vi <alt> + .

執行前一個指令

!!

搜尋最近曾執行過的指令

<ctrl> + r
(reverse-i-search): 輸入指令開頭的前幾個字元

搜尋所有曾執行過的指令集

$> history | grep "keyword"

Alias

以下內容需要編輯 ~/.bashrc
使用方法

# User specific aliases and functions
alias date-time='date && cal'
常用 Aliases
# Custom specified aliases
alias lastmod="find . -type f -exec stat --format '%Y :%y %n' \"{}\" \; | sort -nr | cut -d: -f2-"
alias vi="vim"
alias grep="grep --color"
自訂 functions
date-time () {
    date && cal
    return
}

export -f date-time

避免誤用 crontab 清除指令

# Avoid crontab deleted by mistake with the command 'crontab -r'.
# shift 1: get rid of the first one argument
crontab() {
    case $* in
      -r* ) shift 1; echo -n "Really delete your crontab? (y/n)? "; read ans; if [ "$ans" = "y" ]; then command crontab -r; else echo "Canceled."; fi;;
        * ) command crontab "$@";;
    esac
}

管理自訂函式

# List All Functions
declare -F

# View a specific function
declare -F function-name

# Delete a specific function
unset -f function-name

Custom Prompt

.bashrc:

# Kali-like Prompt
if $(__git_ps1 2>/dev/null);then
    PS1="\[\033[38;5;209m\]┌──[\[\033[38;5;141m\]\u\[\033[38;5;209m\]@\[\033[38;5;105m\]\h\[\033[38;5;231m\]:\w\[\033[38;5;209m\]]\[\033[33m\]\$(GIT_PS1_SHOWUNTRACKEDFILES=1 GIT_PS1_SHOWDIRTYSTATE=1 __git_ps1)\[\033[00m\]\n\[\033[38;5;209m\]└─\\[\033[38;5;209m\]\\$\[\033[37m\] "
else
    source /usr/share/git-core/contrib/completion/git-prompt.sh
    PS1="\[\033[38;5;209m\]┌──[\[\033[38;5;141m\]\u\[\033[38;5;209m\]@\[\033[38;5;105m\]\h\[\033[38;5;231m\]:\w\[\033[38;5;209m\]]\[\033[33m\]\$(GIT_PS1_SHOWUNTRACKEDFILES=1 GIT_PS1_SHOWDIRTYSTATE=1 __git_ps1)\[\033[00m\]\n\[\033[38;5;209m\]└─\\[\033[38;5;209m\]\\$\[\033[37m\] "
fi
┌──[alang@mint-HX90:~]
└─$ 

其他

Formatting Scripts
# Install shfmt
## On Ubuntu
sudo snap install shfmt
## On Alpine Linux
sudo apk add shfmt
## On FreeBSD
sudo pkg install devel/shfmt

# Format shell programs using Shfmt
## -i flag is the amount of spaces that will be used to intend.
shfmt -i 4 myscript.sh
## With Diff style output
shfmt -d -i 4 myscript.sh
終端機輸出內容轉存一個檔案
# 1. script
# Once you are done with the session, type the 'exit'.
script my.out

# 2. tee
myprogram | tee my.out

常用鍵盤快捷鍵

Key Operation
Ctrl + a Go to the beginning of the line.
Ctrl + e Go to the end of the line.
Alt + b Go left(back) one word.
Alt + f Go right(forward) one word.
Alt + . Use the last word of the previous command.
Ctrl + r Search through the command's history
Ctrl + u Cut the part of the line before the cursor, adding it to the clipboard.
Ctrl + k Cut the part of the line after the cursor, adding it to the clipboard.
Ctrl + l Clear the screen
Ctrl + w 刪除游標前個單字
Ctrl+x,Ctrl+e Edit the current command in your $EDITOR.

Linux_terminal_shortcuts.jpeg

Function Samples

清理目錄裡的 *.gz 檔案

用法: KEEP=100 CleanUp /path/to/dir

CleanUp() {
    dest="$1"
    if [ $dest ];then
        CheckDIR $dest 
        cd $dest
        echo "-> Cleaning up the directory $dest [$(date +'%Y-%m-%d %T')] ..."
        total=$(ls -lt *.gz 2>/dev/null | grep -v "^d" | grep -v "^total" | wc -l)
        echo " [!] The numbers of the existing files: $total"
        tail_num=$(( $total - $KEEP ))
        if [ $tail_num -gt 0 ];then
            for f in $(ls -lt *.gz | grep -v "^d" | grep -v "^total" | tail -$tail_num | awk -F ' ' '{print $9}')
            do
                #echo " [!] Deleted file: $f"
                rm -vf $f
            done
        else
            echo " [!] No need of deletion"
        fi
    fi
}
檢查目錄

用法: CheckDIR /path/to/dir

CheckDIR() {
    echo -n "-> Checking the directory <$1> ...... "
    if [ ! -d  $1 ]
    then
        echo "[Failed]"
        exit 1
    else
        echo "[OK]"
    fi
}
ip 驗證
# Test an IP address for validity:
# Usage:
#      valid_ip IP_ADDRESS
#      if [[ $? -eq 0 ]]; then echo good; else echo bad; fi
#   OR
#      if valid_ip IP_ADDRESS; then echo good; else echo bad; fi
#
function is_ip()
{
    local  ip=$1
    local  stat=1

    if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
        OIFS=$IFS
        IFS='.'
        ip=($ip)
        IFS=$OIFS
        [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
            && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
        stat=$?
    fi
    return $stat
}
驗證數值型資料
Is_number() {
    # Usage: Is_number ${your-number} 0
    # Type: Number with decimal
    # If the number is invalid, return 0.
    # If not specify, return NULL
    local input err_num re
    input=$1
    err_num=$2
    re="^[0-9]+([.][0-9]+)?$"
    if [[ $input =~ $re ]]; then
        echo $input
    else
        echo $err_num
    fi
}

log_utilization_percent=$(Is_number $raw 0)

# Validate for integer
[[ $port =~ ^-?[0-9]+$ ]] && dosomething
# No minus character
[[ $port =~ ^[0-9]+$ ]] && dosomething
分割線(自動調整寬度)
CL () { 
WORDS=$@; termwidth="$(tput cols)"; padding="$(printf '%0.1s' ={1..500})"; printf '%*.*s %s %*.*s\n' 0 "$(((termwidth-2-${#WORDS})/2))" "$padding" "$WORDS" 0 "$(((termwidth-1-${#WORDS})/2))" "$padding"; 
}

CL "This is Seperator Line"
大小寫轉換
ToUpper() {
    echo $1 | tr "[:lower:]" "[:upper:]"
}

ToLower() {
    echo $1 | tr "[:upper:]" "[:lower:]"
} 
URL 字串編碼
function urlencode() {
    local data
    if [[ $# != 1 ]]; then
        echo "Error: No string to urlencode"
        return 1
    fi
    data="$(curl -s -o /dev/null -w %{url_effective} --get --data-urlencode "$1" "")"
    if [[ $? != 3 ]]; then
        echo "Unexpected error" 1>&2
    else
        echo "${data##/?}"
    fi
    return 0
}

urlencode "$1"

function urldecode() {
    # urldecode <string>

    local url_encoded="${1//+/ }"
    printf '%b' "${url_encoded//%/\\x}"
}
函數回傳字串

Example-1: Using Global Variable

function F1()
{
    retval='I like programming'
}

retval='I hate programming'
echo $retval
F1
echo $retval

Example-2: Using Function Command

function F2()
{
    local  retval='Using BASH Function'
    echo "$retval"
}

getval=$(F2)   
echo $getval

Example-3: Using Variable

function F3()
{
    local arg1=$1
    
    if [[ $arg1 != "" ]]; 
    then
        retval="BASH function with variable"
    else
        echo "No Argument"
    fi
}

getval1="Bash Function"
F3 $getval1
echo $retval
getval2=$(F3)
echo $getval2
is_root

用途:檢查是否以 root 帳號執行

is_root()
{
	current_user="`id`"
	case "$current_user" in
		uid=0\(root\)*)
		;;
		*)
			echo "$0: You must have root privileges in order to install ${product_name}"
			echo "su as root and try again"
			echo
			exit 1
		;;
	esac
}

With if-then

if [ ${EUID} -ne 0 ]
then
	exit 1 # this is meant to be run as root
fi

Run_MyCodes
check_os

用途:檢查 OS 版本
範例一

check_os()
{
	if [ -f /etc/redhat-release ];then
		rhel=`grep "^Red Hat Enterprise Linux" /etc/redhat-release`
		centos=`grep "^CentOS" /etc/redhat-release`
		
		release=`cat /etc/redhat-release | cut -d'.' -f1 | awk '{print $NF}'`
		
		if [ ! -z "${rhel}" ];then
			sys="rhel"
		fi

		if [ ! -z "${centos}" ];then
			sys="rhel"
		fi
	fi

	if [ -z "${sys}" -o -z "${release}" ];then
		echo "Can not determine Linux distribution."
		exit 127
	fi
			
	if [ -f /proc/vz/veinfo ];then
	# we are under Virtuozzo
		virtuozzo=1
	else
		virtuozzo=0
	fi
	
	if [ "${VNDEBUG}" ];then
		debugrpm='--rpmverbosity=debug'
	else
		debugrpm=''
	fi
}

範例二

# Detect the platform for different environment.
OS="$(uname)"
case $OS in
    "Linux")
      MAIL_CMD="/bin/mail"
      ;;
    "AIX")
      MAIL_CMD="/usr/bin/mail"
      ;;
    *)
      MAIL_CMD="/bin/mail"
      ;;
esac

範例三

get_linux_distribution ()
{ 
    if [ -f /etc/debian_version ]; then
        DIST="DEBIAN"
        HTTP_DIR="/etc/apache2/"
        HTTP_CONFIG=${HTTP_DIR}"apache2.conf"
        MYSQL_CONFIG="/etc/mysql/mariadb.conf.d/50-server.cnf"
    elif [ -f /etc/redhat-release ]; then
        DIST="CENTOS"
        HTTP_DIR="/etc/httpd/"
        HTTP_CONFIG=${HTTP_DIR}"conf/httpd.conf"
        MYSQL_CONFIG="/etc/my.cnf"
    else
        DIST="OTHER"
        echo 'Installation does not support your distribution'
        exit 1
    fi
}
is_running

用途:檢查自身程序是否已經執行中,如果有,程序立即跳離,可避免同一個 script 重複被執行。

IsRunning() {
    if [ ! -e $script_pid ];then
        RUNCOUNT=0
    else
        RUNPID=$(cat $script_pid)
        RUNCOUNT=$(ps -fp $RUNPID | grep -c `basename $0`)
    fi
    #echo $RUNCOUNT

    if [ $RUNCOUNT -eq 0 ];then
        echo $$ > $script_pid
        return 1  # return False
    else
        return 0  # return True
    fi
}

script_pid="/tmp/$(basename $0).$(whoami).pid"
# Check if the program is already running.
if IsRunning
then
    echo "Abort: The program is already running!"
    exit 1
fi
getopts 指令參數
function usage ()
{
	cat <<- EOT
 
  Usage :  ${0##/*/} [options] -t application -q 1024,1055,1077 -m dpggen -u kroybal -r ods [--] 
 
  Options: 
  -d|debug      Bash Debugging Info
  -m|machine	Database host machine
  -t|table	Starting table
  -u|user	User to connect to DB
  -q|quals	Qualifying records to delete
  -r|relation	Database name
  -s|schema     Logical Partition
  -h|help       Display this message
  -v|version    Display script version
 
	EOT
}    # ----------  end of function usage  ----------

#-----------------------------------------------------------------------
#  Handle command line arguments
#-----------------------------------------------------------------------
 
[[ $# -eq 0 ]] && {
	# no arguments
	usage
	exit 1
}

while getopts ":dhm:q:r:s:t:u:v" opt
do
  case $opt in
 
    d|debug    )  set -x;;
    h|help     )  usage; exit 0   ;;
    t|table    )  table=$OPTARG;;
    q|quals    )  quals=$OPTARG;;
    m|machine  )  host=$OPTARG;;
    u|user     )  user=$OPTARG;;
    r|relation )  db=$OPTARG;;
    s|schema   )  schema=$OPTARG;;
    v|version  )  echo "$0 -- Version $ScriptVersion"; exit 0   ;;
 
    \? )  echo -e "\n  Option does not exist : $OPTARG\n"
          usage; exit 1   ;;
 
  esac    # --- end of case ---
done
shift $(($OPTIND-1))
Usage() {
    echo "Usage: $0 [-t <TEXT-Message>] [-n <phone-number>]";
    echo "For example: $0 -t \"Hello, SuperMan\" -n \"085713 457199\" "
}

while getopts "t:n:" o; do
    case "$o" in
        t)
            t=$OPTARG
            ;;
        n)
            n=$OPTARG
            ;;
        \?)
            echo "Invalid option: -$OPTARG"
            Usage
            exit 1
            ;;
        :)
            echo "Option -$OPTARG requires an argument !"
            Usage
            exit 1
            ;;
    esac
done

if [ $OPTIND -ne 5 ]; then
    echo "Invalid options entered !"
    Usage
    exit 1
fi

常用參數代號

[Option]	[Typical meaning]
-a	All, append
-b	Buffer,block size, batch
-c	Command, check
-d	Debug, delete, directory
-D	Define
-e	Execute, edit
-f	File, force
-h	Headers, help
-i	Initialize
-I	Include
-k	Keep, kill
-l	List, long, load
-m	Message
-n	Number, not
-o	Output
-p	Port, protocol
-q	Quiet
-r	Recurse, reverse
-s	Silent, subject
-t	Tag
-u	User
-v	Verbose
-V	Version
-w	Width, warning
-x	Enable debugging, extract
-y	Yes
-z	Enable compression

Without getopts

while [[ "${1}" != "" ]]; do
	case "${1}" in
    	--vertical)    verticle="true" ;;
        --add-txt)     add_txt="true" ;;
        --replace)     replace="true" ;;
        --verify)      verify="true" ;;
    esac
    
    shift 1
done
Pause 暫停

用法: pause

可用於程式開發及偵錯

pause()
{

	local force_pause=$1

	if [ -z $force_pause ]; then
		if test $NONINTERACTIVE; then
			return 0
		fi
	else
		shift
	fi

	[ $# -ne 0 ] && echo -e $* >&2
	echo -e "Press [Enter] to continue...\c " >&2
	read tmp
	return 0
}
Log 訊息
log() {  # classic logger
    local prefix="[$(date +%Y/%m/%d\ %H:%M:%S)]: "
    echo "${prefix} $@" >&2
} 

log "INFO" "a message"
name=XXXX
tstamp="$(date '+%F_%H%M')"
log_file="$name-output.$tstamp.txt"

timestamp(){
    printf "%s\n" "$(date '+%F %T')  $*" >&2
}

timestamp "Dumping Linux specific command outputs"
timestamp  "Collected $name output to file: $log_file"
Error 錯誤訊息

用途:輸出錯誤訊息

error()
{
	echo -e "Error: $*!" >&2
	return 0
}
die()
{
	echo
	echo "ERROR: $*"
	echo
	echo "Check the error reason, fix and try again."
	echo "You are welcome to open a support ticket at https://help.4psa.com!"
	echo
	rm -f ${repolocal} local$$
	rm -f /var/lock/subsys/vninstaller
	exit 1
}

die() {
    printf "ERR: %s\n" "$1" >&2
    exit 1
}
[ -e "$SRC" ] || die "Wal colors not found, exiting script. Have you executed Wal before?"


check_deps()
{
	which yum >/dev/null 2>&1
	if [ $? -gt 0 ];then
		echo "yum binary can not be found."
		die "It's not in the system PATH or in a standard location."
	fi
	
	which ifconfig >/dev/null 2>&1
    if [ $? -gt 0 ];then
		yum -y install net-tools >/dev/null 2>&1
		if [ $? -gt 0 ];then	
			echo "ifconfig binary can not be found and package net-tools could not be installed."
			die "It's not in the system PATH or in a standard location."
		fi
	fi
} 
詢問 Yes/No

用法: getyn "Would you like to install WANPIPE now? [y]"

$NONINTERACTIVE  如果程式執行時,不想讓 user 做任何的輸入時,可以為 1
NONINTERACTIVE=1 ,或者 = 空白,以顯示相關的提示訊息

prompt()
{
	if test $NONINTERACTIVE; then
		return 0
	fi

	echo -ne "$*" >&2
	read CMD rest
	return 0
}

getyn()
{
	if test $NONINTERACTIVE; then
		return 0
	fi

	while prompt "$* (y/n) "
	do	case $CMD in
		[yY])	return 0
			;;
		[nN])	return 1
			;;
		*)	echo -e "\nPlease answer y or n" >&2
			;;
		esac
	done
} 
字串加密 (hash/encryption)
_Decrypt() {
    # Encrypt the string by following the command.
    # echo "your password" | openssl enc -base64
    # NOTE: The openssl requires to be installed.
    #
    # Decrypt the hash code by following the command.
    # _Decrypt "your-hash-key"
    #
    local hash
    hash="$1"
    if which openssl >/dev/null 2>&1;then
        if [ -n $hash ];then
            echo -n "$hash" | openssl enc -base64 -d
        fi
    fi
    return 0
}
# Encrypt
echo -n "123456" | openssl enc -aes-256-cbc -a -pass pass:ThisIsPassword

U2FsdGVkX188Af3yumHOyI19WPbltgRQnPDACmtRniQ=

# Decrypt
echo -n "U2FsdGVkX188Af3yumHOyI19WPbltgRQnPDACmtRniQ=" | openssl enc -aes-256-cbc -a -pass pass:ThisIsPassword -d

123456
# Hashing with openssl
echo -n "Hello, World" | openssl dgst -sha256

帳密資訊雜湊處裡 (hash + salt)

#!/bin/bash
## Create:
salt=12345_

protocol=sha1sum

read -p "Enter login: " username
read -p "Password: " -s pass1
read -p "Repeat: " -s pass2

if [ "$pass1" != "$pass2" ]; then echo "Pass missmatch"; exit 1; else password=$pass1; fi

echo -en "$username " >> ./mypasswd
echo -e "${salt}${password}" | $protocol | awk '{print $1}' >> ./mypasswd
#!/bin/bash
## Read:
salt=12345_ #(samesalt)
protocol=sha1sum

read -p "Enter username: " username
read -p "Enter password: " -s password

if [ `grep $username ./mypasswd | awk '{print $2}'` != `echo "${salt}${password}" | $protocol | awk '{print $1}'` ]; then echo -e "wrong username or password"; exit 127; else echo -e "login successfull"; fi
# With openssl
echo 'mypassword' | openssl passwd -stdin
echo 'mypassword' | openssl passwd -salt "sAlT" -stdin

# Toe see mode details
openssl passwd -h
批次轉檔
# set q here 
q=""
helper(){
    local p="$*"
    for f in $p
    do
        echo "Working on $f"
        ffmpeg -nostdin -vaapi_device /dev/dri/renderD128 -i "$f" -vf format=nv12,hwupload -c:v hevc_vaapi -f mkv -rc_mode 1 -qp "$q" "${f%.*}.HEVC.mkv"
    done
}

helper '/tmp/r/*.mp4'
helper '/tmp/r/*.avi'
helper '/path/to/*.mkv'
亂數密碼
genpasswd() 
{
    length=$1
    [ "$length" == "" ] && length=16
    tr -dc A-Za-z0-9_ < /dev/urandom | head -c ${length} | xargs
}
password=$(genpasswd)

if [ -e "/root/passwordMysql.log" ] && [ ! -z "/root/passwordMysql.log" ]
then
    password=$(awk '{print $1}' /root/passwordMysql.log)
fi

touch /root/passwordMysql.log
echo "$password" > /root/passwordMysql.log
對話框 whiptail

訊息視窗

whiptail --title "Example Dialog" --msgbox "This is an example of a message box. You must hit OK to continue." 8 78

Yes/No

if (whiptail --title "Example Dialog" --yesno "This is an example of a yes/no box." 8 78); then
    echo "User selected Yes, exit status was $?."
else
    echo "User selected No, exit status was $?."
fi

輸入視窗

COLOR=$(whiptail --inputbox "What is your favorite Color?" 8 39 Blue --title "Example Dialog" 3>&1 1>&2 2>&3)

exitstatus=$?
if [ $exitstatus = 0 ]; then
    echo "User selected Ok and entered " $COLOR
else
    echo "User selected Cancel."
fi

echo "(Exit status was $exitstatus)"

A file

echo "Welcome to Bash $BASH_VERSION" > test_textbox
#                  filename height width
whiptail --textbox test_textbox 12 80

密碼遮蔽

PASSWORD=$(whiptail --passwordbox "please enter your secret password" 8 78 --title "password dialog" 3>&1 1>&2 2>&3)

exitstatus=$?
if [ $exitstatus == 0 ]; then
    echo "User selected Ok and entered " $PASSWORD
else
    echo "User selected Cancel."
fi

echo "(Exit status was $exitstatus)"

選單

whiptail --title "Menu example" --menu "Choose an option" 25 78 16 \
"<-- Back" "Return to the main menu." \
"Add User" "Add a user to the system." \
"Modify User" "Modify an existing user." \
"List Users" "List all users on the system." \
"Add Group" "Add a user group to the system." \
"Modify Group" "Modify a group and its list of members." \
"List Groups" "List all groups on the system."

複選清單

whiptail --title "Check list example" --checklist \
"Choose user's permissions" 20 78 4 \
"NET_OUTBOUND" "Allow connections to other hosts" ON \
"NET_INBOUND" "Allow connections from other hosts" OFF \
"LOCAL_MOUNT" "Allow mounting of local devices" OFF \
"REMOTE_MOUNT" "Allow mounting of remote devices" OFF

單選清單

whiptail --title "Radio list example" --radiolist \
"Choose user's permissions" 20 78 4 \
"NET_OUTBOUND" "Allow connections to other hosts" ON \
"NET_INBOUND" "Allow connections from other hosts" OFF \
"LOCAL_MOUNT" "Allow mounting of local devices" OFF \
"REMOTE_MOUNT" "Allow mounting of remote devices" OFF

進度條

#!/bin/bash
{
    for ((i = 0 ; i <= 100 ; i+=5)); do
        sleep 0.1
        echo $i
    done
} | whiptail --gauge "Please wait while we are sleeping..." 6 50 0
檢查指令或套件
function color
{
    # 定义颜色和样式变量
    reset="\033[0m"
    bold="\033[1m"
    underline="\033[4m"
    inverse="\033[7m"

    # 定义前景色变量
    redx="\e[1;31m"
    black="\033[30m"
    red="\033[1;31m"
    green="\033[32m"
    yellow="\033[33m"
    orange="\033[1;93m"
    blue="\033[1;34m"
    purple="\033[35m"
    cyan="\033[36m"
    white="\033[1;37m"

    # 定义背景色变量
    bg_black="\033[40m"
    bg_red="\033[41m"
    bg_green="\033[42m"
    bg_yellow="\033[43m"
    bg_blue="\033[44m"
    bg_purple="\033[45m"
    bg_cyan="\033[46m"
    bg_white="\033[47m"

}

function os_name
{
    if [ -e /etc/os-release ]; then
        # Get the name of the current Linux distribution
        os_name=$(grep PRETTY_NAME /etc/os-release | cut -d= -f2 | tr -d '"')
        if [[ "$os_name" == *"Debian"* ]]; then
            OSNAME="Debian"
            OSTYPE="T_Debian"
        elif [[ "$os_name" == *"CentOS"* ]]; then
            OSNAME="CentOS"
            OSTYPE="T_RedHat"
        elif [[ "$os_name" == *"Ubuntu"* ]]; then
            OSNAME="Ubuntu"
            OSTYPE="T_Debian"
        elif [[ "$os_name" == *"Kali"* ]]; then
            OSNAME="Kali"
            OSTYPE="T_Debian"
        else
            OSNAME="Unknown distribution"
            OSTYPE="T_RedHat"
        fi
    fi
}

# NOTE: 不適用於 command 與 package 不同名稱的例子
function i
{
    color
    os_name
    # os
    if [[ $OSTYPE == "T_Debian" ]]; then
        OS_APP="apt-get"
    else
        OS_APP="yum"
    fi
    local package=$1
    if ! command -v "$package" &> /dev/null; then
        printf "$package   ${redx} uninstalled ${reset} \n"
        read -p "是否安装 $package? (Y/n): " choice
        choice=${choice:-y}
        case "$choice" in
            y|Y )
                echo "正在安装 $package..."
                sudo $OS_APP update
                sudo $OS_APP install -y "$package"
                ;;
            * )
                printf "exit\n"
                exit 1
                ;;
        esac
    else
        printf "$package   ${green} installed ${reset} \n"
    fi
}

i "rsync"
# 'which' 與 'command' 擇一使用
function chk_commands
{
    if ! which curl &> /dev/null; then
        printf "WARNING curl 指令不存在。\n"
    elif ! command -v rsync &> /dev/null; then
        printf "WARNING rsync 指令不存在。\n"
    fi
}

function chk_command
{
    local cmd
    cmd=$1
    if ! command -v $cmd &> /dev/null; then
        printf "WARNING $cmd 指令不存在。\n"
    fi
}

chk_command "rsync"
is_alert
value=$1
threshold=10
is_alert=$(awk "BEGIN {print ($value > 10)}")
if (( is_alert )); then
    echo "Alert: The value $value has been exceed the threshold $threshold !"
else
    echo "Everything is fine."
fi

Learning


dialog

dialog1.png

selection.sh

#!/bin/bash
# while-menu-dialog: a menu driven system information program

DIALOG_CANCEL=1
DIALOG_ESC=255
HEIGHT=0
WIDTH=0

display_result() {
  dialog --title "$1" \
    --no-collapse \
    --msgbox "$result" 0 0
}

while true; do
  exec 3>&1
  selection=$(dialog \
    --backtitle "System Information" \
    --title "Menu" \
    --clear \
    --cancel-label "Exit" \
    --menu "Please select:" $HEIGHT $WIDTH 4 \
    "1" "Display System Information" \
    "2" "Display Disk Space" \
    "3" "Display Home Space Utilization" \
    2>&1 1>&3)
  exit_status=$?
  exec 3>&-
  case $exit_status in
    $DIALOG_CANCEL)
      clear
      echo "Program terminated."
      exit
      ;;
    $DIALOG_ESC)
      clear
      echo "Program aborted." >&2
      exit 1
      ;;
  esac
  case $selection in
    0 )
      clear
      echo "Program terminated."
      ;;
    1 )
      result=$(echo "Hostname: $HOSTNAME"; uptime)
      display_result "System Information"
      ;;
    2 )
      result=$(df -h)
      display_result "Disk Space"
      ;;
    3 )
      if [[ $(id -u) -eq 0 ]]; then
        result=$(du -sh /home/* 2> /dev/null)
        display_result "Home Space Utilization (All Users)"
      else
        result=$(du -sh $HOME 2> /dev/null)
        display_result "Home Space Utilization ($USER)"
      fi
      ;;
  esac
done

zenity

zenity.webp

who_is.sh

#!/bin/bash
# Get domain name
_zenity="/usr/bin/zenity"
_out="/tmp/whois.output.$$"
domain=$(${_zenity} --title  "Enter domain" \
	            --entry --text "Enter the domain you would like to see whois info" )
 
if [ $? -eq 0 ]
then
  # Display a progress dialog while searching whois database
  whois $domain  | tee >(${_zenity} --width=200 --height=100 \
  				    --title="whois" --progress \
			            --pulsate --text="Searching domain info..." \
                                    --auto-kill --auto-close \
                                    --percentage=10) >${_out}
 
  # Display back output
  ${_zenity} --width=800 --height=600  \
	     --title "Whois info for $domain" \
	     --text-info --filename="${_out}"
else
  ${_zenity} --error \
	     --text="No input provided"
fi

More Tutorials

Array 陣列

Learning
Samples
#!/bin/bash
str_list=("aaa" "bbb" "ccc" "ddd")
echo ${#str_list[@]}
for i in ${str_list[@]}
do
  echo "$i"
done
# Script to split a string based on the delimiter
my_string="Ubuntu;Linux Mint;Debian;Arch;Fedora"
IFS=';' read -ra my_array <<< "$my_string"

#Print the split string
for i in "${my_array[@]}"
do
    echo $i
done
# Script to split a string based on the delimiter
my_string="Ubuntu;Linux Mint;Debian;Arch;Fedora"  
my_array=($(echo $my_string | tr ";" "\n"))

#Print the split string
for i in "${my_array[@]}"
do
    echo $i
done

移除陣列內容

# Delete the 3th element of the array
unset str_list[3]

 

Learning SHELL

應用範例

String Manipulation 字串處理

字串長度
my_string="abhishek"
echo "length is ${#my_string}"

Using expr

str="my string"
length=$(expr length "$str")
echo "Length of my string is $length"

Using awk

echo "my string" | awk '{print length}'

Using wc

str="my string"
length=$(echo -n "my string" | wc -m)
echo "Length of my string is $length"
字串比對

Using wildcards

#!/bin/bash

STR='GNU/Linux is an operating system'
SUB='Linux'
if [[ "$STR" == *"$SUB"* ]]; then
  echo "It's there."
fi

Using case

#!/bin/bash

STR='GNU/Linux is an operating system'
SUB='Linux'

case $STR in

  *"$SUB"*)
    echo -n "It's there."
    ;;
esac

Using Regex

#!/bin/bash

STR='GNU/Linux is an operating system'
SUB='Linux'

if [[ "$STR" =~ .*"$SUB".* ]]; then
  echo "It's there."
fi

Using Grep

#!/bin/bash

STR='GNU/Linux is an operating system'
SUB='Linux'

if grep -q "$SUB" <<< "$STR"; then
  echo "It's there"
fi
字串合併
str1="hand"
str2="book"
str3=$str1$str2
字元提取
foss="Fedora is a free operating system"
echo ${foss:0:6}
_admin_ip="202.54.1.33|MUM_VPN_GATEWAY 23.1.2.3|DEL_VPN_GATEWAY 13.1.2.3|SG_VPN_GATEWAY"
for e in $_admin_ip
do
   ufw allow from "${e%%|*}" to any port 22 proto tcp comment 'Open SSH port for ${e##*|}'
done
字元取代
foss="Fedora is a free operating system"
echo ${foss/Fedora/Ubuntu}
Summary: String Manipulation and Expanding Variables
${parameter:-defaultValue} Get default shell variables value
${parameter:=defaultValue} Set default shell variables value
${parameter:?"Error Message"} Display an error message if parameter is not set
${#var} Find the length of the string
${var%pattern} Remove from shortest rear (end) pattern
${var%%pattern} Remove from longest rear (end) pattern
${var:num1:num2} Substring
${var#pattern} Remove from shortest front pattern
${var##pattern} Remove from longest front pattern
${var/pattern/string} Find and replace (only replace first occurrence)

${var//pattern/string}

echo "${PATH//:/$'\n'}"

Find and replace all occurrences
${!prefix*} Expands to the names of variables whose names begin with prefix.

${var,}

${var,pattern}

Convert first character to lowercase.

${var,,}

${var,,pattern}

Convert all characters to lowercase.

${var^}

${var^pattern}

Convert first character to uppercase.

${var^^}

${var^^pattern}

Convert all character to uppercase.
Cheatsheet: String Manipulation

bash_string.jpeg

String To Integer
# $((string))
sum=$((3+6))
echo $sum

a=11
b=3
c=$(($a+$b))
echo $c

Alternate method: expr

Note that it is not a “native” Bash procedure, as you need to have coreutils installed (by default on Ubuntu) as a separate package.

a=5; b=3; c=2; d=1
expr $a + $b \* $c - $d
去除變數的末端字元 .XXX
# Operator "#" means "delete from the left, to the first case of what follows."
x="This is my test string."
echo ${x#* }

is my test string.

# Operator "##" means "delete from the left, to the last case of what follows."
x="This is my test string."
echo ${x##* }

string.

# Operator "%" means "delete from the right, to the first case of what follows."
x="This is my test string."
echo ${x% *}

This is my test

# Operator "%%" means "delete from the right, to the last case of what follows."
x="This is my test string."
$ echo ${x%% *}

This

 

loop

for loop
for i in var1 var2 var3; do echo $i; done
for ((i=1;i<=10;i++)); do echo $i; done
for i in $(ls *.log); do echo $i; done  

for t in {1..10};do  echo $t; done
1
2
3
4
5
6
7
8
9
10

for t in {1..10..2};do  echo $t; done
1
3
5
7
9

## 列出目前資料夾的所有目錄名稱
for dir in */;do echo "${dir%/}"; done

## Infinite Loop
for (( ; ; ))
do
   echo "infinite loops [ hit CTRL+C to stop]"
done
while loop
## 讀取檔案內容
cat tables_name.lst | while read sch tab;do
> echo "export to $tab.ixf of ixf messages export_$tab.msg select * from $sch.$tab"
> done

## while loop 的應用
## 每 3 秒監看 /worktmp 的磁碟使用狀況
while true;do
> df -h | grep /worktmp
> sleep 3
> done

## 管理 DB2 時常用
while read s t;do
> db2 "select count(*) from $s.$t"
> done < tables.lst > count-tables.out

## 如果是 CSV
while IFS=, read s t;do
> db2 "select count(*) from $s.$t"
> done < tables.csv > count-tables.out

## 迴圈計數 Loop Counter
x=1
while [ $x -le 5 ]
do
  echo "Welcome $x times"
  x=$(( $x + 1 )) 或者 x=$[$x + 1]
done

## 內容排序後導入迴圈
## <(sort -r lost.txt) 檔案內容預處理
while IFS='.' read -r a b c d e;do
> echo "$a"; 
> done < <(sort -r lost.txt)

## 讀取整行
filename=$1
while read line; do
# reading each line
echo $line
done < $filename

## Reading file by omitting backslash escape
while read -r line; do
# Reading each line
echo $line
done < company2.txt
until loop
until [[ $a -eq 1 ]]
do
    echo "Value of a is $a"
    let a--
done
Infinite loop 無限迴圈
# while
while true
do
    echo "Hi"
    sleep 2s #will run every 2 sec
done

# for
for (( ;;  ))
do
    echo "Hi buddy"
    sleep 2s

done
break and continue
count=0
until false
do
  ((count++))
  if [[ $count -eq 5 ]]
  then
    continue
  elif [[ $count -ge 10 ]]
  then
    break
  fi
  echo "Counter = $count"
done
Counter = 1
Counter = 2
Counter = 3
Counter = 4
Counter = 6
Counter = 7
Counter = 8
Counter = 9
For loop with array elements
DB_AWS_ZONE=('us-east-2a' 'us-west-1a' 'eu-central-1a')
 
for zone in "${DB_AWS_ZONE[@]}"
do
  echo "Creating rds (DB) server in $zone, please wait ..."
  aws rds create-db-instance \
  --availability-zone "$zone"
  --allocated-storage 20 --db-instance-class db.m1.small \
  --db-instance-identifier test-instance \
  --engine mariadb \
  --master-username my_user_name \
  --master-user-password my_password_here
done
Loop with a shell variable
_admin_ip="202.54.1.33|MUM_VPN_GATEWAY 23.1.2.3|DEL_VPN_GATEWAY 13.1.2.3|SG_VPN_GATEWAY"
for e in $_admin_ip
do
   ufw allow from "${e%%|*}" to any port 22 proto tcp comment 'Open SSH port for ${e##*|}'
done


顯示進度 progress

範例:簡易版
echo -ne '>>>                       [20%]\r'
# some task
sleep 2
echo -ne '>>>>>>>                   [40%]\r'
# some task
sleep 2
echo -ne '>>>>>>>>>>>>>>            [60%]\r'
# some task
sleep 2
echo -ne '>>>>>>>>>>>>>>>>>>>>>>>   [80%]\r'
# some task
sleep 2
echo -ne '>>>>>>>>>>>>>>>>>>>>>>>>>>[100%]\r'
echo -ne '\n'
範例:顯示進度條
#!/bin/bash
#
# inprogress.sh
# Description: this script may show some simple motion in foreground
# when main program in progress.
# Author: A-Lang (alang[dot]hsu[at]gmail[dot]com)

trap "kill $BG_PID;echo;exit" 1 2 3 15

function waiting
{
    INTERVAL=0.5
    TT='->——
    –>—–
    —>—-
    —->—
    —–>–
    ——>-
    ——->
    ——<-
    —–<–
    —-<—
    —<—-
    –<—–
    -<——
    <——-';
    echo -ne "Please waiting… \n"
    while true
    do
        for j in $TT
        do
            echo -ne "Please be patient, $j\r"
            sleep $INTERVAL
        done

    done
}

# Start the waiting, where place your main program.
waiting &
BG_PID=$!

sleep 10

# End of the waiting
kill $BG_PID
echo
範例:旋轉符號
#!/usr/bin/env bash

spinner()
{
    local pid=$!
    local delay=${1:1}
    local spinstr='|/-\'
    while [ "$(ps a | awk '{print $1}' | grep $pid)" ]
    do
        local temp=${spinstr#?}
        printf " [%c] " "$spinstr"
        local spinstr=$temp${spinstr%"$temp"}
        sleep $delay
        printf "\b\b\b\b\b"
    done
    printf "    \b\b\b\b"
}

# Run your program here
#(a_long_running_task) &
sleep 10 &
spinner 0.75
範例:整合 LED 的顯示
#!/bin/bash

echo "Starting to format the disk...."

# Start to blink the LED
/root/bin/led.sh b
/root/bin/led.sh blink &
led_pid=$!

# Main codes
echo "formating hdisk2..."
sleep 10

# When the main codes is ending up
# Stop to blink the LED
kill -9 $led_pid
wait $led_pid 2>/dev/null
sleep 2

# Making sure the LED is ON with Blue
/root/bin/led.sh b
echo "done"

led.sh 用來控制 LED 的閃燈

wait .... 這行用來隱藏 killed ... 的文字訊息

 

find

檔案名稱
# Find all *.bak files in the current directory and removes them with confirmation
find . -type f -name "*.bak" -exec rm -i {} \;

# find all *.out, *.bak and *.core files and then delete them
find /projectA/ \( -name '∗.bak' -o -name '*.core' \) -exec rm "{}" \;

檔名有包含空白, 換行符號等特殊字元時

find . -type f -name "*.err" -print0 | xargs -I {} -0 rm -v "{}"

排除特定附檔名

find . ! -name '*.mp3' -type f | xargs cp -t Misc/
檔案大小

找出大於 1000 MB 的檔案

find / -type f -size +1000M
find . -type f -size +50000k -exec ls -lh {} \; | awk '{ print $9 ": " $5 }'

列出空間使用最高的檔案與目錄的清單

find . -type f -exec wc -c {} \; | sort -nr | head

# show all files/folders in current directory on current filesystem ordered by size
find . -maxdepth 1 -mindepth 1 \( -type d -o -type f \) \( -exec mountpoint -q {} \; -o -exec du -smx {} \; \) | sed "s|./||" | sort -n

# With du
du -xak . |sort -n | tail -50 
du -xah . | sort -rh | head -10
檔案時間

搬移指定日期範圍的檔案
將修改日期是 2012 - 2014 的所有檔案搬移至 /export_data/2012-2014

touch --date "2012-01-01" /tmp/start
touch --date "2015-01-01" /tmp/end

find /home/ams/ipdr -type f -newer /tmp/start -not -newer /tmp/end > 2012-2014.lst
find /home/ams/ipdr -type f -newer /tmp/start -not -newer /tmp/end -exec cp -a {}  /export_data/2012-2014 \;
find /home/ams/ipdr -type f -newer /tmp/start -not -newer /tmp/end -exec rm -f {} \;

列出 2018-03-20 至 2018-04-10 期間曾修改過的檔案

find ~/ -xdev -newermt 2018-03-20 \! -newermt 2018-04-10 -type f -ls

列出 365天以前/20分鐘以前 的所有檔案

find ./ -ctime +365 -ls
find ./ -cmin +20 -ls

找尋3日內曾異動過的檔案,並轉換 Excel format(csv)

find . -xdev -mtime -3 -ls > found.list
sed 's/^[ ]*//g;s/[ ][ ]*/   /g' found.list | awk '{print $3","$5","$6","$7","$8" "$9","$10","$11}' > found.list.csv

列出剛剛異動過的檔案

# 最近 5 分鐘異動過的檔案(包含權限與新增檔案)
find dir -cmin -5

# 最近 10 分鐘修改過的檔案
find dir -mmin 10

目錄 data 內有許多以日期命名的小 log,需要將去年的所有 log 搬到新目錄 archive-2013
註:排除目錄 ./archive-2013

cd data/
mkdir archive-2013
find ./ -path "./archive-2013" -prune -o -name "dcdb_13*" -exec mv {} archive-2013 \;

要排除多個目錄時可以改成
find ./  \( -path "./dir1" -o -path "./dir2" \) -prune -o -name "dcdb_13*" -exec mv {} archive-XXXX \;

按時間先後作排序

find /istrpt/arlog/ -name "*.LOG" -print | xargs ls -lt
檔案權限

列出 0777 的目錄

find . -type d -perm 0777 -ls

指定權限的檔案

find ~/UbuntuMint -type f -perm 600
find ~/UbuntuMint -type f -perm 111
find ~/UbuntuMint -type f -perm 644

# Use the "-" minus prefix before the file permissions to list all 
# the files that possess at least read and write permissions for the file owner.
find ~/UbuntuMint -type f -perm -600

# This "/" will retrieve files within the "UbuntuMint" directory where 
# at least one of the categories (owner, group, or others) meets the specified permission bits
find ~/UbuntuMint -type f -perm /600

find ~/UbuntuMint -type f -perm u=rw,g=r,o=r
find ~/UbuntuMint -type f -perm u+rw,g+r,o+r
整合其他指令
# use find to list sha256sum files only
find . -type f -exec sha256sum {} +

# search for and get rid of duplicate .jpg files
find . -type f -name '*.jpg' -exec sha256sum {} + | sort -uk1,1
特定帳號的檔案
# -xdev 需指定目錄,可以避免 NAS/proc 等目錄
# NOTE: 使用 / 目錄,會使 xdev 失效
find /sp1 /home -xdev -user i00001
排除隱藏目錄檔案
# Exclude the hidden files
find /home -xdev -name "[!.]*" -type f -user i00001

# Exclude the hidden files and directories
find /home -xdev \( ! -path '*/.*' \) -type f -user i00001
find /home -xdev \( ! -regex '.*/\..*' \) -type f -user i00001
定期清除 Archive Log Files
# For Cron on Linux
*/30 * * * * find /istfdc/FDCDB/arclog/istfdc/FDCDB -name "*.LOG" -cmin +60 -exec rm -f {} \;

# For Cron on AIX
50 09 * * * find /istit1/archiveLog/smdb -name "*.LOG" -ctime +5 -exec rm -f {} \\;

uniq

uniq 與 sort 一起使用時,可以控制那些重複的資訊。

基本操作
$ cat file2
ChhatrapatiShahuMaharaj
Dr.B.R.Ambedkar
Budhha
Dr.B.R.Ambedkar
Budhha
Dr.B.R.Ambedkar
Budhha

$ uniq file2
ChhatrapatiShahuMaharaj
Dr.B.R.Ambedkar
Budhha
Dr.B.R.Ambedkar
Budhha
Dr.B.R.Ambedkar
Budhha

$ sort file2
Budhha
Budhha
Budhha
ChhatrapatiShahuMaharaj
Dr.B.R.Ambedkar
Dr.B.R.Ambedkar
Dr.B.R.Ambedkar

$ sort file2 | uniq
Budhha
ChhatrapatiShahuMaharaj
Dr.B.R.Ambedkar
With -c, --count option
# using the -c option to count the repeated lines
$ sort file2 | uniq -c
    3 Budhha
    1 ChhatrapatiShahuMaharaj
    3 Dr.B.R.Ambedkar
With -d, --repeated option
# The -d option prints only lines that are repeated.
$ sort file2 | uniq -d
Budhha
Dr.B.R.Ambedkar

# using the -c option to cross-check whether the -d option is only printing the repeated lines or not
$ sort file2 | uniq -cd
    3 Budhha
    3 Dr.B.R.Ambedkar
With -D, --all-repeated option
# The -D option prints repeated lines and discards the non-duplicate lines
$ sort file2 | uniq -D
Budhha
Budhha
Budhha
Dr.B.R.Ambedkar
Dr.B.R.Ambedkar
Dr.B.R.Ambedkar
With -u, --unique option
# the -u option prints unique lines i.e., non-duplicate lines
$ sort file2 | uniq -u
ChhatrapatiShahuMaharaj
With -i, --ignore-case option
# Using the -i option, we can ignore the case sensitivity of characters
$ cat file3
aaaa
aaaa
AAAA
AAAA
bbbb
BBBB

$ uniq file3
aaaa
AAAA
bbbb
BBBB

$ uniq -i file3
aaaa
bbbb
With -f, --skip-fields=N
# skip some fields to filter duplicate lines
$ cat file5
Amit aaaa
Ajit aaaa
Advi bbbb
Kaju bbbb

$ uniq file5
Amit aaaa
Ajit aaaa
Advi bbbb
Kaju bbbb

$ uniq -f 1 file5
Amit aaaa
Advi bbbb
With -s, --skip-char=N option
# skip characters
$ cat file4
22aa
33aa
44bb
55bb

$ uniq file4
22aa
33aa
44bb
55bb

$ uniq -s 2 file4
22aa
44bb
With -w, --check-chars=N option
# consider characters as well using the -w option
$ cat file6
aa12
aa34
bb56
bb78

$ uniq file6
aa12
aa34
bb56
bb78

$ uniq -w 2 file6
aa12
bb56

 

 

date

Shell 範例
# Sample
today=$(date +%Y/%m/%d)
NOW=`date "+%Y/%m/%d %H:%M:%S"`

eval `date "+day=%d; month=%m; year=%Y"`
BKNAME="cacti-backup-$year-$month-$day.tar.gz"

# Sample
# %T time; same as %H:%M:%S
NOWD=$(date +"%F")   # YYYY-MM-DD
NOWT=$(date +"%T")  # H:M:S

#
eggs="202/07/29"
days="20 days"
echo "Eggs expiry date $(date -d "${eggs}+${days}")"
將 Date 轉換為 EPOCH 時間格式
## Date to Epoch
# 用 perl 可用於 AIX
perl -e 'use Time::Local; print timelocal(0,25,1,11,11,2008), "\n";'

# On Linux
# Current date
date +%s
# For specified date
date --date="2023/11/1" +%s

For AIX

Time2Epoch() {
    local dtime yyyy mm dd HH MM SS
    dtime="$1"
    yyyy=$(echo "$dtime" | cut -d ' ' -f1 | cut -d '/' -f1)
    mm=$(echo "$dtime" | cut -d ' ' -f1 | cut -d '/' -f2)
    dd=$(echo "$dtime" | cut -d ' ' -f1 | cut -d '/' -f3)
    HH=$(echo "$dtime" | cut -d ' ' -f2 | cut -d ':' -f1)
    MM=$(echo "$dtime" | cut -d ' ' -f2 | cut -d ':' -f2)
    SS=$(echo "$dtime" | cut -d ' ' -f2 | cut -d ':' -f3)

    echo $(perl -e 'use Time::Local; ($yyyy, $mm, $dd, $HH, $MM, $SS)=@ARGV; $tm=timelocal($SS, $MM, $HH, $dd, $mm - 1, $yyyy); print "$tm";' $yyyy $mm $dd $HH $MM $SS)
    return 
}

Epoch2Date() {
    local epochtime datetime
    epochtime=$1
    datetime=
    if [ -n "$epochtime" ]
    then
        datetime=$(perl -le 'print scalar localtime $ARGV[0]' ${epochtime})
    fi
    echo $datetime
}

常用日期格式
date +"%D %T"
02/04/21 01:42:52

date "+%y-%m-%d_%H%M%S"
17-05-22_105503

date "+%Y/%m/%d"
2015/07/17

date "+%F"
2015-07-17

# Display past date
date --date="49 days ago"
日  9月 11 16:17:02 CST 2016

date --date="1 month ago"
五  9月 30 16:17:34 CST 2016

# Display future date
date --date="next fri"
五  2月  5 00:00:00 CST 2021

date --date='TZ="America/New_York" 10:00 next fri'
五  2月  5 23:00:00 CST 2021
計算程式執行所花費的時間
start=`date "+%Y/%m/%d %H:%M:%S"`
start_s=$(date -d "$start" +%s)

Sleep 20

end=`date "+%Y/%m/%d %H:%M:%S"`
end_s=$(date -d "$end" +%s)
diff=$((end_s - start_s))

Parallel and Multi-thread

GNU Parallel

Official: https://www.gnu.org/software/parallel/ 
Download: http://ftp.gnu.org/gnu/parallel/

Install

# CentOS 7
yum group install "Development Tools"
wget http://ftp.gnu.org/gnu/parallel/parallel-latest.tar.bz2
tar xjf parallel-latest.tar.bz2
cd parallel-*
./configure --prefix=/usr/local
make
make install

Don't change the --prefix if you want to use Man to view the manual of the command.

Use case: The common use case I have is to bzip a large file using Parallels. Files with 40 millions rows (8 GB) are compressed to 400MB bz2 files.

cat largefile.csv | /usr/local/bin/parallel --pipe -k bzip2 --best > largefile.bz2

Use case: my custom shell

One-liner)

# ./gen_mm_log_insert.v5.sh <input-file> <output-dir>
cat files.lst | parallel -j3 "./gen_mm_log_insert.v5.sh raws/{} output/"

Shell Script)

num_processes=3
ls $locdir/mmsevent* | /usr/local/bin/parallel -j $num_processes "./gen_mm_log_insert.v5.sh {} $outputdir"

ls $locdir/mmsevent* | /usr/local/bin/parallel -j $num_processes 'a={}; name=${a##*/};' \
'./gen_mm_log_insert.v5.sh {} "'$outputdir'" 2>"'$logdir'/err.${name}.log"'
With Bash
# Multiple files  *.lst with lots of file paths such as
# cat 1.lst
# /path/to/file1 
# /path/to/file2
# /path/to/file3
#

files="$@"
## An arbitrary limiting factor so that there are some free processes
## in case I want to run something else
num_processes=3
#
echo "Parallel Processing with $num_processes threads (`date "+%F %T"`)"
for lst in $files
do
    for f in $(cat $lst | sed '/^#/d')
    do
        ((i=i%num_processes)); ((i++==0)) && wait
        yourshell.sh $f &
    done
    sleep 120
done

 

 

mail

Installation
sudo apt install mailutils        # [On Debian, Ubuntu and Mint]
sudo yum install mailx            # [On RHEL/CentOS/Fedora and Rocky Linux/AlmaLinux]
sudo emerge -a mail-client/mailx  # [On Gentoo Linux]
sudo pacman -S mailutils          # [On Arch Linux]
sudo zypper install mailutils     # [On OpenSUSE]  
Sending a mail
echo "MAIL BODY" | mail -s 'MAIL SUBJECT' receiver@domain_name

# Enter DOT or Ctrl-d to exit the CLI
mail -s "Test mail" receiver@domain_name
Body with HTML

NOTE: Doesn't work with Notes mail client

echo "<b>HTML Message goes here</b>" | mail -s "$(echo -e "This is the subject\nContent-Type: text/html")" receiver@domain_name

Perl

Tutorials

lsof

Network Port

What process is using the port 50000

lsof -i :50000

lsof -i :https

lsof -i tcp:25

lsof -i udp:53

lsof -i :1-1024

lsof -i udp
Mount Point

What process is using the mount-point

lsof | grep /path/to/mount-point

lsof +f -- /path/to/dir

lsof +D /path/to/dir

# List all NFS files
lsof -N
Process

What files and directories are using by the process

lsof -p `pidof <app_name>`

lsof -c apache

lsof | grep apache

# list Parent PID of processes
lsof -p [PID] -R
User
# List files opened by processes belonging to a specific user
lsof -u [user-name]

lsof -a -i -u www-data
IP & Domain

List files based on their Internet address

# For IPv4
lsof -i 4

# For IPv6
lsof -i 6
File
lsof -t [file-name]
lsof -t /usr/lib/x86_64-linux-gnu/libpcre2-8.so.0.9.0

# Finding open files in a specific directory under Linux or Unix
lsof +D /path/to/dir/

# list any open files which have been removed or deleted
lsof -nP +L1
lsof -nP +L1 +D /path/to/dir/
Memory
lsof -d mem


算術

let
#!/usr/bin/env bash

let NUMBER1=10
let NUMBER2=3

# Addition => + operator
let ADD=$NUMBER1+$NUMBER2
echo "Addition of two numbers : ${ADD}"

# Subtraction => - operator
let SUB=$NUMBER1-$NUMBER2
echo "Subtraction of two numbers : ${SUB}"

# Multiply => * operator
let MUL=$NUMBER1*$NUMBER2
echo "Multiply two numbers : ${MUL}"

# Divide => / operator
let DIV=$NUMBER1/$NUMBER2
echo "Division of two numbers : ${DIV}"

# Remainder => % operator
let REM=$NUMBER1%$NUMBER2
echo "Remainder of two numbers : ${REM}"

# Exponent => ** operator
let EXPO=$NUMBER1**$NUMBER2
echo "Exponent of two numbers : ${EXPO}"

# post increment and post decrement operations
let variable++
let variable--
Double Brackets

雙括號,必須是整數型態

((NUMBER2++)
((NUMBER1--))

(( NUMBER2 = NUMBER2 + 10 ))
(( NUMBER2 += 10 )) # Shorthand
expr

必須是整數型態

expr 10 + 3 # Addition
expr 10 - 3 # Subtraction
expr 10 * 3 # Multiply
expr 10 / 3 # Divide
expr 10 % 3 # Remainder
bc
# Add
$ echo "10 + 100" | bc
=> 110

$ echo "10.15 + 11.20" | bc
21.35

# Subtract
$ echo "100 - 25" | bc
=> 75

$ echo "100 - 25.5" | bc
=> 74.5

# Multiply
$ echo "10 * 5" | bc
=> 50

$ echo "10.10 * 4" | bc
=> 40.40

# without scale
echo "10.10 / 4" | bc
=> 2

# with scale
echo "scale=2;10.10 / 4" | bc
=> 2.52

$ echo "2.2^4" | bc
=> 23.4
awk
$ awk "BEGIN {print 23 * 4.5 }"
=> 103.5

$ awk "BEGIN{print int(10.111) }"
=> 10

$ awk "BEGIN{print sqrt(10) }"
=> 3.16228

# Since this is a CSV file, I am setting the field separator to(-F “,”). 
# Here the entire second column is first added and divided by the NR(number of records).
$ awk -F "," '{sum+=$2} END { print "average value from column 2 = ",sum/NR}' data.csv
Rounding 取整數
除法取整數方法 X / Y (3 / 2) = 1.5
無條件捨去 (floor rouding) X / Y , (3 / 2) = 1
無條件進位 (ceiling rouding) (X + Y – 1) / Y , (3 + 2 - 1) / 2 = 2
四捨五入 (half-up rouding) (X + Y / 2) / Y , (3 + 2 / 2) / 2 = 2
# 將 float 轉成 integer
## Way #1
float_num = 12.63333
round_num = $(printf "%.0f" $float_num)
# round_num is 12

## Way #2
float_num = 12.63333
round_num = ${float_num%%.*}
# round_num is 12

# 整數相除 3/2 後取整數值 (無條件捨去)
X=3
Y=2
echo $(( X / Y ))
# Output 1

# 整數相除 3/2 後取整數值 (四捨五入)
echo $(( (X + Y / 2) / Y ))
# Output 2


Diagrams

Linux Shell Lifecycle

linux_shell_lifecycle.jpeg

ANSI Color

高亮文字
highlight() {
    mode=$1
    string="$2"
    case "$mode" in
      red)          echo -e "\033[0;101m${string}\033[0m";;
      green)        echo -e "\033[0;102m${string}\033[0m";;
      black_yellow) echo  -e "\033[33;5;7m${string}\033[0m";;
      *)            echo $string
    esac
}

highlight "green" "Version 1.0.0"
Scripts
echo -e "\033[0;0m -- 0 » no style --\033[0;0m"
echo -e "\033[0;1m -- 1 » white:bolded --\033[0;0m"
echo -e "\033[0;2m -- 2 » white:dimmed --\033[0;0m"
echo -e "\033[0;3m -- 3 » white:italic --\033[0;0m"
echo -e "\033[0;4m -- 4 » white:underline --\033[0;0m"
echo -e "\033[0;5m -- 5 » white:slow blink --\033[0;0m"
echo -e "\033[0;6m -- 6 » white:rapid blink --\033[0;0m"
echo -e "\033[0;7m -- 7 » white background aka 'invert' --\033[0;0m"
echo -e "\033[0;8m -- 8 » hidden --\033[0;0m"
echo -e "\033[0;9m -- 9 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;10m -- 10 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;11m -- 11 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;12m -- 12 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;13m -- 13 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;14m -- 14 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;15m -- 15 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;16m -- 16 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;17m -- 17 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;18m -- 18 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;19m -- 19 » white 'alternate font' --\033[0;0m"
echo -e "\033[0;20m -- 20 » white:Gothic --\033[0;0m"
echo -e "\033[0;21m -- 20 » white:double underline --\033[0;0m"
echo -e "\033[0;22m -- 20 » white:normal intensity --\033[0;0m"
echo -e "\033[0;23m -- 20 » white:Neither italic, nor blackletter --\033[0;0m"
echo -e "\033[0;24m -- 20 » white:Not underlined --\033[0;0m"
echo -e "\033[0;25m -- 20 » white:Not blinking --\033[0;0m"
echo -e "\033[0;26m -- 20 » white:Proportional spacing --\033[0;0m"
echo -e "\033[0;27m -- 20 » white:Not reversed --\033[0;0m"
echo -e "\033[0;28m -- 20 » white:Reveal --\033[0;0m"
echo -e "\033[0;29m -- 20 » white:Not crossed out --\033[0;0m"
echo -e "\033[0;30m -- 30 » black --\033[0;0m (-- 30 » black --) "
echo -e "\033[0;31m -- 31 » red --\033[0;0m"
echo -e "\033[0;32m -- 32 » green --\033[0;0m"
echo -e "\033[0;33m -- 33 » yellow --\033[0;0m"
echo -e "\033[0;34m -- 34 » blue --\033[0;0m"
echo -e "\033[0;35m -- 35 » magenta --\033[0;0m"
echo -e "\033[0;36m -- 36 » teal --\033[0;0m"
echo -e "\033[0;37m -- 37 » grey --\033[0;0m"
echo -e "\033[0;38m -- 38 » white --\033[0;0m"
echo -e "\033[0;39m -- 39 » white/default color --\033[0;0m"
echo -e "\033[0;40m -- 40 » black background? --\033[0;0m"
echo -e "\033[0;41m -- 41 » red background --\033[0;0m"
echo -e "\033[0;42m -- 42 » green background --\033[0;0m"
echo -e "\033[0;43m -- 43 » yellow background --\033[0;0m"
echo -e "\033[0;44m -- 44 » blue background --\033[0;0m"
echo -e "\033[0;45m -- 45 » magenta background --\033[0;0m"
echo -e "\033[0;46m -- 46 » teal background --\033[0;0m"
echo -e "\033[0;47m -- 47 » grey background --\033[0;0m"
echo -e "\033[0;48m -- 48 » Set background color --\033[0;0m"
echo -e "\033[0;49m -- 49 » default background --\033[0;0m"
echo -e "\033[0;50m -- 50 » Disable proportional spacing --\033[0;0m"
echo -e "\033[0;51m -- 51 » Framed --\033[0;0m"
echo -e "\033[0;52m -- 52 » Encircled --\033[0;0m"
echo -e "\033[0;53m -- 53 » Overlined --\033[0;0m"
echo -e "\033[0;54m -- 54 » Neither framed nor encircled --\033[0;0m"
echo -e "\033[0;55m -- 55 » Not overlined --\033[0;0m"
echo -e "\033[0;56m -- 56 » undefined --\033[0;0m"
echo -e "\033[0;57m -- 57 » undefined --\033[0;0m"
echo -e "\033[0;58m -- 58 » Set underline color --\033[0;0m"
echo -e "\033[0;59m -- 59 » Default underline color --\033[0;0m"
echo -e "\033[0;60m -- 60 » Ideogram underline or right side line --\033[0;0m"
echo -e "\033[0;61m -- 61 » Ideogram double underline, or double line on the right side --\033[0;0m"
echo -e "\033[0;62m -- 62 » Ideogram overline or left side line --\033[0;0m"
echo -e "\033[0;63m -- 63 » Ideogram double overline, or double line on the left side --\033[0;0m"
echo -e "\033[0;64m -- 64 » Ideogram stress marking --\033[0;0m"
echo -e "\033[0;65m -- 65 » No ideogram attributes --\033[0;0m"
echo -e "\033[0;66m -- 66 » undefined --\033[0;0m"
echo -e "\033[0;67m -- 67 » undefined --\033[0;0m"
echo -e "\033[0;68m -- 68 » undefined --\033[0;0m"
echo -e "\033[0;69m -- 69 » undefined --\033[0;0m"
echo -e "\033[0;70m -- 70 » undefined --\033[0;0m"
echo -e "\033[0;71m -- 71 » undefined --\033[0;0m"
echo -e "\033[0;72m -- 72 » undefined --\033[0;0m"
echo -e "\033[0;73m -- 73 » Superscript --\033[0;0m"
echo -e "\033[0;74m -- 74 » Subscript --\033[0;0m"
echo -e "\033[0;75m -- 75 » Neither superscript nor subscript --\033[0;0m"
echo -e "\033[0;76m -- 76 » undefined --\033[0;0m"
echo -e "\033[0;77m -- 77 » undefined --\033[0;0m"
echo -e "\033[0;78m -- 78 » undefined --\033[0;0m"
echo -e "\033[0;79m -- 79 » undefined --\033[0;0m"
echo -e "\033[0;80m -- 80 » undefined --\033[0;0m"
echo -e "\033[0;81m -- 81 » undefined --\033[0;0m"
echo -e "\033[0;82m -- 82 » undefined --\033[0;0m"
echo -e "\033[0;83m -- 83 » undefined --\033[0;0m"
echo -e "\033[0;84m -- 84 » undefined --\033[0;0m"
echo -e "\033[0;85m -- 85 » undefined --\033[0;0m"
echo -e "\033[0;86m -- 86 » undefined --\033[0;0m"
echo -e "\033[0;87m -- 87 » undefined --\033[0;0m"
echo -e "\033[0;88m -- 88 » undefined --\033[0;0m"
echo -e "\033[0;89m -- 89 » undefined --\033[0;0m"
echo -e "\033[0;90m -- 90 » Set bright foreground color - grey --\033[0;0m"
echo -e "\033[0;91m -- 91 » Set bright foreground color - red --\033[0;0m"
echo -e "\033[0;92m -- 92 » Set bright foreground color - green --\033[0;0m"
echo -e "\033[0;93m -- 93 » Set bright foreground color - yellow --\033[0;0m"
echo -e "\033[0;94m -- 94 » Set bright foreground color - blue --\033[0;0m"
echo -e "\033[0;95m -- 95 » Set bright foreground color - magenta --\033[0;0m"
echo -e "\033[0;96m -- 96 » Set bright foreground color - teal --\033[0;0m"
echo -e "\033[0;97m -- 97 » Set bright foreground color - white --\033[0;0m"
echo -e "\033[0;98m -- 98 » undefined --\033[0;0m"
echo -e "\033[0;99m -- 99 » undefined --\033[0;0m"
echo -e "\033[0;100m -- 100 » Set bright background color - grey --\033[0;0m"
echo -e "\033[0;101m -- 101 » Set bright background color - red --\033[0;0m"
echo -e "\033[0;102m -- 102 » Set bright background color - green --\033[0;0m"
echo -e "\033[0;103m -- 103 » Set bright background color - yellow --\033[0;0m"
echo -e "\033[0;104m -- 104 » Set bright background color - blue --\033[0;0m"
echo -e "\033[0;105m -- 105 » Set bright background color - magenta --\033[0;0m"
echo -e "\033[0;106m -- 106 » Set bright background color - teal --\033[0;0m"
echo -e "\033[0;107m -- 107 » Set bright background color - white --\033[0;0m"
echo -e "\033[0;108m -- 108 » undefined --\033[0;0m"
echo -e "\033[0;109m -- 109 » undefined --\033[0;0m"

Cheat Sheets

PDF
Bash Parameter

bash_parameter.jpg

Bash Loops

bash_loops.jpg

Bash Basics

bash_basics.jpg

Bash Brackets

bash_brackets.jpg