Skip to main content

範例與常用技巧

檔頭常用宣告
# quickly syntax
set -euo pipefail

# let script exit if a command fails
set -o errexit 
# OR
set -e

# let script exit if an unsed variable is used
set -o nounset
# OR
set -u

# This setting prevents errors in a pipeline from being masked.
set -o pipefail

# for Debug
set -x

# setting IFS
IFS=$'\n\t'

How set -o pipefail works

$ grep some-string /non/existent/file | sort
grep: /non/existent/file: No such file or directory
$ echo $?
0

$ set -o pipefail
$ grep some-string /non/existent/file | sort
grep: /non/existent/file: No such file or directory
$ echo $?
2

How the IFS  works

#!/bin/bash
names=(
  "Aaron Maxwell"
  "Wayne Gretzky"
  "David Beckham"
  "Anderson da Silva"
)

echo "With default IFS value..."
for name in ${names[@]}; do
  echo "$name"
done

echo ""
echo "With strict-mode IFS value..."
IFS=$'\n\t'
for name in ${names[@]}; do
  echo "$name"
done

############### Output #############
With default IFS value...
Aaron
Maxwell
Wayne
Gretzky
David
Beckham
Anderson
da
Silva

With strict-mode IFS value...
Aaron Maxwell
Wayne Gretzky
David Beckham
Anderson da Silva
檔案的目錄位置
# 指定的檔案
readlink -f <file.name>

# 目前檔案
WORKDIR=$(readlink -f "$0") ;目前檔案的絕對路經
WORKDIR=$( cd $( dirname "$0" ) && pwd )
Script 檔案名稱
$ echo $0
./test.sh

$ echo `basename $0`
test.sh
刪除 * 天前的舊備份檔
cd $BACKUP_DIR
for backup in `find . -ctime +$BACKUP_KEEP_DAYS -name "cacti.*.tar.gz"`; do rm -f $backup; done;
輸出多行的文字訊息
cat <<EOF
Welcome .....
	
Here are the messages that you want to show up
		
EOF
將執行後的所有輸出訊息導入一個檔案

Sample #1

#!/bin/sh
LOG="my.log"
(
....
) 2>&1 | tee -a $LOG

不適用在內容裡有 python 指令的訊息輸入，輸入等待的畫面會無法顯示。

Sample #2

temp=$(mktemp)
exec &> ${temp}

echo "All outputs will be saved into the file ${temp}."

Sample #3

#!/bin/bash
set -eu 
exec 3>&1 4>&2
trap 'exec 2>&4 1>&3' 0 1 2 3
exec 1>/path/to/script.log 2>&1

# rest of the script below
dnf -y in foo bar
# firewall rules goes here 
....
...
快速修改大量檔案的副檔名
## *.old -> *.new
for fname in $(ls *.old);do echo "mv $fname ->"; echo $(echo $fname |sed 's/.old/.new/');mv $fname $(echo $fname | sed 's/.old/.new/');done
快速新增或列出多個指定目錄
mkdir {AAA,BBB,CCC}
快速備份設定檔
cp my.cfg{,.bak}
計算指定目錄底下的檔案總數量

用 find

# 計算目前目錄底下所有檔案數
find . -type f | wc -l 

# 計算第一層所有子目錄的檔案數並排序清單
find . -maxdepth 1 -type d -print0 | xargs -0 -I {} sh -c 'echo $(find {} -type f | wc -l) {}' | sort -n
NOTE: 第一層子目錄名稱不可包含空格

用 rsync

rsync --stats --dry-run -ax /usr /tmp

Number of files: 326,373 (reg: 211,698, dir: 24,284, link: 90,391)
Number of created files: 326,373 (reg: 211,698, dir: 24,284, link: 90,391)
Number of deleted files: 0
Number of regular files transferred: 211,698
Total file size: 7,180,685,730 bytes
Total transferred file size: 7,178,524,818 bytes

NOTE: 這指令實際不會作檔案複製，/tmp 只是一個假目錄，回傳結果是 reg: 211698 就是檔案總數

用 tree

tree /mydir -a | tail -n 1

5 directories, 56 files

NOTE: 當目錄底下有包含 symbolic link 也會被計算
在 SHELL 內執行另一個外部 SHELL 或指令

1. 使用 pipe line

echo"md5sum $X > $X.sum "| bash

2. 使用 eval

get_arch="uname -p"
if [ "`eval "$get_arch"`" = "i686" ]; then
....
fi
命令模式提示字元的路徑名稱太長

加上這變數

PROMPT_DIRTRIM=2
處理 CSV 檔
#!/bin/bash
INPUT=data.cvs
OLDIFS=$IFS
IFS=','
[ ! -f $INPUT ] && { echo "$INPUT file not found"; exit 99; }
while read flname dob ssn tel status
do
	echo "Name : $flname"
	echo "DOB : $dob"
	echo "SSN : $ssn"
	echo "Telephone : $tel"
	echo "Status : $status"
done < $INPUT
IFS=$OLDIFS

CSV and JSON

# JSON to CSV
cat df.json | jq -r '.[]| join(",")'
cat bingbot.json | jq -r '.prefixes[] | {cidr: .ipv4Prefix, comment: "BingBot"} | join(",")' > bingbot.csv
JSON 檔
建立暫存檔
tmpfile1=$(mktemp)
tmpfile2="/tmp/$(basename $0).$$.tmp"
Get my public IP
curl ifconfig.me
curl ifconfig.me/ip
curl ifconfig.co
curl checkip.amazonaws.com
curl icanhazip.com
curl ipecho.net/plain

dig +short myip.opendns.com @resolver1.opendns.com
dig TXT +short o-o.myaddr.l.google.com @ns1.google.com
dig TXT +short o-o.myaddr.l.google.com @ns1.google.com | awk -F'"' '{print $2}'
主機 IP
# On Linux
hostname -I
hostip=$(/sbin/ip a | awk '/eth[012]:|ens192:|bond0:/,/^$/' | grep -E "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | head -1 | awk -F " " '{print $2}' | cut -d"/" -f1)

# On AIX
hostip=$( ifconfig -a | grep inet | awk '{print $2}' | head -1 )
hostip=$( ifconfig -a | grep inet | awk '{print $2}' | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p' | head -1 )
找出最大的目錄或檔案
# Way 1
du -a /var | sort -n -r | head -n 10

# Way 2
cd /path/to/some/where
du -hsx * | sort -rh | head -10
du -hsx -- * | sort -rh | head -10

# Way 3
find /path/to/dir/ -printf '%s %p\n'| sort -nr | head -10
find . -printf '%s %p\n'| sort -nr | head -10
## Skip directories and only display files
find /path/to/search/ -type f -printf '%s %p\n'| sort -nr | head -10

# Create a shell alias
## shell alias ##  
alias ducks='du -cks * | sort -rn | head'
## deal with special files names ##
alias ducks='du -cks -- * | sort -rn | head'
shuf: 隨機排序
curl -s https://www.imdb.com/list/ls020046354/export | cut -d ',' -f 6  | shuf
tree: 顯示專案目錄的檔案樹
tree --dirsfirst --filelimit 10 --sort=name

# Display size of files
tree -s

# Display permissions of files
tree -p

# Display directory only
tree -d

# Display till a certain level/depth
tree -L 1

# List only those files that match pattern given
tree -P *screenshot*
sort : 資料排序
sort -t ',' -k5,5 -k1,1 -k9,9 -k3,3 -k11,11 my.csv
  •  -t 分隔符號
  • -k5,5 排序第 5 欄，以字串類型排列
  • 欄位排序先後依序為第 5, 1, 9, 3, 11 欄
  • 欄位排序若要以數值方式來排，改成 -k5,5n
timeout : 自動停止執行
timeout 10 tail -f /var/log/httpd/access.log
timeout 5m ping 8.8.8.8
timeout 300 tcpdump -n -w data.pcap

# Sending specific signal
# To get a list of all available signals, use the command kill -l .
timeout -s SIGKILL ping 8.8.8.8
variables : 變數
變數 說明
 $0  腳本檔名
 $1  第1個參數
 $2  第2個參數
 ${10}  第10個參數 #10
 $#  參數的個數
 $*  顯示所有參數 (作為一個字串)
 $@  顯示所有參數 (每個為一個獨立字串)
 ${$*}  傳遞到腳本中的參數的個數
 ${$@}  傳遞到腳本中的參數的個數
 $?  腳本結束後的傳回值
 $$  腳本的程序 ID
 $_  前個命令的最後一個參數
 $!  最後一個執行程序的 ID
u=${1:-root}  如果 $1 未指定，就賦予值 root
u=${USER:-foo}  如果 $USER 未指定，就賦予值 foo
len=${#var}  計算 $var 字串的長度

如果 $2 未指定或空值，輸出錯誤訊息

${varName?Error varName is not defined}
${varName:?Error varName is not defined or is empty}

建立目錄並且切換至目錄

mkdir my-dir && cd $_

 

cut: 切割文字
# AAA = BBB, 取出 BBB
cut -d= -f2

# 111 2222 33 444444 555, 取出 33 以後的所有內容
cut -d ' ' -f3-

# 檢視超長的文字
head -n 1 data.csv | wc  # 計算行的字元數
head data.csv | cut -c -30  # 列出前幾行的前 30 個字元, 如果要顯示後 30 個字元，可以用 30-
tr: 置換
# aaa bbb ccc
# 換成
# aaa
# bbb
# ccc
echo "aaa bbb ccc" | tr " " "\n"
完整複製 Home 目錄

由於 User 的 Home 目錄內有許多隱藏檔，若要完整複製它們，有兩個方法：

方法一：可以複製成一個新目錄

cd /home
cp -a user1/ user1_new/

方法二：複製到一個現有目錄內

cd /home
cp -a user1/.[^.]* user1_new/

建立一個新的 Home 目錄

cp -r /etc/skel /home/user1
chown -R user1.group1 /home/user1
chmod 0700 /home/user1
xargs

接入(Pipe) 不支援 stdin 指令的替代方法

printf: 格式化輸出
  • %s 字串
  • %d 整數
printf "%-40s ..................%s\n" "Disable the service $1" "[$2]"

Disable the service apmd                 ..................[OK]
Disable the service bluetooth            ..................[OK]
Disable the service hidd                 ..................[OK]
Disable the service cups                 ..................[OK]
Disable the service firstboot            ..................[OK]
Disable the service readahead_early      ..................[OK]

printf "%40s ..................%s\n" "Disable the service $1" "[$2]"

                Disable the service apmd ..................[OK]
           Disable the service bluetooth ..................[OK]
                Disable the service hidd ..................[OK]
                Disable the service cups ..................[OK]
           Disable the service firstboot ..................[OK]
     Disable the service readahead_early ..................[OK]
印出一個長符號
printf -- '-%.0s' {1..80}
printf -- '=%.0s' {1..80}
ping: 掃描一個 IP 範圍
{ for p in {1..254}; do ping -c1 -w1 10.22.9.$p & done } | grep "64 bytes"
nice: Reduce CPU and Disk load of backup scripts
# Reduce the I/O priority of the /usr/local/bin/backup.sh script so that it does not interfere with other processes
# The -n parameter must be between 0 and 7, where lower numbers mean higher priority
/usr/bin/ionice -c2 -n7 /usr/local/bin/backup.sh

# To reduce the CPU priority, use the command nice
# The -n parameter can range from -20 to 19, where lower numbers mean higher priority
/usr/bin/nice -n 19 /usr/local/bin/backup.sh

# Nice and ionice can also be combined, to run a script at low I/O and CPU priority
/usr/bin/nice -n 19 /usr/bin/ionice -c2 -n7 /usr/local/bin/backup.sh
Hex to ASCII
# hex = 54657374696e672031203220330
# ascii = Testing 1 2 3

# xxd
echo 54657374696e672031203220330 | xxd -r -p && echo ''

# printf
printf '\x54\x65\x73\x74\x69\x6e\x67\x20\x31\x20\x32\x20\x33\x0' && echo ''

# sed
echo -n 54657374696e67203120322033 | sed 's/\([0-9A-F]\{2\}\)/\\\\\\x\1/gI' | xargs printf && echo ''
隨機密碼
genpasswd() { 
	local l=$1
       	[ "$l" == "" ] && l=16
      	tr -dc A-Za-z0-9_ < /dev/urandom | head -c ${l} | xargs 
}
tr -dc A-Za-z0-9_ < /dev/urandom | head -c 16 | xargs

# Generate more than one
tr -dc A-Za-z0-9_ < /dev/urandom | fold -16 | head -5
密碼加密

Install mkpasswd

sudo apt install whois      # [On Debian, Ubuntu and Mint]
sudo yum install expect     # [On RHEL/CentOS/Fedora and Rocky/AlmaLinux]
sudo emerge -a whois        # [On Gentoo Linux]
sudo apk add mkpasswd       # [On Alpine Linux]
sudo pacman -S whois        # [On Arch Linux]
sudo zypper install whois   # [On OpenSUSE]

Usage

# Generate MD5 Password Hash With a Salt
mkpasswd -m md5 -S UMsalt12 UbuntuMintPassword

# Generate SHA-256 Password Hash Without Salt
mkpasswd -m sha-256 UbuntuMintPassword

# Generate SHA-256 Password Hash With Salt and Number of Rounds
mkpasswd -m sha-512 -S UMsalt12 -R 10000 UbuntuMintPassword

# Generate SHA-256 Hash from Password File
echo "UbuntuMintPassword" | mkpasswd -m sha-256 -s

ls: 進階技巧
# Find the biggest zip file
ls -lSrh ~/Downloads/*.zip
stat: 檔案屬性
❯ stat --printf='Name: %n\nPermissions: %a\n' my.log
Name: my.log
Permissions: 777

❯ stat --format="%F" my.log
regular file

# Symlink
❯ stat ~/bin/FoxitReader
  File: /home/alang/bin/FoxitReader -> /home/alang/opt/foxitsoftware/foxitreader/FoxitReader.sh
  Size: 56        	Blocks: 0          IO Block: 4096   symbolic link
Device: 10302h/66306d	Inode: 787474      Links: 1
Access: (0777/lrwxrwxrwx)  Uid: ( 1000/   alang)   Gid: ( 1000/   alang)
Access: 2023-07-16 10:26:13.412193581 +0800
Modify: 2023-02-26 12:13:50.234374171 +0800
Change: 2023-02-26 12:13:50.234374171 +0800
 Birth: 2023-02-26 12:13:50.234374171 +0800

❯ stat -L ~/bin/FoxitReader
  File: /home/alang/bin/FoxitReader
  Size: 120       	Blocks: 8          IO Block: 4096   regular file
Device: 10302h/66306d	Inode: 1457222     Links: 1
Access: (0755/-rwxr-xr-x)  Uid: ( 1000/   alang)   Gid: ( 1000/   alang)
Access: 2023-07-02 14:34:09.957428285 +0800
Modify: 2023-02-26 12:13:50.246374250 +0800
Change: 2023-02-26 12:13:50.246374250 +0800
 Birth: 2023-02-26 12:13:48.530362986 +0800

Back to top